Update from Google.
--
MOE_MIGRATED_REVID=85702957
diff --git a/src/BUILD b/src/BUILD
new file mode 100644
index 0000000..7235123
--- /dev/null
+++ b/src/BUILD
@@ -0,0 +1,104 @@
+# Packaging
+
+genrule(
+ name = "client-info-file",
+ outs = ["client_info"],
+ cmd = "touch $@",
+ executable = 1,
+)
+
+md5_cmd = "set -e -o pipefail && cat $(SRCS) | %s | awk '{ print $$1; }' > $@"
+
+# TODO(bazel-team): find a better way to handle dylib extensions.
+filegroup(
+ name = "libunix",
+ srcs = select({
+ ":darwin": ["//src/main/native:libunix.dylib"],
+ "//conditions:default": ["//src/main/native:libunix.so"],
+ }),
+ visibility = ["//src/test/java:__pkg__"],
+)
+
+genrule(
+ name = "install_base_key-file",
+ srcs = [
+ "//src/main/java:bazel-main_deploy.jar",
+ "//src/main/cpp:client",
+ ":libunix",
+ "//src/main/tools:build-runfiles",
+ "//src/main/tools:process-wrapper",
+ ":namespace-sandbox",
+ "client_info",
+ "//src/main/tools:build_interface_so",
+ ],
+ outs = ["install_base_key"],
+ cmd = select({
+ ":darwin": md5_cmd % "/sbin/md5",
+ "//conditions:default": md5_cmd % "md5sum",
+ }),
+)
+
+genrule(
+ name = "java-version",
+ outs = ["java.version"],
+ cmd = "echo 1.8 >$@",
+)
+
+genrule(
+ name = "package-zip",
+ srcs = [
+ "//src/main/java:bazel-main_deploy.jar",
+ # The jar must the first in the zip file because the client launcher
+ # looks for the first entry in the zip file for the java server.
+ "//src/main/cpp:client",
+ ":libunix",
+ "//src/main/tools:build-runfiles",
+ "//src/main/tools:process-wrapper",
+ ":namespace-sandbox",
+ "client_info",
+ "//src/main/tools:build_interface_so",
+ "install_base_key",
+ ":java-version",
+ ],
+ outs = ["package.zip"],
+ # Terrible hack to remove timestamps in the zip file
+ cmd = "mkdir $(@D)/package-zip && " +
+ "cp $(SRCS) $(@D)/package-zip && " +
+ "touch -t 198001010000.00 $(@D)/package-zip/* && " +
+ "zip -qj $@ $(@D)/package-zip/* && " +
+ "rm -fr $(@D)/package-zip",
+)
+
+genrule(
+ name = "bazel-bin",
+ srcs = [
+ "//src/main/cpp:client",
+ "package-zip",
+ ],
+ outs = ["bazel"],
+ cmd = "cat $(location //src/main/cpp:client) package-zip > $@ && zip -qA $@",
+ executable = 1,
+ output_to_bindir = 1,
+)
+
+filegroup(
+ name = "tools",
+ srcs = [
+ "//src/java_tools/buildjar:JavaBuilder_deploy.jar",
+ "//src/java_tools/singlejar:SingleJar_deploy.jar",
+ "//third_party/ijar",
+ ],
+)
+
+filegroup(
+ name = "namespace-sandbox",
+ srcs = [
+ "//src/main/tools:namespace-sandbox",
+ ],
+)
+
+config_setting(
+ name = "darwin",
+ values = {"cpu": "darwin"},
+ visibility = ["//visibility:public"],
+)
diff --git a/src/java_tools/buildjar/BUILD b/src/java_tools/buildjar/BUILD
new file mode 100644
index 0000000..8b61435
--- /dev/null
+++ b/src/java_tools/buildjar/BUILD
@@ -0,0 +1,14 @@
+package(default_visibility = ["//src:__pkg__"])
+
+java_binary(
+ name = "JavaBuilder",
+ srcs = glob(["java/**/*.java"]),
+ main_class = "com.google.devtools.build.buildjar.BazelJavaBuilder",
+ deps = [
+ "//src/main/protobuf:proto_deps",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ "//third_party:protobuf",
+ "//tools/jdk:langtools-neverlink",
+ ],
+)
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java
new file mode 100644
index 0000000..15349a1
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java
@@ -0,0 +1,268 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.Files;
+import com.google.devtools.build.buildjar.javac.JavacRunner;
+import com.google.devtools.build.buildjar.javac.JavacRunnerImpl;
+
+import com.sun.tools.javac.main.Main.Result;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.List;
+import java.util.zip.ZipEntry;
+
+/**
+ * A command line interface to compile a java_library rule using in-process
+ * javac. This allows us to spawn multiple java_library compilations on a
+ * single machine or distribute Java compilations to multiple machines.
+ */
+public abstract class AbstractJavaBuilder extends AbstractLibraryBuilder {
+
+ /** The name of the protobuf meta file. */
+ private static final String PROTOBUF_META_NAME = "protobuf.meta";
+
+ /** Enables more verbose output from the compiler. */
+ protected boolean debug = false;
+
+ @Override
+ protected boolean keepFileDuringCleanup(File file) {
+ return false;
+ }
+
+ /**
+ * Flush the buffers of this JavaBuilder
+ */
+ @SuppressWarnings("unused") // IOException
+ public synchronized void flush(OutputStream err) throws IOException {
+ }
+
+ /**
+ * Shut this JavaBuilder down
+ */
+ @SuppressWarnings("unused") // IOException
+ public synchronized void shutdown(OutputStream err) throws IOException {
+ }
+
+ /**
+ * Prepares a compilation run and sets everything up so that the source files
+ * in the build request can be compiled. Invokes compileSources to do the
+ * actual compilation.
+ *
+ * @param build A JavaLibraryBuildRequest request object describing what to
+ * compile
+ * @param err PrintWriter for logging any diagnostic output
+ */
+ public void compileJavaLibrary(final JavaLibraryBuildRequest build, final OutputStream err)
+ throws IOException {
+ prepareSourceCompilation(build);
+
+ final String[] message = { null };
+ final JavacRunner javacRunner = new JavacRunnerImpl(build.getPlugins());
+ runWithLargeStack(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ internalCompileJavaLibrary(build, javacRunner, err);
+ } catch (JavacException e) {
+ message[0] = e.getMessage();
+ } catch (Exception e) {
+ // Some exceptions have a null message, yet the stack trace is useful
+ e.printStackTrace();
+ message[0] = "java compilation threw exception: " + e.getMessage();
+ }
+ }
+ }, 4L * 1024 * 1024); // 4MB stack
+
+ if (message[0] != null) {
+ throw new IOException("Error compiling java source: " + message[0]);
+ }
+ }
+
+ /**
+ * Compiles the java files of the java library specified in the build request.<p>
+ * The compilation consists of two parts:<p>
+ * First, javac is invoked directly to compile the java files in the build request.<p>
+ * Second, additional processing is done to the .class files that came out of the compile.<p>
+ *
+ * @param build A JavaLibraryBuildRequest request object describing what to compile
+ * @param err OutputStream for logging any diagnostic output
+ */
+ private void internalCompileJavaLibrary(JavaLibraryBuildRequest build, JavacRunner javacRunner,
+ OutputStream err) throws IOException, JavacException {
+ // result may not be null, in case somebody changes the set of source files
+ // to the empty set
+ Result result = Result.OK;
+ if (!build.getSourceFiles().isEmpty()) {
+ PrintWriter javacErrorOutputWriter = new PrintWriter(err);
+ try {
+ result = compileSources(build, javacRunner, javacErrorOutputWriter);
+ } finally {
+ javacErrorOutputWriter.flush();
+ }
+ }
+
+ if (!result.isOK()) {
+ throw new JavacException(result);
+ }
+ runClassPostProcessing(build);
+ }
+
+ /**
+ * Build a jar file containing source files that were generated by an annotation processor.
+ */
+ public abstract void buildGensrcJar(JavaLibraryBuildRequest build, OutputStream err)
+ throws IOException;
+
+ @VisibleForTesting
+ protected void runClassPostProcessing(JavaLibraryBuildRequest build)
+ throws IOException {
+ for (AbstractPostProcessor postProcessor : build.getPostProcessors()) {
+ postProcessor.initialize(build);
+ postProcessor.processRequest();
+ }
+ }
+
+ /**
+ * Compiles the java files specified in 'JavaLibraryBuildRequest'.
+ * Implementations can try to avoid recompiling the java files. Whenever
+ * this function is invoked, it is guaranteed that the build request
+ * contains files to compile.
+ *
+ * @param build A JavaLibraryBuildRequest request object describing what to
+ * compile
+ * @param err PrintWriter for logging any diagnostic output
+ * @return the exit status of the java compiler.
+ */
+ abstract Result compileSources(JavaLibraryBuildRequest build, JavacRunner javacRunner,
+ PrintWriter err) throws IOException;
+
+ /**
+ * Perform the build.
+ */
+ public void run(JavaLibraryBuildRequest build, PrintStream err)
+ throws IOException {
+ boolean successful = false;
+ try {
+ compileJavaLibrary(build, err);
+ buildJar(build);
+ if (!build.getProcessors().isEmpty()) {
+ if (build.getGeneratedSourcesOutputJar() != null) {
+ buildGensrcJar(build, err);
+ }
+ }
+ successful = true;
+ } finally {
+ build.getDependencyModule().emitUsedClasspath(build.getClassPath());
+ build.getDependencyModule().emitDependencyInformation(build.getClassPath(), successful);
+ shutdown(err);
+ }
+ }
+
+ // Utility functions
+
+ /**
+ * Runs "run" in another thread (whose lifetime is contained within the
+ * activation of this function call) using a stack size of 'stackSize' bytes.
+ * Unchecked exceptions thrown by the Runnable will be re-thrown in the main
+ * thread.
+ */
+ private static void runWithLargeStack(final Runnable run, long stackSize) {
+ final Throwable[] unchecked = { null };
+ Thread t = new Thread(null, run, "runWithLargeStack", stackSize);
+ t.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ unchecked[0] = e;
+ }
+ });
+ t.start();
+ boolean wasInterrupted = false;
+ for (;;) {
+ try {
+ t.join(0);
+ break;
+ } catch (InterruptedException e) {
+ wasInterrupted = true;
+ }
+ }
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ if (unchecked[0] instanceof Error) {
+ throw (Error) unchecked[0];
+ } else if (unchecked[0] instanceof RuntimeException) {
+ throw (RuntimeException) unchecked[0];
+ }
+ }
+
+ /**
+ * A SourceJarEntryListener that collects protobuf meta data files from the
+ * source jar files.
+ */
+ private static class ProtoMetaFileCollector implements SourceJarEntryListener {
+
+ private final String sourceDir;
+ private final String outputDir;
+ private final ByteArrayOutputStream buffer;
+
+ public ProtoMetaFileCollector(String sourceDir, String outputDir) {
+ this.sourceDir = sourceDir;
+ this.outputDir = outputDir;
+ this.buffer = new ByteArrayOutputStream();
+ }
+
+ @Override
+ public void onEntry(ZipEntry entry) throws IOException {
+ String entryName = entry.getName();
+ if (!entryName.equals(PROTOBUF_META_NAME)) {
+ return;
+ }
+ Files.copy(new File(sourceDir, PROTOBUF_META_NAME), buffer);
+ }
+
+ /**
+ * Writes the combined the meta files into the output directory. Delete the
+ * stalling meta file if no meta file is collected.
+ */
+ @Override
+ public void finish() throws IOException {
+ File outputFile = new File(outputDir, PROTOBUF_META_NAME);
+ if (buffer.size() > 0) {
+ try (OutputStream outputStream = new FileOutputStream(outputFile)) {
+ buffer.writeTo(outputStream);
+ }
+ } else if (outputFile.exists()) {
+ // Delete stalled meta file.
+ outputFile.delete();
+ }
+ }
+ }
+
+ @Override
+ protected List<SourceJarEntryListener> getSourceJarEntryListeners(
+ JavaLibraryBuildRequest build) {
+ List<SourceJarEntryListener> result = super.getSourceJarEntryListeners(build);
+ result.add(new ProtoMetaFileCollector(
+ build.getTempDir(), build.getClassDir()));
+ return result;
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java
new file mode 100644
index 0000000..cf1c985
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java
@@ -0,0 +1,295 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteStreams;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+/**
+ * Base class for java_library builders.
+ *
+ * <p>Implements common functionality like source files preparation and
+ * output jar creation.
+ */
+public abstract class AbstractLibraryBuilder extends CommonJavaLibraryProcessor {
+
+ /**
+ * Prepares a compilation run. This involves cleaning up temporary dircectories and
+ * writing the classpath files.
+ */
+ protected void prepareSourceCompilation(JavaLibraryBuildRequest build) throws IOException {
+ File classDirectory = new File(build.getClassDir());
+ if (classDirectory.exists()) {
+ try {
+ // Necessary for local builds in order to discard previous outputs
+ cleanupOutputDirectory(classDirectory);
+ } catch (IOException e) {
+ throw new IOException("Cannot clean output directory '" + classDirectory + "'", e);
+ }
+ }
+ classDirectory.mkdirs();
+
+ setUpSourceJars(build);
+ }
+
+ public void buildJar(JavaLibraryBuildRequest build) throws IOException {
+ JarCreator jar = new JarCreator(build.getOutputJar());
+ jar.setNormalize(true);
+ jar.setCompression(build.compressJar());
+
+ // The easiest way to handle resource jars is to unpack them into the class directory, just
+ // before we start zipping it up.
+ for (String resourceJar : build.getResourceJars()) {
+ setUpSourceJar(new File(resourceJar), build.getClassDir(),
+ new ArrayList<SourceJarEntryListener>());
+ }
+
+ jar.addDirectory(build.getClassDir());
+
+ jar.addRootEntries(build.getRootResourceFiles());
+ addResourceEntries(jar, build.getResourceFiles());
+ addMessageEntries(jar, build.getMessageFiles());
+
+ jar.execute();
+ }
+
+ /**
+ * Adds a collection of resource entries. Each entry is a string composed of a
+ * pair of parts separated by a colon ':'. The name of the resource comes from
+ * the second part, and the path to the resource comes from the whole string
+ * with the colon replaced by a slash '/'.
+ * <pre>
+ * prefix:name => (name, prefix/name)
+ * </pre>
+ */
+ private static void addResourceEntries(JarCreator jar, Collection<String> resources)
+ throws IOException {
+ for (String resource : resources) {
+ int colon = resource.indexOf(':');
+ if (colon < 0) {
+ throw new IOException("" + resource + ": Illegal resource entry.");
+ }
+ String prefix = resource.substring(0, colon);
+ String name = resource.substring(colon + 1);
+ String path = colon > 0 ? prefix + "/" + name : name;
+ addEntryWithParents(jar, name, path);
+ }
+ }
+
+ private static void addMessageEntries(JarCreator jar, List<String> messages)
+ throws IOException {
+ for (String message : messages) {
+ int colon = message.indexOf(':');
+ if (colon < 0) {
+ throw new IOException("" + message + ": Illegal message entry.");
+ }
+ String prefix = message.substring(0, colon);
+ String name = message.substring(colon + 1);
+ String path = colon > 0 ? prefix + "/" + name : name;
+ File messageFile = new File(path);
+ // Ignore empty messages. They get written by the translation importer
+ // when there is no translation for a particular language.
+ if (messageFile.length() != 0L) {
+ addEntryWithParents(jar, name, path);
+ }
+ }
+ }
+
+ /**
+ * Adds an entry to the jar, making sure that all the parent dirs up to the
+ * base of {@code entry} are also added.
+ *
+ * @param entry the PathFragment of the entry going into the Jar file
+ * @param file the PathFragment of the input file for the entry
+ */
+ @VisibleForTesting
+ static void addEntryWithParents(JarCreator creator, String entry, String file) {
+ while ((entry != null) && creator.addEntry(entry, file)) {
+ entry = new File(entry).getParent();
+ file = new File(file).getParent();
+ }
+ }
+
+ /**
+ * Internal interface which will listen on each entry of the source jar
+ * files during the source jar setup process.
+ */
+ protected interface SourceJarEntryListener {
+ void onEntry(ZipEntry entry) throws IOException;
+ void finish() throws IOException;
+ }
+
+ protected List<SourceJarEntryListener> getSourceJarEntryListeners(JavaLibraryBuildRequest build) {
+ List<SourceJarEntryListener> result = new ArrayList<>();
+ result.add(new SourceJavaFileCollector(build));
+ return result;
+ }
+
+ /**
+ * A SourceJarEntryListener that collects a lists of source Java files from
+ * the source jar files.
+ */
+ private static class SourceJavaFileCollector implements SourceJarEntryListener {
+ private final List<String> sources;
+ private final JavaLibraryBuildRequest build;
+
+ public SourceJavaFileCollector(JavaLibraryBuildRequest build) {
+ this.sources = new ArrayList<>();
+ this.build = build;
+ }
+
+ @Override
+ public void onEntry(ZipEntry entry) {
+ String entryName = entry.getName();
+ if (entryName.endsWith(".java")) {
+ sources.add(build.getTempDir() + File.separator + entryName);
+ }
+ }
+
+ @Override
+ public void finish() {
+ build.getSourceFiles().addAll(sources);
+ }
+ }
+
+ /**
+ * Extracts the all source jars from the build request into the temporary
+ * directory specified in the build request. Empties the temporary directory,
+ * if it exists.
+ */
+ private void setUpSourceJars(JavaLibraryBuildRequest build) throws IOException {
+ String sourcesDir = build.getTempDir();
+
+ File sourceDirFile = new File(sourcesDir);
+ if (sourceDirFile.exists()) {
+ cleanupDirectory(sourceDirFile, true);
+ }
+
+ if (build.getSourceJars().isEmpty()) {
+ return;
+ }
+
+ List<SourceJarEntryListener> listeners = getSourceJarEntryListeners(build);
+ for (String sourceJar : build.getSourceJars()) {
+ setUpSourceJar(new File(sourceJar), sourcesDir, listeners);
+ }
+ for (SourceJarEntryListener listener : listeners) {
+ listener.finish();
+ }
+ }
+
+ /**
+ * Extracts the source jar into the directory sourceDir. Calls each of the
+ * SourceJarEntryListeners for each non-directory entry to do additional work.
+ */
+ private void setUpSourceJar(File sourceJar, String sourceDir,
+ List<SourceJarEntryListener> listeners)
+ throws IOException {
+ try (ZipFile zipFile = new ZipFile(sourceJar)) {
+ Enumeration<? extends ZipEntry> zipEntries = zipFile.entries();
+ while (zipEntries.hasMoreElements()) {
+ ZipEntry currentEntry = zipEntries.nextElement();
+ String entryName = currentEntry.getName();
+ File outputFile = new File(sourceDir, entryName);
+
+ outputFile.getParentFile().mkdirs();
+
+ if (currentEntry.isDirectory()) {
+ outputFile.mkdir();
+ } else {
+ // Copy the data from the zip file to the output file.
+ try (InputStream in = zipFile.getInputStream(currentEntry);
+ OutputStream out = new FileOutputStream(outputFile)) {
+ ByteStreams.copy(in, out);
+ }
+
+ for (SourceJarEntryListener listener : listeners) {
+ listener.onEntry(currentEntry);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Recursively cleans up the files beneath the specified output directory.
+ * Does not follow symbolic links. Throws IOException if any deletion fails.
+ *
+ * Will delete all empty directories.
+ *
+ * @param dir the directory to clean up.
+ * @return true if the directory itself was removed as well.
+ */
+ boolean cleanupOutputDirectory(File dir) throws IOException {
+ return cleanupDirectory(dir, false);
+ }
+
+ /**
+ * Recursively cleans up the files beneath the specified output directory.
+ * Does not follow symbolic links. Throws IOException if any deletion fails.
+ * If removeEverything is false, keeps .class files if keepClassFilesDuringCleanup()
+ * returns true, and also keeps all flags.xml files.
+ * If removeEverything is true, removes everything.
+ * Will delete all empty directories.
+ *
+ * @param dir the directory to clean up.
+ * @param removeEverything whether to remove all files, or keep flags.xml/.class files.
+ * @return true if the directory itself was removed as well.
+ */
+ private boolean cleanupDirectory(File dir, boolean removeEverything) throws IOException {
+ boolean isEmpty = true;
+ File[] files = dir.listFiles();
+ if (files == null) { return false; } // avoid race condition
+ for (File file : files) {
+ if (file.isDirectory()) {
+ isEmpty &= cleanupDirectory(file, removeEverything);
+ } else if (!removeEverything && keepClassFilesDuringCleanup() &&
+ file.getName().endsWith(".class")) {
+ isEmpty = false;
+ } else if (!removeEverything && keepFileDuringCleanup(file)) {
+ isEmpty = false;
+ } else {
+ file.delete();
+ }
+ }
+ if (isEmpty) {
+ dir.delete();
+ }
+ return isEmpty;
+ }
+
+ protected abstract boolean keepFileDuringCleanup(File file);
+
+ /**
+ * Returns true if cleaning the output directory should remove all
+ * .class files in the output directory.
+ */
+ protected boolean keepClassFilesDuringCleanup() {
+ return false;
+ }
+
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractPostProcessor.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractPostProcessor.java
new file mode 100644
index 0000000..e8e608d
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractPostProcessor.java
@@ -0,0 +1,119 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import com.google.common.base.Preconditions;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A processor to apply additional steps to the compiled java classes. It can be used to add code
+ * coverage instrumentation for instance.
+ */
+public abstract class AbstractPostProcessor {
+ private static final Map<String, AbstractPostProcessor> postProcessors = new HashMap<>();
+
+ /**
+ * Declares a post processor with a name. This name serves as the command line argument to
+ * reference a processor.
+ *
+ * @param name the command line name of the processor
+ * @param postProcessor the post processor object
+ */
+ public static void addPostProcessor(String name, AbstractPostProcessor postProcessor) {
+ postProcessors.put(name, postProcessor);
+ }
+
+ private String workingDir = null;
+ private JavaLibraryBuildRequest build = null;
+
+ /**
+ * Sets the command line arguments for this processor.
+ *
+ * @param arguments the list of arguments
+ *
+ * @throws InvalidCommandLineException when the list of arguments for this processors is
+ * incorrect.
+ */
+ public abstract void setCommandLineArguments(List<String> arguments)
+ throws InvalidCommandLineException;
+
+ /**
+ * This initializer is outside of the constructor so the arguments are not passed to the
+ * descendants.
+ */
+ void initialize(JavaLibraryBuildRequest build) {
+ this.build = build;
+ }
+
+ protected String workingPath(String name) {
+ Preconditions.checkNotNull(this.build);
+ return workingDir != null && name.length() > 0 && name.charAt(0) != '/'
+ ? workingDir + File.separator + name
+ : name;
+ }
+
+ protected boolean shouldCompressJar() {
+ return build.compressJar();
+ }
+
+ protected String getBuildClassDir() {
+ return build.getClassDir();
+ }
+
+ /**
+ * Main interface method of the post processor.
+ */
+ public abstract void processRequest() throws IOException;
+
+ /**
+ * Create an {@link AbstractPostProcessor} using reflection.
+ *
+ * @param processorName the name of the processor to instantiate. It should exist in the list of
+ * post processors added with the {@link #addPostProcessor(String, AbstractPostProcessor)}
+ * method.
+ * @param arguments the list of arguments that should be passed to the processor during
+ * instantiation.
+ * @throws InvalidCommandLineException on error creating the processor
+ */
+ static AbstractPostProcessor create(String processorName, List<String> arguments)
+ throws InvalidCommandLineException {
+ AbstractPostProcessor processor = postProcessors.get(processorName);
+ if (processor == null) {
+ throw new InvalidCommandLineException("No such processor '" + processorName + "'");
+ }
+ processor.setCommandLineArguments(arguments);
+ return processor;
+ }
+
+ /**
+ * Recursively delete the given file, it is unsafe.
+ *
+ * @param file the file to recursively remove
+ */
+ protected static void recursiveRemove(File file) {
+ if (file.isDirectory()) {
+ for (File f : file.listFiles()) {
+ recursiveRemove(f);
+ }
+ } else if (file.exists()) {
+ file.delete();
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/BazelJavaBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/BazelJavaBuilder.java
new file mode 100644
index 0000000..4cd1dfe
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/BazelJavaBuilder.java
@@ -0,0 +1,42 @@
+// Copyright 2007-2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * The JavaBuilder main called by bazel.
+ */
+public abstract class BazelJavaBuilder {
+
+ private static final String CMDNAME = "BazelJavaBuilder";
+
+ /**
+ * The main method of the BazelJavaBuilder.
+ */
+ public static void main(String[] args) {
+ try {
+ JavaLibraryBuildRequest build = JavaLibraryBuildRequest.parse(Arrays.asList(args));
+ AbstractJavaBuilder builder = build.getDependencyModule().reduceClasspath()
+ ? new ReducedClasspathJavaLibraryBuilder()
+ : new SimpleJavaLibraryBuilder();
+ builder.run(build, System.err);
+ } catch (IOException | InvalidCommandLineException e) {
+ System.err.println(CMDNAME + " threw exception : " + e.getMessage());
+ System.exit(1);
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/CommonJavaLibraryProcessor.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/CommonJavaLibraryProcessor.java
new file mode 100644
index 0000000..dae8d97
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/CommonJavaLibraryProcessor.java
@@ -0,0 +1,65 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import com.sun.tools.javac.main.Main.Result;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Superclass for all JavaBuilder processor classes
+ * involved in compiling and processing java code.
+ */
+public abstract class CommonJavaLibraryProcessor {
+
+ /**
+ * Exception used to represent failed javac invocation.
+ */
+ static final class JavacException extends Exception {
+ public JavacException(Result result) {
+ super("java compilation returned status " + result);
+ if (result.isOK()) {
+ throw new IllegalArgumentException();
+ }
+ }
+ }
+
+ /**
+ * Creates the initial set of arguments to javac from the Build
+ * configuration supplied. This set of arguments should be extended
+ * by the code invoking it.
+ *
+ * @param build The build request for the initial set of arguments is needed
+ * @return The list of initial arguments
+ */
+ protected List<String> createInitialJavacArgs(JavaLibraryBuildRequest build,
+ String classPath) {
+ List<String> args = new ArrayList<>();
+ if (!classPath.isEmpty()) {
+ args.add("-cp");
+ args.add(classPath);
+ }
+ args.add("-d");
+ args.add(build.getClassDir());
+
+ // Add an empty source path to prevent javac from sucking in source files
+ // from .jar files on the classpath.
+ args.add("-sourcepath");
+ args.add(":");
+
+ return args;
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/InvalidCommandLineException.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/InvalidCommandLineException.java
new file mode 100644
index 0000000..a84c460
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/InvalidCommandLineException.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+/**
+ * Exception to be thrown on command line parsing errors
+ */
+public class InvalidCommandLineException extends Exception {
+
+ public InvalidCommandLineException(String message) {
+ super(message);
+ }
+
+ public InvalidCommandLineException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
\ No newline at end of file
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarCreator.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarCreator.java
new file mode 100644
index 0000000..ead7ceb
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarCreator.java
@@ -0,0 +1,200 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+/**
+ * A class for creating Jar files. Allows normalization of Jar entries by setting their timestamp to
+ * the DOS epoch. All Jar entries are sorted alphabetically.
+ */
+public class JarCreator extends JarHelper {
+
+ // Map from Jar entry names to files. Use TreeMap so we can establish a canonical order for the
+ // entries regardless in what order they get added.
+ private final Map<String, String> jarEntries = new TreeMap<>();
+ private String manifestFile;
+ private String mainClass;
+
+ public JarCreator(String fileName) {
+ super(fileName);
+ }
+
+ /**
+ * Adds an entry to the Jar file, normalizing the name.
+ *
+ * @param entryName the name of the entry in the Jar file
+ * @param fileName the name of the input file for the entry
+ * @return true iff a new entry was added
+ */
+ public boolean addEntry(String entryName, String fileName) {
+ if (entryName.startsWith("/")) {
+ entryName = entryName.substring(1);
+ } else if (entryName.startsWith("./")) {
+ entryName = entryName.substring(2);
+ }
+ return jarEntries.put(entryName, fileName) == null;
+ }
+
+ /**
+ * Adds the contents of a directory to the Jar file. All files below this
+ * directory will be added to the Jar file using the name relative to the
+ * directory as the name for the Jar entry.
+ *
+ * @param directory the directory to add to the jar
+ */
+ public void addDirectory(String directory) {
+ addDirectory(null, new File(directory));
+ }
+
+ /**
+ * Adds the contents of a directory to the Jar file. All files below this
+ * directory will be added to the Jar file using the prefix and the name
+ * relative to the directory as the name for the Jar entry. Always uses '/' as
+ * the separator char for the Jar entries.
+ *
+ * @param prefix the prefix to prepend to every Jar entry name found below the
+ * directory
+ * @param directory the directory to add to the Jar
+ */
+ private void addDirectory(String prefix, File directory) {
+ File[] files = directory.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ String entryName = prefix != null ? prefix + "/" + file.getName() : file.getName();
+ jarEntries.put(entryName, file.getAbsolutePath());
+ if (file.isDirectory()) {
+ addDirectory(entryName, file);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds a collection of entries to the jar, each with a given source path, and with
+ * the resulting file in the root of the jar.
+ * <pre>
+ * some/long/path.foo => (path.foo, some/long/path.foo)
+ * </pre>
+ */
+ public void addRootEntries(Collection<String> entries) {
+ for (String entry : entries) {
+ jarEntries.put(new File(entry).getName(), entry);
+ }
+ }
+
+ /**
+ * Sets the main.class entry for the manifest. A value of <code>null</code>
+ * (the default) will omit the entry.
+ *
+ * @param mainClass the fully qualified name of the main class
+ */
+ public void setMainClass(String mainClass) {
+ this.mainClass = mainClass;
+ }
+
+ /**
+ * Sets filename for the manifest content. If this is set the manifest will be
+ * read from this file otherwise the manifest content will get generated on
+ * the fly.
+ *
+ * @param manifestFile the filename of the manifest file.
+ */
+ public void setManifestFile(String manifestFile) {
+ this.manifestFile = manifestFile;
+ }
+
+ private byte[] manifestContent() throws IOException {
+ Manifest manifest;
+ if (manifestFile != null) {
+ FileInputStream in = new FileInputStream(manifestFile);
+ manifest = new Manifest(in);
+ } else {
+ manifest = new Manifest();
+ }
+ Attributes attributes = manifest.getMainAttributes();
+ attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ Attributes.Name createdBy = new Attributes.Name("Created-By");
+ if (attributes.getValue(createdBy) == null) {
+ attributes.put(createdBy, "blaze");
+ }
+ if (mainClass != null) {
+ attributes.put(Attributes.Name.MAIN_CLASS, mainClass);
+ }
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ manifest.write(out);
+ return out.toByteArray();
+ }
+
+ /**
+ * Executes the creation of the Jar file.
+ *
+ * @throws IOException if the Jar cannot be written or any of the entries
+ * cannot be read.
+ */
+ public void execute() throws IOException {
+ out = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(jarFile)));
+
+ // Create the manifest entry in the Jar file
+ writeManifestEntry(manifestContent());
+ try {
+ for (Map.Entry<String, String> entry : jarEntries.entrySet()) {
+ copyEntry(entry.getKey(), new File(entry.getValue()));
+ }
+ } finally {
+ out.closeEntry();
+ out.close();
+ }
+ }
+
+ /**
+ * A simple way to create Jar file using the JarCreator class.
+ */
+ public static void main(String[] args) {
+ if (args.length < 1) {
+ System.err.println("usage: CreateJar output [root directories]");
+ System.exit(1);
+ }
+ String output = args[0];
+ JarCreator createJar = new JarCreator(output);
+ for (int i = 1; i < args.length; i++) {
+ createJar.addDirectory(args[i]);
+ }
+ createJar.setCompression(true);
+ createJar.setNormalize(true);
+ createJar.setVerbose(true);
+ long start = System.currentTimeMillis();
+ try {
+ createJar.execute();
+ } catch (IOException e) {
+ e.printStackTrace();
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+ long stop = System.currentTimeMillis();
+ System.err.println((stop - start) + "ms.");
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarHelper.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarHelper.java
new file mode 100644
index 0000000..0832082
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JarHelper.java
@@ -0,0 +1,201 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+
+/**
+ * A simple helper class for creating Jar files. All Jar entries are sorted alphabetically. Allows
+ * normalization of Jar entries by setting the timestamp of non-.class files to the DOS epoch.
+ * Timestamps of .class files are set to the DOS epoch + 2 seconds (The zip timestamp granularity)
+ * Adjusting the timestamp for .class files is neccessary since otherwise javac will recompile java
+ * files if both the java file and its .class file are present.
+ */
+public class JarHelper {
+
+ public static final String MANIFEST_DIR = "META-INF/";
+ public static final String MANIFEST_NAME = JarFile.MANIFEST_NAME;
+ public static final String SERVICES_DIR = "META-INF/services/";
+
+ public static final long DOS_EPOCH_IN_JAVA_TIME = 315561600000L;
+
+ // ZIP timestamps have a resolution of 2 seconds.
+ // see http://www.info-zip.org/FAQ.html#limits
+ public static final long MINIMUM_TIMESTAMP_INCREMENT = 2000L;
+
+ // The name of the Jar file we want to create
+ protected final String jarFile;
+
+ // The properties to describe how to create the Jar
+ protected boolean normalize;
+ protected int storageMethod = JarEntry.DEFLATED;
+ protected boolean verbose = false;
+
+ // The state needed to create the Jar
+ protected final Set<String> names = new HashSet<>();
+ protected JarOutputStream out;
+
+ public JarHelper(String filename) {
+ jarFile = filename;
+ }
+
+ /**
+ * Enables or disables the Jar entry normalization.
+ *
+ * @param normalize If true the timestamps of Jar entries will be set to the
+ * DOS epoch.
+ */
+ public void setNormalize(boolean normalize) {
+ this.normalize = normalize;
+ }
+
+ /**
+ * Enables or disables compression for the Jar file entries.
+ *
+ * @param compression if true enables compressions for the Jar file entries.
+ */
+ public void setCompression(boolean compression) {
+ storageMethod = compression ? JarEntry.DEFLATED : JarEntry.STORED;
+ }
+
+ /**
+ * Enables or disables verbose messages.
+ *
+ * @param verbose if true enables verbose messages.
+ */
+ public void setVerbose(boolean verbose) {
+ this.verbose = verbose;
+ }
+
+ /**
+ * Returns the normalized timestamp for a jar entry based on its name.
+ * This is necessary since javac will, when loading a class X, prefer a
+ * source file to a class file, if both files have the same timestamp.
+ * Therefore, we need to adjust the timestamp for class files to slightly
+ * after the normalized time.
+ * @param name The name of the file for which we should return the
+ * normalized timestamp.
+ * @return the time for a new Jar file entry in milliseconds since the epoch.
+ */
+ private long normalizedTimestamp(String name) {
+ if (name.endsWith(".class")) {
+ return DOS_EPOCH_IN_JAVA_TIME + MINIMUM_TIMESTAMP_INCREMENT;
+ } else {
+ return DOS_EPOCH_IN_JAVA_TIME;
+ }
+ }
+
+ /**
+ * Returns the time for a new Jar file entry in milliseconds since the epoch.
+ * Uses {@link JarCreator#DOS_EPOCH_IN_JAVA_TIME} for normalized entries,
+ * {@link System#currentTimeMillis()} otherwise.
+ *
+ * @param filename The name of the file for which we are entering the time
+ * @return the time for a new Jar file entry in milliseconds since the epoch.
+ */
+ protected long newEntryTimeMillis(String filename) {
+ return normalize ? normalizedTimestamp(filename) : System.currentTimeMillis();
+ }
+
+ /**
+ * Writes an entry with specific contents to the jar. Directory entries must
+ * include the trailing '/'.
+ */
+ protected void writeEntry(JarOutputStream out, String name, byte[] content) throws IOException {
+ if (names.add(name)) {
+ // Create a new entry
+ JarEntry entry = new JarEntry(name);
+ entry.setTime(newEntryTimeMillis(name));
+ int size = content.length;
+ entry.setSize(size);
+ if (size == 0) {
+ entry.setMethod(JarEntry.STORED);
+ entry.setCrc(0);
+ out.putNextEntry(entry);
+ } else {
+ entry.setMethod(storageMethod);
+ if (storageMethod == JarEntry.STORED) {
+ entry.setCrc(Hashing.crc32().hashBytes(content).padToLong());
+ }
+ out.putNextEntry(entry);
+ out.write(content);
+ }
+ out.closeEntry();
+ }
+ }
+
+ /**
+ * Writes a standard Java manifest entry into the JarOutputStream. This
+ * includes the directory entry for the "META-INF" directory
+ *
+ * @param content the Manifest content to write to the manifest entry.
+ * @throws IOException
+ */
+ protected void writeManifestEntry(byte[] content) throws IOException {
+ writeEntry(out, MANIFEST_DIR, new byte[]{});
+ writeEntry(out, MANIFEST_NAME, content);
+ }
+
+ /**
+ * Copies file or directory entries from the file system into the jar.
+ * Directory entries will be detected and their names automatically '/'
+ * suffixed.
+ */
+ protected void copyEntry(String name, File file) throws IOException {
+ if (!names.contains(name)) {
+ if (!file.exists()) {
+ throw new FileNotFoundException(file.getAbsolutePath() + " (No such file or directory)");
+ }
+ boolean isDirectory = file.isDirectory();
+ if (isDirectory && !name.endsWith("/")) {
+ name = name + '/'; // always normalize directory names before checking set
+ }
+ if (names.add(name)) {
+ if (verbose) {
+ System.err.println("adding " + file);
+ }
+ // Create a new entry
+ long size = isDirectory ? 0 : file.length();
+ JarEntry outEntry = new JarEntry(name);
+ long newtime = normalize ? normalizedTimestamp(name) : file.lastModified();
+ outEntry.setTime(newtime);
+ outEntry.setSize(size);
+ if (size == 0L) {
+ outEntry.setMethod(JarEntry.STORED);
+ outEntry.setCrc(0);
+ out.putNextEntry(outEntry);
+ } else {
+ outEntry.setMethod(storageMethod);
+ if (storageMethod == JarEntry.STORED) {
+ outEntry.setCrc(Files.hash(file, Hashing.crc32()).padToLong());
+ }
+ out.putNextEntry(outEntry);
+ Files.copy(file, out);
+ }
+ out.closeEntry();
+ }
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java
new file mode 100644
index 0000000..9b899e5
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java
@@ -0,0 +1,516 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+import com.google.devtools.build.buildjar.javac.plugins.dependency.DependencyModule;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * All the information needed to perform a single Java library build operation.
+ */
+public final class JavaLibraryBuildRequest {
+ private boolean compressJar;
+
+ private final List<String> sourceFiles;
+ private final ImmutableList<String> sourceJars;
+ private final ImmutableList<String> messageFiles;
+ private final ImmutableList<String> resourceFiles;
+ private final ImmutableList<String> resourceJars;
+ /** Resource files that should be put in the root of the output jar. */
+ private final ImmutableList<String> rootResourceFiles;
+
+ private final String classPath;
+
+ private final String processorPath;
+ private final List<String> processorNames;
+
+ private final String classDir;
+ private final String tempDir;
+
+ private final String outputJar;
+
+ // Post processors
+ private final ImmutableList<AbstractPostProcessor> postProcessors;
+
+ private final ImmutableList<String> javacOpts;
+
+ /**
+ * Where to store source files generated by annotation processors.
+ */
+ private final String sourceGenDir;
+
+ /**
+ * The path to an output jar for source files generated by annotation processors.
+ */
+ private final String generatedSourcesOutputJar;
+
+ /**
+ * Repository for all dependency-related information.
+ */
+ private final DependencyModule dependencyModule;
+
+ /**
+ * List of plugins that are given to javac.
+ */
+ private final ImmutableList<BlazeJavaCompilerPlugin> plugins;
+
+ private JavaLibraryBuildRequest(
+ boolean compressJar,
+ List<String> sourceFiles, ImmutableList<String> sourceJars,
+ ImmutableList<String> messageFiles, ImmutableList<String> resourceFiles,
+ ImmutableList<String> resourceJars, ImmutableList<String> rootResourceFiles,
+ String classPath, String processorPath, List<String> processorNames, String classDir,
+ String tempDir, String outputJar, ImmutableList<AbstractPostProcessor> postProcessors,
+ ImmutableList<String> javacOpts, String sourceGenDir, String generatedSourcesOutputJar,
+ DependencyModule dependencyModule, ImmutableList<BlazeJavaCompilerPlugin> plugins) {
+ this.compressJar = compressJar;
+ this.sourceFiles = sourceFiles;
+ this.sourceJars = sourceJars;
+ this.messageFiles = messageFiles;
+ this.resourceFiles = resourceFiles;
+ this.resourceJars = resourceJars;
+ this.rootResourceFiles = rootResourceFiles;
+ this.classPath = classPath;
+ this.processorPath = processorPath;
+ this.processorNames = processorNames;
+ this.classDir = classDir;
+ this.tempDir = tempDir;
+ this.outputJar = outputJar;
+ this.postProcessors = postProcessors;
+ this.javacOpts = javacOpts;
+ this.sourceGenDir = sourceGenDir;
+ this.generatedSourcesOutputJar = generatedSourcesOutputJar;
+ this.dependencyModule = dependencyModule;
+ this.plugins = plugins;
+ }
+
+ public boolean compressJar() {
+ return compressJar;
+ }
+
+ public List<String> getSourceFiles() {
+ // TODO(bazel-team): This is being modified after parsing to add files from source jars.
+ return sourceFiles;
+ }
+
+ public ImmutableList<String> getSourceJars() {
+ return sourceJars;
+ }
+
+ public ImmutableList<String> getMessageFiles() {
+ return messageFiles;
+ }
+
+ public ImmutableList<String> getResourceFiles() {
+ return resourceFiles;
+ }
+
+ public ImmutableList<String> getResourceJars() {
+ return resourceJars;
+ }
+
+ public ImmutableList<String> getRootResourceFiles() {
+ return rootResourceFiles;
+ }
+
+ public String getClassPath() {
+ return classPath;
+ }
+
+ public String getProcessorPath() {
+ return processorPath;
+ }
+
+ public List<String> getProcessors() {
+ // TODO(bazel-team): This might be modified by a JavaLibraryBuilder to enable specific
+ // annotation processors.
+ return processorNames;
+ }
+
+ public String getClassDir() {
+ return classDir;
+ }
+
+ public String getTempDir() {
+ return tempDir;
+ }
+
+ public String getOutputJar() {
+ return outputJar;
+ }
+
+ public ImmutableList<String> getJavacOpts() {
+ return javacOpts;
+ }
+
+ public String getSourceGenDir() {
+ return sourceGenDir;
+ }
+
+ public String getGeneratedSourcesOutputJar() {
+ return generatedSourcesOutputJar;
+ }
+
+ public ImmutableList<AbstractPostProcessor> getPostProcessors() {
+ return postProcessors;
+ }
+
+ public ImmutableList<BlazeJavaCompilerPlugin> getPlugins() {
+ return plugins;
+ }
+
+ public DependencyModule getDependencyModule() {
+ return dependencyModule;
+ }
+
+ /**
+ * Parses the list of arguments into a {@link JavaLibraryBuildRequest}. The returned
+ * {@link JavaLibraryBuildRequest} object can be then used to configure the compilation itself.
+ *
+ * @throws IOException if the argument list contains file (with the @ prefix) and reading that
+ * file failed.
+ * @throws InvalidCommandLineException on any command line error
+ */
+ public static JavaLibraryBuildRequest parse(List<String> args) throws IOException,
+ InvalidCommandLineException {
+ return new JavaLibraryBuildRequest.Builder(args).build();
+ }
+
+ /**
+ * Builds a {@link JavaLibraryBuildRequest}.
+ */
+ public static final class Builder {
+ private boolean compressJar;
+
+ private final ImmutableList.Builder<String> sourceFiles = ImmutableList.builder();
+ private final ImmutableList.Builder<String> sourceJars = ImmutableList.builder();
+ private final ImmutableList.Builder<String> messageFiles = ImmutableList.builder();
+ private final ImmutableList.Builder<String> resourceFiles = ImmutableList.builder();
+ private final ImmutableList.Builder<String> resourceJars = ImmutableList.builder();
+ private final ImmutableList.Builder<String> rootResourceFiles = ImmutableList.builder();
+
+ private String classPath;
+
+ private String processorPath = "";
+ private final List<String> processorNames = new ArrayList<>();
+
+ // Since the default behavior of this tool with no arguments is
+ // "rm -fr <classDir>", let's not default to ".", shall we?
+ private String classDir = "classes";
+ private String tempDir = "_tmp";
+
+ private String outputJar;
+
+ private final ImmutableList.Builder<AbstractPostProcessor> postProcessors =
+ ImmutableList.builder();
+
+ private String ruleKind;
+ private String targetLabel;
+
+ private ImmutableList.Builder<String> javacOpts = ImmutableList.builder();
+
+ private String sourceGenDir;
+
+ private String generatedSourcesOutputJar;
+
+ private final DependencyModule dependencyModule;
+
+ private final ImmutableList.Builder<BlazeJavaCompilerPlugin> plugins = ImmutableList.builder();
+
+ /**
+ * Constructs a build from a list of command args. Sets the same JavacRunner
+ * for both compilation and annotation processing.
+ *
+ * @param args the list of command line args
+ * @throws InvalidCommandLineException on any command line error
+ */
+ public Builder(List<String> args) throws InvalidCommandLineException, IOException {
+ dependencyModule = processCommandlineArgs(expandArguments(args));
+ plugins.add(dependencyModule.getPlugin());
+ }
+
+ /**
+ * Constructs a build from a list of command args. Sets the same JavacRunner
+ * for both compilation and annotation processing.
+ *
+ * @param args the list of command line args
+ * @param extraPlugins extraneous plugins to use in addition to the strict dependency module.
+ * @throws InvalidCommandLineException on any command line error
+ */
+ public Builder(List<String> args, List<BlazeJavaCompilerPlugin> extraPlugins)
+ throws InvalidCommandLineException, IOException {
+ this(args);
+ plugins.addAll(extraPlugins);
+ }
+
+ public ImmutableList<String> getJavacOpts() {
+ return javacOpts.build();
+ }
+
+ public void setJavacOpts(List<String> javacOpts) {
+ this.javacOpts = ImmutableList.<String>builder().addAll(javacOpts);
+ }
+
+ public JavaLibraryBuildRequest build() {
+ ArrayList<String> sourceFiles = new ArrayList<>(this.sourceFiles.build());
+ ImmutableList<String> sourceJars = this.sourceJars.build();
+ ImmutableList<String> messageFiles = this.messageFiles.build();
+ ImmutableList<String> resourceFiles = this.resourceFiles.build();
+ ImmutableList<String> resourceJars = this.resourceJars.build();
+ ImmutableList<String> rootResourceFiles = this.rootResourceFiles.build();
+ ImmutableList<AbstractPostProcessor> postProcessors = this.postProcessors.build();
+ ImmutableList<String> javacOpts = this.javacOpts.build();
+ ImmutableList<BlazeJavaCompilerPlugin> plugins = this.plugins.build();
+ return new JavaLibraryBuildRequest(compressJar, sourceFiles, sourceJars, messageFiles,
+ resourceFiles, resourceJars, rootResourceFiles, classPath, processorPath, processorNames,
+ classDir, tempDir, outputJar, postProcessors, javacOpts, sourceGenDir,
+ generatedSourcesOutputJar, dependencyModule, plugins);
+ }
+
+ /**
+ * Processes the command line arguments.
+ *
+ * @throws InvalidCommandLineException on an invalid option being passed.
+ */
+ private DependencyModule processCommandlineArgs(Deque<String> argQueue)
+ throws InvalidCommandLineException {
+ DependencyModule.Builder builder = new DependencyModule.Builder();
+ for (String arg = argQueue.pollFirst(); arg != null; arg = argQueue.pollFirst()) {
+ switch (arg) {
+ case "--javacopts":
+ // Collect additional arguments to javac.
+ // Assumes that javac options do not start with "--".
+ // otherwise we have to do something like adding a "--"
+ // terminator to the passed arguments.
+ collectFlagArguments(javacOpts, argQueue, "--");
+ break;
+ case "--direct_dependency": {
+ String jar = getArgument(argQueue, arg);
+ String target = getArgument(argQueue, arg);
+ builder.addDirectMapping(jar, target);
+ break;
+ }
+ case "--indirect_dependency": {
+ String jar = getArgument(argQueue, arg);
+ String target = getArgument(argQueue, arg);
+ builder.addIndirectMapping(jar, target);
+ break;
+ }
+ case "--strict_java_deps":
+ builder.setStrictJavaDeps(getArgument(argQueue, arg));
+ break;
+ case "--output_deps":
+ builder.setOutputDepsFile(getArgument(argQueue, arg));
+ break;
+ case "--output_deps_proto":
+ builder.setOutputDepsProtoFile(getArgument(argQueue, arg));
+ break;
+ case "--deps_artifacts":
+ ImmutableList.Builder<String> depsArtifacts = ImmutableList.builder();
+ collectFlagArguments(depsArtifacts, argQueue, "--");
+ builder.addDepsArtifacts(depsArtifacts.build());
+ break;
+ case "--reduce_classpath":
+ builder.setReduceClasspath();
+ break;
+ case "--sourcegendir":
+ sourceGenDir = getArgument(argQueue, arg);
+ break;
+ case "--generated_sources_output":
+ generatedSourcesOutputJar = getArgument(argQueue, arg);
+ break;
+ default:
+ processArg(arg, argQueue);
+ }
+ }
+ builder.setRuleKind(ruleKind);
+ builder.setTargetLabel(targetLabel);
+ return builder.build();
+ }
+
+ /**
+ * Pre-processes an argument list, expanding options &at;filename to read in
+ * the content of the file and add it to the list of arguments.
+ *
+ * @param args the List of arguments to pre-process.
+ * @return the List of pre-processed arguments.
+ * @throws IOException if one of the files containing options cannot be read.
+ */
+ private static Deque<String> expandArguments(List<String> args) throws IOException {
+ Deque<String> expanded = new ArrayDeque<>(args.size());
+ for (String arg : args) {
+ expandArgument(expanded, arg);
+ }
+ return expanded;
+ }
+
+ /**
+ * Expands a single argument, expanding options &at;filename to read in the content of the file
+ * and add it to the list of processed arguments. The &at; itself can be escaped with &at;&at;.
+ *
+ * @param arg the argument to pre-process.
+ * @return the list of pre-processed arguments.
+ * @throws IOException if one of the files containing options cannot be read.
+ */
+ private static void expandArgument(Deque<String> expanded, String arg)
+ throws IOException {
+ if (arg.startsWith("@") && !arg.startsWith("@@")) {
+ for (String line : Files.readAllLines(Paths.get(arg.substring(1)), UTF_8)) {
+ if (line.length() > 0) {
+ expandArgument(expanded, line);
+ }
+ }
+ } else {
+ expanded.add(arg);
+ }
+ }
+
+ /**
+ * Collects the arguments for a command line flag until it finds a flag that starts with the
+ * terminatorPrefix.
+ *
+ * @param output where to put the collected flag arguments.
+ * @param args
+ * @param terminatorPrefix the terminator prefix to stop collecting of argument flags.
+ */
+ private static void collectFlagArguments(
+ ImmutableList.Builder<String> output, Deque<String> args, String terminatorPrefix) {
+ for (String arg = args.pollFirst(); arg != null; arg = args.pollFirst()) {
+ if (arg.startsWith(terminatorPrefix)) {
+ args.addFirst(arg);
+ break;
+ }
+ output.add(arg);
+ }
+ }
+
+ /**
+ * Collects the arguments for the --processors command line flag until it finds a flag that
+ * starts with the terminatorPrefix.
+ *
+ * @param output where to put the collected flag arguments.
+ * @param args
+ * @param terminatorPrefix the terminator prefix to stop collecting of argument flags.
+ */
+ private static void collectProcessorArguments(
+ List<String> output, Deque<String> args, String terminatorPrefix)
+ throws InvalidCommandLineException {
+ for (String arg = args.pollFirst(); arg != null; arg = args.pollFirst()) {
+ if (arg.startsWith(terminatorPrefix)) {
+ args.addFirst(arg);
+ break;
+ }
+ if (arg.contains(",")) {
+ throw new InvalidCommandLineException("processor argument may not contain commas: "
+ + arg);
+ }
+ output.add(arg);
+ }
+ }
+
+ private static String getArgument(Deque<String> args, String arg)
+ throws InvalidCommandLineException {
+ try {
+ return args.remove();
+ } catch (NoSuchElementException e) {
+ throw new InvalidCommandLineException(arg + ": missing argument");
+ }
+ }
+
+ private void processArg(String arg, Deque<String> args)
+ throws InvalidCommandLineException {
+ switch (arg) {
+ case "--sources":
+ collectFlagArguments(sourceFiles, args, "-");
+ break;
+ case "--source_jars":
+ collectFlagArguments(sourceJars, args, "-");
+ break;
+ case "--messages":
+ collectFlagArguments(messageFiles, args, "-");
+ break;
+ case "--resources":
+ collectFlagArguments(resourceFiles, args, "-");
+ break;
+ case "--resource_jars":
+ collectFlagArguments(resourceJars, args, "-");
+ break;
+ case "--classpath_resources":
+ collectFlagArguments(rootResourceFiles, args, "-");
+ break;
+ case "--classpath":
+ classPath = getArgument(args, arg);
+ break;
+ case "--processorpath":
+ processorPath = getArgument(args, arg);
+ break;
+ case "--processors":
+ collectProcessorArguments(processorNames, args, "-");
+ break;
+ case "--output":
+ outputJar = getArgument(args, arg);
+ break;
+ case "--classdir":
+ classDir = getArgument(args, arg);
+ break;
+ case "--tempdir":
+ tempDir = getArgument(args, arg);
+ break;
+ case "--gendir":
+ // TODO(bazel-team) - remove when Bazel no longer passes this flag to buildjar.
+ getArgument(args, arg);
+ break;
+ case "--post_processor":
+ addExternalPostProcessor(postProcessors, args, arg);
+ break;
+ case "--compress_jar":
+ compressJar = true;
+ break;
+ case "--rule_kind":
+ ruleKind = getArgument(args, arg);
+ break;
+ case "--target_label":
+ targetLabel = getArgument(args, arg);
+ break;
+ default:
+ throw new InvalidCommandLineException("unknown option : '" + arg + "'");
+ }
+ }
+
+ private void addExternalPostProcessor(ImmutableList.Builder<AbstractPostProcessor> output,
+ Deque<String> args, String arg) throws InvalidCommandLineException {
+ String processorName = getArgument(args, arg);
+ ImmutableList.Builder<String> arguments = ImmutableList.builder();
+ collectFlagArguments(arguments, args, "--");
+ // TODO(bazel-team): there is no check than the same post processor is not added twice.
+ // We should either forbid multiple add of the same post processor or use a processor factory
+ // to allow multiple add of the same post processor. Anyway, this binary is invoked by Blaze
+ // and not manually.
+ output.add(AbstractPostProcessor.create(processorName, arguments.build()));
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/ReducedClasspathJavaLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/ReducedClasspathJavaLibraryBuilder.java
new file mode 100644
index 0000000..ebc1f0f
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/ReducedClasspathJavaLibraryBuilder.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import com.google.devtools.build.buildjar.javac.JavacRunner;
+
+import com.sun.tools.javac.main.Main.Result;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * A variant of SimpleJavaLibraryBuilder that attempts to reduce the compile-time classpath right
+ * before invoking the compiler, based on extra information from provided .jdeps files. This mode is
+ * enabled via the --reduce_classpath flag, only when Blaze runs with --experimental_java_classpath.
+ *
+ * <p>A fall-back mechanism detects whether javac fails because the classpath is incorrectly
+ * discarding required entries, and re-attempts to compile with the full classpath.
+ */
+public class ReducedClasspathJavaLibraryBuilder extends SimpleJavaLibraryBuilder {
+
+ /**
+ * Attempts to minimize the compile-time classpath before invoking javac, falling back to a
+ * regular compile.
+ *
+ * @param build A JavaLibraryBuildRequest request object describing what to compile
+ * @return result code of the javac compilation
+ * @throws IOException clean-up up the output directory fails
+ */
+ @Override
+ Result compileSources(JavaLibraryBuildRequest build, JavacRunner javacRunner, PrintWriter err)
+ throws IOException {
+ // Minimize classpath, but only if we're actually compiling some sources (some invocations of
+ // JavaBuilder are only building resource jars).
+ String compressedClasspath = build.getClassPath();
+ if (!build.getSourceFiles().isEmpty()) {
+ compressedClasspath = build.getDependencyModule()
+ .computeStrictClasspath(build.getClassPath(), build.getClassDir());
+ }
+ String[] javacArguments = makeJavacArguments(build, compressedClasspath);
+
+ // Compile!
+ StringWriter javacOutput = new StringWriter();
+ PrintWriter javacOutputWriter = new PrintWriter(javacOutput);
+ Result result = javacRunner.invokeJavac(javacArguments, javacOutputWriter);
+ javacOutputWriter.close();
+
+ // If javac errored out because of missing entries on the classpath, give it another try.
+ // TODO(bazel-team): check performance impact of additional retries.
+ if (!result.isOK() && hasRecognizedError(javacOutput.toString())) {
+ if (debug) {
+ err.println("warning: [transitive] Target uses transitive classpath to compile.");
+ }
+
+ // Reset output directories
+ prepareSourceCompilation(build);
+
+ // Fall back to the regular compile, but add extra checks to catch transitive uses
+ javacArguments = makeJavacArguments(build);
+ result = javacRunner.invokeJavac(javacArguments, err);
+ } else {
+ err.print(javacOutput.getBuffer());
+ }
+ return result;
+ }
+
+ private boolean hasRecognizedError(String javacOutput) {
+ return javacOutput.contains("error: cannot access")
+ || javacOutput.contains("error: cannot find symbol")
+ || javacOutput.contains("com.sun.tools.javac.code.Symbol$CompletionFailure");
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/SimpleJavaLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/SimpleJavaLibraryBuilder.java
new file mode 100644
index 0000000..714542f
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/SimpleJavaLibraryBuilder.java
@@ -0,0 +1,137 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.buildjar.javac.JavacRunner;
+
+import com.sun.tools.javac.main.Main.Result;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An implementation of the JavaBuilder that uses in-process javac to compile java files.
+ */
+class SimpleJavaLibraryBuilder extends AbstractJavaBuilder {
+
+ @Override
+ Result compileSources(JavaLibraryBuildRequest build, JavacRunner javacRunner, PrintWriter err)
+ throws IOException {
+ String[] javacArguments = makeJavacArguments(build, build.getClassPath());
+ return javacRunner.invokeJavac(javacArguments, err);
+ }
+
+ @Override
+ protected void prepareSourceCompilation(JavaLibraryBuildRequest build) throws IOException {
+ super.prepareSourceCompilation(build);
+
+ // Create sourceGenDir if necessary.
+ if (build.getSourceGenDir() != null) {
+ File sourceGenDir = new File(build.getSourceGenDir());
+ if (sourceGenDir.exists()) {
+ try {
+ cleanupOutputDirectory(sourceGenDir);
+ } catch (IOException e) {
+ throw new IOException("Cannot clean output directory '" + sourceGenDir + "'", e);
+ }
+ }
+ sourceGenDir.mkdirs();
+ }
+ }
+
+ /**
+ * For the build configuration 'build', construct a command line that
+ * can be used for a javac invocation.
+ */
+ protected String[] makeJavacArguments(JavaLibraryBuildRequest build) {
+ return makeJavacArguments(build, build.getClassPath());
+ }
+
+ /**
+ * For the build configuration 'build', construct a command line that
+ * can be used for a javac invocation.
+ */
+ protected String[] makeJavacArguments(JavaLibraryBuildRequest build, String classPath) {
+ List<String> javacArguments = createInitialJavacArgs(build, classPath);
+
+ javacArguments.addAll(getAnnotationProcessingOptions(build));
+
+ for (String option : build.getJavacOpts()) {
+ if (option.startsWith("-J")) { // ignore the VM options.
+ continue;
+ }
+ if (option.equals("-processor") || option.equals("-processorpath")) {
+ throw new IllegalStateException(
+ "Using " + option + " in javacopts is no longer supported."
+ + " Use a java_plugin() rule instead.");
+ }
+ javacArguments.add(option);
+ }
+
+ javacArguments.addAll(build.getSourceFiles());
+ return javacArguments.toArray(new String[0]);
+ }
+
+ /**
+ * Given a JavaLibraryBuildRequest, computes the javac options for the annotation processing
+ * requested.
+ */
+ private List<String> getAnnotationProcessingOptions(JavaLibraryBuildRequest build) {
+ List<String> args = new ArrayList<>();
+
+ // Javac treats "-processorpath ''" as setting the processor path to an empty list,
+ // whereas omitting the option is treated as not having a processor path (which causes
+ // processor path searches to fallback to the class path).
+ args.add("-processorpath");
+ args.add(
+ build.getProcessorPath().isEmpty() ? "" : build.getProcessorPath());
+
+ if (!build.getProcessors().isEmpty() && !build.getSourceFiles().isEmpty()) {
+ // ImmutableSet.copyOf maintains order
+ ImmutableSet<String> deduplicatedProcessorNames = ImmutableSet.copyOf(build.getProcessors());
+ args.add("-processor");
+ args.add(Joiner.on(',').join(deduplicatedProcessorNames));
+
+ // Set javac output directory for generated sources.
+ if (build.getSourceGenDir() != null) {
+ args.add("-s");
+ args.add(build.getSourceGenDir());
+ }
+ } else {
+ // This is necessary because some jars contain discoverable annotation processors that
+ // previously didn't run, and they break builds if the "-proc:none" option is not passed to
+ // javac.
+ args.add("-proc:none");
+ }
+
+ return args;
+ }
+
+ @Override
+ public void buildGensrcJar(JavaLibraryBuildRequest build, OutputStream err)
+ throws IOException {
+ JarCreator jar = new JarCreator(build.getGeneratedSourcesOutputJar());
+ jar.setNormalize(true);
+ jar.setCompression(build.compressJar());
+ jar.addDirectory(build.getSourceGenDir());
+ jar.execute();
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavaCompiler.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavaCompiler.java
new file mode 100644
index 0000000..9494ce7
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavaCompiler.java
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac;
+
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+
+import com.sun.tools.javac.comp.AttrContext;
+import com.sun.tools.javac.comp.CompileStates.CompileState;
+import com.sun.tools.javac.comp.Env;
+import com.sun.tools.javac.main.JavaCompiler;
+import com.sun.tools.javac.util.Context;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+
+/**
+ * An extended version of the javac compiler, providing support for
+ * composable static analyses via a plugin mechanism. BlazeJavaCompiler
+ * keeps a list of plugins and calls callback methods in those plugins
+ * after certain compiler phases. The plugins perform the actual static
+ * analyses.
+ */
+public class BlazeJavaCompiler extends JavaCompiler {
+
+ /**
+ * A list of plugins to run at particular points in the compile
+ */
+ private final List<BlazeJavaCompilerPlugin> plugins = new ArrayList<>();
+
+ public BlazeJavaCompiler(Context context, Iterable<BlazeJavaCompilerPlugin> plugins) {
+ super(context);
+
+ // initialize all plugins
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ plugin.init(context, log, this);
+ this.plugins.add(plugin);
+ }
+ }
+
+ /**
+ * Adds an initialization hook to the Context, such that each subsequent
+ * request for a JavaCompiler (i.e., a lookup for 'compilerKey' of our
+ * superclass, JavaCompiler) will actually construct and return our version
+ * of BlazeJavaCompiler. It's necessary since many new JavaCompilers may
+ * be requested for later stages of the compilation (annotation processing),
+ * within the same Context. And it's the preferred way for extending behavior
+ * within javac, per the documentation in {@link Context}.
+ */
+ public static void preRegister(final Context context,
+ final Iterable<BlazeJavaCompilerPlugin> plugins) {
+ context.put(compilerKey, new Context.Factory<JavaCompiler>() {
+ @Override
+ public JavaCompiler make(Context c) {
+ return new BlazeJavaCompiler(c, plugins);
+ }
+ });
+ }
+
+ @Override
+ public Env<AttrContext> attribute(Env<AttrContext> env) {
+ Env<AttrContext> result = super.attribute(env);
+
+ // Iterate over all plugins, calling their postAttribute methods
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ plugin.postAttribute(result);
+ }
+
+ return result;
+ }
+
+ @Override
+ protected void flow(Env<AttrContext> env, Queue<Env<AttrContext>> results) {
+ if (compileStates.isDone(env, CompileState.FLOW)) {
+ super.flow(env, results);
+ return;
+ }
+ super.flow(env, results);
+ // Iterate over all plugins, calling their postFlow methods
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ plugin.postFlow(env);
+ }
+ }
+
+ @Override
+ public void close() {
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ plugin.finish();
+ }
+ plugins.clear();
+ super.close();
+ }
+
+ /**
+ * Testing purposes only. Returns true if the collection of plugins in
+ * this instance contains one of the provided type.
+ */
+ boolean pluginsContain(Class<? extends BlazeJavaCompilerPlugin> klass) {
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ if (klass.isInstance(plugin)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacLog.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacLog.java
new file mode 100644
index 0000000..6c0ee49
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacLog.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac;
+
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.JCDiagnostic;
+import com.sun.tools.javac.util.Log;
+
+/**
+ * Log class for our custom patched javac.
+ *
+ * <p> This log class tweaks the standard javac log class so
+ * that it drops all non-errors after the first error that
+ * gets reported. By doing this, we
+ * ensure that all warnings are listed before all errors in javac's
+ * output. This makes life easier for everybody.
+ */
+public class BlazeJavacLog extends Log {
+
+ private boolean hadError = false;
+
+ /**
+ * Registers a custom BlazeJavacLog for the given context and -Werror spec.
+ *
+ * @param context Context
+ * @param warningsAsErrors Werror value
+ */
+ public static void preRegister(final Context context) {
+ context.put(logKey, new Context.Factory<Log>() {
+ @Override
+ public Log make(Context c) {
+ return new BlazeJavacLog(c);
+ }
+ });
+ }
+
+ public static BlazeJavacLog instance(Context context) {
+ return (BlazeJavacLog) context.get(logKey);
+ }
+
+ BlazeJavacLog(Context context) {
+ super(context);
+ }
+
+ /**
+ * Returns true if we should display the note diagnostic
+ * passed in as argument, and false if we should discard
+ * it.
+ */
+ private boolean shouldDisplayNote(JCDiagnostic diag) {
+ String noteCode = diag.getCode();
+ return noteCode == null ||
+ (!noteCode.startsWith("compiler.note.deprecated") &&
+ !noteCode.startsWith("compiler.note.unchecked"));
+ }
+
+ @Override
+ protected void writeDiagnostic(JCDiagnostic diag) {
+ switch (diag.getKind()) {
+ case NOTE:
+ if (shouldDisplayNote(diag)) {
+ super.writeDiagnostic(diag);
+ }
+ break;
+ case ERROR:
+ hadError = true;
+ super.writeDiagnostic(diag);
+ break;
+ default:
+ if (!hadError) {
+ // Do not print further warnings if an error has occured.
+ super.writeDiagnostic(diag);
+ }
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java
new file mode 100644
index 0000000..9283588
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java
@@ -0,0 +1,202 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.buildjar.InvalidCommandLineException;
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin.PluginException;
+
+import com.sun.source.util.TaskEvent;
+import com.sun.source.util.TaskListener;
+import com.sun.tools.javac.api.JavacTaskImpl;
+import com.sun.tools.javac.api.JavacTool;
+import com.sun.tools.javac.api.MultiTaskListener;
+import com.sun.tools.javac.main.Main;
+import com.sun.tools.javac.main.Main.Result;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Options;
+import com.sun.tools.javac.util.PropagatedException;
+
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.processing.Processor;
+import javax.tools.DiagnosticListener;
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+
+/**
+ * Main class for our custom patched javac.
+ *
+ * <p> This main class tweaks the standard javac log class by changing the
+ * compiler's context to use our custom log class. This custom log class
+ * modifies javac's output to list all errors after all warnings.
+ */
+public class BlazeJavacMain {
+
+ /**
+ * Compose {@link com.sun.tools.javac.main.Main} and perform custom setup before deferring to
+ * its compile() method.
+ *
+ * Historically BlazeJavacMain extended javac's Main and overrode methods to get the desired
+ * custom behaviour. That approach created incompatibilities when upgrading to newer versions of
+ * javac, so composition is preferred.
+ */
+ @VisibleForTesting
+ private List<BlazeJavaCompilerPlugin> plugins;
+ private final PrintWriter errOutput;
+ private final String compilerName;
+
+ public BlazeJavacMain(PrintWriter errOutput, List<BlazeJavaCompilerPlugin> plugins) {
+ this.compilerName = "blaze javac";
+ this.errOutput = errOutput;
+ this.plugins = plugins;
+ }
+
+ /**
+ * Installs the BlazeJavaCompiler within the provided context. Enables
+ * plugins based on field values.
+ *
+ * @param context JavaCompiler's associated Context
+ */
+ void setupBlazeJavaCompiler(Context context) {
+ preRegister(context, plugins);
+ }
+
+ public void preRegister(Context context, List<BlazeJavaCompilerPlugin> plugins) {
+ this.plugins = plugins;
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ plugin.initializeContext(context);
+ }
+
+ BlazeJavacLog.preRegister(context);
+ BlazeJavaCompiler.preRegister(context, plugins);
+ }
+
+ public Result compile(String[] argv) {
+ // set up a fresh Context with our custom bindings for JavaCompiler
+ Context context = new Context();
+ // disable faulty Zip optimizations
+ Options options = Options.instance(context);
+ options.put("useOptimizedZip", "false");
+
+ // enable Java 8-style type inference features
+ //
+ // This is currently duplicated in JAVABUILDER. That's deliberate for now, because
+ // (1) JavaBuilder's integration test coverage for default options isn't very good, and
+ // (2) the migration from JAVABUILDER to java_toolchain configs is in progress so blaze
+ // integration tests for defaults options are also not trustworthy.
+ //
+ // TODO(bazel-team): removed duplication with JAVABUILDER
+ options.put("usePolyAttribution", "true");
+ options.put("useStrictMethodClashCheck", "true");
+ options.put("useStructuralMostSpecificResolution", "true");
+ options.put("useGraphInference", "true");
+
+ String[] processedArgs;
+
+ try {
+ processedArgs = processPluginArgs(argv);
+ } catch (InvalidCommandLineException e) {
+ errOutput.println(e.getMessage());
+ return Result.CMDERR;
+ }
+
+ setupBlazeJavaCompiler(context);
+ return compile(processedArgs, context);
+ }
+
+ @VisibleForTesting
+ public Result compile(String[] argv, Context context) {
+ enableEndPositions(context);
+ try {
+ return new Main(compilerName, errOutput).compile(argv, context);
+ } catch (PropagatedException e) {
+ if (e.getCause() instanceof PluginException) {
+ PluginException pluginException = (PluginException) e.getCause();
+ errOutput.println(pluginException.getMessage());
+ return pluginException.getResult();
+ }
+ e.printStackTrace(errOutput);
+ return Result.ABNORMAL;
+ }
+ }
+
+ // javac9 removes the ability to pass lists of {@link JavaFileObject}s or {@link Processors}s to
+ // it's 'Main' class (i.e. the entry point for command-line javac). Having BlazeJavacMain
+ // continue to call into javac's Main has the nice property that it keeps JavaBuilder's
+ // behaviour closer to stock javac, but it makes it harder to write integration tests. This class
+ // provides a compile method that accepts file objects and processors, but it isn't
+ // guaranteed to behave exactly the same way as JavaBuilder does when used from the command-line.
+ // TODO(bazel-team): either stop using Main and commit to using the the API for everything, or
+ // re-write integration tests for JavaBuilder to use the real compile() method.
+ @VisibleForTesting
+ @Deprecated
+ public Result compile(
+ String[] argv,
+ Context context,
+ JavaFileManager fileManager,
+ DiagnosticListener<? super JavaFileObject> diagnosticListener,
+ List<JavaFileObject> javaFileObjects,
+ Iterable<? extends Processor> processors) {
+
+ JavacTool tool = JavacTool.create();
+ JavacTaskImpl task = (JavacTaskImpl) tool.getTask(
+ errOutput,
+ fileManager,
+ diagnosticListener,
+ Arrays.asList(argv),
+ null,
+ javaFileObjects,
+ context);
+ if (processors != null) {
+ task.setProcessors(processors);
+ }
+
+ try {
+ return task.doCall();
+ } catch (PluginException e) {
+ errOutput.println(e.getMessage());
+ return e.getResult();
+ }
+ }
+
+ private static final TaskListener EMPTY_LISTENER = new TaskListener() {
+ @Override public void started(TaskEvent e) {}
+ @Override public void finished(TaskEvent e) {}
+ };
+
+ /**
+ * Convinces javac to run in 'API mode', and collect end position information needed by
+ * error-prone.
+ */
+ private static void enableEndPositions(Context context) {
+ MultiTaskListener.instance(context).add(EMPTY_LISTENER);
+ }
+
+ /**
+ * Processes Plugin-specific arguments and removes them from the args array.
+ */
+ @VisibleForTesting
+ String[] processPluginArgs(String[] args) throws InvalidCommandLineException {
+ List<String> processedArgs = Arrays.asList(args);
+ for (BlazeJavaCompilerPlugin plugin : plugins) {
+ processedArgs = plugin.processArgs(processedArgs);
+ }
+ return processedArgs.toArray(new String[processedArgs.size()]);
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunner.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunner.java
new file mode 100644
index 0000000..6cd4b9a
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunner.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac;
+
+import com.sun.tools.javac.main.Main.Result;
+
+import java.io.PrintWriter;
+
+/**
+ * The JavacRunner is a type that can be used to invoke
+ * javac and provides a convenient hook for modifications.
+ * It is split in two parts: An interface "JavacRunner" and
+ * an implementation of that interface, "JavacRunnerImpl".
+ *
+ * The type is split in two parts to allow us to load
+ * the implementation multiple times in different classloaders.
+ * This is neccessary, as a single javac can not run multiple
+ * times in parallel. By using different classloaders to load
+ * different copies of javac in different JavacRunnerImpls,
+ * we can run them in parallel.
+ *
+ * However, since each JavacRunnerImpl will then be loaded
+ * in a different classloader, we then would not be able to
+ * refer to it by simply declaring a type as "JavacRunnerImpl",
+ * as this refers to the JavacRunnerImpl type loaded with the
+ * default classloader. Therefore, we'd have to address each
+ * of the different JavacRunnerImpls as "Object" and invoke
+ * its method via reflection.
+ *
+ * We can circumvent this problem by declaring an interface
+ * that JavacRunnerImpl implements (i.e. JavacRunner).
+ * If we always load this super-interface in the default
+ * classloader, and make each JavacRunnerImpl (loaded in its
+ * own classloader) implement it, we can refer to the
+ * JavacRunnerImpls as "JavacRunner"s in the main program.
+ * That way, we can avoid using reflection and "Object"
+ * to deal with the different JavacRunnerImpls.
+ */
+public interface JavacRunner {
+
+ Result invokeJavac(String[] args, PrintWriter output);
+
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunnerImpl.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunnerImpl.java
new file mode 100644
index 0000000..a791ad6
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/JavacRunnerImpl.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac;
+
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+
+import com.sun.tools.javac.main.Main.Result;
+
+import java.io.PrintWriter;
+import java.util.List;
+
+/**
+ * This class wraps a single invocation of Javac. We
+ * invoke javac statically but wrap it with a synchronization.
+ * This is because the same javac cannot be invoked multiple
+ * times in parallel.
+ */
+public class JavacRunnerImpl implements JavacRunner {
+
+ private final List<BlazeJavaCompilerPlugin> plugins;
+
+ /**
+ * Passes extra information to BlazeJavacMain in case strict Java
+ * dependencies are enforced.
+ */
+ public JavacRunnerImpl(List<BlazeJavaCompilerPlugin> plugins) {
+ this.plugins = plugins;
+ }
+
+ @Override
+ public synchronized Result invokeJavac(String[] args, PrintWriter output) {
+ return new BlazeJavacMain(output, plugins).compile(args);
+ }
+
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/BlazeJavaCompilerPlugin.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/BlazeJavaCompilerPlugin.java
new file mode 100644
index 0000000..e42713f
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/BlazeJavaCompilerPlugin.java
@@ -0,0 +1,135 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+
+package com.google.devtools.build.buildjar.javac.plugins;
+
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import com.google.devtools.build.buildjar.InvalidCommandLineException;
+
+import com.sun.tools.javac.comp.AttrContext;
+import com.sun.tools.javac.comp.Env;
+import com.sun.tools.javac.main.JavaCompiler;
+import com.sun.tools.javac.main.Main.Result;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Log;
+import com.sun.tools.javac.util.PropagatedException;
+
+import java.util.List;
+
+/**
+ * An interface for additional static analyses that need access to the javac compiler's AST at
+ * specific points in the compilation process. This class provides callbacks after the attribute and
+ * flow phases of the javac compilation process. A static analysis may be implemented by subclassing
+ * this abstract class and performing the analysis in the callback methods. The analysis may then be
+ * registered with the BlazeJavaCompiler to be run during the compilation process. See
+ * {@link com.google.devtools.build.buildjar.javac.plugins.dependency.StrictJavaDepsPlugin} for an
+ * example.
+ */
+public abstract class BlazeJavaCompilerPlugin {
+
+ /**
+ * Allows plugins to pass errors through javac.Main to BlazeJavacMain and cleanly shut down the
+ * compiler.
+ */
+ public static final class PluginException extends RuntimeException {
+ private final Result result;
+ private final String message;
+
+ /** The compiler's exit status. */
+ public Result getResult() {
+ return result;
+ }
+
+ /** The message that will be printed to stderr before shutting down. */
+ @Override
+ public String getMessage() {
+ return message;
+ }
+
+ private PluginException(Result result, String message) {
+ this.result = result;
+ this.message = message;
+ }
+ }
+
+ /**
+ * Pass an error through javac.Main to BlazeJavacMain and cleanly shut down the compiler.
+ */
+ protected static Exception throwError(Result result, String message) {
+ // Javac re-throws exceptions wrapped by PropagatedException.
+ throw new PropagatedException(new PluginException(result, message));
+ }
+
+ protected Context context;
+ protected Log log;
+ protected JavaCompiler compiler;
+
+ /**
+ * Preprocess the command-line flags that were passed to javac. This is called before
+ * {@link #init(Context, Log, JavaCompiler)} and {@link #initializeContext(Context)}.
+ *
+ * @param args The command-line flags that javac was invoked with.
+ * @throws InvalidCommandLineException if the arguments are invalid
+ * @returns The flags that do not belong to this plugin.
+ */
+ public List<String> processArgs(List<String> args) throws InvalidCommandLineException {
+ return args;
+ }
+
+ /**
+ * Called after all plugins have processed arguments and can be used to customize the Java
+ * compiler context.
+ */
+ public void initializeContext(Context context) {
+ this.context = context;
+ }
+
+ /**
+ * Performs analysis actions after the attribute phase of the javac compiler.
+ * The attribute phase performs symbol resolution on the parse tree.
+ *
+ * @param env The attributed parse tree (after symbol resolution)
+ */
+ public void postAttribute(Env<AttrContext> env) {}
+
+ /**
+ * Performs analysis actions after the flow phase of the javac compiler.
+ * The flow phase performs dataflow checks, such as finding unreachable
+ * statements.
+ *
+ * @param env The attributed parse tree (after symbol resolution)
+ */
+ public void postFlow(Env<AttrContext> env) {}
+
+ /**
+ * Performs analysis actions when the compiler is done and is about to wipe
+ * clean its internal data structures (such as the symbol table).
+ */
+ public void finish() {}
+
+ /**
+ * Initializes the plugin. Called by
+ * {@link com.google.devtools.build.buildjar.javac.BlazeJavaCompiler}'s constructor.
+ *
+ * @param context The Context object from the enclosing BlazeJavaCompiler instance
+ * @param log The Log object from the enclosing BlazeJavaCompiler instance
+ * @param compiler The enclosing BlazeJavaCompiler instance
+ */
+ public void init(Context context, Log log, JavaCompiler compiler) {
+ this.context = context;
+ this.log = log;
+ this.compiler = compiler;
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/DependencyModule.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/DependencyModule.java
new file mode 100644
index 0000000..31964e5
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/DependencyModule.java
@@ -0,0 +1,441 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac.plugins.dependency;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+import com.google.devtools.build.lib.view.proto.Deps;
+import com.google.devtools.build.lib.view.proto.Deps.Dependency.Kind;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Wrapper class for managing dependencies on top of
+ * {@link com.google.devtools.build.buildjar.javac.BlazeJavaCompiler}. If strict_java_deps is
+ * enabled, it keeps two maps between jar names (as they appear on the classpath) and their
+ * originating targets, one for direct dependencies and the other for transitive (indirect)
+ * dependencies, and enables the {@link StrictJavaDepsPlugin} to perform the actual checks. The
+ * plugin also collects dependency information during compilation, and DependencyModule generates a
+ * .jdeps artifact summarizing the discovered dependencies.
+ */
+public final class DependencyModule {
+
+ public static enum StrictJavaDeps {
+ /** Legacy behavior: Silently allow referencing transitive dependencies. */
+ OFF(false),
+ /** Warn about transitive dependencies being used directly. */
+ WARN(true),
+ /** Fail the build when transitive dependencies are used directly. */
+ ERROR(true);
+
+ private boolean enabled;
+
+ StrictJavaDeps(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ /** Convenience method for just checking if it's not OFF */
+ public boolean isEnabled() {
+ return enabled;
+ }
+ }
+
+ private StrictJavaDeps strictJavaDeps = StrictJavaDeps.OFF;
+ private final Map<String, String> directJarsToTargets;
+ private final Map<String, String> indirectJarsToTargets;
+ private boolean strictClasspathMode;
+ private final Set<String> depsArtifacts;
+ private final String ruleKind;
+ private final String targetLabel;
+ private final String outputDepsFile;
+ private final String outputDepsProtoFile;
+ private final Set<String> usedClasspath;
+ private final Map<String, Deps.Dependency> explicitDependenciesMap;
+ private final Map<String, Deps.Dependency> implicitDependenciesMap;
+ Set<String> requiredClasspath;
+
+ DependencyModule(StrictJavaDeps strictJavaDeps,
+ Map<String, String> directJarsToTargets,
+ Map<String, String> indirectJarsToTargets,
+ boolean strictClasspathMode,
+ Set<String> depsArtifacts,
+ String ruleKind,
+ String targetLabel,
+ String outputDepsFile,
+ String outputDepsProtoFile) {
+ this.strictJavaDeps = strictJavaDeps;
+ this.directJarsToTargets = directJarsToTargets;
+ this.indirectJarsToTargets = indirectJarsToTargets;
+ this.strictClasspathMode = strictClasspathMode;
+ this.depsArtifacts = depsArtifacts;
+ this.ruleKind = ruleKind;
+ this.targetLabel = targetLabel;
+ this.outputDepsFile = outputDepsFile;
+ this.outputDepsProtoFile = outputDepsProtoFile;
+ this.explicitDependenciesMap = new HashMap<>();
+ this.implicitDependenciesMap = new HashMap<>();
+ this.usedClasspath = new HashSet<>();
+ }
+
+ /**
+ * Returns a plugin to be enabled in the compiler.
+ */
+ public BlazeJavaCompilerPlugin getPlugin() {
+ return new StrictJavaDepsPlugin(this);
+ }
+
+ /**
+ * Writes the true, used compile-time classpath to the deps file, if specified.
+ */
+ public void emitUsedClasspath(String classpath) throws IOException {
+ if (outputDepsFile != null) {
+ try (BufferedWriter out = new BufferedWriter(new FileWriter(outputDepsFile))) {
+ // Filter using the original classpath, to preserve ordering.
+ for (String entry : classpath.split(":")) {
+ if (usedClasspath.contains(entry)) {
+ out.write(entry);
+ out.newLine();
+ }
+ }
+ } catch (IOException ex) {
+ throw new IOException("Cannot write dependencies to " + outputDepsFile, ex);
+ }
+ }
+ }
+
+ /**
+ * Writes dependency information to the deps file in proto format, if specified.
+ *
+ * This is a replacement for {@link #emitUsedClasspath} above, which only outputs the used
+ * classpath. We collect more precise dependency information to allow Blaze to analyze both
+ * strict and unused dependencies based on the new deps.proto.
+ */
+ public void emitDependencyInformation(String classpath, boolean successful) throws IOException {
+ if (outputDepsProtoFile == null) {
+ return;
+ }
+
+ try (BufferedOutputStream out =
+ new BufferedOutputStream(new FileOutputStream(outputDepsProtoFile))) {
+ buildDependenciesProto(classpath, successful).writeTo(out);
+ } catch (IOException ex) {
+ throw new IOException("Cannot write dependencies to " + outputDepsProtoFile, ex);
+ }
+ }
+
+ @VisibleForTesting
+ Deps.Dependencies buildDependenciesProto(String classpath, boolean successful) {
+ Deps.Dependencies.Builder deps = Deps.Dependencies.newBuilder();
+ if (targetLabel != null) {
+ deps.setRuleLabel(targetLabel);
+ }
+ deps.setSuccess(successful);
+ // Filter using the original classpath, to preserve ordering.
+ for (String entry : classpath.split(":")) {
+ if (explicitDependenciesMap.containsKey(entry)) {
+ deps.addDependency(explicitDependenciesMap.get(entry));
+ } else if (implicitDependenciesMap.containsKey(entry)) {
+ deps.addDependency(implicitDependenciesMap.get(entry));
+ }
+ }
+ return deps.build();
+ }
+
+ /**
+ * Returns whether strict dependency checks (strictJavaDeps) are enabled.
+ */
+ public boolean isStrictDepsEnabled() {
+ return strictJavaDeps.isEnabled();
+ }
+
+ /**
+ * Returns the mapping for jars of direct dependencies. The keys are full
+ * paths (as seen on the classpath), and the values are build target names.
+ */
+ public Map<String, String> getDirectMapping() {
+ return directJarsToTargets;
+ }
+
+ /**
+ * Returns the mapping for jars of indirect dependencies. The keys are full
+ * paths (as seen on the classpath), and the values are build target names.
+ */
+ public Map<String, String> getIndirectMapping() {
+ return indirectJarsToTargets;
+ }
+
+ /**
+ * Returns the strict dependency checking (strictJavaDeps) setting.
+ */
+ public StrictJavaDeps getStrictJavaDeps() {
+ return strictJavaDeps;
+ }
+
+ /**
+ * Returns the map collecting precise explicit dependency information.
+ */
+ public Map<String, Deps.Dependency> getExplicitDependenciesMap() {
+ return explicitDependenciesMap;
+ }
+
+ /**
+ * Returns the map collecting precise implicit dependency information.
+ */
+ public Map<String, Deps.Dependency> getImplicitDependenciesMap() {
+ return implicitDependenciesMap;
+ }
+
+ /**
+ * Returns the type (rule kind) of the originating target.
+ */
+ public String getRuleKind() {
+ return ruleKind;
+ }
+
+ /**
+ * Returns the name (label) of the originating target.
+ */
+ public String getTargetLabel() {
+ return targetLabel;
+ }
+
+ /**
+ * Returns the file name collecting dependency information.
+ */
+ public String getOutputDepsFile() {
+ return outputDepsFile;
+ }
+
+ @VisibleForTesting
+ Set<String> getUsedClasspath() {
+ return usedClasspath;
+ }
+
+ /**
+ * Returns whether classpath reduction is enabled for this invocation.
+ */
+ public boolean reduceClasspath() {
+ return strictClasspathMode;
+ }
+
+ /**
+ * Computes a reduced compile-time classpath from the union of direct dependencies and their
+ * dependencies, as listed in the associated .deps artifacts.
+ */
+ public String computeStrictClasspath(String originalClasspath, String classDir) {
+ if (!strictClasspathMode) {
+ return originalClasspath;
+ }
+
+ // Classpath = direct deps + runtime direct deps + their .deps
+ requiredClasspath = new HashSet<>(directJarsToTargets.keySet());
+
+ for (String depsArtifact : depsArtifacts) {
+ collectDependenciesFromArtifact(depsArtifact);
+ }
+
+ // Filter the initial classpath and keep the original order, with classDir as the last entry.
+ StringBuilder sb = new StringBuilder();
+ String[] originalClasspathEntries = originalClasspath.split(":");
+
+ for (String entry : originalClasspathEntries) {
+ if (requiredClasspath.contains(entry)) {
+ sb.append(entry).append(":");
+ }
+ }
+ sb.append(classDir);
+ return sb.toString();
+ }
+
+ @VisibleForTesting
+ void setStrictClasspath(Set<String> strictClasspath) {
+ this.requiredClasspath = strictClasspath;
+ }
+
+ /**
+ * Updates {@link #requiredClasspath} to include dependencies from the given output artifact.
+ *
+ * During the .deps migration from text to proto format, this method will try to handle both.
+ * Blaze can thus switch the .deps artifacts independently.
+ */
+ private void collectDependenciesFromArtifact(String path) {
+ // Try reading in proto format first
+ try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(path))) {
+ Deps.Dependencies deps = Deps.Dependencies.parseFrom(bis);
+ // Sanity check to make sure we have a valid proto, not a text file that happened to match.
+ if (!deps.hasRuleLabel()) {
+ throw new IOException("Text file");
+ }
+ for (Deps.Dependency dep : deps.getDependencyList()) {
+ if (dep.getKind() == Kind.EXPLICIT || dep.getKind() == Kind.IMPLICIT) {
+ requiredClasspath.add(dep.getPath());
+ }
+ }
+ } catch (IOException ex) {
+ // TODO(bazel-team): Remove this fallback to text format when Blaze is ready.
+ try (BufferedReader reader = new BufferedReader(new FileReader(path))) {
+ for (String dep = reader.readLine(); dep != null; dep = reader.readLine()) {
+ requiredClasspath.add(dep);
+ }
+ } catch (IOException exc) {
+ // At this point we can give up altogether
+ exc.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Builder for {@link DependencyModule}.
+ */
+ public static class Builder {
+
+ private StrictJavaDeps strictJavaDeps = StrictJavaDeps.OFF;
+ private final Map<String, String> directJarsToTargets = new HashMap<>();
+ private final Map<String, String> indirectJarsToTargets = new HashMap<>();
+ private final Set<String> depsArtifacts = new HashSet<>();
+ private String ruleKind;
+ private String targetLabel;
+ private String outputDepsFile;
+ private String outputDepsProtoFile;
+ private boolean strictClasspathMode = false;
+
+ /**
+ * Constructs the DependencyModule, guaranteeing that the maps are
+ * never null (they may be empty), and the default strictJavaDeps setting
+ * is OFF.
+ *
+ * @return an instance of DependencyModule
+ */
+ public DependencyModule build() {
+ return new DependencyModule(strictJavaDeps, directJarsToTargets, indirectJarsToTargets,
+ strictClasspathMode, depsArtifacts, ruleKind, targetLabel, outputDepsFile,
+ outputDepsProtoFile);
+ }
+
+ /**
+ * Sets the strictness level for dependency checking.
+ *
+ * @param strictJavaDeps level, as specified by {@link StrictJavaDeps}
+ * @return this Builder instance
+ */
+ public Builder setStrictJavaDeps(String strictJavaDeps) {
+ this.strictJavaDeps = StrictJavaDeps.valueOf(strictJavaDeps);
+ return this;
+ }
+
+ /**
+ * Sets the type (rule kind) of the originating target.
+ *
+ * @param ruleKind kind, such as the rule kind of a RuleConfiguredTarget
+ * @return this Builder instance
+ */
+ public Builder setRuleKind(String ruleKind) {
+ this.ruleKind = ruleKind;
+ return this;
+ }
+
+ /**
+ * Sets the name (label) of the originating target.
+ *
+ * @param targetLabel label, such as the label of a RuleConfiguredTarget
+ * @return this Builder instance
+ */
+ public Builder setTargetLabel(String targetLabel) {
+ this.targetLabel = targetLabel;
+ return this;
+ }
+
+ /**
+ * Adds a direct mapping to the existing map for direct dependencies.
+ *
+ * @param jar path of jar artifact, as seen on classpath
+ * @param target full name of build target providing the jar
+ * @return this Builder instance
+ */
+ public Builder addDirectMapping(String jar, String target) {
+ directJarsToTargets.put(jar, target);
+ return this;
+ }
+
+ /**
+ * Adds an indirect mapping to the existing map for indirect dependencies.
+ *
+ * @param jar path of jar artifact, as seen on classpath
+ * @param target full name of build target providing the jar
+ * @return this Builder instance
+ */
+ public Builder addIndirectMapping(String jar, String target) {
+ indirectJarsToTargets.put(jar, target);
+ return this;
+ }
+
+ /**
+ * Sets the name of the file that will contain dependency information.
+ *
+ * @param outputDepsFile output file name for dependency information
+ * @return this Builder instance
+ */
+ public Builder setOutputDepsFile(String outputDepsFile) {
+ this.outputDepsFile = outputDepsFile;
+ return this;
+ }
+
+ /**
+ * Sets the name of the file that will contain dependency information in the protocol buffer
+ * format.
+ *
+ * @param outputDepsProtoFile output file name for dependency information
+ * @return this Builder instance
+ */
+ public Builder setOutputDepsProtoFile(String outputDepsProtoFile) {
+ this.outputDepsProtoFile = outputDepsProtoFile;
+ return this;
+ }
+
+ /**
+ * Adds a collection of dependency artifacts to use when reducing the compile-time classpath.
+ *
+ * @param depsArtifacts dependency artifacts
+ * @return this Builder instance
+ */
+ public Builder addDepsArtifacts(Collection<String> depsArtifacts) {
+ this.depsArtifacts.addAll(depsArtifacts);
+ return this;
+ }
+
+ /**
+ * Requests compile-time classpath reduction based on provided dependency artifacts.
+ *
+ * @return this Builder instance
+ */
+ public Builder setReduceClasspath() {
+ this.strictClasspathMode = true;
+ return this;
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/ImplicitDependencyExtractor.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/ImplicitDependencyExtractor.java
new file mode 100644
index 0000000..df7190d
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/ImplicitDependencyExtractor.java
@@ -0,0 +1,191 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac.plugins.dependency;
+
+import com.google.devtools.build.lib.view.proto.Deps;
+
+import com.sun.tools.javac.code.Symbol.ClassSymbol;
+import com.sun.tools.javac.code.Symtab;
+import com.sun.tools.javac.file.ZipArchive;
+import com.sun.tools.javac.file.ZipFileIndexArchive;
+import com.sun.tools.javac.util.Context;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.lang.model.util.SimpleTypeVisitor7;
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardLocation;
+
+/**
+ * A lightweight mechanism for extracting compile-time dependencies from javac, by performing a scan
+ * of the symbol table after compilation finishes. It only includes dependencies from jar files,
+ * which can be interface jars or regular third_party jars, matching the compilation model of Blaze.
+ * Note that JDK8 may provide support for extracting per-class, finer-grained dependencies, and if
+ * that implementation has reasonable overhead it may be a future option.
+ */
+public class ImplicitDependencyExtractor {
+
+ /** Set collecting dependencies names, used for the text output (soon to be removed) */
+ private final Set<String> depsSet;
+ /** Map collecting dependency information, used for the proto output */
+ private final Map<String, Deps.Dependency> depsMap;
+ private final TypeVisitor typeVisitor = new TypeVisitor();
+ private final JavaFileManager fileManager;
+
+ /**
+ * ImplicitDependencyExtractor does not guarantee any ordering of the reported
+ * dependencies. Clients should preserve the original classpath ordering
+ * if trying to minimize their classpaths using this information.
+ */
+ public ImplicitDependencyExtractor(Set<String> depsSet, Map<String, Deps.Dependency> depsMap,
+ JavaFileManager fileManager) {
+ this.depsSet = depsSet;
+ this.depsMap = depsMap;
+ this.fileManager = fileManager;
+ }
+
+ /**
+ * Collects the implicit dependencies of the given set of ClassSymbol roots.
+ * As we're interested in differentiating between symbols that were just
+ * resolved vs. symbols that were fully completed by the compiler, we start
+ * the analysis by finding all the implicit dependencies reachable from the
+ * given set of roots. For completeness, we then walk the symbol table
+ * associated with the given context and collect the jar files of the
+ * remaining class symbols found there.
+ *
+ * @param context compilation context
+ * @param roots root classes in the implicit dependency collection
+ */
+ public void accumulate(Context context, Set<ClassSymbol> roots) {
+ Symtab symtab = Symtab.instance(context);
+ if (symtab.classes == null) {
+ return;
+ }
+
+ // Collect transitive references for root types
+ for (ClassSymbol root : roots) {
+ root.type.accept(typeVisitor, null);
+ }
+
+ Set<JavaFileObject> platformClasses = getPlatformClasses(fileManager);
+
+ // Collect all other partially resolved types
+ for (ClassSymbol cs : symtab.classes.values()) {
+ if (cs.classfile != null) {
+ collectJarOf(cs.classfile, platformClasses);
+ } else if (cs.sourcefile != null) {
+ collectJarOf(cs.sourcefile, platformClasses);
+ }
+ }
+ }
+
+ /**
+ * Collect the set of classes on the compilation bootclasspath.
+ *
+ * <p>TODO(bazel-team): this needs some work. JavaFileManager.list() is slower than
+ * StandardJavaFileManager.getLocation() and doesn't get cached. Additionally, tracking all
+ * classes in the bootclasspath requires a much bigger set than just tracking a list of jars.
+ * However, relying on the context containing a StandardJavaFileManager is brittle (e.g. Lombok
+ * wraps the file-manager in a ForwardingJavaFileManager.)
+ */
+ public static HashSet<JavaFileObject> getPlatformClasses(JavaFileManager fileManager) {
+ HashSet<JavaFileObject> result = new HashSet<JavaFileObject>();
+ Iterable<JavaFileObject> files;
+ try {
+ files = fileManager.list(
+ StandardLocation.PLATFORM_CLASS_PATH, "", EnumSet.of(JavaFileObject.Kind.CLASS), true);
+ } catch (IOException e) {
+ throw new IOError(e);
+ }
+ for (JavaFileObject file : files) {
+ result.add(file);
+ }
+ return result;
+ }
+
+ /**
+ * Attempts to add the jar associated with the given JavaFileObject, if any,
+ * to the collection, filtering out jars on the compilation bootclasspath.
+ *
+ * @param reference JavaFileObject representing a class or source file
+ * @param platformClasses classes on javac's bootclasspath
+ */
+ private void collectJarOf(JavaFileObject reference, Set<JavaFileObject> platformClasses) {
+ reference = unwrapFileObject(reference);
+ if (reference instanceof ZipArchive.ZipFileObject ||
+ reference instanceof ZipFileIndexArchive.ZipFileIndexFileObject) {
+ // getName() will return something like com/foo/libfoo.jar(Bar.class)
+ String name = reference.getName().split("\\(")[0];
+ // Filter out classes in rt.jar
+ if (!platformClasses.contains(reference)) {
+ depsSet.add(name);
+ if (!depsMap.containsKey(name)) {
+ depsMap.put(name, Deps.Dependency.newBuilder()
+ .setKind(Deps.Dependency.Kind.IMPLICIT)
+ .setPath(name)
+ .build());
+ }
+ }
+ }
+ }
+
+
+ private static class TypeVisitor extends SimpleTypeVisitor7<Void, Void> {
+ // TODO(bazel-team): Override the visitor methods we're interested in.
+ }
+
+ private static final Class<?> WRAPPED_JAVA_FILE_OBJECT =
+ getClassOrDie("com.sun.tools.javac.api.ClientCodeWrapper$WrappedJavaFileObject");
+
+ private static final java.lang.reflect.Field UNWRAP_FIELD =
+ getFieldOrDie(
+ getClassOrDie("com.sun.tools.javac.api.ClientCodeWrapper$WrappedFileObject"),
+ "clientFileObject");
+
+ private static Class<?> getClassOrDie(String name) {
+ try {
+ return Class.forName(name);
+ } catch (ClassNotFoundException e) {
+ throw new LinkageError(e.getMessage());
+ }
+ }
+
+ private static java.lang.reflect.Field getFieldOrDie(Class<?> clazz, String name) {
+ try {
+ java.lang.reflect.Field field = clazz.getDeclaredField(name);
+ field.setAccessible(true);
+ return field;
+ } catch (ReflectiveOperationException e) {
+ throw new LinkageError(e.getMessage());
+ }
+ }
+
+ public static JavaFileObject unwrapFileObject(JavaFileObject file) {
+ if (!file.getClass().equals(WRAPPED_JAVA_FILE_OBJECT)) {
+ return file;
+ }
+ try {
+ return (JavaFileObject) UNWRAP_FIELD.get(file);
+ } catch (ReflectiveOperationException e) {
+ throw new LinkageError(e.getMessage());
+ }
+ }
+}
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/StrictJavaDepsPlugin.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/StrictJavaDepsPlugin.java
new file mode 100644
index 0000000..1bdc73b
--- /dev/null
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/dependency/StrictJavaDepsPlugin.java
@@ -0,0 +1,397 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.buildjar.javac.plugins.dependency;
+
+import static com.google.devtools.build.buildjar.javac.plugins.dependency.DependencyModule.StrictJavaDeps.ERROR;
+import static com.google.devtools.build.buildjar.javac.plugins.dependency.ImplicitDependencyExtractor.getPlatformClasses;
+import static com.google.devtools.build.buildjar.javac.plugins.dependency.ImplicitDependencyExtractor.unwrapFileObject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+import com.google.devtools.build.lib.view.proto.Deps;
+import com.google.devtools.build.lib.view.proto.Deps.Dependency;
+
+import com.sun.tools.javac.code.Flags;
+import com.sun.tools.javac.code.Kinds;
+import com.sun.tools.javac.code.Symbol;
+import com.sun.tools.javac.code.Symbol.ClassSymbol;
+import com.sun.tools.javac.comp.AttrContext;
+import com.sun.tools.javac.comp.Env;
+import com.sun.tools.javac.file.ZipArchive;
+import com.sun.tools.javac.file.ZipFileIndexArchive;
+import com.sun.tools.javac.main.JavaCompiler;
+import com.sun.tools.javac.tree.JCTree;
+import com.sun.tools.javac.tree.TreeScanner;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Log;
+import com.sun.tools.javac.util.Log.WriterKind;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.text.MessageFormat;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeSet;
+
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+
+/**
+ * A plugin for BlazeJavaCompiler that checks for types referenced directly
+ * in the source, but included through transitive dependencies. To get this
+ * information, we hook into the type attribution phase of the BlazeJavaCompiler
+ * (thus the overhead is another tree scan with the classic visitor). The
+ * constructor takes a map from jar names to target names, only for the jars that
+ * come from transitive dependencies (Blaze computes this information).
+ */
+public final class StrictJavaDepsPlugin extends BlazeJavaCompilerPlugin {
+
+ @VisibleForTesting
+ static String targetMapping =
+ "com/google/devtools/build/buildjar/javac/resources/target.properties";
+
+ private static final String FIX_MESSAGE =
+ "%s** Command to add missing strict dependencies:%s\n"
+ + " add_dep %s%s\n\n";
+
+ private static final boolean USE_COLOR = true;
+ private ImplicitDependencyExtractor implicitDependencyExtractor;
+ private CheckingTreeScanner checkingTreeScanner;
+ private DependencyModule dependencyModule;
+
+ /** Marks seen compilation toplevels and their import sections */
+ private final Set<JCTree.JCCompilationUnit> toplevels;
+ /** Marks seen ASTs */
+ private final Set<JCTree> trees;
+
+ /** Computed missing dependencies */
+ private final Set<String> missingTargets;
+
+ private static Properties targetMap;
+
+ private PrintWriter errWriter;
+
+ /**
+ * On top of javac, we keep Blaze-specific information in the form of two
+ * maps. Both map jars (exactly as they appear on the classpath) to target
+ * names, one is used for direct dependencies, the other for the transitive
+ * dependencies.
+ *
+ * <p>This enables the detection of dependency issues. For instance, when a
+ * type com.Foo is referenced in the source and it's coming from an indirect
+ * dependency, we emit a warning flagging that dependency. Also, we can check
+ * whether the direct dependencies were actually necessary, i.e. if their
+ * associated jars were used at all for looking up class definitions.
+ */
+ public StrictJavaDepsPlugin(DependencyModule dependencyModule) {
+ this.dependencyModule = dependencyModule;
+ toplevels = new HashSet<>();
+ trees = new HashSet<>();
+ targetMap = new Properties();
+ missingTargets = new TreeSet<>();
+ }
+
+ @Override
+ public void init(Context context, Log log, JavaCompiler compiler) {
+ super.init(context, log, compiler);
+ errWriter = log.getWriter(WriterKind.ERROR);
+ JavaFileManager fileManager = context.get(JavaFileManager.class);
+ implicitDependencyExtractor = new ImplicitDependencyExtractor(
+ dependencyModule.getUsedClasspath(), dependencyModule.getImplicitDependenciesMap(),
+ fileManager);
+ checkingTreeScanner = context.get(CheckingTreeScanner.class);
+ if (checkingTreeScanner == null) {
+ Set<JavaFileObject> platformClasses = getPlatformClasses(fileManager);
+ checkingTreeScanner = new CheckingTreeScanner(
+ dependencyModule, log, missingTargets, platformClasses);
+ context.put(CheckingTreeScanner.class, checkingTreeScanner);
+ }
+ initTargetMap();
+ }
+
+ private void initTargetMap() {
+ try (InputStream is = getClass().getClassLoader().getResourceAsStream(targetMapping)) {
+ if (is != null) {
+ targetMap.load(is);
+ }
+ } catch (IOException ex) {
+ log.warning("Error loading Strict Java Deps mapping file: " + targetMapping, ex);
+ }
+ }
+
+ /**
+ * We want to make another pass over the AST and "type-check" the usage
+ * of direct/transitive dependencies after the type attribution phase.
+ */
+ @Override
+ public void postAttribute(Env<AttrContext> env) {
+ // We want to generate warnings/errors as if we were javac, and in order to
+ // use the internal log properly, we need to set its current source file
+ // information. The useSource call does just that, and is a common pattern
+ // from JavaCompiler: set source to current file and save the previous
+ // value, do work and generate warnings, reset source.
+ JavaFileObject prev = log.useSource(
+ env.enclClass.sym.sourcefile != null
+ ? env.enclClass.sym.sourcefile
+ : env.toplevel.sourcefile);
+ if (trees.add(env.tree)) {
+ checkingTreeScanner.scan(env.tree);
+ }
+ if (toplevels.add(env.toplevel)) {
+ checkingTreeScanner.scan(env.toplevel.getImports());
+ }
+ log.useSource(prev);
+ }
+
+ @Override
+ public void finish() {
+ implicitDependencyExtractor.accumulate(context, checkingTreeScanner.getSeenClasses());
+
+ if (!missingTargets.isEmpty()) {
+ StringBuilder missingTargetsStr = new StringBuilder();
+ for (String target : missingTargets) {
+ missingTargetsStr.append(target);
+ missingTargetsStr.append(" ");
+ }
+ errWriter.print(String.format(FIX_MESSAGE,
+ USE_COLOR ? "\033[35m\033[1m" : "",
+ USE_COLOR ? "\033[0m" : "",
+ missingTargetsStr.toString(),
+ dependencyModule.getTargetLabel()));
+ }
+ }
+
+ /**
+ * An AST visitor that implements our strict_java_deps checks. For now, it
+ * only emits warnings for types loaded from jar files provided by transitive
+ * (indirect) dependencies. Each type is considered only once, so at most one
+ * warning is generated for it.
+ */
+ private static class CheckingTreeScanner extends TreeScanner {
+
+ private static final String transitiveDepMessage =
+ "[strict] Using type {0} from an indirect dependency (TOOL_INFO: \"{1}\"). "
+ + "See command below **";
+
+ /** Lookup for jars coming from transitive dependencies */
+ private final Map<String, String> indirectJarsToTargets;
+
+ /** All error reporting is done through javac's log, */
+ private final Log log;
+
+ /** The strict_java_deps mode */
+ private final DependencyModule.StrictJavaDeps strictJavaDepsMode;
+
+ /** Missing targets */
+ private final Set<String> missingTargets;
+
+ /** Collect seen direct dependencies and their associated information */
+ private final Map<String, Deps.Dependency> directDependenciesMap;
+
+ /** We only emit one warning/error per class symbol */
+ private final Set<ClassSymbol> seenClasses = new HashSet<>();
+ private final Set<String> seenTargets = new HashSet<>();
+
+ /** The set of classes on the compilation bootclasspath. */
+ private final Set<JavaFileObject> platformClasses;
+
+ public CheckingTreeScanner(DependencyModule dependencyModule, Log log,
+ Set<String> missingTargets, Set<JavaFileObject> platformClasses) {
+ this.indirectJarsToTargets = dependencyModule.getIndirectMapping();
+ this.strictJavaDepsMode = dependencyModule.getStrictJavaDeps();
+ this.log = log;
+ this.missingTargets = missingTargets;
+ this.directDependenciesMap = dependencyModule.getExplicitDependenciesMap();
+ this.platformClasses = platformClasses;
+ }
+
+ Set<ClassSymbol> getSeenClasses() {
+ return seenClasses;
+ }
+
+ /**
+ * Checks an AST node denoting a class type against direct/transitive
+ * dependencies.
+ */
+ private void checkTypeLiteral(JCTree node) {
+ if (node == null || node.type.tsym == null) {
+ return;
+ }
+
+ Symbol.TypeSymbol sym = node.type.tsym;
+ String jarName = getJarName(sym.enclClass(), platformClasses);
+
+ // If this type symbol comes from a class file loaded from a jar, check
+ // whether that jar was a direct dependency and error out otherwise.
+ if (jarName != null && seenClasses.add(sym.enclClass())) {
+ collectExplicitDependency(jarName, node, sym);
+ }
+ }
+
+ /**
+ * Marks the provided dependency as a direct/explicit dependency. Additionally, if
+ * strict_java_deps is enabled, it emits a [strict] compiler warning/error (behavior to be soon
+ * replaced by the more complete Blaze implementation).
+ */
+ private void collectExplicitDependency(String jarName, JCTree node, Symbol.TypeSymbol sym) {
+ if (strictJavaDepsMode.isEnabled()) {
+ // Does it make sense to emit a warning/error for this pair of (type, target)?
+ // We want to emit only one error/warning per target.
+ String target = indirectJarsToTargets.get(jarName);
+ if (target != null && seenTargets.add(target)) {
+ String canonicalTargetName = canonicalizeTarget(target);
+ missingTargets.add(canonicalTargetName);
+ if (strictJavaDepsMode == ERROR) {
+ log.error(node.pos, "proc.messager",
+ MessageFormat.format(transitiveDepMessage, sym, canonicalTargetName));
+ } else {
+ log.warning(node.pos, "proc.messager",
+ MessageFormat.format(transitiveDepMessage, sym, canonicalTargetName));
+ }
+ }
+ }
+
+ if (!directDependenciesMap.containsKey(jarName)) {
+ // Also update the dependency proto
+ Dependency dep = Dependency.newBuilder()
+ .setPath(jarName)
+ .setKind(Dependency.Kind.EXPLICIT)
+ .build();
+ directDependenciesMap.put(jarName, dep);
+ }
+ }
+
+ @Override
+ public void visitMethodDef(JCTree.JCMethodDecl method) {
+ if ((method.mods.flags & Flags.GENERATEDCONSTR) != 0) {
+ // If this is the constructor for an anonymous inner class, refrain from checking the
+ // compiler-generated method signature. Don't skip scanning the method body though, there
+ // might have been an anonymous initializer which still needs to be checked.
+ scan(method.body);
+ } else {
+ super.visitMethodDef(method);
+ }
+ }
+
+ /**
+ * Visits an identifier in the AST. We only care about type symbols.
+ */
+ @Override
+ public void visitIdent(JCTree.JCIdent tree) {
+ if (tree.sym != null && tree.sym.kind == Kinds.TYP) {
+ checkTypeLiteral(tree);
+ }
+ }
+
+ /**
+ * Visits a field selection in the AST. We care because in some cases types
+ * may appear fully qualified and only inside a field selection
+ * (e.g., "com.foo.Bar.X", we want to catch the reference to Bar).
+ */
+ @Override
+ public void visitSelect(JCTree.JCFieldAccess tree) {
+ scan(tree.selected);
+ if (tree.sym != null && tree.sym.kind == Kinds.TYP) {
+ checkTypeLiteral(tree);
+ }
+ }
+
+ /**
+ * Visits an import statement. Static imports must not be omitted, as they
+ * are the only place we'll see the containing class references.
+ */
+ @Override
+ public void visitImport(JCTree.JCImport tree) {
+ if (tree.isStatic()) {
+ scan(tree.getQualifiedIdentifier());
+ }
+ }
+ }
+
+ /**
+ * Returns the canonical version of the target name. Package private for testing.
+ */
+ static String canonicalizeTarget(String target) {
+ String replacement = targetMap.getProperty(target);
+ if (replacement != null) {
+ return replacement;
+ }
+ int colonIndex = target.indexOf(':');
+ if (colonIndex == -1) {
+ // No ':' in target, nothing to do.
+ return target;
+ }
+ int lastSlash = target.lastIndexOf('/', colonIndex);
+ if (lastSlash == -1) {
+ // No '/' or target is actually a filename in label format, return unmodified.
+ return target;
+ }
+ String packageName = target.substring(lastSlash + 1, colonIndex);
+ String suffix = target.substring(colonIndex + 1);
+ if (packageName.equals(suffix)) {
+ // target ends in "/something:something", canonicalize.
+ return target.substring(0, colonIndex);
+ }
+ return target;
+ }
+
+ /**
+ * Returns the name of the jar file from which the given class symbol was
+ * loaded, if available, and null otherwise. Implicitly filters out jars
+ * from the compilation bootclasspath.
+ * @param platformClasses classes on javac's bootclasspath
+ */
+ static String getJarName(ClassSymbol classSymbol, Set<JavaFileObject> platformClasses) {
+ if (classSymbol != null) {
+ // Ignore symbols that appear in the sourcepath:
+ if (haveSourceForSymbol(classSymbol)) {
+ return null;
+ }
+ JavaFileObject classfile = unwrapFileObject(classSymbol.classfile);
+ if (classfile instanceof ZipArchive.ZipFileObject
+ || classfile instanceof ZipFileIndexArchive.ZipFileIndexFileObject) {
+ String name = classfile.getName();
+ // Here name will be something like blaze-out/.../com/foo/libfoo.jar(Bar.class)
+ String jarName = name.split("\\(")[0];
+ if (!platformClasses.contains(classfile)) {
+ return jarName;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns true if the given classSymbol corresponds to one of the sources being compiled.
+ */
+ private static boolean haveSourceForSymbol(ClassSymbol classSymbol) {
+ if (classSymbol.sourcefile == null) {
+ return false;
+ }
+
+ try {
+ // The classreader uses metadata to populate the symbol's sourcefile with a fake file object.
+ // Call getLastModified() to check if it's a real file:
+ classSymbol.sourcefile.getLastModified();
+ } catch (UnsupportedOperationException e) {
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git a/src/java_tools/singlejar/BUILD b/src/java_tools/singlejar/BUILD
new file mode 100644
index 0000000..463c506
--- /dev/null
+++ b/src/java_tools/singlejar/BUILD
@@ -0,0 +1,32 @@
+package(default_visibility = ["//src:__pkg__"])
+
+java_library(
+ name = "libSingleJar",
+ srcs = glob(["java/**/*.java"]),
+ deps = [
+ "//src/main/java:shell",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
+
+java_binary(
+ name = "SingleJar",
+ main_class = "com.google.devtools.build.singlejar.SingleJar",
+ runtime_deps = [":libSingleJar"],
+)
+
+java_test(
+ name = "tests",
+ srcs = glob(["javatests/**/*.java"]),
+ args = ["com.google.devtools.build.singlejar.SingleJarTests"],
+ deps = [
+ ":libSingleJar",
+ "//src/main/java:shell",
+ "//src/test/java:testutil",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
diff --git a/src/java_tools/singlejar/README b/src/java_tools/singlejar/README
new file mode 100644
index 0000000..da92eb7
--- /dev/null
+++ b/src/java_tools/singlejar/README
@@ -0,0 +1,2 @@
+SingleJar is a tool used to combine multiple jar file into a single one. It is used by Bazel to
+build java binaries that are self-contained.
\ No newline at end of file
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ConcatenateStrategy.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ConcatenateStrategy.java
new file mode 100644
index 0000000..7dfb31f
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ConcatenateStrategy.java
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * A strategy that merges a set of files by concatenating them. This is used
+ * for services files. By default, this class automatically adds a newline
+ * character {@code '\n'} between files if the previous file did not end with one.
+ *
+ * <p>Note: automatically inserting newline characters differs from the
+ * original behavior. Use {@link #ConcatenateStrategy(boolean)} to turn this
+ * behavior off.
+ */
+@NotThreadSafe
+public final class ConcatenateStrategy implements CustomMergeStrategy {
+
+ // The strategy assumes that files are generally small. This is a first guess
+ // about the size of the files.
+ private static final int BUFFER_SIZE = 4096;
+
+ private final byte[] buffer = new byte[BUFFER_SIZE];
+ private byte lastByteCopied = '\n';
+ private final boolean appendNewLine;
+
+ ConcatenateStrategy() {
+ this(true);
+ }
+
+ /**
+ * @param appendNewLine Whether to add a newline character between files if
+ * the previous file did not end with one.
+ */
+ ConcatenateStrategy(boolean appendNewLine) {
+ this.appendNewLine = appendNewLine;
+ }
+
+ @Override
+ public void merge(InputStream in, OutputStream out) throws IOException {
+ if (appendNewLine && lastByteCopied != '\n') {
+ out.write('\n');
+ lastByteCopied = '\n';
+ }
+ int bytesRead;
+ while ((bytesRead = in.read(buffer)) != -1) {
+ out.write(buffer, 0, bytesRead);
+ lastByteCopied = buffer[bytesRead - 1];
+ }
+ }
+
+ @Override
+ public void finish(OutputStream out) {
+ // No need to do anything. All the data was already written.
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/CopyEntryFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/CopyEntryFilter.java
new file mode 100644
index 0000000..586c378
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/CopyEntryFilter.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import java.io.IOException;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A filter which invokes {@link StrategyCallback#copy} for every entry. As a
+ * result, the first entry for every given name is copied and further entries
+ * with the same name are skipped.
+ */
+@Immutable
+public final class CopyEntryFilter implements ZipEntryFilter {
+
+ @Override
+ public void accept(String filename, StrategyCallback callback) throws IOException {
+ callback.copy(null);
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/DefaultJarEntryFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/DefaultJarEntryFilter.java
new file mode 100644
index 0000000..fd26c60
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/DefaultJarEntryFilter.java
@@ -0,0 +1,119 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.jar.JarFile;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A default filter for JAR files. It merges all services files in the {@code META-INF/services/}
+ * directory. The original {@code MANIFEST} files are skipped, as are JAR signing files. Anything
+ * not in the supplied path filter, an arbitrary predicate, is also skipped. To use this filter
+ * properly, a new {@code MANIFEST} file should be explicitly added to the combined ZIP file.
+ */
+@Immutable
+public class DefaultJarEntryFilter implements ZipEntryFilter {
+
+ /** An interface to restrict which files are copied over and which are not. */
+ public static interface PathFilter {
+ /**
+ * Returns true if an entry with the given name may be copied over.
+ */
+ boolean allowed(String path);
+ }
+
+ /** A filter that allows any path. */
+ public static final PathFilter ANY_PATH = new PathFilter() {
+ @Override
+ public boolean allowed(String path) {
+ return true;
+ }
+ };
+
+ // ZIP timestamps have a resolution of 2 seconds, so this is the next timestamp after 1/1/1980.
+ // This is only Visible for testing.
+ static final Date DOS_EPOCH_PLUS_2_SECONDS =
+ new GregorianCalendar(1980, 0, 1, 0, 0, 2).getTime();
+
+ // Merge all files with a name in here:
+ private static final String SERVICES_DIR = "META-INF/services/";
+
+ // Merge all spring.handlers files.
+ private static final String SPRING_HANDLERS = "META-INF/spring.handlers";
+
+ // Merge all spring.schemas files.
+ private static final String SPRING_SCHEMAS = "META-INF/spring.schemas";
+
+ // Ignore all files with this name:
+ private static final String MANIFEST_NAME = JarFile.MANIFEST_NAME;
+
+ // Merge all protobuf extension registries.
+ private static final String PROTOBUF_META = "protobuf.meta";
+
+ protected final Date date;
+ protected final Date classDate;
+ protected PathFilter allowedPaths;
+
+ public DefaultJarEntryFilter(boolean normalize, PathFilter allowedPaths) {
+ this.date = normalize ? ZipCombiner.DOS_EPOCH : null;
+ this.classDate = normalize ? DOS_EPOCH_PLUS_2_SECONDS : null;
+ this.allowedPaths = allowedPaths;
+ }
+
+ public DefaultJarEntryFilter(boolean normalize) {
+ this(normalize, ANY_PATH);
+ }
+
+ public DefaultJarEntryFilter() {
+ this(true);
+ }
+
+ @Override
+ public void accept(String filename, StrategyCallback callback) throws IOException {
+ if (!allowedPaths.allowed(filename)) {
+ callback.skip();
+ } else if (filename.equals(SPRING_HANDLERS)) {
+ callback.customMerge(date, new ConcatenateStrategy());
+ } else if (filename.equals(SPRING_SCHEMAS)) {
+ callback.customMerge(date, new ConcatenateStrategy());
+ } else if (filename.startsWith(SERVICES_DIR)) {
+ // Merge all services files.
+ callback.customMerge(date, new ConcatenateStrategy());
+ } else if (filename.equals(MANIFEST_NAME) || filename.endsWith(".SF")
+ || filename.endsWith(".DSA") || filename.endsWith(".RSA")) {
+ // Ignore existing manifests and any .SF, .DSA or .RSA jar signing files.
+ // TODO(bazel-team): I think we should be stricter and only skip signing
+ // files from the META-INF/ directory.
+ callback.skip();
+ } else if (filename.endsWith(".class")) {
+ // Copy .class files over, but 2 seconds ahead of the dos epoch. If it finds both source and
+ // class files on the classpath, javac prefers the source file, if the class file is not newer
+ // than the source file. Since we normalize the timestamps, we need to provide timestamps for
+ // class files that are newer than those for the corresponding source files.
+ callback.copy(classDate);
+ } else if (filename.equals(PROTOBUF_META)) {
+ // Merge all protobuf meta data without inserting newlines,
+ // since the file is in protobuf binary format.
+ callback.customMerge(date, new ConcatenateStrategy(false));
+ } else {
+ // Copy all other files over.
+ callback.copy(date);
+ }
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java
new file mode 100644
index 0000000..2e8cb75
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+/**
+ * A holder class for extra data in a ZIP entry.
+ *
+ * <p>Note: This class performs no defensive copying of the byte array, so the
+ * byte array passed into this class or returned from this class may not be
+ * modified.
+ */
+final class ExtraData {
+
+ private final short id;
+ private final byte[] data;
+
+ public ExtraData(short id, byte[] data) {
+ this.id = id;
+ this.data = data;
+ }
+
+ public short getId() {
+ return id;
+ }
+
+ public byte[] getData() {
+ return data;
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java
new file mode 100644
index 0000000..a9c8ee3
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Provides utilities for using ZipCombiner to pack up Jar files.
+ */
+public final class JarUtils {
+ private static final String MANIFEST_DIRECTORY = "META-INF/";
+ private static final short MAGIC_JAR_ID = (short) 0xCAFE;
+ private static final ExtraData[] MAGIC_JAR_ID_EXTRA_ENTRIES =
+ new ExtraData[] { new ExtraData(MAGIC_JAR_ID, new byte[0]) };
+
+ /**
+ * Adds META-INF directory through ZipCombiner with the given date and the
+ * magic jar ID.
+ *
+ * @throws IOException if {@link ZipCombiner#addDirectory(String, Date, ExtraData[])}
+ * throws an IOException.
+ */
+ public static void addMetaInf(ZipCombiner combiner, Date date) throws IOException {
+ combiner.addDirectory(MANIFEST_DIRECTORY, date, MAGIC_JAR_ID_EXTRA_ENTRIES);
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java
new file mode 100644
index 0000000..0da6e33
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * An implementation based on java.io.
+ */
+public final class JavaIoFileSystem implements SimpleFileSystem {
+
+ @Override
+ public InputStream getInputStream(String filename) throws IOException {
+ return new FileInputStream(filename);
+ }
+
+ @Override
+ public OutputStream getOutputStream(String filename) throws IOException {
+ return new FileOutputStream(filename);
+ }
+
+ @Override
+ public boolean delete(String filename) {
+ return new File(filename).delete();
+ }
+}
\ No newline at end of file
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/OptionFileExpander.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/OptionFileExpander.java
new file mode 100644
index 0000000..dafeb9d
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/OptionFileExpander.java
@@ -0,0 +1,127 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A utility class to parse option files and expand them.
+ */
+@Immutable
+final class OptionFileExpander {
+
+ /**
+ * An interface that allows injecting different implementations for reading
+ * files. This is mostly used for testing.
+ */
+ interface OptionFileProvider {
+
+ /**
+ * Opens a file for reading and returns an input stream.
+ */
+ InputStream getInputStream(String filename) throws IOException;
+ }
+
+ private final OptionFileProvider fileSystem;
+
+ /**
+ * Creates an instance with the given option file provider.
+ */
+ public OptionFileExpander(OptionFileProvider fileSystem) {
+ this.fileSystem = fileSystem;
+ }
+
+ /**
+ * Pre-processes an argument list, expanding options of the form &at;filename
+ * to read in the content of the file and add it to the list of arguments.
+ *
+ * @param args the List of arguments to pre-process.
+ * @return the List of pre-processed arguments.
+ * @throws IOException if one of the files containing options cannot be read.
+ */
+ public List<String> expandArguments(List<String> args) throws IOException {
+ List<String> expanded = new ArrayList<>(args.size());
+ for (String arg : args) {
+ expandArgument(arg, expanded);
+ }
+ return expanded;
+ }
+
+ /**
+ * Expands a single argument, expanding options &at;filename to read in
+ * the content of the file and add it to the list of processed arguments.
+ *
+ * @param arg the argument to pre-process.
+ * @param expanded the List of pre-processed arguments.
+ * @throws IOException if one of the files containing options cannot be read.
+ */
+ private void expandArgument(String arg, List<String> expanded) throws IOException {
+ if (arg.startsWith("@")) {
+ InputStream in = fileSystem.getInputStream(arg.substring(1));
+ try {
+ // TODO(bazel-team): This code doesn't handle escaped newlines correctly.
+ // ShellUtils doesn't support them either.
+ for (String line : readAllLines(new InputStreamReader(in, ISO_8859_1))) {
+ List<String> parsedTokens = new ArrayList<>();
+ try {
+ ShellUtils.tokenize(parsedTokens, line);
+ } catch (TokenizationException e) {
+ throw new IOException("Could not tokenize parameter file!", e);
+ }
+ for (String token : parsedTokens) {
+ expandArgument(token, expanded);
+ }
+ }
+ InputStream inToClose = in;
+ in = null;
+ inToClose.close();
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ // Ignore the exception. It can only occur if an exception already
+ // happened and in that case, we want to preserve the original one.
+ }
+ }
+ }
+ } else {
+ expanded.add(arg);
+ }
+ }
+
+ private List<String> readAllLines(Reader in) throws IOException {
+ List<String> result = new ArrayList<>();
+ BufferedReader reader = new BufferedReader(in);
+ String line;
+ while ((line = reader.readLine()) != null) {
+ result.add(line);
+ }
+ return result;
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/PrefixListPathFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/PrefixListPathFilter.java
new file mode 100644
index 0000000..a6e30d4
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/PrefixListPathFilter.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import com.google.devtools.build.singlejar.DefaultJarEntryFilter.PathFilter;
+
+import java.util.List;
+
+/**
+ * A predicate used to filter jar entries according to a list of path prefixes.
+ */
+final class PrefixListPathFilter implements PathFilter {
+ private final List<String> prefixes;
+
+ public PrefixListPathFilter(List<String> prefixes) {
+ this.prefixes = prefixes;
+ }
+
+ @Override
+ public boolean allowed(String path) {
+ for (String prefix : prefixes) {
+ if (path.startsWith(prefix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java
new file mode 100644
index 0000000..844f12b
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import com.google.devtools.build.singlejar.OptionFileExpander.OptionFileProvider;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * A simple virtual file system interface. It's much simpler than the Blaze
+ * virtual file system and only to be used inside this package.
+ */
+public interface SimpleFileSystem extends OptionFileProvider {
+
+ @Override
+ InputStream getInputStream(String filename) throws IOException;
+
+ /**
+ * Opens a file for output and returns an output stream. If a file of that
+ * name already exists, it is overwritten.
+ */
+ OutputStream getOutputStream(String filename) throws IOException;
+
+ /**
+ * Delete the file with the given name and return whether deleting it was
+ * successfull.
+ */
+ boolean delete(String filename);
+}
\ No newline at end of file
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java
new file mode 100644
index 0000000..4551fd1
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java
@@ -0,0 +1,401 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import com.google.devtools.build.singlejar.DefaultJarEntryFilter.PathFilter;
+import com.google.devtools.build.singlejar.ZipCombiner.OutputMode;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * An application that emulates the existing SingleJar tool, using the {@link
+ * ZipCombiner} class.
+ */
+@NotThreadSafe
+public class SingleJar {
+
+ private static final byte NEWLINE_BYTE = (byte) '\n';
+ private static final String MANIFEST_FILENAME = JarFile.MANIFEST_NAME;
+ private static final String BUILD_DATA_FILENAME = "build-data.properties";
+
+ private final SimpleFileSystem fileSystem;
+
+ /** The input jar files we want to combine into the output jar. */
+ private final List<String> inputJars = new ArrayList<>();
+
+ /** Additional resources to be added to the output jar. */
+ private final List<String> resources = new ArrayList<>();
+
+ /** Additional class path resources to be added to the output jar. */
+ private final List<String> classpathResources = new ArrayList<>();
+
+ /** The name of the output Jar file. */
+ private String outputJar;
+
+ /** A filter for what jar entries to include */
+ private PathFilter allowedPaths = DefaultJarEntryFilter.ANY_PATH;
+
+ /** Extra manifest contents. */
+ private String extraManifestContent;
+ /** The main class - this is put into the manifest and also into the build info. */
+ private String mainClass;
+
+ /**
+ * Warn about duplicate resource files, and skip them. Default behavior is to
+ * give an error message.
+ */
+ private boolean warnDuplicateFiles = false;
+
+ /** Indicates whether to set all timestamps to a fixed value. */
+ private boolean normalize = false;
+ private OutputMode outputMode = OutputMode.FORCE_STORED;
+
+ /** Whether to include build-data.properties file */
+ protected boolean includeBuildData = true;
+
+ /** List of build information properties files */
+ protected List<String> buildInformationFiles = new ArrayList<String>();
+
+ /** Extraneous build informations (key=value) */
+ protected List<String> buildInformations = new ArrayList<String>();
+
+ /** The (optional) native executable that will be prepended to this JAR. */
+ private String launcherBin = null;
+
+ // Only visible for testing.
+ protected SingleJar(SimpleFileSystem fileSystem) {
+ this.fileSystem = fileSystem;
+ }
+
+ /**
+ * Creates a manifest and returns an input stream for its contents.
+ */
+ private InputStream createManifest() throws IOException {
+ Manifest manifest = new Manifest();
+ Attributes attributes = manifest.getMainAttributes();
+ attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ attributes.put(new Attributes.Name("Created-By"), "blaze-singlejar");
+ if (mainClass != null) {
+ attributes.put(Attributes.Name.MAIN_CLASS, mainClass);
+ }
+ if (extraManifestContent != null) {
+ ByteArrayInputStream in = new ByteArrayInputStream(extraManifestContent.getBytes("UTF8"));
+ manifest.read(in);
+ }
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ manifest.write(out);
+ return new ByteArrayInputStream(out.toByteArray());
+ }
+
+ private InputStream createBuildData() throws IOException {
+ Properties properties = mergeBuildData();
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ properties.store(outputStream, null);
+ byte[] output = outputStream.toByteArray();
+ // Properties#store() adds a timestamp comment as first line, delete it.
+ output = stripFirstLine(output);
+ return new ByteArrayInputStream(output);
+ }
+
+ static byte[] stripFirstLine(byte[] output) {
+ int i = 0;
+ while (i < output.length && output[i] != NEWLINE_BYTE) {
+ i++;
+ }
+ if (i < output.length) {
+ output = Arrays.copyOfRange(output, i + 1, output.length);
+ } else {
+ output = new byte[0];
+ }
+ return output;
+ }
+
+ private Properties mergeBuildData() throws IOException {
+ Properties properties = new Properties();
+ for (String fileName : buildInformationFiles) {
+ InputStream file = fileSystem.getInputStream(fileName);
+ if (file != null) {
+ properties.load(file);
+ }
+ }
+
+ // extra properties
+ for (String info : buildInformations) {
+ String[] split = info.split("=", 2);
+ String key = split[0];
+ String value = "";
+ if (split.length > 1) {
+ value = split[1];
+ }
+ properties.put(key, value);
+ }
+
+ // finally add generic information
+ // TODO(bazel-team) do we need to resolve the path to be absolute or canonical?
+ properties.put("build.target", outputJar);
+ if (mainClass != null) {
+ properties.put("main.class", mainClass);
+ }
+ return properties;
+ }
+
+ private String getName(String filename) {
+ int index = filename.lastIndexOf('/');
+ return index < 0 ? filename : filename.substring(index + 1);
+ }
+
+ // Only visible for testing.
+ protected int run(List<String> args) throws IOException {
+ List<String> expandedArgs = new OptionFileExpander(fileSystem).expandArguments(args);
+ processCommandlineArgs(expandedArgs);
+ InputStream buildInfo = createBuildData();
+
+ ZipCombiner combiner = null;
+ try {
+ combiner = new ZipCombiner(outputMode, createEntryFilter(normalize, allowedPaths),
+ fileSystem.getOutputStream(outputJar));
+ if (launcherBin != null) {
+ combiner.prependExecutable(fileSystem.getInputStream(launcherBin));
+ }
+ Date date = normalize ? ZipCombiner.DOS_EPOCH : null;
+
+ // Add a manifest file.
+ JarUtils.addMetaInf(combiner, date);
+ combiner.addFile(MANIFEST_FILENAME, date, createManifest());
+
+ if (includeBuildData) {
+ // Add the build data file.
+ combiner.addFile(BUILD_DATA_FILENAME, date, buildInfo);
+ }
+
+ // Copy the resources to the top level of the jar file.
+ for (String classpathResource : classpathResources) {
+ String entryName = getName(classpathResource);
+ if (warnDuplicateFiles && combiner.containsFile(entryName)) {
+ System.err.println("File " + entryName + " clashes with a previous file");
+ continue;
+ }
+ combiner.addFile(entryName, date, fileSystem.getInputStream(classpathResource));
+ }
+
+ // Copy the resources into the jar file.
+ for (String resource : resources) {
+ String from, to;
+ int i = resource.indexOf(':');
+ if (i < 0) {
+ to = from = resource;
+ } else {
+ from = resource.substring(0, i);
+ to = resource.substring(i + 1);
+ }
+ if (warnDuplicateFiles && combiner.containsFile(to)) {
+ System.err.println("File " + from + " at " + to + " clashes with a previous file");
+ continue;
+ }
+ combiner.addFile(to, date, fileSystem.getInputStream(from));
+ }
+
+ // Copy the jars into the jar file.
+ for (String inputJar : inputJars) {
+ InputStream in = fileSystem.getInputStream(inputJar);
+ try {
+ combiner.addZip(inputJar, in);
+ InputStream inToClose = in;
+ in = null;
+ inToClose.close();
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ // Preserve original exception.
+ }
+ }
+ }
+ }
+
+ // Close the output file. If something goes wrong here, delete the file.
+ combiner.close();
+ combiner = null;
+ } finally {
+ // This part is only executed if an exception occurred.
+ if (combiner != null) {
+ try {
+ // We may end up calling close twice, but that's ok.
+ combiner.close();
+ } catch (IOException e) {
+ // There's already an exception in progress - this won't add any
+ // additional information.
+ }
+ // Ignore return value - there's already an exception in progress.
+ fileSystem.delete(outputJar);
+ }
+ }
+ return 0;
+ }
+
+ protected ZipEntryFilter createEntryFilter(boolean normalize, PathFilter allowedPaths) {
+ return new DefaultJarEntryFilter(normalize, allowedPaths);
+ }
+
+ /**
+ * Collects the arguments for a command line flag until it finds a flag that
+ * starts with the terminatorPrefix.
+ *
+ * @param args
+ * @param startIndex the start index in the args to collect the flag arguments
+ * from
+ * @param flagArguments the collected flag arguments
+ * @param terminatorPrefix the terminator prefix to stop collecting of
+ * argument flags
+ * @return the index of the first argument that started with the
+ * terminatorPrefix
+ */
+ private static int collectFlagArguments(List<String> args, int startIndex,
+ List<String> flagArguments, String terminatorPrefix) {
+ startIndex++;
+ while (startIndex < args.size()) {
+ String name = args.get(startIndex);
+ if (name.startsWith(terminatorPrefix)) {
+ return startIndex - 1;
+ }
+ flagArguments.add(name);
+ startIndex++;
+ }
+ return startIndex;
+ }
+
+ /**
+ * Returns a single argument for a command line option.
+ *
+ * @throws IOException if no more arguments are available
+ */
+ private static String getArgument(List<String> args, int i, String arg) throws IOException {
+ if (i + 1 < args.size()) {
+ return args.get(i + 1);
+ }
+ throw new IOException(arg + ": missing argument");
+ }
+
+ /**
+ * Processes the command line arguments.
+ *
+ * @throws IOException if one of the files containing options cannot be read
+ */
+ protected void processCommandlineArgs(List<String> args) throws IOException {
+ List<String> manifestLines = new ArrayList<>();
+ List<String> prefixes = new ArrayList<>();
+ for (int i = 0; i < args.size(); i++) {
+ String arg = args.get(i);
+ if (arg.equals("--sources")) {
+ i = collectFlagArguments(args, i, inputJars, "--");
+ } else if (arg.equals("--resources")) {
+ i = collectFlagArguments(args, i, resources, "--");
+ } else if (arg.equals("--classpath_resources")) {
+ i = collectFlagArguments(args, i, classpathResources, "--");
+ } else if (arg.equals("--deploy_manifest_lines")) {
+ i = collectFlagArguments(args, i, manifestLines, "--");
+ } else if (arg.equals("--build_info_file")) {
+ buildInformationFiles.add(getArgument(args, i, arg));
+ i++;
+ } else if (arg.equals("--extra_build_info")) {
+ buildInformations.add(getArgument(args, i, arg));
+ i++;
+ } else if (arg.equals("--main_class")) {
+ mainClass = getArgument(args, i, arg);
+ i++;
+ } else if (arg.equals("--output")) {
+ outputJar = getArgument(args, i, arg);
+ i++;
+ } else if (arg.equals("--compression")) {
+ outputMode = OutputMode.FORCE_DEFLATE;
+ } else if (arg.equals("--dont_change_compression")) {
+ outputMode = OutputMode.DONT_CARE;
+ } else if (arg.equals("--normalize")) {
+ normalize = true;
+ } else if (arg.equals("--include_prefixes")) {
+ i = collectFlagArguments(args, i, prefixes, "--");
+ } else if (arg.equals("--exclude_build_data")) {
+ includeBuildData = false;
+ } else if (arg.equals("--warn_duplicate_resources")) {
+ warnDuplicateFiles = true;
+ } else if (arg.equals("--java_launcher")) {
+ launcherBin = getArgument(args, i, arg);
+ i++;
+ } else {
+ throw new IOException("unknown option : '" + arg + "'");
+ }
+ }
+ if (!manifestLines.isEmpty()) {
+ setExtraManifestContent(joinWithNewlines(manifestLines));
+ }
+ if (!prefixes.isEmpty()) {
+ setPathPrefixes(prefixes);
+ }
+ }
+
+ private String joinWithNewlines(Iterable<String> lines) {
+ StringBuilder result = new StringBuilder();
+ Iterator<String> it = lines.iterator();
+ if (it.hasNext()) {
+ result.append(it.next());
+ }
+ while (it.hasNext()) {
+ result.append('\n');
+ result.append(it.next());
+ }
+ return result.toString();
+ }
+
+ private void setExtraManifestContent(String extraManifestContent) {
+ // The manifest content has to be terminated with a newline character
+ if (!extraManifestContent.endsWith("\n")) {
+ extraManifestContent = extraManifestContent + '\n';
+ }
+ this.extraManifestContent = extraManifestContent;
+ }
+
+ private void setPathPrefixes(List<String> prefixes) throws IOException {
+ if (prefixes.isEmpty()) {
+ throw new IOException(
+ "Empty set of path prefixes; cowardly refusing to emit an empty jar file");
+ }
+ allowedPaths = new PrefixListPathFilter(prefixes);
+ }
+
+ public static void main(String[] args) {
+ try {
+ SingleJar singlejar = new SingleJar(new JavaIoFileSystem());
+ System.exit(singlejar.run(Arrays.asList(args)));
+ } catch (IOException e) {
+ System.err.println("SingleJar threw exception : " + e.getMessage());
+ System.exit(1);
+ }
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java
new file mode 100644
index 0000000..d38c6d4
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java
@@ -0,0 +1,1643 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy;
+import com.google.devtools.build.singlejar.ZipEntryFilter.StrategyCallback;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.CRC32;
+import java.util.zip.DataFormatException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * An object that combines multiple ZIP files into a single file. It only
+ * supports a subset of the ZIP format, specifically:
+ * <ul>
+ * <li>It only supports STORE and DEFLATE storage methods.</li>
+ * <li>There may be no data before the first file or between files.</li>
+ * <li>It ignores any data after the last file.</li>
+ * </ul>
+ *
+ * <p>These restrictions are also present in the JDK implementations
+ * {@link java.util.jar.JarInputStream}, {@link java.util.zip.ZipInputStream},
+ * though they are not documented there.
+ *
+ * <p>IMPORTANT NOTE: Callers must call {@link #finish()} or {@link #close()}
+ * at the end of processing to ensure that the output buffers are flushed and
+ * the ZIP file is complete.
+ *
+ * <p>This class performs only rudimentary data checking. If the input files
+ * are damaged, the output will likely also be damaged.
+ *
+ * <p>Also see:
+ * <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP format</a>
+ */
+@NotThreadSafe
+public final class ZipCombiner implements AutoCloseable {
+
+ /**
+ * A Date set to the 1/1/1980, 00:00:00, the minimum value that can be stored
+ * in a ZIP file.
+ */
+ public static final Date DOS_EPOCH = new GregorianCalendar(1980, 0, 1, 0, 0, 0).getTime();
+
+ private static final int DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE = 1048576; // 1 MB for each block
+
+ // The following constants are ZIP-specific.
+ private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50;
+ private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50;
+ private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50;
+ private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50;
+
+ private static final int FILE_HEADER_BUFFER_SIZE = 30;
+
+ private static final int VERSION_TO_EXTRACT_OFFSET = 4;
+ private static final int GENERAL_PURPOSE_FLAGS_OFFSET = 6;
+ private static final int COMPRESSION_METHOD_OFFSET = 8;
+ private static final int MTIME_OFFSET = 10;
+ private static final int MDATE_OFFSET = 12;
+ private static final int CRC32_OFFSET = 14;
+ private static final int COMPRESSED_SIZE_OFFSET = 18;
+ private static final int UNCOMPRESSED_SIZE_OFFSET = 22;
+ private static final int FILENAME_LENGTH_OFFSET = 26;
+ private static final int EXTRA_LENGTH_OFFSET = 28;
+
+ private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 46;
+
+ // Set if the size, compressed size and CRC are set to zero, and present in
+ // the data descriptor after the data.
+ private static final int SIZE_MASKED_FLAG = 1 << 3;
+
+ private static final int STORED_METHOD = 0;
+ private static final int DEFLATE_METHOD = 8;
+
+ private static final int VERSION_STORED = 10; // Version 1.0
+ private static final int VERSION_DEFLATE = 20; // Version 2.0
+
+ private static final long MAXIMUM_DATA_SIZE = 0xffffffffL;
+
+ // This class relies on the buffer to have sufficient space for a complete
+ // file name. 2^16 is the maximum number of bytes in a file name.
+ private static final int BUFFER_SIZE = 65536;
+
+ /** An empty entry used to skip files that have already been copied (or skipped). */
+ private static final FileEntry COPIED_FILE_ENTRY = new FileEntry(null, null, 0);
+
+ /** An empty entry used to mark files that have already been renamed. */
+ private static final FileEntry RENAMED_FILE_ENTRY = new FileEntry(null, null, 0);
+
+ /** A zero length array of ExtraData. */
+ public static final ExtraData[] NO_EXTRA_ENTRIES = new ExtraData[0];
+
+ /**
+ * Whether to compress or decompress entries.
+ */
+ public enum OutputMode {
+
+ /**
+ * Output entries using any method.
+ */
+ DONT_CARE,
+
+ /**
+ * Output all entries using DEFLATE method, except directory entries. It is
+ * always more efficient to store directory entries uncompressed.
+ */
+ FORCE_DEFLATE,
+
+ /**
+ * Output all entries using STORED method.
+ */
+ FORCE_STORED;
+ }
+
+ // A two-element enum for copyOrSkip type methods.
+ private static enum SkipMode {
+
+ /**
+ * Copy the read data to the output stream.
+ */
+ COPY,
+
+ /**
+ * Do not write anything to the output stream.
+ */
+ SKIP;
+ }
+
+ /**
+ * Stores internal information about merges or skips.
+ */
+ private static final class FileEntry {
+
+ /** If null, the file should be skipped. Otherwise, it should be merged. */
+ private final CustomMergeStrategy mergeStrategy;
+ private final ByteArrayOutputStream outputBuffer;
+ private final int dosTime;
+
+ private FileEntry(CustomMergeStrategy mergeStrategy, ByteArrayOutputStream outputBuffer,
+ int dosTime) {
+ this.mergeStrategy = mergeStrategy;
+ this.outputBuffer = outputBuffer;
+ this.dosTime = dosTime;
+ }
+ }
+
+ /**
+ * The directory entry info used for files whose extra directory entry info is not given
+ * explicitly. It uses {@code -1} for {@link DirectoryEntryInfo#withMadeByVersion(short)}, which
+ * indicates it will be set to the same version as "needed to extract."
+ *
+ * <p>The {@link DirectoryEntryInfo#withExternalFileAttribute(int)} value is set to {@code 0},
+ * whose meaning depends on the value of {@code madeByVersion}, but is usually a reasonable
+ * default.
+ */
+ public static final DirectoryEntryInfo DEFAULT_DIRECTORY_ENTRY_INFO =
+ new DirectoryEntryInfo((short) -1, 0);
+
+ /**
+ * Contains information related to a zip entry that is stored in the central directory record.
+ * This does not contain all the information stored in the central directory record, only the
+ * information that can be customized and is not automatically calculated or detected.
+ */
+ public static final class DirectoryEntryInfo {
+ private final short madeByVersion;
+ private final int externalFileAttribute;
+
+ private DirectoryEntryInfo(short madeByVersion, int externalFileAttribute) {
+ this.madeByVersion = madeByVersion;
+ this.externalFileAttribute = externalFileAttribute;
+ }
+
+ /**
+ * This will be written as "made by" version in the central directory.
+ * If -1 (default) then "made by" will be the same to version "needed to extract".
+ */
+ public DirectoryEntryInfo withMadeByVersion(short madeByVersion) {
+ return new DirectoryEntryInfo(madeByVersion, externalFileAttribute);
+ }
+
+ /**
+ * This will be written as external file attribute. The meaning of this depends upon the value
+ * set with {@link #withMadeByVersion(short)}. If that value indicates a Unix source, then this
+ * value has the file mode and permission bits in the upper two bytes (e.g. possibly
+ * {@code 0100644} for a regular file).
+ */
+ public DirectoryEntryInfo withExternalFileAttribute(int externalFileAttribute) {
+ return new DirectoryEntryInfo(madeByVersion, externalFileAttribute);
+ }
+ }
+
+ /**
+ * The central directory, which is grown as required; instead of using a single large buffer, we
+ * store a sequence of smaller buffers. With a single large buffer, whenever we grow the buffer by
+ * 2x, we end up requiring 3x the memory temporarily, which can lead to OOM problems even if there
+ * would still be enough memory.
+ *
+ * <p>The invariants for the fields are as follows:
+ * <ul>
+ * <li>All blocks must have the same size.
+ * <li>The list of blocks must contain all blocks, including the current block (even if empty).
+ * <li>The current block offset must apply to the last block in the list, which is
+ * simultaneously the current block.
+ * <li>The current block may only be {@code null} if the list is empty.
+ * </ul>
+ */
+ private static final class CentralDirectory {
+ private final int blockSize; // We allow this to be overridden for testing.
+ private List<byte[]> blockList = new ArrayList<>();
+ private byte[] currentBlock;
+ private int currentBlockOffset = 0;
+ private int size = 0;
+
+ CentralDirectory(int centralDirectoryBlockSize) {
+ this.blockSize = centralDirectoryBlockSize;
+ }
+
+ /**
+ * Appends the given data to the central directory and returns the start
+ * offset within the central directory to allow back-patching.
+ */
+ int writeToCentralDirectory(byte[] b, int off, int len) {
+ checkArgument(len >= 0);
+ int offsetStarted = size;
+ while (len > 0) {
+ if (currentBlock == null
+ || currentBlockOffset >= currentBlock.length) {
+ currentBlock = new byte[blockSize];
+ currentBlockOffset = 0;
+ blockList.add(currentBlock);
+ }
+ int maxCopy = Math.min(blockSize - currentBlockOffset, len);
+ System.arraycopy(b, off, currentBlock, currentBlockOffset, maxCopy);
+ off += maxCopy;
+ len -= maxCopy;
+ size += maxCopy;
+ currentBlockOffset += maxCopy;
+ }
+ return offsetStarted;
+ }
+
+ /** Calls through to {@link #writeToCentralDirectory(byte[], int, int)}. */
+ int writeToCentralDirectory(byte[] b) {
+ return writeToCentralDirectory(b, 0, b.length);
+ }
+
+ /**
+ * Writes an unsigned int in little-endian byte order to the central directory at the
+ * given offset. Does not perform range checking.
+ */
+ void setUnsignedInt(int offset, int value) {
+ blockList.get(cdIndex(offset + 0))[cdOffset(offset + 0)] = (byte) (value & 0xff);
+ blockList.get(cdIndex(offset + 1))[cdOffset(offset + 1)] = (byte) ((value >> 8) & 0xff);
+ blockList.get(cdIndex(offset + 2))[cdOffset(offset + 2)] = (byte) ((value >> 16) & 0xff);
+ blockList.get(cdIndex(offset + 3))[cdOffset(offset + 3)] = (byte) ((value >> 24) & 0xff);
+ }
+
+ private int cdIndex(int offset) {
+ return offset / blockSize;
+ }
+
+ private int cdOffset(int offset) {
+ return offset % blockSize;
+ }
+
+ /**
+ * Writes the central directory to the given output stream and returns the size, i.e., the
+ * number of bytes written.
+ */
+ int writeTo(OutputStream out) throws IOException {
+ for (int i = 0; i < blockList.size() - 1; i++) {
+ out.write(blockList.get(i));
+ }
+ if (currentBlock != null) {
+ out.write(currentBlock, 0, currentBlockOffset);
+ }
+ return size;
+ }
+ }
+
+ /**
+ * An output stream that counts how many bytes were written.
+ */
+ private static final class ByteCountingOutputStream extends FilterOutputStream {
+ private long bytesWritten = 0L;
+
+ ByteCountingOutputStream(OutputStream out) {
+ super(out);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ bytesWritten += len;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ bytesWritten++;
+ }
+ }
+
+ private final OutputMode mode;
+ private final ZipEntryFilter entryFilter;
+
+ private final ByteCountingOutputStream out;
+
+ // An input buffer to allow reading blocks of data. Keeping it here avoids
+ // another copy operation that would be required by the BufferedInputStream.
+ // The valid data is between bufferOffset and bufferOffset+bufferLength (exclusive).
+ private final byte[] buffer = new byte[BUFFER_SIZE];
+ private int bufferOffset = 0;
+ private int bufferLength = 0;
+
+ private String currentInputFile;
+
+ // An intermediate buffer for the file header data. Keeping it here avoids
+ // creating a new buffer for every entry.
+ private final byte[] headerBuffer = new byte[FILE_HEADER_BUFFER_SIZE];
+
+ // An intermediate buffer for a central directory entry. Keeping it here
+ // avoids creating a new buffer for every entry.
+ private final byte[] directoryEntryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE];
+
+ // The Inflater is a class member to avoid creating a new instance for every
+ // entry in the ZIP file.
+ private final Inflater inflater = new Inflater(true);
+
+ // The contents of this buffer are never read. The Inflater is only used to
+ // determine the length of the compressed data, and the buffer is a throw-
+ // away buffer for the decompressed data.
+ private final byte[] inflaterBuffer = new byte[BUFFER_SIZE];
+
+ private final Map<String, FileEntry> fileNames = new HashMap<>();
+
+ private final CentralDirectory centralDirectory;
+ private int fileCount = 0;
+
+ private boolean finished = false;
+
+ // Package private for testing.
+ ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out,
+ int centralDirectoryBlockSize) {
+ this.mode = mode;
+ this.entryFilter = entryFilter;
+ this.out = new ByteCountingOutputStream(new BufferedOutputStream(out));
+ this.centralDirectory = new CentralDirectory(centralDirectoryBlockSize);
+ }
+
+ /**
+ * Creates a new instance with the given parameters. The {@code entryFilter}
+ * is called for every entry in the ZIP files and the combined ZIP file is
+ * written to {@code out}. The output mode determines whether entries must be
+ * written in compressed or decompressed form. Note that the result is
+ * invalid if an exception is thrown from any of the methods in this class,
+ * and before a call to {@link #close} or {@link #finish}.
+ */
+ public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) {
+ this(mode, entryFilter, out, DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE);
+ }
+
+ /**
+ * Creates a new instance with the given parameters and the DONT_CARE mode.
+ */
+ public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) {
+ this(OutputMode.DONT_CARE, entryFilter, out);
+ }
+
+ /**
+ * Creates a new instance with the {@link CopyEntryFilter} as the filter and
+ * the given mode and output stream.
+ */
+ public ZipCombiner(OutputMode mode, OutputStream out) {
+ this(mode, new CopyEntryFilter(), out);
+ }
+
+ /**
+ * Creates a new instance with the {@link CopyEntryFilter} as the filter, the
+ * DONT_CARE mode and the given output stream.
+ */
+ public ZipCombiner(OutputStream out) {
+ this(OutputMode.DONT_CARE, new CopyEntryFilter(), out);
+ }
+
+ /**
+ * Returns whether the output zip already contains a file or directory with
+ * the given name.
+ */
+ public boolean containsFile(String filename) {
+ return fileNames.containsKey(filename);
+ }
+
+ /**
+ * Makes a write call to the output stream, and updates the current offset.
+ */
+ private void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ }
+
+ /** Calls through to {@link #write(byte[], int, int)}. */
+ private void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ /**
+ * Reads at least one more byte into the internal buffer. This method must
+ * only be called when more data is necessary to correctly decode the ZIP
+ * format.
+ *
+ * <p>This method automatically compacts the existing data in the buffer by
+ * moving it to the beginning of the buffer.
+ *
+ * @throws EOFException if no more data is available from the input stream
+ * @throws IOException if the underlying stream throws one
+ */
+ private void readMoreData(InputStream in) throws IOException {
+ if ((bufferLength > 0) && (bufferOffset > 0)) {
+ System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength);
+ }
+ if (bufferLength >= buffer.length) {
+ // The buffer size is specifically chosen to avoid this situation.
+ throw new AssertionError("Internal error: buffer overrun.");
+ }
+ bufferOffset = 0;
+ int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength);
+ if (bytesRead <= 0) {
+ throw new EOFException();
+ }
+ bufferLength += bytesRead;
+ }
+
+ /**
+ * Reads data until the buffer is filled with at least {@code length} bytes.
+ *
+ * @throws IllegalArgumentException if not 0 <= length <= buffer.length
+ * @throws IOException if the underlying input stream throws one or the end
+ * of the input stream is reached before the required
+ * number of bytes is read
+ */
+ private void readFully(InputStream in, int length) throws IOException {
+ checkArgument(length >= 0, "length too small: %s", length);
+ checkArgument(length <= buffer.length, "length too large: %s", length);
+ while (bufferLength < length) {
+ readMoreData(in);
+ }
+ }
+
+ /**
+ * Reads an unsigned short in little-endian byte order from the buffer at the
+ * given offset. Does not perform range checking.
+ */
+ private int getUnsignedShort(byte[] source, int offset) {
+ int a = source[offset + 0] & 0xff;
+ int b = source[offset + 1] & 0xff;
+ return (b << 8) | a;
+ }
+
+ /**
+ * Reads an unsigned int in little-endian byte order from the buffer at the
+ * given offset. Does not perform range checking.
+ */
+ private long getUnsignedInt(byte[] source, int offset) {
+ int a = source[offset + 0] & 0xff;
+ int b = source[offset + 1] & 0xff;
+ int c = source[offset + 2] & 0xff;
+ int d = source[offset + 3] & 0xff;
+ return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL;
+ }
+
+ /**
+ * Writes an unsigned short in little-endian byte order to the buffer at the
+ * given offset. Does not perform range checking.
+ */
+ private void setUnsignedShort(byte[] target, int offset, short value) {
+ target[offset + 0] = (byte) (value & 0xff);
+ target[offset + 1] = (byte) ((value >> 8) & 0xff);
+ }
+
+ /**
+ * Writes an unsigned int in little-endian byte order to the buffer at the
+ * given offset. Does not perform range checking.
+ */
+ private void setUnsignedInt(byte[] target, int offset, int value) {
+ target[offset + 0] = (byte) (value & 0xff);
+ target[offset + 1] = (byte) ((value >> 8) & 0xff);
+ target[offset + 2] = (byte) ((value >> 16) & 0xff);
+ target[offset + 3] = (byte) ((value >> 24) & 0xff);
+ }
+
+ /**
+ * Copies or skips {@code length} amount of bytes from the input stream to the
+ * output stream. If the internal buffer is not empty, those bytes are copied
+ * first. When the method returns, there may be more bytes remaining in the
+ * buffer.
+ *
+ * @throws IOException if the underlying stream throws one
+ */
+ private void copyOrSkipData(InputStream in, long length, SkipMode skip) throws IOException {
+ checkArgument(length >= 0);
+ while (length > 0) {
+ if (bufferLength == 0) {
+ readMoreData(in);
+ }
+ int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength;
+ if (skip == SkipMode.COPY) {
+ write(buffer, bufferOffset, bytesToWrite);
+ }
+ bufferOffset += bytesToWrite;
+ bufferLength -= bytesToWrite;
+ length -= bytesToWrite;
+ }
+ }
+
+ /**
+ * Copies or skips {@code length} amount of bytes from the input stream to the
+ * output stream. If the internal buffer is not empty, those bytes are copied
+ * first. When the method returns, there may be more bytes remaining in the
+ * buffer. In addition to writing to the output stream, it also writes to the
+ * central directory.
+ *
+ * @throws IOException if the underlying stream throws one
+ */
+ private void forkOrSkipData(InputStream in, long length, SkipMode skip) throws IOException {
+ checkArgument(length >= 0);
+ while (length > 0) {
+ if (bufferLength == 0) {
+ readMoreData(in);
+ }
+ int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength;
+ if (skip == SkipMode.COPY) {
+ write(buffer, bufferOffset, bytesToWrite);
+ centralDirectory.writeToCentralDirectory(buffer, bufferOffset, bytesToWrite);
+ }
+ bufferOffset += bytesToWrite;
+ bufferLength -= bytesToWrite;
+ length -= bytesToWrite;
+ }
+ }
+
+ /**
+ * A mutable integer reference value to allow returning two values from a
+ * method.
+ */
+ private static class MutableInt {
+
+ private int value;
+
+ MutableInt(int initialValue) {
+ this.value = initialValue;
+ }
+
+ public void setValue(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+ }
+
+ /**
+ * Uses the inflater to decompress some data into the given buffer. This
+ * method performs no error checking on the input parameters and also does
+ * not update the buffer parameters of the input buffer (such as bufferOffset
+ * and bufferLength). It's only here to avoid code duplication.
+ *
+ * <p>The Inflater may not be in the finished state when this method is
+ * called.
+ *
+ * <p>This method returns 0 if it read data and reached the end of the
+ * DEFLATE stream without producing output. In that case, {@link
+ * Inflater#finished} is guaranteed to return true.
+ *
+ * @throws IOException if the underlying stream throws an IOException or if
+ * illegal data is encountered
+ */
+ private int inflateData(InputStream in, byte[] dest, int off, int len, MutableInt consumed)
+ throws IOException {
+ // Defend against Inflater.finished() returning true.
+ consumed.setValue(0);
+ int bytesProduced = 0;
+ int bytesConsumed = 0;
+ while ((bytesProduced == 0) && !inflater.finished()) {
+ inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed);
+ int remainingBefore = inflater.getRemaining();
+ try {
+ bytesProduced = inflater.inflate(dest, off, len);
+ } catch (DataFormatException e) {
+ throw new IOException("Invalid deflate stream in ZIP file.", e);
+ }
+ bytesConsumed += remainingBefore - inflater.getRemaining();
+ consumed.setValue(bytesConsumed);
+ if (bytesProduced == 0) {
+ if (inflater.needsDictionary()) {
+ // The DEFLATE algorithm as used in the ZIP file format does not
+ // require an additional dictionary.
+ throw new AssertionError("Inflater unexpectedly requires a dictionary.");
+ } else if (inflater.needsInput()) {
+ readMoreData(in);
+ } else if (inflater.finished()) {
+ return 0;
+ } else {
+ // According to the Inflater specification, this cannot happen.
+ throw new AssertionError("Inflater unexpectedly produced no output.");
+ }
+ }
+ }
+ return bytesProduced;
+ }
+
+ /**
+ * Copies or skips data from the input stream to the output stream. To
+ * determine the length of the data, the data is decompressed with the
+ * DEFLATE algorithm, which stores the length implicitly as part of the
+ * compressed data, using a combination of end markers and length indicators.
+ *
+ * @see <a href="http://www.ietf.org/rfc/rfc1951.txt">RFC 1951</a>
+ *
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private long copyOrSkipDeflateData(InputStream in, SkipMode skip) throws IOException {
+ long bytesCopied = 0;
+ inflater.reset();
+ MutableInt consumedBytes = new MutableInt(0);
+ while (!inflater.finished()) {
+ // Neither the uncompressed data nor the length of it is used. The
+ // decompression is only required to determine the correct length of the
+ // compressed data to copy.
+ inflateData(in, inflaterBuffer, 0, inflaterBuffer.length, consumedBytes);
+ int bytesRead = consumedBytes.getValue();
+ if (skip == SkipMode.COPY) {
+ write(buffer, bufferOffset, bytesRead);
+ }
+ bufferOffset += bytesRead;
+ bufferLength -= bytesRead;
+ bytesCopied += bytesRead;
+ }
+ return bytesCopied;
+ }
+
+ /**
+ * Returns a 32-bit integer containing a ZIP-compatible encoding of the given
+ * date. Only dates between 1980 and 2107 (inclusive) are supported.
+ *
+ * <p>The upper 16 bits contain the year, month, and day. The lower 16 bits
+ * contain the hour, minute, and second. The resolution of the second field
+ * is only 4 bits, which means that the only even second values can be
+ * stored - this method rounds down to the nearest even value.
+ *
+ * @throws IllegalArgumentException if the given date is outside the
+ * supported range
+ */
+ // Only visible for testing.
+ static int dateToDosTime(Date date) {
+ Calendar calendar = new GregorianCalendar();
+ calendar.setTime(date);
+ int year = calendar.get(Calendar.YEAR);
+ if (year < 1980) {
+ throw new IllegalArgumentException("date must be in or after 1980");
+ }
+ // The ZIP format only provides 7 bits for the year.
+ if (year > 2107) {
+ throw new IllegalArgumentException("date must before 2107");
+ }
+ int month = calendar.get(Calendar.MONTH) + 1; // Months from Calendar are zero-based.
+ int day = calendar.get(Calendar.DAY_OF_MONTH);
+ int hour = calendar.get(Calendar.HOUR_OF_DAY);
+ int minute = calendar.get(Calendar.MINUTE);
+ int second = calendar.get(Calendar.SECOND);
+ return ((year - 1980) << 25) | (month << 21) | (day << 16)
+ | (hour << 11) | (minute << 5) | (second >> 1);
+ }
+
+ /**
+ * Fills the directory entry, using the information from the header buffer,
+ * and writes it to the central directory. It returns the offset into the
+ * central directory that can be used for patching the entry. Requires that
+ * the entire entry header is present in {@link #headerBuffer}. It also uses
+ * the {@link ByteCountingOutputStream#bytesWritten}, so it must be called
+ * just before the header is written to the output stream.
+ *
+ * @throws IOException if the current offset is too large for the ZIP format
+ */
+ private int fillDirectoryEntryBuffer(
+ DirectoryEntryInfo directoryEntryInfo) throws IOException {
+ // central file header signature
+ setUnsignedInt(directoryEntryBuffer, 0, CENTRAL_DIRECTORY_MARKER);
+ short version = (short) getUnsignedShort(headerBuffer, VERSION_TO_EXTRACT_OFFSET);
+ short curMadeMyVersion = (directoryEntryInfo.madeByVersion == -1)
+ ? version : directoryEntryInfo.madeByVersion;
+ setUnsignedShort(directoryEntryBuffer, 4, curMadeMyVersion); // version made by
+ // version needed to extract
+ setUnsignedShort(directoryEntryBuffer, 6, version);
+ // general purpose bit flag
+ setUnsignedShort(directoryEntryBuffer, 8,
+ (short) getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET));
+ // compression method
+ setUnsignedShort(directoryEntryBuffer, 10,
+ (short) getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET));
+ // last mod file time, last mod file date
+ setUnsignedShort(directoryEntryBuffer, 12,
+ (short) getUnsignedShort(headerBuffer, MTIME_OFFSET));
+ setUnsignedShort(directoryEntryBuffer, 14,
+ (short) getUnsignedShort(headerBuffer, MDATE_OFFSET));
+ // crc-32
+ setUnsignedInt(directoryEntryBuffer, 16, (int) getUnsignedInt(headerBuffer, CRC32_OFFSET));
+ // compressed size
+ setUnsignedInt(directoryEntryBuffer, 20,
+ (int) getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET));
+ // uncompressed size
+ setUnsignedInt(directoryEntryBuffer, 24,
+ (int) getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET));
+ // file name length
+ setUnsignedShort(directoryEntryBuffer, 28,
+ (short) getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET));
+ // extra field length
+ setUnsignedShort(directoryEntryBuffer, 30,
+ (short) getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET));
+ setUnsignedShort(directoryEntryBuffer, 32, (short) 0); // file comment length
+ setUnsignedShort(directoryEntryBuffer, 34, (short) 0); // disk number start
+ setUnsignedShort(directoryEntryBuffer, 36, (short) 0); // internal file attributes
+ setUnsignedInt(directoryEntryBuffer, 38, directoryEntryInfo.externalFileAttribute);
+ if (out.bytesWritten >= MAXIMUM_DATA_SIZE) {
+ throw new IOException("Unable to handle files bigger than 2^32 bytes.");
+ }
+ // relative offset of local header
+ setUnsignedInt(directoryEntryBuffer, 42, (int) out.bytesWritten);
+ fileCount++;
+ return centralDirectory.writeToCentralDirectory(directoryEntryBuffer);
+ }
+
+ /**
+ * Fix the directory entry with the correct crc32, compressed size, and
+ * uncompressed size.
+ */
+ private void fixDirectoryEntry(int offset, long crc32, long compressedSize,
+ long uncompressedSize) {
+ // The constants from the top don't apply here, because this is the central directory entry.
+ centralDirectory.setUnsignedInt(offset + 16, (int) crc32); // crc-32
+ centralDirectory.setUnsignedInt(offset + 20, (int) compressedSize); // compressed size
+ centralDirectory.setUnsignedInt(offset + 24, (int) uncompressedSize); // uncompressed size
+ }
+
+ /**
+ * (Un)Compresses and copies the current ZIP file entry. Requires that the
+ * entire entry header is present in {@link #headerBuffer}. It currently
+ * drops the extra data in the process.
+ *
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private void modifyAndCopyEntry(String filename, InputStream in, int dosTime)
+ throws IOException {
+ final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET);
+ final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET);
+ final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET);
+ final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET);
+ // TODO(bazel-team): Read and copy the extra data if present.
+
+ forkOrSkipData(in, fileNameLength, SkipMode.SKIP);
+ forkOrSkipData(in, extraFieldLength, SkipMode.SKIP);
+ if (method == STORED_METHOD) {
+ long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET);
+ copyStreamToEntry(filename, new FixedLengthInputStream(in, compressedSize), dosTime,
+ NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO);
+ } else if (method == DEFLATE_METHOD) {
+ inflater.reset();
+ copyStreamToEntry(filename, new DeflateInputStream(in), dosTime, NO_EXTRA_ENTRIES, false,
+ DEFAULT_DIRECTORY_ENTRY_INFO);
+ if ((flags & SIZE_MASKED_FLAG) != 0) {
+ copyOrSkipData(in, 16, SkipMode.SKIP);
+ }
+ } else {
+ throw new AssertionError("This should have been checked in validateHeader().");
+ }
+ }
+
+ /**
+ * Copies or skips the current ZIP file entry. Requires that the entire entry
+ * header is present in {@link #headerBuffer}. It uses the current mode to
+ * decide whether to compress or decompress the entry.
+ *
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date,
+ DirectoryEntryInfo directoryEntryInfo) throws IOException {
+ copyOrSkipEntry(filename, in, skip, date, directoryEntryInfo, false);
+ }
+
+ /**
+ * Renames and otherwise copies the current ZIP file entry. Requires that the entire
+ * entry header is present in {@link #headerBuffer}. It uses the current mode to
+ * decide whether to compress or decompress the entry.
+ *
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private void renameEntry(String filename, InputStream in, Date date,
+ DirectoryEntryInfo directoryEntryInfo) throws IOException {
+ copyOrSkipEntry(filename, in, SkipMode.COPY, date, directoryEntryInfo, true);
+ }
+
+ /**
+ * Copies or skips the current ZIP file entry. Requires that the entire entry
+ * header is present in {@link #headerBuffer}. It uses the current mode to
+ * decide whether to compress or decompress the entry.
+ *
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date,
+ DirectoryEntryInfo directoryEntryInfo, boolean rename) throws IOException {
+ final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET);
+
+ // We can cast here, because the result is only treated as a bitmask.
+ int dosTime = date == null ? (int) getUnsignedInt(headerBuffer, MTIME_OFFSET)
+ : dateToDosTime(date);
+ if (skip == SkipMode.COPY) {
+ if ((mode == OutputMode.FORCE_DEFLATE) && (method == STORED_METHOD)
+ && !filename.endsWith("/")) {
+ modifyAndCopyEntry(filename, in, dosTime);
+ return;
+ } else if ((mode == OutputMode.FORCE_STORED) && (method == DEFLATE_METHOD)) {
+ modifyAndCopyEntry(filename, in, dosTime);
+ return;
+ }
+ }
+
+ int directoryOffset = copyOrSkipEntryHeader(filename, in, date, directoryEntryInfo,
+ skip, rename);
+
+ copyOrSkipEntryData(filename, in, skip, directoryOffset);
+ }
+
+ /**
+ * Copies or skips the header of an entry, including filename and extra data.
+ * Requires that the entire entry header is present in {@link #headerBuffer}.
+ *
+ * @returns the enrty offset in the central directory
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private int copyOrSkipEntryHeader(String filename, InputStream in, Date date,
+ DirectoryEntryInfo directoryEntryInfo, SkipMode skip, boolean rename)
+ throws IOException {
+ final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET);
+ final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET);
+
+ byte[] fileNameAsBytes = null;
+ if (rename) {
+ // If the entry is renamed, we patch the filename length in the buffer
+ // before it's copied, and before writing to the central directory.
+ fileNameAsBytes = filename.getBytes(UTF_8);
+ checkArgument(fileNameAsBytes.length <= 65535,
+ "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length);
+ setUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET, (short) fileNameAsBytes.length);
+ }
+
+ int directoryOffset = 0;
+ if (skip == SkipMode.COPY) {
+ if (date != null) {
+ int dosTime = dateToDosTime(date);
+ setUnsignedShort(headerBuffer, MTIME_OFFSET, (short) dosTime); // lower 16 bits
+ setUnsignedShort(headerBuffer, MDATE_OFFSET, (short) (dosTime >> 16)); // upper 16 bits
+ }
+ // Call this before writing the data out, so that we get the correct offset.
+ directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo);
+ write(headerBuffer, 0, FILE_HEADER_BUFFER_SIZE);
+ }
+ if (!rename) {
+ forkOrSkipData(in, fileNameLength, skip);
+ } else {
+ forkOrSkipData(in, fileNameLength, SkipMode.SKIP);
+ write(fileNameAsBytes);
+ centralDirectory.writeToCentralDirectory(fileNameAsBytes);
+ }
+ forkOrSkipData(in, extraFieldLength, skip);
+ return directoryOffset;
+ }
+
+ /**
+ * Copy or skip the data of an entry. Requires that the
+ * entire entry header is present in {@link #headerBuffer}.
+ *
+ * @throws IOException if the underlying stream throws an IOException
+ */
+ private void copyOrSkipEntryData(String filename, InputStream in, SkipMode skip,
+ int directoryOffset) throws IOException {
+ final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET);
+ final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET);
+ if ((flags & SIZE_MASKED_FLAG) != 0) {
+ // The compressed data size is unknown.
+ if (method != DEFLATE_METHOD) {
+ throw new AssertionError("This should have been checked in validateHeader().");
+ }
+ copyOrSkipDeflateData(in, skip);
+ // The flags indicate that a data descriptor must follow the data.
+ readFully(in, 16);
+ if (getUnsignedInt(buffer, bufferOffset) != DATA_DESCRIPTOR_MARKER) {
+ throw new IOException("Missing data descriptor for " + filename + " in " + currentInputFile
+ + ".");
+ }
+ long crc32 = getUnsignedInt(buffer, bufferOffset + 4);
+ long compressedSize = getUnsignedInt(buffer, bufferOffset + 8);
+ long uncompressedSize = getUnsignedInt(buffer, bufferOffset + 12);
+ if (skip == SkipMode.COPY) {
+ fixDirectoryEntry(directoryOffset, crc32, compressedSize, uncompressedSize);
+ }
+ copyOrSkipData(in, 16, skip);
+ } else {
+ // The size value is present in the header, so just copy that amount.
+ long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET);
+ copyOrSkipData(in, compressedSize, skip);
+ }
+ }
+
+ /**
+ * An input stream that reads a fixed number of bytes from the given input
+ * stream before it returns end-of-input. It uses the local buffer, so it
+ * can't be static.
+ */
+ private class FixedLengthInputStream extends InputStream {
+
+ private final InputStream in;
+ private long remainingBytes;
+ private final byte[] singleByteBuffer = new byte[1];
+
+ FixedLengthInputStream(InputStream in, long remainingBytes) {
+ this.in = in;
+ this.remainingBytes = remainingBytes;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int bytesRead = read(singleByteBuffer, 0, 1);
+ return (bytesRead == -1) ? -1 : singleByteBuffer[0];
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ checkArgument(len >= 0);
+ checkArgument(off >= 0);
+ checkArgument(off + len <= b.length);
+ if (remainingBytes == 0) {
+ return -1;
+ }
+ if (bufferLength == 0) {
+ readMoreData(in);
+ }
+ int bytesToCopy = len;
+ if (remainingBytes < bytesToCopy) {
+ bytesToCopy = (int) remainingBytes;
+ }
+ if (bufferLength < bytesToCopy) {
+ bytesToCopy = bufferLength;
+ }
+ System.arraycopy(buffer, bufferOffset, b, off, bytesToCopy);
+ bufferOffset += bytesToCopy;
+ bufferLength -= bytesToCopy;
+ remainingBytes -= bytesToCopy;
+ return bytesToCopy;
+ }
+ }
+
+ /**
+ * An input stream that reads from a given input stream, decoding that data
+ * according to the DEFLATE algorithm. The DEFLATE data stream implicitly
+ * contains its own end-of-input marker. It uses the local buffer, so it
+ * can't be static.
+ */
+ private class DeflateInputStream extends InputStream {
+
+ private final InputStream in;
+ private final byte[] singleByteBuffer = new byte[1];
+ private final MutableInt consumedBytes = new MutableInt(0);
+
+ DeflateInputStream(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int bytesRead = read(singleByteBuffer, 0, 1);
+ // Do an unsigned cast on the byte from the buffer if it exists.
+ return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff);
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (inflater.finished()) {
+ return -1;
+ }
+ int length = inflateData(in, b, off, len, consumedBytes);
+ int bytesRead = consumedBytes.getValue();
+ bufferOffset += bytesRead;
+ bufferLength -= bytesRead;
+ return length == 0 ? -1 : length;
+ }
+ }
+
+ /**
+ * Handles a custom merge operation with the given strategy. This method
+ * creates an appropriate input stream and hands it to the strategy for
+ * processing. Requires that the entire entry header is present in {@link
+ * #headerBuffer}.
+ *
+ * @throws IOException if one of the underlying stream throws an IOException,
+ * if the ZIP entry data is inconsistent, or if the
+ * implementation cannot handle the compression method
+ * given in the ZIP entry
+ */
+ private void handleCustomMerge(final InputStream in, CustomMergeStrategy mergeStrategy,
+ ByteArrayOutputStream outputBuffer) throws IOException {
+ final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET);
+ final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET);
+ final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET);
+
+ final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET);
+ final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET);
+
+ copyOrSkipData(in, fileNameLength, SkipMode.SKIP);
+ copyOrSkipData(in, extraFieldLength, SkipMode.SKIP);
+ if (method == STORED_METHOD) {
+ mergeStrategy.merge(new FixedLengthInputStream(in, compressedSize), outputBuffer);
+ } else if (method == DEFLATE_METHOD) {
+ inflater.reset();
+ // TODO(bazel-team): Defend against the mergeStrategy not reading the complete input.
+ mergeStrategy.merge(new DeflateInputStream(in), outputBuffer);
+ if ((flags & SIZE_MASKED_FLAG) != 0) {
+ copyOrSkipData(in, 16, SkipMode.SKIP);
+ }
+ } else {
+ throw new AssertionError("This should have been checked in validateHeader().");
+ }
+ }
+
+ /**
+ * Implementation of the strategy callback.
+ */
+ private class TheStrategyCallback implements StrategyCallback {
+
+ private String filename;
+ private final InputStream in;
+
+ // Use an atomic boolean to make sure that only a single call goes
+ // through, even if there are multiple concurrent calls. Paranoid
+ // defensive programming.
+ private final AtomicBoolean callDone = new AtomicBoolean();
+
+ TheStrategyCallback(String filename, InputStream in) {
+ this.filename = filename;
+ this.in = in;
+ }
+
+ // Verify that this is the first call and throw an exception if not.
+ private void checkCall() {
+ checkState(callDone.compareAndSet(false, true), "The callback was already called once.");
+ }
+
+ @Override
+ public void copy(Date date) throws IOException {
+ checkCall();
+ if (!containsFile(filename)) {
+ fileNames.put(filename, COPIED_FILE_ENTRY);
+ copyOrSkipEntry(filename, in, SkipMode.COPY, date, DEFAULT_DIRECTORY_ENTRY_INFO);
+ } else { // can't copy, name already used for renamed entry
+ copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+ }
+
+ @Override
+ public void rename(String newName, Date date) throws IOException {
+ checkCall();
+ if (!containsFile(newName)) {
+ fileNames.put(newName, RENAMED_FILE_ENTRY);
+ renameEntry(newName, in, date, DEFAULT_DIRECTORY_ENTRY_INFO);
+ } else {
+ copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+ filename = newName;
+ }
+
+ @Override
+ public void skip() throws IOException {
+ checkCall();
+ if (!containsFile(filename)) {// don't overwrite possible RENAMED_FILE_ENTRY value
+ fileNames.put(filename, COPIED_FILE_ENTRY);
+ }
+ copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+
+ @Override
+ public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException {
+ checkCall();
+ ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream();
+ fileNames.put(filename, new FileEntry(strategy, outputBuffer, dateToDosTime(date)));
+ handleCustomMerge(in, strategy, outputBuffer);
+ }
+ }
+
+ /**
+ * Validates that the current entry obeys all the restrictions of this implementation.
+ *
+ * @throws IOException if the current entry doesn't obey the restrictions
+ */
+ private void validateHeader() throws IOException {
+ // We only handle DEFLATE and STORED, like java.util.zip.
+ final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET);
+ if ((method != DEFLATE_METHOD) && (method != STORED_METHOD)) {
+ throw new IOException("Unable to handle compression methods other than DEFLATE!");
+ }
+
+ // If the method is STORED, then the size must be available in the header.
+ final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET);
+ if ((method == STORED_METHOD) && ((flags & SIZE_MASKED_FLAG) != 0)) {
+ throw new IOException("If the method is STORED, then the size must be available in the"
+ + " header!");
+ }
+
+ // If the method is STORED, the compressed and uncompressed sizes must be equal.
+ final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET);
+ final long uncompressedSize = getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET);
+ if ((method == STORED_METHOD) && (compressedSize != uncompressedSize)) {
+ throw new IOException("Compressed and uncompressed sizes for STORED entry differ!");
+ }
+
+ // The compressed or uncompressed size being set to 0xffffffff is a strong indicator that the
+ // ZIP file is in ZIP64 mode, which supports files larger than 2^32.
+ // TODO(bazel-team): Support the ZIP64 extension.
+ if ((compressedSize == MAXIMUM_DATA_SIZE) || (uncompressedSize == MAXIMUM_DATA_SIZE)) {
+ throw new IOException("Unable to handle ZIP64 compressed files.");
+ }
+ }
+
+ /**
+ * Reads a file entry from the input stream, calls the entryFilter to
+ * determine what to do with the entry, and performs the requested operation.
+ * Returns true if the input stream contained another entry.
+ *
+ * @throws IOException if one of the underlying stream throws an IOException,
+ * if the ZIP contains unsupported, inconsistent or
+ * incomplete data or if the filter throws an IOException
+ */
+ private boolean handleNextEntry(final InputStream in) throws IOException {
+ // Just try to read the complete header and fail if it didn't work.
+ try {
+ readFully(in, FILE_HEADER_BUFFER_SIZE);
+ } catch (EOFException e) {
+ return false;
+ }
+
+ System.arraycopy(buffer, bufferOffset, headerBuffer, 0, FILE_HEADER_BUFFER_SIZE);
+ bufferOffset += FILE_HEADER_BUFFER_SIZE;
+ bufferLength -= FILE_HEADER_BUFFER_SIZE;
+ if (getUnsignedInt(headerBuffer, 0) != LOCAL_FILE_HEADER_MARKER) {
+ return false;
+ }
+ validateHeader();
+
+ final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET);
+ readFully(in, fileNameLength);
+ // TODO(bazel-team): If I read the spec correctly, this should be UTF-8 rather than ISO-8859-1.
+ final String filename = new String(buffer, bufferOffset, fileNameLength, ISO_8859_1);
+
+ FileEntry handler = fileNames.get(filename);
+ // The handler is null if this is the first time we see an entry with this filename,
+ // or if all previous entries with this name were renamed by the filter (and we can
+ // pretend we didn't encounter the name yet).
+ // If the handler is RENAMED_FILE_ENTRY, a previous entry was renamed as filename,
+ // in which case the filter should now be invoked for this name for the first time,
+ // giving the filter a chance to choose an unique name.
+ if (handler == null || handler == RENAMED_FILE_ENTRY) {
+ TheStrategyCallback callback = new TheStrategyCallback(filename, in);
+ entryFilter.accept(filename, callback);
+ if (fileNames.get(callback.filename) == null && fileNames.get(filename) == null) {
+ throw new IllegalStateException();
+ }
+ } else if (handler.mergeStrategy == null) {
+ copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO);
+ } else {
+ handleCustomMerge(in, handler.mergeStrategy, handler.outputBuffer);
+ }
+ return true;
+ }
+
+ /**
+ * Clears the internal buffer.
+ */
+ private void clearBuffer() {
+ bufferOffset = 0;
+ bufferLength = 0;
+ }
+
+ /**
+ * Copies another ZIP file into the output. If multiple entries with the same
+ * name are present, the first such entry is copied, but the others are
+ * ignored. This is also true for multiple invocations of this method. The
+ * {@code inputName} parameter is used to provide better error messages in the
+ * case of a failure to decode the ZIP file.
+ *
+ * @throws IOException if one of the underlying stream throws an IOException,
+ * if the ZIP contains unsupported, inconsistent or
+ * incomplete data or if the filter throws an IOException
+ */
+ public void addZip(String inputName, InputStream in) throws IOException {
+ if (finished) {
+ throw new IllegalStateException();
+ }
+ if (in == null) {
+ throw new NullPointerException();
+ }
+ clearBuffer();
+ currentInputFile = inputName;
+ while (handleNextEntry(in)) {/*handleNextEntry has side-effect.*/}
+ }
+
+ public void addZip(InputStream in) throws IOException {
+ addZip(null, in);
+ }
+
+ private void copyStreamToEntry(String filename, InputStream in, int dosTime,
+ ExtraData[] extraDataEntries, boolean compress, DirectoryEntryInfo directoryEntryInfo)
+ throws IOException {
+ fileNames.put(filename, COPIED_FILE_ENTRY);
+
+ byte[] fileNameAsBytes = filename.getBytes(UTF_8);
+ checkArgument(fileNameAsBytes.length <= 65535,
+ "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length);
+
+ // Note: This method can be called with an input stream that uses the buffer field of this
+ // class. We use a local buffer here to avoid conflicts.
+ byte[] localBuffer = new byte[4096];
+
+ byte[] uncompressedData = null;
+ if (!compress) {
+ ByteArrayOutputStream temp = new ByteArrayOutputStream();
+ int bytesRead;
+ while ((bytesRead = in.read(localBuffer)) != -1) {
+ temp.write(localBuffer, 0, bytesRead);
+ }
+ uncompressedData = temp.toByteArray();
+ }
+ byte[] extraData = null;
+ if (extraDataEntries.length != 0) {
+ int totalLength = 0;
+ for (ExtraData extra : extraDataEntries) {
+ int length = extra.getData().length;
+ if (totalLength > 0xffff - 4 - length) {
+ throw new IOException("Total length of extra data too big.");
+ }
+ totalLength += length + 4;
+ }
+ extraData = new byte[totalLength];
+ int position = 0;
+ for (ExtraData extra : extraDataEntries) {
+ byte[] data = extra.getData();
+ setUnsignedShort(extraData, position + 0, extra.getId());
+ setUnsignedShort(extraData, position + 2, (short) data.length);
+ System.arraycopy(data, 0, extraData, position + 4, data.length);
+ position += data.length + 4;
+ }
+ }
+
+ // write header
+ Arrays.fill(headerBuffer, (byte) 0);
+ setUnsignedInt(headerBuffer, 0, LOCAL_FILE_HEADER_MARKER); // file header signature
+ if (compress) {
+ setUnsignedShort(headerBuffer, 4, (short) VERSION_DEFLATE); // version to extract
+ setUnsignedShort(headerBuffer, 6, (short) SIZE_MASKED_FLAG); // general purpose bit flag
+ setUnsignedShort(headerBuffer, 8, (short) DEFLATE_METHOD); // compression method
+ } else {
+ setUnsignedShort(headerBuffer, 4, (short) VERSION_STORED); // version to extract
+ setUnsignedShort(headerBuffer, 6, (short) 0); // general purpose bit flag
+ setUnsignedShort(headerBuffer, 8, (short) STORED_METHOD); // compression method
+ }
+ setUnsignedShort(headerBuffer, 10, (short) dosTime); // mtime
+ setUnsignedShort(headerBuffer, 12, (short) (dosTime >> 16)); // mdate
+ if (uncompressedData != null) {
+ CRC32 crc = new CRC32();
+ crc.update(uncompressedData);
+ setUnsignedInt(headerBuffer, 14, (int) crc.getValue()); // crc32
+ setUnsignedInt(headerBuffer, 18, uncompressedData.length); // compressed size
+ setUnsignedInt(headerBuffer, 22, uncompressedData.length); // uncompressed size
+ } else {
+ setUnsignedInt(headerBuffer, 14, 0); // crc32
+ setUnsignedInt(headerBuffer, 18, 0); // compressed size
+ setUnsignedInt(headerBuffer, 22, 0); // uncompressed size
+ }
+ setUnsignedShort(headerBuffer, 26, (short) fileNameAsBytes.length); // file name length
+ if (extraData != null) {
+ setUnsignedShort(headerBuffer, 28, (short) extraData.length); // extra field length
+ } else {
+ setUnsignedShort(headerBuffer, 28, (short) 0); // extra field length
+ }
+
+ // This call works for both compressed or uncompressed entries.
+ int directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo);
+ write(headerBuffer);
+ write(fileNameAsBytes);
+ centralDirectory.writeToCentralDirectory(fileNameAsBytes);
+ if (extraData != null) {
+ write(extraData);
+ centralDirectory.writeToCentralDirectory(extraData);
+ }
+
+ // write data
+ if (uncompressedData != null) {
+ write(uncompressedData);
+ } else {
+ try (DeflaterOutputStream deflaterStream = new DeflaterOutputStream()) {
+ int bytesRead;
+ while ((bytesRead = in.read(localBuffer)) != -1) {
+ deflaterStream.write(localBuffer, 0, bytesRead);
+ }
+ deflaterStream.finish();
+
+ // write data descriptor
+ Arrays.fill(headerBuffer, (byte) 0);
+ setUnsignedInt(headerBuffer, 0, DATA_DESCRIPTOR_MARKER);
+ setUnsignedInt(headerBuffer, 4, deflaterStream.getCRC()); // crc32
+ setUnsignedInt(headerBuffer, 8, deflaterStream.getCompressedSize()); // compressed size
+ setUnsignedInt(headerBuffer, 12, deflaterStream.getUncompressedSize()); // uncompressed size
+ write(headerBuffer, 0, 16);
+ fixDirectoryEntry(directoryOffset, deflaterStream.getCRC(),
+ deflaterStream.getCompressedSize(), deflaterStream.getUncompressedSize());
+ }
+ }
+ }
+
+ /**
+ * Adds a new entry into the output, by reading the input stream until it
+ * returns end of stream. Equivalent to
+ * {@link #addFile(String, Date, InputStream, DirectoryEntryInfo)}, but uses
+ * {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the file's directory entry.
+ */
+ public void addFile(String filename, Date date, InputStream in) throws IOException {
+ addFile(filename, date, in, DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+
+ /**
+ * Adds a new entry into the output, by reading the input stream until it
+ * returns end of stream. This method does not call {@link
+ * ZipEntryFilter#accept}.
+ *
+ * @throws IOException if one of the underlying streams throws an IOException
+ * or if the input stream returns more data than
+ * supported by the ZIP format
+ * @throws IllegalStateException if an entry with the given name already
+ * exists
+ * @throws IllegalArgumentException if the given file name is longer than
+ * supported by the ZIP format
+ */
+ public void addFile(String filename, Date date, InputStream in,
+ DirectoryEntryInfo directoryEntryInfo) throws IOException {
+ checkNotFinished();
+ if (in == null) {
+ throw new NullPointerException();
+ }
+ if (filename == null) {
+ throw new NullPointerException();
+ }
+ checkState(!fileNames.containsKey(filename),
+ "jar already contains a file named %s", filename);
+ int dosTime = dateToDosTime(date != null ? date : new Date());
+ copyStreamToEntry(filename, in, dosTime, NO_EXTRA_ENTRIES,
+ mode != OutputMode.FORCE_STORED, // Always compress if we're allowed to.
+ directoryEntryInfo);
+ }
+
+ /**
+ * Adds a new directory entry into the output. This method does not call
+ * {@link ZipEntryFilter#accept}. Uses {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the added
+ * directory entry.
+ *
+ * @throws IOException if one of the underlying streams throws an IOException
+ * @throws IllegalStateException if an entry with the given name already
+ * exists
+ * @throws IllegalArgumentException if the given file name is longer than
+ * supported by the ZIP format
+ */
+ public void addDirectory(String filename, Date date, ExtraData[] extraDataEntries)
+ throws IOException {
+ checkNotFinished();
+ checkArgument(filename.endsWith("/")); // Can also throw NPE.
+ checkState(!fileNames.containsKey(filename),
+ "jar already contains a directory named %s", filename);
+ int dosTime = dateToDosTime(date != null ? date : new Date());
+ copyStreamToEntry(filename, new ByteArrayInputStream(new byte[0]), dosTime, extraDataEntries,
+ false, // Never compress directory entries.
+ DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+
+ /**
+ * Adds a new directory entry into the output. This method does not call
+ * {@link ZipEntryFilter#accept}.
+ *
+ * @throws IOException if one of the underlying streams throws an IOException
+ * @throws IllegalStateException if an entry with the given name already
+ * exists
+ * @throws IllegalArgumentException if the given file name is longer than
+ * supported by the ZIP format
+ */
+ public void addDirectory(String filename, Date date)
+ throws IOException {
+ addDirectory(filename, date, NO_EXTRA_ENTRIES);
+ }
+
+ /**
+ * A deflater output stream that also counts uncompressed and compressed
+ * numbers of bytes and computes the CRC so that the data descriptor marker
+ * is written correctly.
+ *
+ * <p>Not static, so it can access the write() methods.
+ */
+ private class DeflaterOutputStream extends OutputStream {
+
+ private final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true);
+ private final CRC32 crc = new CRC32();
+ private final byte[] outputBuffer = new byte[4096];
+ private long uncompressedBytes = 0;
+ private long compressedBytes = 0;
+
+ @Override
+ public void write(int b) throws IOException {
+ byte[] buf = new byte[] { (byte) (b & 0xff) };
+ write(buf, 0, buf.length);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ checkNotFinished();
+ uncompressedBytes += len;
+ crc.update(b, off, len);
+ deflater.setInput(b, off, len);
+ while (!deflater.needsInput()) {
+ deflate();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ deflater.end();
+ }
+
+ /**
+ * Writes out the remaining buffered data without closing the output
+ * stream.
+ */
+ public void finish() throws IOException {
+ checkNotFinished();
+ deflater.finish();
+ while (!deflater.finished()) {
+ deflate();
+ }
+ if ((compressedBytes >= MAXIMUM_DATA_SIZE) || (uncompressedBytes >= MAXIMUM_DATA_SIZE)) {
+ throw new IOException("Too much data for ZIP entry.");
+ }
+ }
+
+ private void deflate() throws IOException {
+ int length = deflater.deflate(outputBuffer);
+ ZipCombiner.this.write(outputBuffer, 0, length);
+ compressedBytes += length;
+ }
+
+ public int getCRC() {
+ return (int) crc.getValue();
+ }
+
+ public int getCompressedSize() {
+ return (int) compressedBytes;
+ }
+
+ public int getUncompressedSize() {
+ return (int) uncompressedBytes;
+ }
+
+ private void checkNotFinished() {
+ if (deflater.finished()) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ /**
+ * Writes any remaining output data to the output stream and also creates the
+ * merged entries by calling the {@link CustomMergeStrategy} implementations
+ * given back from the ZIP entry filter.
+ *
+ * @throws IOException if the output stream or the filter throws an
+ * IOException
+ * @throws IllegalStateException if this method was already called earlier
+ */
+ public void finish() throws IOException {
+ checkNotFinished();
+ finished = true;
+ for (Map.Entry<String, FileEntry> entry : fileNames.entrySet()) {
+ String filename = entry.getKey();
+ CustomMergeStrategy mergeStrategy = entry.getValue().mergeStrategy;
+ ByteArrayOutputStream outputBuffer = entry.getValue().outputBuffer;
+ int dosTime = entry.getValue().dosTime;
+ if (mergeStrategy == null) {
+ // Do nothing.
+ } else {
+ mergeStrategy.finish(outputBuffer);
+ copyStreamToEntry(filename, new ByteArrayInputStream(outputBuffer.toByteArray()), dosTime,
+ NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+ }
+
+ // Write central directory.
+ if (out.bytesWritten >= MAXIMUM_DATA_SIZE) {
+ throw new IOException("Unable to handle files bigger than 2^32 bytes.");
+ }
+ int startOfCentralDirectory = (int) out.bytesWritten;
+ int centralDirectorySize = centralDirectory.writeTo(out);
+
+ // end of central directory signature
+ setUnsignedInt(directoryEntryBuffer, 0, END_OF_CENTRAL_DIRECTORY_MARKER);
+ // number of this disk
+ setUnsignedShort(directoryEntryBuffer, 4, (short) 0);
+ // number of the disk with the start of the central directory
+ setUnsignedShort(directoryEntryBuffer, 6, (short) 0);
+ // total number of entries in the central directory on this disk
+ setUnsignedShort(directoryEntryBuffer, 8, (short) fileCount);
+ // total number of entries in the central directory
+ setUnsignedShort(directoryEntryBuffer, 10, (short) fileCount);
+ // size of the central directory
+ setUnsignedInt(directoryEntryBuffer, 12, centralDirectorySize);
+ // offset of start of central directory with respect to the starting disk number
+ setUnsignedInt(directoryEntryBuffer, 16, startOfCentralDirectory);
+ // .ZIP file comment length
+ setUnsignedShort(directoryEntryBuffer, 20, (short) 0);
+ write(directoryEntryBuffer, 0, 22);
+
+ out.flush();
+ }
+
+ private void checkNotFinished() {
+ if (finished) {
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Writes any remaining output data to the output stream and closes it.
+ *
+ * @throws IOException if the output stream or the filter throws an
+ * IOException
+ */
+ @Override
+ public void close() throws IOException {
+ if (!finished) {
+ finish();
+ }
+ out.close();
+ }
+
+ /**
+ * Turns this JAR file into an executable JAR by prepending an executable.
+ * JAR files are placed at the end of a file, and executables are placed
+ * at the beginning, so a file can be both, if desired.
+ *
+ * @param launcherIn The InputStream, from which the launcher is read.
+ * @throws NullPointerException if launcherIn is null
+ * @throws IOException if reading from launcherIn or writing to the output
+ * stream throws an IOException.
+ */
+ public void prependExecutable(InputStream launcherIn) throws IOException {
+ if (launcherIn == null) {
+ throw new NullPointerException("No launcher specified");
+ }
+ byte[] buf = new byte[BUFFER_SIZE];
+ int bytesRead;
+ while ((bytesRead = launcherIn.read(buf)) > 0) {
+ out.write(buf, 0, bytesRead);
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ */
+ private static void checkArgument(boolean expression,
+ @Nullable String errorMessageTemplate,
+ @Nullable Object... errorMessageArgs) {
+ if (!expression) {
+ throw new IllegalArgumentException(String.format(errorMessageTemplate, errorMessageArgs));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ */
+ private static void checkArgument(boolean expression) {
+ if (!expression) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving state.
+ */
+ private static void checkState(boolean expression,
+ @Nullable String errorMessageTemplate,
+ @Nullable Object... errorMessageArgs) {
+ if (!expression) {
+ throw new IllegalStateException(String.format(errorMessageTemplate, errorMessageArgs));
+ }
+ }
+}
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipEntryFilter.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipEntryFilter.java
new file mode 100644
index 0000000..ab5a24a
--- /dev/null
+++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipEntryFilter.java
@@ -0,0 +1,119 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Date;
+
+/**
+ * A custom filter for entries when combining multiple ZIP files (or even just
+ * copying a single ZIP file).
+ *
+ * <p>Implementations of this interface must be thread-safe. The {@link
+ * #accept} method may be called concurrently by multiple threads.
+ */
+public interface ZipEntryFilter {
+
+ /**
+ * Strategy for a custom merge operation. The current file and all additional
+ * file are passed to the strategy object via {@link #merge}, which merges
+ * the files. At the end of the ZIP combination, {@link #finish} is called,
+ * which then writes the merged single entry of that name.
+ *
+ * <p>Implementations of this interface are not required to be thread-safe.
+ * Thread-safety is achieved by creating multiple instances. Each instance
+ * that is separately passed to {@link StrategyCallback#customMerge} is
+ * guaranteed not to be called by two threads at the same time.
+ */
+ interface CustomMergeStrategy {
+
+ /**
+ * Merges another file into the current state. This method is called for
+ * every file entry of the same name.
+ */
+ void merge(InputStream in, OutputStream out) throws IOException;
+
+ /**
+ * Outputs the merged result into the given output stream. This method is
+ * only called once when no further file of the same name is available.
+ */
+ void finish(OutputStream out) throws IOException;
+ }
+
+ /**
+ * A callback interface for the {@link ZipEntryFilter#accept} method. Use
+ * this interface to indicate the type of processing for the given file name.
+ * For every file name, exactly one of the methods must be called once. A
+ * second method call throws {@link IllegalStateException}.
+ *
+ * <p>There is no guarantee that the callback will perform the requested
+ * operation at the time of the invocation. An implementation may choose to
+ * defer the operation to an arbitrary later time.
+ *
+ * <p>IMPORTANT NOTE: Do not implement this interface. It will be modified to
+ * support future extensions, and all implementations in this package will be
+ * updated. If you violate this advice, your code will break.
+ */
+ interface StrategyCallback {
+
+ /**
+ * Skips the current entry and all entries with the same name.
+ */
+ void skip() throws IOException;
+
+ /**
+ * Copies the current entry and skips all further entries with the same
+ * name. If {@code date} is non-null, then the timestamp of the entry is
+ * overwritten with the given value.
+ */
+ void copy(Date date) throws IOException;
+
+ /**
+ * Renames and copies the current entry, and skips all further entries with
+ * the same name. If {@code date} is non-null, then the timestamp of the entry
+ * is overwritten with the given value.
+ */
+ void rename(String filename, Date date) throws IOException;
+
+ /**
+ * Merges this and all further entries with the same name with the given
+ * {@link CustomMergeStrategy}. This method must never be called twice with
+ * the same object. If {@code date} is non-null, then the timestamp of the
+ * generated entry is set to the given value; otherwise, it is set to the
+ * current time.
+ */
+ void customMerge(Date date, CustomMergeStrategy strategy) throws IOException;
+ }
+
+ /**
+ * Determines the policy with which to handle the ZIP file entry with the
+ * given name and calls the appropriate method on the callback interface
+ * {@link StrategyCallback}. For every unique name in the set of all ZIP file
+ * entries, this method is called exactly once and the result is used for all
+ * entries of the same name. Except, if an entry is renamed, the original name
+ * is not considered as having been encountered yet.
+ *
+ * <p>Implementations should use the filename to distinguish the desired
+ * processing, call one method on the callback interface and return
+ * immediately after that call.
+ *
+ * <p>There is no guarantee that the callback will perform the requested
+ * operation at the time of the invocation. An implementation may choose to
+ * defer the operation to an arbitrary later time.
+ */
+ void accept(String filename, StrategyCallback callback) throws IOException;
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ConcatenateStrategyTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ConcatenateStrategyTest.java
new file mode 100644
index 0000000..af03729
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ConcatenateStrategyTest.java
@@ -0,0 +1,75 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * Unit tests for {@link ConcatenateStrategy}.
+ */
+@RunWith(JUnit4.class)
+public class ConcatenateStrategyTest {
+
+ private String merge(String... inputs) throws IOException {
+ return mergeInternal(true, inputs);
+ }
+
+ private String mergeNoNewLine(String... inputs) throws IOException {
+ return mergeInternal(false, inputs);
+ }
+
+ private String mergeInternal(boolean appendNewLine, String... inputs) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ConcatenateStrategy strategy = new ConcatenateStrategy(appendNewLine);
+ for (String input : inputs) {
+ strategy.merge(new ByteArrayInputStream(input.getBytes(UTF_8)), out);
+ }
+ strategy.finish(out);
+ return new String(out.toByteArray(), UTF_8);
+ }
+
+ @Test
+ public void testSingleInput() throws IOException {
+ assertEquals("a", merge("a"));
+ assertEquals("a", mergeNoNewLine("a"));
+ }
+
+ @Test
+ public void testTwoInputs() throws IOException {
+ assertEquals("a\nb", merge("a\n", "b"));
+ assertEquals("a\nb", mergeNoNewLine("a\n", "b"));
+ }
+
+ @Test
+ public void testAutomaticNewline() throws IOException {
+ assertEquals("a\nb", merge("a", "b"));
+ assertEquals("ab", mergeNoNewLine("a", "b"));
+ }
+
+ @Test
+ public void testAutomaticNewlineAndEmptyFile() throws IOException {
+ assertEquals("a\nb", merge("a", "", "b"));
+ assertEquals("ab", mergeNoNewLine("a", "", "b"));
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/CopyEntryFilterTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/CopyEntryFilterTest.java
new file mode 100644
index 0000000..8eeb8d0
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/CopyEntryFilterTest.java
@@ -0,0 +1,39 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Unit tests for {@link CopyEntryFilter}.
+ */
+@RunWith(JUnit4.class)
+public class CopyEntryFilterTest {
+
+ @Test
+ public void testSingleInput() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new CopyEntryFilter().accept("abc", callback);
+ assertEquals(Arrays.asList("copy"), callback.calls);
+ }
+
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/DefaultJarEntryFilterTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/DefaultJarEntryFilterTest.java
new file mode 100644
index 0000000..6240cfb
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/DefaultJarEntryFilterTest.java
@@ -0,0 +1,101 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.jar.JarFile;
+
+/**
+ * Unit tests for {@link DefaultJarEntryFilter}.
+ */
+@RunWith(JUnit4.class)
+public class DefaultJarEntryFilterTest {
+
+ private static final Date DOS_EPOCH = ZipCombiner.DOS_EPOCH;
+
+ @Test
+ public void testSingleInput() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept("abc", callback);
+ assertEquals(Arrays.asList("copy"), callback.calls);
+ assertEquals(Arrays.asList(DOS_EPOCH), callback.dates);
+ }
+
+ @Test
+ public void testProtobufExtensionsInput() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept("protobuf.meta", callback);
+ assertEquals(Arrays.asList("customMerge"), callback.calls);
+ assertEquals(Arrays.asList(DOS_EPOCH), callback.dates);
+ }
+
+ @Test
+ public void testManifestInput() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept(JarFile.MANIFEST_NAME, callback);
+ assertEquals(Arrays.asList("skip"), callback.calls);
+ }
+
+ @Test
+ public void testServiceInput() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept("META-INF/services/any.service", callback);
+ assertEquals(Arrays.asList("customMerge"), callback.calls);
+ assertEquals(Arrays.asList(DOS_EPOCH), callback.dates);
+ }
+
+ @Test
+ public void testSpringHandlers() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept("META-INF/spring.handlers", callback);
+ assertEquals(Arrays.asList("customMerge"), callback.calls);
+ assertEquals(Arrays.asList(DOS_EPOCH), callback.dates);
+ }
+
+ @Test
+ public void testSpringSchemas() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept("META-INF/spring.schemas", callback);
+ assertEquals(Arrays.asList("customMerge"), callback.calls);
+ assertEquals(Arrays.asList(DOS_EPOCH), callback.dates);
+ }
+
+ @Test
+ public void testClassInput() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ new DefaultJarEntryFilter().accept("a.class", callback);
+ assertEquals(Arrays.asList("copy"), callback.calls);
+ assertEquals(Arrays.asList(DefaultJarEntryFilter.DOS_EPOCH_PLUS_2_SECONDS), callback.dates);
+ }
+
+ @Test
+ public void testOtherSkippedInputs() throws IOException {
+ RecordingCallback callback = new RecordingCallback();
+ ZipEntryFilter filter = new DefaultJarEntryFilter();
+ filter.accept("a.SF", callback);
+ filter.accept("a.DSA", callback);
+ filter.accept("a.RSA", callback);
+ assertEquals(Arrays.asList("skip", "skip", "skip"), callback.calls);
+ assertEquals(Arrays.<Date>asList(), callback.dates);
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/FakeZipFile.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/FakeZipFile.java
new file mode 100644
index 0000000..0156cbc
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/FakeZipFile.java
@@ -0,0 +1,265 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Receiver;
+import com.google.devtools.build.singlejar.SingleJarTest.EntryMode;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * A fake zip file to assert that a given {@link ZipInputStream} contains
+ * specified entries in a specified order. Just for unit testing.
+ */
+public final class FakeZipFile {
+
+ private static void assertSameByteArray(byte[] expected, byte[] actual) {
+ if (expected == null) {
+ assertNull(actual);
+ } else {
+ assertArrayEquals(expected, actual);
+ }
+ }
+
+ private static byte[] readZipEntryContent(ZipInputStream zipInput) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int bytesCopied;
+ while ((bytesCopied = zipInput.read(buffer)) != -1) {
+ out.write(buffer, 0, bytesCopied);
+ }
+ return out.toByteArray();
+ }
+
+ private static final class PlainByteValidator implements Receiver<byte[]> {
+ private final byte[] expected;
+
+ private PlainByteValidator(String expected) {
+ this.expected = expected == null ? new byte[0] : expected.getBytes(UTF_8);
+ }
+
+ @Override
+ public void accept(byte[] object) {
+ assertSameByteArray(expected, object);
+ }
+
+ }
+
+ private static final class FakeZipEntry {
+
+ private final String name;
+ private final Receiver<byte[]> content;
+ private final Date date;
+ private final byte[] extra;
+ private final EntryMode mode;
+
+ private FakeZipEntry(String name, Date date, String content, byte[] extra, EntryMode mode) {
+ this.name = name;
+ this.date = date;
+ this.content = new PlainByteValidator(content);
+ this.extra = extra;
+ this.mode = mode;
+ }
+
+ private FakeZipEntry(String name, Date date, Receiver<byte[]> content, byte[] extra,
+ EntryMode mode) {
+ this.name = name;
+ this.date = date;
+ this.content = content;
+ this.extra = extra;
+ this.mode = mode;
+ }
+
+ public void assertNext(ZipInputStream zipInput) throws IOException {
+ ZipEntry zipEntry = zipInput.getNextEntry();
+ assertNotNull(zipEntry);
+ switch (mode) {
+ case EXPECT_DEFLATE:
+ assertEquals(ZipEntry.DEFLATED, zipEntry.getMethod());
+ break;
+ case EXPECT_STORED:
+ assertEquals(ZipEntry.STORED, zipEntry.getMethod());
+ break;
+ default:
+ // we don't care.
+ break;
+ }
+ assertEquals(name, zipEntry.getName());
+ if (date != null) {
+ assertEquals(date.getTime(), zipEntry.getTime());
+ }
+ assertSameByteArray(extra, zipEntry.getExtra());
+ content.accept(readZipEntryContent(zipInput));
+ }
+ }
+
+ private final List<FakeZipEntry> entries = new ArrayList<>();
+
+ public FakeZipFile addEntry(String name, String content) {
+ entries.add(new FakeZipEntry(name, null, content, null, EntryMode.DONT_CARE));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, String content, boolean compressed) {
+ entries.add(new FakeZipEntry(name, null, content, null,
+ compressed ? EntryMode.EXPECT_DEFLATE : EntryMode.EXPECT_STORED));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, Date date, String content) {
+ entries.add(new FakeZipEntry(name, date, content, null, EntryMode.DONT_CARE));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, Date date, String content, boolean compressed) {
+ entries.add(new FakeZipEntry(name, date, content, null,
+ compressed ? EntryMode.EXPECT_DEFLATE : EntryMode.EXPECT_STORED));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, Receiver<byte[]> content) {
+ entries.add(new FakeZipEntry(name, null, content, null, EntryMode.DONT_CARE));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, Receiver<byte[]> content, boolean compressed) {
+ entries.add(new FakeZipEntry(name, null, content, null,
+ compressed ? EntryMode.EXPECT_DEFLATE : EntryMode.EXPECT_STORED));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, Date date, Receiver<byte[]> content) {
+ entries.add(new FakeZipEntry(name, date, content, null, EntryMode.DONT_CARE));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, Date date, Receiver<byte[]> content,
+ boolean compressed) {
+ entries.add(new FakeZipEntry(name, date, content, null,
+ compressed ? EntryMode.EXPECT_DEFLATE : EntryMode.EXPECT_STORED));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, byte[] extra) {
+ entries.add(new FakeZipEntry(name, null, (String) null, extra, EntryMode.DONT_CARE));
+ return this;
+ }
+
+ public FakeZipFile addEntry(String name, byte[] extra, boolean compressed) {
+ entries.add(new FakeZipEntry(name, null, (String) null, extra,
+ compressed ? EntryMode.EXPECT_DEFLATE : EntryMode.EXPECT_STORED));
+ return this;
+ }
+
+ private byte[] preamble = null;
+
+ public FakeZipFile addPreamble(byte[] contents) {
+ preamble = Arrays.copyOf(contents, contents.length);
+ return this;
+ }
+
+ private int getUnsignedShort(byte[] source, int offset) {
+ int a = source[offset + 0] & 0xff;
+ int b = source[offset + 1] & 0xff;
+ return (b << 8) | a;
+ }
+
+ public void assertSame(byte[] data) throws IOException {
+ int offset = 0;
+ int length = data.length;
+ if (preamble != null) {
+ offset += preamble.length;
+ length -= offset;
+ byte[] maybePreamble = Arrays.copyOfRange(data, 0, offset);
+ assertTrue(Arrays.equals(preamble, maybePreamble));
+ }
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(data, offset, length));
+ for (FakeZipEntry entry : entries) {
+ entry.assertNext(zipInput);
+ }
+ assertNull(zipInput.getNextEntry());
+ // Verify that the end of central directory data is correct.
+ // This assumes that the end of directory is at the end of input and that there is no zip file
+ // comment.
+ int count = getUnsignedShort(data, data.length-14);
+ assertEquals(entries.size(), count);
+ count = getUnsignedShort(data, data.length-12);
+ assertEquals(entries.size(), count);
+ }
+
+ /**
+ * Assert that {@code expected} is the same zip file as {@code actual}. It is similar to
+ * {@link org.junit.Assert#assertArrayEquals(byte[], byte[])} but should use a more
+ * helpful error message.
+ */
+ public static void assertSame(byte[] expected, byte[] actual) throws IOException {
+ // First parse the zip files, then compare to have explicit comparison messages.
+ ZipInputStream expectedZip = new ZipInputStream(new ByteArrayInputStream(expected));
+ ZipInputStream actualZip = new ZipInputStream(new ByteArrayInputStream(actual));
+ StringBuffer actualFileList = new StringBuffer();
+ StringBuffer expectedFileList = new StringBuffer();
+ Map<String, ZipEntry> actualEntries = new HashMap<String, ZipEntry>();
+ Map<String, ZipEntry> expectedEntries = new HashMap<String, ZipEntry>();
+ Map<String, byte[]> actualEntryContents = new HashMap<String, byte[]>();
+ Map<String, byte[]> expectedEntryContents = new HashMap<String, byte[]>();
+ parseZipEntry(expectedZip, expectedFileList, expectedEntries, expectedEntryContents);
+ parseZipEntry(actualZip, actualFileList, actualEntries, actualEntryContents);
+ // Compare the ordered file list first.
+ assertEquals(expectedFileList.toString(), actualFileList.toString());
+
+ // Then compare each entry.
+ for (String name : expectedEntries.keySet()) {
+ ZipEntry expectedEntry = expectedEntries.get(name);
+ ZipEntry actualEntry = actualEntries.get(name);
+ assertEquals("Time differs for " + name, expectedEntry.getTime(), actualEntry.getTime());
+ assertArrayEquals("Extraneous content differs for " + name,
+ expectedEntry.getExtra(), actualEntry.getExtra());
+ assertArrayEquals("Content differs for " + name,
+ expectedEntryContents.get(name), actualEntryContents.get(name));
+ }
+
+ // Finally do a binary array comparison to be sure that test fails if files are different in
+ // some way we don't test.
+ assertArrayEquals(expected, actual);
+ }
+
+ private static void parseZipEntry(ZipInputStream expectedZip, StringBuffer expectedFileList,
+ Map<String, ZipEntry> expectedEntries, Map<String, byte[]> expectedEntryContents)
+ throws IOException {
+ ZipEntry expectedEntry;
+ while ((expectedEntry = expectedZip.getNextEntry()) != null) {
+ expectedFileList.append(expectedEntry.getName()).append("\n");
+ expectedEntries.put(expectedEntry.getName(), expectedEntry);
+ expectedEntryContents.put(expectedEntry.getName(), readZipEntryContent(expectedZip));
+ }
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java
new file mode 100644
index 0000000..8fec585
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java
@@ -0,0 +1,88 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * FileSystem for testing. FileSystem supports exactly one one OutputStream for filename
+ * specified in constructor.
+ * Workflow for using this class in tests are following:
+ * <ul>
+ * <li> Construct with exactly one outputFile. </li>
+ * <li> add some input files using method addFile </li>
+ * <li> check content of outputFile calling toByteArray </li>
+ * </ul>
+ */
+public final class MockSimpleFileSystem implements SimpleFileSystem {
+
+ private final String outputFileName;
+ private ByteArrayOutputStream out;
+ private final Map<String, byte[]> files = new HashMap<>();
+
+ public MockSimpleFileSystem(String outputFileName) {
+ this.outputFileName = outputFileName;
+ }
+
+ public void addFile(String name, byte[] content) {
+ files.put(name, content);
+ }
+
+ public void addFile(String name, String content) {
+ files.put(name, content.getBytes(UTF_8));
+ }
+
+ @Override
+ public OutputStream getOutputStream(String filename) {
+ assertEquals(outputFileName, filename);
+ assertNull(out);
+ out = new ByteArrayOutputStream();
+ return out;
+ }
+
+ @Override
+ public InputStream getInputStream(String filename) throws IOException {
+ byte[] data = files.get(filename);
+ if (data == null) {
+ throw new FileNotFoundException();
+ }
+ return new ByteArrayInputStream(data);
+ }
+
+ @Override
+ public boolean delete(String filename) {
+ assertEquals(outputFileName, filename);
+ assertNotNull(out);
+ out = null;
+ return true;
+ }
+
+ public byte[] toByteArray() {
+ assertNotNull(out);
+ return out.toByteArray();
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/OptionFileExpanderTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/OptionFileExpanderTest.java
new file mode 100644
index 0000000..eea87a5
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/OptionFileExpanderTest.java
@@ -0,0 +1,87 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.singlejar.OptionFileExpander.OptionFileProvider;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for {@link OptionFileExpander}.
+ */
+@RunWith(JUnit4.class)
+public class OptionFileExpanderTest {
+
+ private static class StoredOptionFileProvider implements OptionFileProvider {
+
+ private Map<String, byte[]> availableFiles = new HashMap<>();
+
+ void addFile(String filename, String content) {
+ availableFiles.put(filename, content.getBytes(UTF_8));
+ }
+
+ @Override
+ public InputStream getInputStream(String filename) throws IOException {
+ byte[] result = availableFiles.get(filename);
+ if (result == null) {
+ throw new FileNotFoundException();
+ }
+ return new ByteArrayInputStream(result);
+ }
+ }
+
+ @Test
+ public void testNoExpansion() throws IOException {
+ OptionFileExpander expander = new OptionFileExpander(new StoredOptionFileProvider());
+ assertEquals(Arrays.asList("--some", "option", "list"),
+ expander.expandArguments(Arrays.asList("--some", "option", "list")));
+ }
+
+ @Test
+ public void testExpandSimpleOptionsFile() throws IOException {
+ StoredOptionFileProvider provider = new StoredOptionFileProvider();
+ provider.addFile("options", "--some option list");
+ OptionFileExpander expander = new OptionFileExpander(provider);
+ assertEquals(Arrays.asList("--some", "option", "list"),
+ expander.expandArguments(Arrays.asList("@options")));
+ }
+
+ @Test
+ public void testIllegalOptionsFile() {
+ StoredOptionFileProvider provider = new StoredOptionFileProvider();
+ provider.addFile("options", "'missing apostrophe");
+ OptionFileExpander expander = new OptionFileExpander(provider);
+ try {
+ expander.expandArguments(Arrays.asList("@options"));
+ fail();
+ } catch (IOException e) {
+ // Expected exception.
+ }
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/PrefixListPathFilterTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/PrefixListPathFilterTest.java
new file mode 100644
index 0000000..32b5eae
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/PrefixListPathFilterTest.java
@@ -0,0 +1,54 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.singlejar.DefaultJarEntryFilter.PathFilter;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests {@link PrefixListPathFilter}.
+ */
+@RunWith(JUnit4.class)
+public class PrefixListPathFilterTest {
+ private PathFilter filter;
+
+ @Test
+ public void testPrefixList() {
+ filter = new PrefixListPathFilter(ImmutableList.of("dir1", "dir/subdir"));
+ assertIncluded("dir1/file1");
+ assertExcluded("dir2/file1");
+ assertIncluded("dir/subdir/file1");
+ assertExcluded("dir2/subdir/file1");
+ assertExcluded("dir/othersub/file1");
+ assertExcluded("dir3/file1");
+ }
+
+ private void assertExcluded(String path) {
+ assertFalse(path + " should have been excluded, but was included",
+ filter.allowed(path));
+ }
+
+ private void assertIncluded(String path) {
+ assertTrue(path + " should have been included but was not",
+ filter.allowed(path));
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/RecordingCallback.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/RecordingCallback.java
new file mode 100644
index 0000000..a3f69ff
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/RecordingCallback.java
@@ -0,0 +1,56 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+
+import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy;
+import com.google.devtools.build.singlejar.ZipEntryFilter.StrategyCallback;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * A helper implementation of {@link StrategyCallback} that records callback
+ * invocations as string.
+ */
+public final class RecordingCallback implements StrategyCallback {
+
+ public final List<String> calls = new ArrayList<>();
+ public final List<Date> dates = new ArrayList<>();
+
+ @Override
+ public void copy(Date date) {
+ calls.add("copy");
+ dates.add(date);
+ }
+
+ @Override
+ public void rename(String filename, Date date) {
+ calls.add("rename");
+ dates.add(date);
+ }
+
+ @Override
+ public void customMerge(Date date, CustomMergeStrategy strategy) {
+ calls.add("customMerge");
+ dates.add(date);
+ }
+
+ @Override
+ public void skip() {
+ calls.add("skip");
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java
new file mode 100644
index 0000000..dbff155
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java
@@ -0,0 +1,634 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Receiver;
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.jar.JarFile;
+
+/**
+ * Unit tests for {@link SingleJar}.
+ */
+@RunWith(JUnit4.class)
+public class SingleJarTest {
+
+ public static final byte[] EXTRA_FOR_META_INF = new byte[] {(byte) 0xFE, (byte) 0xCA, 0x00, 0x00};
+
+ static final Joiner LINE_JOINER = Joiner.on("\r\n");
+ static final Joiner LINEFEED_JOINER = Joiner.on("\n");
+
+ static enum EntryMode {
+ DONT_CARE, EXPECT_DEFLATE, EXPECT_STORED;
+ }
+
+ public static final class BuildInfoValidator implements Receiver<byte[]> {
+ private final List<String> buildInfoLines;
+
+ public BuildInfoValidator(List<String> buildInfoLines) {
+ this.buildInfoLines = buildInfoLines;
+ }
+
+ @Override
+ public void accept(byte[] content) {
+ String actualBuildInfo = new String(content, StandardCharsets.UTF_8);
+ List<String> expectedBuildInfos = new ArrayList<>();
+ for (String line : buildInfoLines) { // the character : is escaped
+ expectedBuildInfos.add(line.replace(":", "\\:"));
+ }
+ Collections.sort(expectedBuildInfos);
+ String[] actualBuildInfos = actualBuildInfo.split("\n");
+ Arrays.sort(actualBuildInfos);
+ assertEquals(LINEFEED_JOINER.join(expectedBuildInfos),
+ LINEFEED_JOINER.join(actualBuildInfos));
+ }
+
+ }
+
+ // Manifest file line ordering is dependent of the ordering in HashMap (Attributes class) so
+ // we do a sorted comparison for Manifest.
+ public static final class ManifestValidator implements Receiver<byte[]> {
+ private final List<String> manifestLines;
+
+ public ManifestValidator(List<String> manifestLines) {
+ this.manifestLines = new ArrayList<String>(manifestLines);
+ Collections.sort(this.manifestLines);
+ }
+
+ public ManifestValidator(String... manifestLines) {
+ this.manifestLines = Arrays.asList(manifestLines);
+ Collections.sort(this.manifestLines);
+ }
+
+ @Override
+ public void accept(byte[] content) {
+ String actualManifest = new String(content, StandardCharsets.UTF_8);
+ String[] actualManifestLines = actualManifest.trim().split("\r\n");
+ Arrays.sort(actualManifestLines);
+ assertEquals(LINEFEED_JOINER.join(manifestLines), LINEFEED_JOINER.join(actualManifestLines));
+ }
+
+ }
+
+ private BuildInfoValidator redactedBuildData(String outputJar) {
+ return new BuildInfoValidator(ImmutableList.of("build.target=" + outputJar));
+ }
+
+ private BuildInfoValidator redactedBuildData(String outputJar, String mainClass) {
+ return new BuildInfoValidator(
+ ImmutableList.of("build.target=" + outputJar, "main.class=" + mainClass));
+ }
+
+ static List<String> getBuildInfo() {
+ return ImmutableList.of("build.build_id=11111-222-33333",
+ "build.version=12659499",
+ "build.location=user@machine.domain.com:/home/user/source",
+ "build.target=output.jar",
+ "build.time=Fri Jan 2 02:17:36 1970 (123456)",
+ "build.timestamp=Fri Jan 2 02:17:36 1970 (123456)",
+ "build.timestamp.as.int=123456"
+ );
+ }
+
+ private byte[] sampleZip() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", "Hello World!");
+ return factory.toByteArray();
+ }
+
+ private byte[] sampleUncompressedZip() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", "Hello World!", false);
+ return factory.toByteArray();
+ }
+
+ private byte[] sampleZipWithSF() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.SF", "Hello World!");
+ return factory.toByteArray();
+ }
+
+ private byte[] sampleZipWithSubdirs() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("dir1/file1", "contents11");
+ factory.addFile("dir1/file2", "contents12");
+ factory.addFile("dir2/file1", "contents21");
+ factory.addFile("dir3/file1", "contents31");
+ return factory.toByteArray();
+ }
+
+ private void assertStripFirstLine(String expected, String testCase) {
+ byte[] result = SingleJar.stripFirstLine(testCase.getBytes(StandardCharsets.UTF_8));
+ assertEquals(expected, new String(result));
+ }
+
+ @Test
+ public void testStripFirstLine() {
+ assertStripFirstLine("", "");
+ assertStripFirstLine("", "no linefeed");
+ assertStripFirstLine(LINEFEED_JOINER.join("toto", "titi"),
+ LINEFEED_JOINER.join("# timestamp comment", "toto", "titi"));
+ assertStripFirstLine(LINE_JOINER.join("toto", "titi"),
+ LINE_JOINER.join("# timestamp comment", "toto", "titi"));
+ }
+
+ @Test
+ public void testEmptyJar() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ // Test that two identical calls at different time actually returns identical results
+ @Test
+ public void testDeterministicJar() throws IOException, InterruptedException {
+ MockSimpleFileSystem mockFs1 = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar1 = new SingleJar(mockFs1);
+ singleJar1.run(ImmutableList.of("--output", "output.jar", "--extra_build_info", "toto=titi",
+ "--normalize"));
+ Thread.sleep(1000); // ensure that we are not at the same seconds
+
+ MockSimpleFileSystem mockFs2 = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar2 = new SingleJar(mockFs2);
+ singleJar2.run(ImmutableList.of("--output", "output.jar", "--extra_build_info", "toto=titi",
+ "--normalize"));
+
+ FakeZipFile.assertSame(mockFs1.toByteArray(), mockFs2.toByteArray());
+ }
+
+ @Test
+ public void testExtraManifestContent() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--deploy_manifest_lines",
+ "Main-Class: SomeClass", "X-Other: Duh"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar",
+ "Main-Class: SomeClass",
+ "X-Other: Duh"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testMultipleExtraManifestContent() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--deploy_manifest_lines", "X-Other: Duh",
+ "--output", "output.jar",
+ "--deploy_manifest_lines", "Main-Class: SomeClass"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar",
+ "Main-Class: SomeClass",
+ "X-Other: Duh"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testMainClass() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--main_class", "SomeClass"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar",
+ "Main-Class: SomeClass"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar", "SomeClass"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ // These four tests test all combinations of compressed/uncompressed input and output.
+ @Test
+ public void testSimpleZip() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("test.jar", sampleZip());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "test.jar"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"), false)
+ .addEntry("build-data.properties", redactedBuildData("output.jar"), false)
+ .addEntry("hello.txt", "Hello World!", false);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testSimpleZipExpectCompressedOutput() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("test.jar", sampleZip());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "test.jar",
+ "--compression"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"), true)
+ .addEntry("build-data.properties", redactedBuildData("output.jar"), true)
+ .addEntry("hello.txt", "Hello World!", true);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testSimpleUncompressedZip() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("test.jar", sampleUncompressedZip());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "test.jar"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(ImmutableList.of(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar")), false)
+ .addEntry("build-data.properties", redactedBuildData("output.jar"), false)
+ .addEntry("hello.txt", "Hello World!", false);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testSimpleUncompressedZipExpectCompressedOutput() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("test.jar", sampleUncompressedZip());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "test.jar",
+ "--compression"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"), true)
+ .addEntry("build-data.properties", redactedBuildData("output.jar"), true)
+ .addEntry("hello.txt", "Hello World!", true);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ // Integration test for option file expansion.
+ @Test
+ public void testOptionFile() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("input.jar", sampleZip());
+ mockFs.addFile("options", "--output output.jar --sources input.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("@options"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"))
+ .addEntry("hello.txt", "Hello World!");
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testSkipsSignatureFiles() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("input.jar", sampleZipWithSF());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "input.jar"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testSkipsUsingInputPrefixes() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("input.jar", sampleZipWithSubdirs());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources",
+ "input.jar", "--include_prefixes", "dir1", "dir2"));
+
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"))
+ .addEntry("dir1/file1", "contents11")
+ .addEntry("dir1/file2", "contents12")
+ .addEntry("dir2/file1", "contents21");
+
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testSkipsUsingMultipleInputPrefixes() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("input.jar", sampleZipWithSubdirs());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--include_prefixes", "dir2",
+ "--sources", "input.jar", "--include_prefixes", "dir1"));
+
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar"))
+ .addEntry("dir1/file1", "contents11")
+ .addEntry("dir1/file2", "contents12")
+ .addEntry("dir2/file1", "contents21");
+
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testNormalize() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("input.jar", sampleZip());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "input.jar",
+ "--normalize"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, ZipCombiner.DOS_EPOCH, new ManifestValidator(
+ "Manifest-Version: 1.0", "Created-By: blaze-singlejar"), false)
+ .addEntry("build-data.properties", ZipCombiner.DOS_EPOCH,
+ redactedBuildData("output.jar"), false)
+ .addEntry("hello.txt", ZipCombiner.DOS_EPOCH, "Hello World!", false);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testNormalizeAndCompress() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("input.jar", sampleZip());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--sources", "input.jar",
+ "--normalize", "--compression"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, ZipCombiner.DOS_EPOCH, new ManifestValidator(
+ "Manifest-Version: 1.0", "Created-By: blaze-singlejar"), true)
+ .addEntry("build-data.properties", ZipCombiner.DOS_EPOCH,
+ redactedBuildData("output.jar"), true)
+ .addEntry("hello.txt", ZipCombiner.DOS_EPOCH, "Hello World!", true);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testAddBuildInfoProperties() throws IOException {
+ List<String> buildInfo = getBuildInfo();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0", "Created-By: blaze-singlejar"), false)
+ .addEntry("build-data.properties", new BuildInfoValidator(buildInfo),
+ false);
+
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ List<String> args = new ArrayList<String>();
+ args.add("--output");
+ args.add("output.jar");
+ args.addAll(infoPropertyArguments(buildInfo));
+ singleJar.run(args);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ private static List<String> infoPropertyArguments(List<String> buildInfoLines) {
+ List<String> args = new ArrayList<>();
+ for (String s : buildInfoLines) {
+ if (!s.isEmpty()) {
+ args.add("--extra_build_info");
+ args.add(s);
+ }
+ }
+ return args;
+ }
+
+ @Test
+ public void testAddBuildInfoPropertiesFile() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ doTestAddBuildInfoPropertiesFile(mockFs, "output.jar", singleJar);
+ }
+
+ public static void doTestAddBuildInfoPropertiesFile(MockSimpleFileSystem mockFs, String target,
+ SingleJar singleJar) throws IOException {
+ List<String> buildInfo = getBuildInfo();
+ mockFs.addFile("my.properties", makePropertyFileFromBuildInfo(buildInfo));
+ singleJar.run(ImmutableList.of("--output", target, "--build_info_file", "my.properties"));
+
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME,
+ new ManifestValidator("Manifest-Version: 1.0", "Created-By: blaze-singlejar"), false)
+ .addEntry("build-data.properties", new BuildInfoValidator(buildInfo),
+ false);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ private static String makePropertyFileFromBuildInfo(List<String> buildInfo) {
+ return LINEFEED_JOINER.join(buildInfo).replace(":", "\\:");
+ }
+
+ @Test
+ public void testAddBuildInfoPropertiesFiles() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ doTestAddBuildInfoPropertiesFiles(mockFs, "output.jar", singleJar);
+ }
+
+ public static void doTestAddBuildInfoPropertiesFiles(MockSimpleFileSystem mockFs, String target,
+ SingleJar singleJar) throws IOException {
+ List<String> buildInfo = getBuildInfo();
+
+ mockFs.addFile("my1.properties", makePropertyFileFromBuildInfo(buildInfo.subList(0, 4)));
+ mockFs.addFile("my2.properties",
+ makePropertyFileFromBuildInfo(buildInfo.subList(4, buildInfo.size())));
+ singleJar.run(ImmutableList.of("--output", target,
+ "--build_info_file", "my1.properties",
+ "--build_info_file", "my2.properties"));
+
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME,
+ new ManifestValidator("Manifest-Version: 1.0", "Created-By: blaze-singlejar"), false)
+ .addEntry("build-data.properties", new BuildInfoValidator(buildInfo),
+ false);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testAddBuildInfoPropertiesAndFiles() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ doTestAddBuildInfoPropertiesAndFiles(mockFs, "output.jar", singleJar);
+ }
+
+ public static void doTestAddBuildInfoPropertiesAndFiles(MockSimpleFileSystem mockFs,
+ String target, SingleJar singleJar) throws IOException {
+ List<String> buildInfo = getBuildInfo();
+
+ mockFs.addFile("my1.properties", makePropertyFileFromBuildInfo(buildInfo.subList(0, 4)));
+ mockFs.addFile("my2.properties", makePropertyFileFromBuildInfo(
+ buildInfo.subList(4, buildInfo.size())));
+ List<String> args = ImmutableList.<String>builder()
+ .add("--output").add(target)
+ .add("--build_info_file").add("my1.properties")
+ .add("--build_info_file").add("my2.properties")
+ .addAll(infoPropertyArguments(buildInfo.subList(4, buildInfo.size())))
+ .build();
+
+ singleJar.run(args);
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF, false)
+ .addEntry(JarFile.MANIFEST_NAME,
+ new ManifestValidator("Manifest-Version: 1.0", "Created-By: blaze-singlejar"), false)
+ .addEntry("build-data.properties", new BuildInfoValidator(buildInfo),
+ false);
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+
+ @Test
+ public void testExcludeBuildData() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ SingleJar singleJar = new SingleJar(mockFs);
+ doTestExcludeBuildData(mockFs, "output.jar", singleJar);
+ }
+
+ public static void doTestExcludeBuildData(MockSimpleFileSystem mockFs, String target,
+ SingleJar singleJar) throws IOException {
+ singleJar.run(ImmutableList.of("--output", target, "--exclude_build_data"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testResourceMapping() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("a/b/c", "Test");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--exclude_build_data",
+ "--resources", "a/b/c:c/b/a"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("c/b/a", "Test");
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testResourceMappingIdentity() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("a/b/c", "Test");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--exclude_build_data",
+ "--resources", "a/b/c"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("a/b/c", "Test");
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testResourceMappingDuplicateError() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("a/b/c", "Test");
+ SingleJar singleJar = new SingleJar(mockFs);
+ try {
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--exclude_build_data",
+ "--resources", "a/b/c", "a/b/c"));
+ fail();
+ } catch (IllegalStateException e) {
+ assertTrue(e.getMessage().contains("already contains a file named a/b/c"));
+ }
+ }
+
+ @Test
+ public void testResourceMappingDuplicateWarning() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ mockFs.addFile("a/b/c", "Test");
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar", "--exclude_build_data",
+ "--warn_duplicate_resources", "--resources", "a/b/c", "a/b/c"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar"))
+ .addEntry("a/b/c", "Test");
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+
+ @Test
+ public void testCanAddPreamble() throws IOException {
+ MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar");
+ String preamble = "WeThePeople";
+ mockFs.addFile(preamble, preamble.getBytes());
+ SingleJar singleJar = new SingleJar(mockFs);
+ singleJar.run(ImmutableList.of("--output", "output.jar",
+ "--java_launcher", preamble,
+ "--main_class", "SomeClass"));
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addPreamble(preamble.getBytes())
+ .addEntry("META-INF/", EXTRA_FOR_META_INF)
+ .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator(
+ "Manifest-Version: 1.0",
+ "Created-By: blaze-singlejar",
+ "Main-Class: SomeClass"))
+ .addEntry("build-data.properties", redactedBuildData("output.jar", "SomeClass"));
+ expectedResult.assertSame(mockFs.toByteArray());
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTests.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTests.java
new file mode 100644
index 0000000..8b68004
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTests.java
@@ -0,0 +1,27 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+
+import com.google.devtools.build.lib.testutil.ClasspathSuite;
+
+import org.junit.runner.RunWith;
+
+/**
+ * A test-suite builder for this package.
+ */
+@RunWith(ClasspathSuite.class)
+public class SingleJarTests {
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SlowConcatenateStrategy.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SlowConcatenateStrategy.java
new file mode 100644
index 0000000..d1a5091
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SlowConcatenateStrategy.java
@@ -0,0 +1,45 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+
+import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * A strategy that merges a set of files by concatenating them. It inserts no
+ * additional characters and copies bytes one by one. Used for testing.
+ */
+@NotThreadSafe
+final class SlowConcatenateStrategy implements CustomMergeStrategy {
+
+ @Override
+ public void merge(InputStream in, OutputStream out) throws IOException {
+ int nextByte;
+ while ((nextByte = in.read()) != -1) {
+ out.write(nextByte);
+ }
+ }
+
+ @Override
+ public void finish(OutputStream out) {
+ // No need to do anything. All the data was already written.
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java
new file mode 100644
index 0000000..e5345cb
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java
@@ -0,0 +1,936 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.singlejar.ZipCombiner.OutputMode;
+import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * Unit tests for {@link ZipCombiner}.
+ */
+@RunWith(JUnit4.class)
+public class ZipCombinerTest {
+
+ private static final Date DOS_EPOCH = ZipCombiner.DOS_EPOCH;
+
+ private InputStream sampleZip() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", "Hello World!");
+ return factory.toInputStream();
+ }
+
+ private InputStream sampleZip2() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello2.txt", "Hello World 2!");
+ return factory.toInputStream();
+ }
+
+ private InputStream sampleZipWithTwoEntries() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", "Hello World!");
+ factory.addFile("hello2.txt", "Hello World 2!");
+ return factory.toInputStream();
+ }
+
+ private InputStream sampleZipWithOneUncompressedEntry() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", "Hello World!", false);
+ return factory.toInputStream();
+ }
+
+ private InputStream sampleZipWithTwoUncompressedEntries() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", "Hello World!", false);
+ factory.addFile("hello2.txt", "Hello World 2!", false);
+ return factory.toInputStream();
+ }
+
+ private void assertEntry(ZipInputStream zipInput, String filename, long time, byte[] content)
+ throws IOException {
+ ZipEntry zipEntry = zipInput.getNextEntry();
+ assertNotNull(zipEntry);
+ assertEquals(filename, zipEntry.getName());
+ assertEquals(time, zipEntry.getTime());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int bytesCopied;
+ while ((bytesCopied = zipInput.read(buffer)) != -1) {
+ out.write(buffer, 0, bytesCopied);
+ }
+ assertTrue(Arrays.equals(content, out.toByteArray()));
+ }
+
+ private void assertEntry(ZipInputStream zipInput, String filename, byte[] content)
+ throws IOException {
+ assertEntry(zipInput, filename, ZipCombiner.DOS_EPOCH.getTime(), content);
+ }
+
+ private void assertEntry(ZipInputStream zipInput, String filename, String content)
+ throws IOException {
+ assertEntry(zipInput, filename, content.getBytes(ISO_8859_1));
+ }
+
+ private void assertEntry(ZipInputStream zipInput, String filename, Date date, String content)
+ throws IOException {
+ assertEntry(zipInput, filename, date.getTime(), content.getBytes(ISO_8859_1));
+ }
+
+ @Test
+ public void testDateToDosTime() {
+ assertEquals(0x210000, ZipCombiner.dateToDosTime(ZipCombiner.DOS_EPOCH));
+ Calendar calendar = new GregorianCalendar();
+ for (int i = 1980; i <= 2107; i++) {
+ calendar.set(i, 0, 1, 0, 0, 0);
+ int result = ZipCombiner.dateToDosTime(calendar.getTime());
+ assertEquals(i - 1980, result >>> 25);
+ assertEquals(1, (result >> 21) & 0xf);
+ assertEquals(1, (result >> 16) & 0x1f);
+ assertEquals(0, result & 0xffff);
+ }
+ }
+
+ @Test
+ public void testDateToDosTimeFailsForBadValues() {
+ try {
+ Calendar calendar = new GregorianCalendar();
+ calendar.set(1979, 0, 1, 0, 0, 0);
+ ZipCombiner.dateToDosTime(calendar.getTime());
+ fail();
+ } catch (IllegalArgumentException e) {
+ /* Expected exception. */
+ }
+ try {
+ Calendar calendar = new GregorianCalendar();
+ calendar.set(2108, 0, 1, 0, 0, 0);
+ ZipCombiner.dateToDosTime(calendar.getTime());
+ fail();
+ } catch (IllegalArgumentException e) {
+ /* Expected exception. */
+ }
+ }
+
+ @Test
+ public void testCompressedDontCare() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("hello.txt", "Hello World!", true);
+ expectedResult.assertSame(out.toByteArray());
+ }
+
+ @Test
+ public void testCompressedForceDeflate() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_DEFLATE, out);
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("hello.txt", "Hello World!", true);
+ expectedResult.assertSame(out.toByteArray());
+ }
+
+ @Test
+ public void testCompressedForceStored() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_STORED, out);
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("hello.txt", "Hello World!", false);
+ expectedResult.assertSame(out.toByteArray());
+ }
+
+ @Test
+ public void testUncompressedDontCare() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(sampleZipWithOneUncompressedEntry());
+ singleJar.close();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("hello.txt", "Hello World!", false);
+ expectedResult.assertSame(out.toByteArray());
+ }
+
+ @Test
+ public void testUncompressedForceDeflate() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_DEFLATE, out);
+ singleJar.addZip(sampleZipWithOneUncompressedEntry());
+ singleJar.close();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("hello.txt", "Hello World!", true);
+ expectedResult.assertSame(out.toByteArray());
+ }
+
+ @Test
+ public void testUncompressedForceStored() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_STORED, out);
+ singleJar.addZip(sampleZipWithOneUncompressedEntry());
+ singleJar.close();
+ FakeZipFile expectedResult = new FakeZipFile()
+ .addEntry("hello.txt", "Hello World!", false);
+ expectedResult.assertSame(out.toByteArray());
+ }
+
+ @Test
+ public void testCopyTwoEntries() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testCopyTwoUncompressedEntries() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testCombine() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(sampleZip());
+ singleJar.addZip(sampleZip2());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testDuplicateEntry() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(sampleZip());
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // Returns an input stream that can only read one byte at a time.
+ private InputStream slowRead(final InputStream in) {
+ return new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return in.read();
+ }
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ Preconditions.checkArgument(b != null);
+ Preconditions.checkArgument((len >= 0) && (off >= 0));
+ Preconditions.checkArgument(len <= b.length - off);
+ if (len == 0) {
+ return 0;
+ }
+ int value = read();
+ if (value == -1) {
+ return -1;
+ }
+ b[off] = (byte) value;
+ return 1;
+ }
+ };
+ }
+
+ @Test
+ public void testDuplicateUncompressedEntryWithSlowRead() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(slowRead(sampleZipWithOneUncompressedEntry()));
+ singleJar.addZip(slowRead(sampleZipWithOneUncompressedEntry()));
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testDuplicateEntryWithSlowRead() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(slowRead(sampleZip()));
+ singleJar.addZip(slowRead(sampleZip()));
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testBadZipFileNoEntry() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(new ByteArrayInputStream(new byte[] { 1, 2, 3, 4 }));
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertNull(zipInput.getNextEntry());
+ }
+
+ private InputStream asStream(String content) {
+ return new ByteArrayInputStream(content.getBytes(UTF_8));
+ }
+
+ @Test
+ public void testAddFile() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addFile("hello.txt", DOS_EPOCH, asStream("Hello World!"));
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testAddFileAndDuplicateZipEntry() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addFile("hello.txt", DOS_EPOCH, asStream("Hello World!"));
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ static final class MergeStrategyPlaceHolder implements CustomMergeStrategy {
+
+ @Override
+ public void finish(OutputStream out) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void merge(InputStream in, OutputStream out) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static final CustomMergeStrategy COPY_PLACEHOLDER = new MergeStrategyPlaceHolder();
+ private static final CustomMergeStrategy SKIP_PLACEHOLDER = new MergeStrategyPlaceHolder();
+
+ /**
+ * A mock implementation that either uses the specified behavior or calls
+ * through to copy.
+ */
+ class MockZipEntryFilter implements ZipEntryFilter {
+
+ private Date date = DOS_EPOCH;
+ private final List<String> calls = new ArrayList<>();
+ // File name to merge strategy map.
+ private final Map<String, CustomMergeStrategy> behavior =
+ new HashMap<>();
+ private final ListMultimap<String, String> renameMap = ArrayListMultimap.create();
+
+ @Override
+ public void accept(String filename, StrategyCallback callback) throws IOException {
+ calls.add(filename);
+ CustomMergeStrategy strategy = behavior.get(filename);
+ if (strategy == null) {
+ callback.copy(null);
+ } else if (strategy == COPY_PLACEHOLDER) {
+ List<String> names = renameMap.get(filename);
+ if (names != null && !names.isEmpty()) {
+ // rename to the next name in list of replacement names.
+ String newName = names.get(0);
+ callback.rename(newName, null);
+ // Unless this is the last replacment names, we pop the used name.
+ // The lastreplacement name applies any additional entries.
+ if (names.size() > 1) {
+ names.remove(0);
+ }
+ } else {
+ callback.copy(null);
+ }
+ } else if (strategy == SKIP_PLACEHOLDER) {
+ callback.skip();
+ } else {
+ callback.customMerge(date, strategy);
+ }
+ }
+ }
+
+ @Test
+ public void testCopyCallsFilter() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt"), mockFilter.calls);
+ }
+
+ @Test
+ public void testDuplicateEntryCallsFilterOnce() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZip());
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt"), mockFilter.calls);
+ }
+
+ @Test
+ public void testMergeStrategy() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new ConcatenateStrategy());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZip());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello.txt", "Hello World!\nHello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testMergeStrategyWithUncompressedFiles() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new ConcatenateStrategy());
+ mockFilter.behavior.put("hello2.txt", SKIP_PLACEHOLDER);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!\nHello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testMergeStrategyWithUncompressedEntriesAndSlowRead() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new ConcatenateStrategy());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(slowRead(sampleZipWithOneUncompressedEntry()));
+ singleJar.addZip(slowRead(sampleZipWithTwoUncompressedEntries()));
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello.txt", "Hello World!\nHello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testMergeStrategyWithSlowCopy() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new SlowConcatenateStrategy());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZip());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello.txt", "Hello World!Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testMergeStrategyWithUncompressedFilesAndSlowCopy() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new SlowConcatenateStrategy());
+ mockFilter.behavior.put("hello2.txt", SKIP_PLACEHOLDER);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ private InputStream specialZipWithMinusOne() {
+ ZipFactory factory = new ZipFactory();
+ factory.addFile("hello.txt", new byte[] {-1});
+ return factory.toInputStream();
+ }
+
+ @Test
+ public void testMergeStrategyWithSlowCopyAndNegativeBytes() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new SlowConcatenateStrategy());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(specialZipWithMinusOne());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", new byte[] { -1 });
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testCopyDateHandling() throws IOException {
+ final Date date = new GregorianCalendar(2009, 8, 2, 0, 0, 0).getTime();
+ ZipEntryFilter mockFilter = new ZipEntryFilter() {
+ @Override
+ public void accept(String filename, StrategyCallback callback) throws IOException {
+ assertEquals("hello.txt", filename);
+ callback.copy(date);
+ }
+ };
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZip());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", date, "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testMergeDateHandling() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", new ConcatenateStrategy());
+ mockFilter.date = new GregorianCalendar(2009, 8, 2, 0, 0, 0).getTime();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZip());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls);
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello2.txt", DOS_EPOCH, "Hello World 2!");
+ assertEntry(zipInput, "hello.txt", mockFilter.date, "Hello World!\nHello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ @Test
+ public void testDuplicateCallThrowsException() throws IOException {
+ ZipEntryFilter badFilter = new ZipEntryFilter() {
+ @Override
+ public void accept(String filename, StrategyCallback callback) throws IOException {
+ // Duplicate callback call.
+ callback.skip();
+ callback.copy(null);
+ }
+ };
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipCombiner singleJar = new ZipCombiner(badFilter, out)) {
+ singleJar.addZip(sampleZip());
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected exception.
+ }
+ }
+
+ @Test
+ public void testNoCallThrowsException() throws IOException {
+ ZipEntryFilter badFilter = new ZipEntryFilter() {
+ @Override
+ public void accept(String filename, StrategyCallback callback) {
+ // No callback call.
+ }
+ };
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipCombiner singleJar = new ZipCombiner(badFilter, out)) {
+ singleJar.addZip(sampleZip());
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected exception.
+ }
+ }
+
+ // This test verifies that if an entry A is renamed as A (identy mapping),
+ // then subsequent entries named A are still subject to filtering.
+ // Note: this is different from a copy, where subsequent entries are skipped.
+ @Test
+ public void testRenameIdentityMapping() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER);
+ mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER);
+ mockFilter.renameMap.put("hello.txt", "hello.txt"); // identity rename, not copy
+ mockFilter.renameMap.put("hello2.txt", "hello2.txt"); // identity rename, not copy
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt",
+ "hello.txt", "hello2.txt").inOrder();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // This test verifies that multiple entries with the same name can be
+ // renamed to unique names.
+ @Test
+ public void testRenameNoConflictMapping() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER);
+ mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER);
+ mockFilter.renameMap.putAll("hello.txt", Arrays.asList("hello1.txt", "hello2.txt"));
+ mockFilter.renameMap.putAll("hello2.txt", Arrays.asList("world1.txt", "world2.txt"));
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt",
+ "hello.txt", "hello2.txt").inOrder();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello1.txt", "Hello World!");
+ assertEntry(zipInput, "world1.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello2.txt", "Hello World!");
+ assertEntry(zipInput, "world2.txt", "Hello World 2!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // This tests verifies that an attempt to rename an entry to a
+ // name already written, results in the entry being skipped, after
+ // calling the filter.
+ @Test
+ public void testRenameSkipUsedName() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER);
+ mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER);
+ mockFilter.renameMap.putAll("hello.txt",
+ Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt"));
+ mockFilter.renameMap.put("hello2.txt", "hello2.txt");
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt",
+ "hello.txt", "hello2.txt", "hello.txt", "hello2.txt").inOrder();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello1.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello3.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // This tests verifies that if an entry has been copied, then
+ // further entries of the same name are skipped (filter not invoked),
+ // and entries renamed to the same name are skipped (after calling filter).
+ @Test
+ public void testRenameAndCopy() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER);
+ mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER);
+ mockFilter.renameMap.putAll("hello.txt",
+ Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt"));
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt",
+ "hello.txt", "hello.txt").inOrder();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello1.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello3.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // This tests verifies that if an entry has been skipped, then
+ // further entries of the same name are skipped (filter not invoked),
+ // and entries renamed to the same name are skipped (after calling filter).
+ @Test
+ public void testRenameAndSkip() throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER);
+ mockFilter.behavior.put("hello2.txt", SKIP_PLACEHOLDER);
+ mockFilter.renameMap.putAll("hello.txt",
+ Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt"));
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.addZip(sampleZipWithTwoEntries());
+ singleJar.close();
+ assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt",
+ "hello.txt", "hello.txt").inOrder();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello1.txt", "Hello World!");
+ assertEntry(zipInput, "hello3.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // This test verifies that renaming works when input and output
+ // disagree on compression method. This is the simple case, where
+ // content is read and rewritten, and no header repair is needed.
+ @Test
+ public void testRenameWithUncompressedFiles () throws IOException {
+ MockZipEntryFilter mockFilter = new MockZipEntryFilter();
+ mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER);
+ mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER);
+ mockFilter.renameMap.putAll("hello.txt",
+ Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt"));
+ mockFilter.renameMap.put("hello2.txt", "hello2.txt");
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(mockFilter, out);
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.addZip(sampleZipWithTwoUncompressedEntries());
+ singleJar.close();
+ assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt",
+ "hello.txt", "hello2.txt", "hello.txt", "hello2.txt").inOrder();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "hello1.txt", "Hello World!");
+ assertEntry(zipInput, "hello2.txt", "Hello World 2!");
+ assertEntry(zipInput, "hello3.txt", "Hello World!");
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // The next two tests check that ZipCombiner can handle a ZIP with an data
+ // descriptor marker in the compressed data, i.e. that it does not scan for
+ // the data descriptor marker. It's unfortunately a bit tricky to create such
+ // a ZIP.
+ private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50;
+ private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50;
+ private static final byte[] DATA_DESCRIPTOR_MARKER_AS_BYTES = new byte[] {
+ 0x50, 0x4b, 0x07, 0x08
+ };
+
+ // Create a ZIP with an data descriptor marker in the DEFLATE content of a
+ // file. To do that, we build the ZIP byte by byte.
+ private InputStream zipWithUnexpectedDataDescriptorMarker() {
+ ByteBuffer out = ByteBuffer.wrap(new byte[200]).order(ByteOrder.LITTLE_ENDIAN);
+ out.clear();
+ // file header
+ out.putInt(LOCAL_FILE_HEADER_MARKER); // file header signature
+ out.putShort((short) 6); // version to extract
+ out.putShort((short) 8); // general purpose bit flag
+ out.putShort((short) ZipOutputStream.DEFLATED); // compression method
+ out.putShort((short) 0); // mtime (00:00:00)
+ out.putShort((short) 0x21); // mdate (1.1.1980)
+ out.putInt(0); // crc32
+ out.putInt(0); // compressed size
+ out.putInt(0); // uncompressed size
+ out.putShort((short) 1); // file name length
+ out.putShort((short) 0); // extra field length
+ out.put((byte) 'a'); // file name
+
+ // file contents
+ out.put((byte) 0x01); // deflated content block is last block and uncompressed
+ out.putShort((short) 4); // uncompressed block length
+ out.putShort((short) ~4); // negated uncompressed block length
+ out.putInt(DATA_DESCRIPTOR_MARKER); // 4 bytes uncompressed data
+
+ // data descriptor
+ out.putInt(DATA_DESCRIPTOR_MARKER); // data descriptor with marker
+ out.putInt((int) ZipFactory.calculateCrc32(DATA_DESCRIPTOR_MARKER_AS_BYTES));
+ out.putInt(9);
+ out.putInt(4);
+ // We omit the central directory here. It's currently not used by
+ // ZipCombiner or by java.util.zip.ZipInputStream, so that shouldn't be a
+ // problem.
+ return new ByteArrayInputStream(out.array());
+ }
+
+ // Check that the created ZIP is correct.
+ @Test
+ public void testZipWithUnexpectedDataDescriptorMarkerIsCorrect() throws IOException {
+ ZipInputStream zipInput = new ZipInputStream(zipWithUnexpectedDataDescriptorMarker());
+ assertEntry(zipInput, "a", DATA_DESCRIPTOR_MARKER_AS_BYTES);
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // Check that ZipCombiner handles the ZIP correctly.
+ @Test
+ public void testZipWithUnexpectedDataDescriptorMarker() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addZip(zipWithUnexpectedDataDescriptorMarker());
+ singleJar.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ assertEntry(zipInput, "a", DATA_DESCRIPTOR_MARKER_AS_BYTES);
+ assertNull(zipInput.getNextEntry());
+ }
+
+ // Create a ZIP with a partial entry.
+ private InputStream zipWithPartialEntry() {
+ ByteBuffer out = ByteBuffer.wrap(new byte[32]).order(ByteOrder.LITTLE_ENDIAN);
+ out.clear();
+ // file header
+ out.putInt(LOCAL_FILE_HEADER_MARKER); // file header signature
+ out.putShort((short) 6); // version to extract
+ out.putShort((short) 0); // general purpose bit flag
+ out.putShort((short) ZipOutputStream.STORED); // compression method
+ out.putShort((short) 0); // mtime (00:00:00)
+ out.putShort((short) 0x21); // mdate (1.1.1980)
+ out.putInt(0); // crc32
+ out.putInt(10); // compressed size
+ out.putInt(10); // uncompressed size
+ out.putShort((short) 1); // file name length
+ out.putShort((short) 0); // extra field length
+ out.put((byte) 'a'); // file name
+
+ // file contents
+ out.put((byte) 0x01);
+ // Unexpected end of file.
+
+ return new ByteArrayInputStream(out.array());
+ }
+
+ @Test
+ public void testBadZipFilePartialEntry() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipCombiner singleJar = new ZipCombiner(out)) {
+ singleJar.addZip(zipWithPartialEntry());
+ fail();
+ } catch (EOFException e) {
+ // Expected exception.
+ }
+ }
+
+ @Test
+ public void testSimpleJarAgainstJavaUtil() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ JarOutputStream jarOut = new JarOutputStream(out);
+ ZipEntry entry;
+ entry = new ZipEntry("META-INF/");
+ entry.setTime(DOS_EPOCH.getTime());
+ entry.setMethod(JarOutputStream.STORED);
+ entry.setSize(0);
+ entry.setCompressedSize(0);
+ entry.setCrc(0);
+ jarOut.putNextEntry(entry);
+ entry = new ZipEntry("META-INF/MANIFEST.MF");
+ entry.setTime(DOS_EPOCH.getTime());
+ entry.setMethod(JarOutputStream.DEFLATED);
+ jarOut.putNextEntry(entry);
+ jarOut.write(new byte[] { 1, 2, 3, 4 });
+ jarOut.close();
+ byte[] javaFile = out.toByteArray();
+ out.reset();
+
+ ZipCombiner singleJar = new ZipCombiner(out);
+ singleJar.addDirectory("META-INF/", DOS_EPOCH,
+ new ExtraData[] { new ExtraData((short) 0xCAFE, new byte[0]) });
+ singleJar.addFile("META-INF/MANIFEST.MF", DOS_EPOCH,
+ new ByteArrayInputStream(new byte[] { 1, 2, 3, 4 }));
+ singleJar.close();
+ byte[] singlejarFile = out.toByteArray();
+
+ new ZipTester(singlejarFile).validate();
+ assertZipFilesEquivalent(singlejarFile, javaFile);
+ }
+
+ void assertZipFilesEquivalent(byte[] x, byte[] y) {
+ assertEquals(x.length, y.length);
+
+ for (int i = 0; i < x.length; i++) {
+ if (x[i] != y[i]) {
+ // Allow general purpose bit 11 (UTF-8 encoding) used in jdk7 to differ
+ assertEquals("at position " + i, 0x08, x[i] ^ y[i]);
+ // Check that x[i] is the second byte of a general purpose bit flag.
+ // Phil Katz, you will never be forgotten.
+ assertTrue(
+ // Local header
+ x[i-7] == 'P' && x[i-6] == 'K' && x[i-5] == 3 && x[i-4] == 4 ||
+ // Central directory header
+ x[i-9] == 'P' && x[i-8] == 'K' && x[i-7] == 1 && x[i-6] == 2);
+ }
+ }
+ }
+
+ /**
+ * Ensures that the code that grows the central directory and the code that patches it is not
+ * obviously broken.
+ */
+ @Test
+ public void testLotsOfFiles() throws IOException {
+ int fileCount = 100;
+ for (int blockSize : new int[] { 1, 2, 3, 4, 10, 1000 }) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipCombiner zipCombiner = new ZipCombiner(
+ OutputMode.DONT_CARE, new CopyEntryFilter(), out, blockSize);
+ for (int i = 0; i < fileCount; i++) {
+ zipCombiner.addFile("hello" + i, DOS_EPOCH, asStream("Hello " + i + "!"));
+ }
+ zipCombiner.close();
+ ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray()));
+ for (int i = 0; i < fileCount; i++) {
+ assertEntry(zipInput, "hello" + i, "Hello " + i + "!");
+ }
+ assertNull(zipInput.getNextEntry());
+ new ZipTester(out.toByteArray()).validate();
+ }
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipFactory.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipFactory.java
new file mode 100644
index 0000000..a6474fa
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipFactory.java
@@ -0,0 +1,106 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * A helper class to create zip files for testing.
+ */
+public class ZipFactory {
+
+ static class Entry {
+ private final String name;
+ private final byte[] content;
+ private final boolean compressed;
+ private Entry(String name, byte[] content, boolean compressed) {
+ this.name = name;
+ this.content = content;
+ this.compressed = compressed;
+ }
+ }
+
+ private final List<Entry> entries = new ArrayList<>();
+
+ // Assumes that content was created locally. Does not perform a defensive copy!
+ private void addEntry(String name, byte[] content, boolean compressed) {
+ entries.add(new Entry(name, content, compressed));
+ }
+
+ public ZipFactory addFile(String name, String content) {
+ addEntry(name, content.getBytes(ISO_8859_1), true);
+ return this;
+ }
+
+ public ZipFactory addFile(String name, byte[] content) {
+ addEntry(name, content.clone(), true);
+ return this;
+ }
+
+ public ZipFactory addFile(String name, String content, boolean compressed) {
+ addEntry(name, content.getBytes(ISO_8859_1), compressed);
+ return this;
+ }
+
+ public ZipFactory addFile(String name, byte[] content, boolean compressed) {
+ addEntry(name, content.clone(), compressed);
+ return this;
+ }
+
+ public byte[] toByteArray() {
+ try {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ZipOutputStream zipper = new ZipOutputStream(out);
+ for (Entry entry : entries) {
+ ZipEntry zipEntry = new ZipEntry(entry.name);
+ if (entry.compressed) {
+ zipEntry.setMethod(ZipEntry.DEFLATED);
+ } else {
+ zipEntry.setMethod(ZipEntry.STORED);
+ zipEntry.setSize(entry.content.length);
+ zipEntry.setCrc(calculateCrc32(entry.content));
+ }
+ zipEntry.setTime(ZipCombiner.DOS_EPOCH.getTime());
+ zipper.putNextEntry(zipEntry);
+ zipper.write(entry.content);
+ zipper.closeEntry();
+ }
+ zipper.close();
+ return out.toByteArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public InputStream toInputStream() {
+ return new ByteArrayInputStream(toByteArray());
+ }
+
+ public static long calculateCrc32(byte[] content) {
+ CRC32 crc = new CRC32();
+ crc.update(content);
+ return crc.getValue();
+ }
+}
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java
new file mode 100644
index 0000000..c1293d6
--- /dev/null
+++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java
@@ -0,0 +1,412 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.singlejar;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.CRC32;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+/**
+ * A helper class to validate zip files and provide reasonable diagnostics (better than what zip
+ * does). We might want to make this into a fully-fledged binary some day.
+ */
+final class ZipTester {
+
+ // The following constants are ZIP-specific.
+ private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50;
+ private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50;
+ private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50;
+ private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50;
+
+ private static final int FILE_HEADER_BUFFER_SIZE = 26; // without marker
+ private static final int DATA_DESCRIPTOR_BUFFER_SIZE = 12; // without marker
+
+ private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 42; // without marker
+ private static final int END_OF_CENTRAL_DIRECTORY_BUFFER_SIZE = 18; // without marker
+
+ // Set if the size, compressed size and CRC are set to zero, and present in
+ // the data descriptor after the data.
+ private static final int SIZE_MASKED_FLAG = 1 << 3;
+
+ private static final int STORED_METHOD = 0;
+ private static final int DEFLATE_METHOD = 8;
+
+ private static final int VERSION_STORED = 10; // Version 1.0
+ private static final int VERSION_DEFLATE = 20; // Version 2.0
+
+ private static class Entry {
+ private final long pos;
+ private final String name;
+ private final int flags;
+ private final int method;
+ private final int dosTime;
+ Entry(long pos, String name, int flags, int method, int dosTime) {
+ this.pos = pos;
+ this.name = name;
+ this.flags = flags;
+ this.method = method;
+ this.dosTime = dosTime;
+ }
+ }
+
+ private final InputStream in;
+ private final byte[] buffer = new byte[1024];
+ private int bufferLength;
+ private int bufferOffset;
+ private long pos;
+
+ private List<Entry> entries = new ArrayList<Entry>();
+
+ public ZipTester(InputStream in) {
+ this.in = in;
+ }
+
+ public ZipTester(byte[] data) {
+ this(new ByteArrayInputStream(data));
+ }
+
+ private void warn(String msg) {
+ System.err.println("WARNING: " + msg);
+ }
+
+ private void readMoreData(String action) throws IOException {
+ if ((bufferLength > 0) && (bufferOffset > 0)) {
+ System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength);
+ }
+ if (bufferLength >= buffer.length) {
+ // The buffer size is specifically chosen to avoid this situation.
+ throw new AssertionError("Internal error: buffer overrun.");
+ }
+ bufferOffset = 0;
+ int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength);
+ if (bytesRead <= 0) {
+ throw new IOException("Unexpected end of file, while " + action);
+ }
+ bufferLength += bytesRead;
+ }
+
+ private int readByte(String action) throws IOException {
+ if (bufferLength == 0) {
+ readMoreData(action);
+ }
+ byte result = buffer[bufferOffset];
+ bufferOffset++; bufferLength--;
+ pos++;
+ return result & 0xff;
+ }
+
+ private long getUnsignedInt(String action) throws IOException {
+ int a = readByte(action);
+ int b = readByte(action);
+ int c = readByte(action);
+ int d = readByte(action);
+ return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL;
+ }
+
+ private void readFully(byte[] buffer, String action) throws IOException {
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) readByte(action);
+ }
+ }
+
+ private void skip(long length, String action) throws IOException {
+ for (long i = 0; i < length; i++) {
+ readByte(action);
+ }
+ }
+
+ private int getUnsignedShort(byte[] source, int offset) {
+ int a = source[offset + 0] & 0xff;
+ int b = source[offset + 1] & 0xff;
+ return (b << 8) | a;
+ }
+
+ private long getUnsignedInt(byte[] source, int offset) {
+ int a = source[offset + 0] & 0xff;
+ int b = source[offset + 1] & 0xff;
+ int c = source[offset + 2] & 0xff;
+ int d = source[offset + 3] & 0xff;
+ return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL;
+ }
+
+ private class DeflateInputStream extends InputStream {
+
+ private final byte[] singleByteBuffer = new byte[1];
+ private int consumedBytes;
+ private final Inflater inflater = new Inflater(true);
+ private long totalBytesRead;
+
+ private int inflateData(byte[] dest, int off, int len)
+ throws IOException {
+ consumedBytes = 0;
+ int bytesProduced = 0;
+ int bytesConsumed = 0;
+ while ((bytesProduced == 0) && !inflater.finished()) {
+ inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed);
+ int remainingBefore = inflater.getRemaining();
+ try {
+ bytesProduced = inflater.inflate(dest, off, len);
+ } catch (DataFormatException e) {
+ throw new IOException("Invalid deflate stream in ZIP file.", e);
+ }
+ bytesConsumed += remainingBefore - inflater.getRemaining();
+ consumedBytes = bytesConsumed;
+ if (bytesProduced == 0) {
+ if (inflater.needsDictionary()) {
+ // The DEFLATE algorithm as used in the ZIP file format does not
+ // require an additional dictionary.
+ throw new AssertionError("Inflater unexpectedly requires a dictionary.");
+ } else if (inflater.needsInput()) {
+ readMoreData("need more data for deflate");
+ } else if (inflater.finished()) {
+ return 0;
+ } else {
+ // According to the Inflater specification, this cannot happen.
+ throw new AssertionError("Inflater unexpectedly produced no output.");
+ }
+ }
+ }
+ return bytesProduced;
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (inflater.finished()) {
+ return -1;
+ }
+ int length = inflateData(b, off, len);
+ totalBytesRead += consumedBytes;
+ bufferLength -= consumedBytes;
+ bufferOffset += consumedBytes;
+ pos += consumedBytes;
+ return length == 0 ? -1 : length;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int bytesRead = read(singleByteBuffer, 0, 1);
+ return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff);
+ }
+ }
+
+ private void readEntry() throws IOException {
+ long entrypos = pos - 4;
+ String entryDesc = "file entry at " + Long.toHexString(entrypos);
+ byte[] entryBuffer = new byte[FILE_HEADER_BUFFER_SIZE];
+ readFully(entryBuffer, "reading file header");
+ int versionToExtract = getUnsignedShort(entryBuffer, 0);
+ int flags = getUnsignedShort(entryBuffer, 2);
+ int method = getUnsignedShort(entryBuffer, 4);
+ int dosTime = (int) getUnsignedInt(entryBuffer, 6);
+ int crc32 = (int) getUnsignedInt(entryBuffer, 10);
+ long compressedSize = getUnsignedInt(entryBuffer, 14);
+ long uncompressedSize = getUnsignedInt(entryBuffer, 18);
+ int filenameLength = getUnsignedShort(entryBuffer, 22);
+ int extraLength = getUnsignedShort(entryBuffer, 24);
+
+ byte[] filename = new byte[filenameLength];
+ readFully(filename, "reading file name");
+ skip(extraLength, "skipping extra data");
+
+ String name = new String(filename, "UTF-8");
+ for (int i = 0; i < filename.length; i++) {
+ if ((filename[i] < ' ') || (filename[i] > 127)) {
+ warn(entryDesc + ": file name has unexpected non-ascii characters");
+ }
+ }
+ entryDesc = "file entry '" + name + "' at " + Long.toHexString(entrypos);
+
+ if ((method != STORED_METHOD) && (method != DEFLATE_METHOD)) {
+ throw new IOException(entryDesc + ": unknown method " + method);
+ }
+ if ((flags != 0) && (flags != SIZE_MASKED_FLAG)) {
+ throw new IOException(entryDesc + ": unknown flags " + flags);
+ }
+ if ((method == STORED_METHOD) && (versionToExtract != VERSION_STORED)) {
+ warn(entryDesc + ": unexpected version to extract for stored entry " + versionToExtract);
+ }
+ if ((method == DEFLATE_METHOD) && (versionToExtract != VERSION_DEFLATE)) {
+// warn(entryDesc + ": unexpected version to extract for deflated entry " + versionToExtract);
+ }
+
+ if (method == STORED_METHOD) {
+ if (compressedSize != uncompressedSize) {
+ throw new IOException(entryDesc + ": stored entries should have identical compressed and "
+ + "uncompressed sizes");
+ }
+ skip(compressedSize, entryDesc + "skipping data");
+ } else {
+ // No OS resources are actually allocated.
+ @SuppressWarnings("resource") DeflateInputStream deflater = new DeflateInputStream();
+ long generatedBytes = 0;
+ byte[] deflated = new byte[1024];
+ int readBytes;
+ CRC32 crc = new CRC32();
+ while ((readBytes = deflater.read(deflated)) > 0) {
+ crc.update(deflated, 0, readBytes);
+ generatedBytes += readBytes;
+ }
+ int actualCrc32 = (int) crc.getValue();
+ long consumedBytes = deflater.totalBytesRead;
+ if (flags == SIZE_MASKED_FLAG) {
+ long id = getUnsignedInt("reading footer marker");
+ if (id != DATA_DESCRIPTOR_MARKER) {
+ throw new IOException(entryDesc + ": expected footer at " + Long.toHexString(pos - 4)
+ + ", but found " + Long.toHexString(id));
+ }
+ byte[] footer = new byte[DATA_DESCRIPTOR_BUFFER_SIZE];
+ readFully(footer, "reading footer");
+ crc32 = (int) getUnsignedInt(footer, 0);
+ compressedSize = getUnsignedInt(footer, 4);
+ uncompressedSize = getUnsignedInt(footer, 8);
+ }
+
+ if (consumedBytes != compressedSize) {
+ throw new IOException(entryDesc + ": amount of compressed data does not match value "
+ + "specified in the zip (specified: " + compressedSize + ", actual: " + consumedBytes
+ + ")");
+ }
+ if (generatedBytes != uncompressedSize) {
+ throw new IOException(entryDesc + ": amount of uncompressed data does not match value "
+ + "specified in the zip (specified: " + uncompressedSize + ", actual: "
+ + generatedBytes + ")");
+ }
+ if (crc32 != actualCrc32) {
+ throw new IOException(entryDesc + ": specified crc checksum does not match actual check "
+ + "sum");
+ }
+ }
+ entries.add(new Entry(entrypos, name, flags, method, dosTime));
+ }
+
+ @SuppressWarnings("unused") // A couple of unused local variables.
+ private void validateCentralDirectoryEntry(Entry entry) throws IOException {
+ long entrypos = pos - 4;
+ String entryDesc = "file directory entry '" + entry.name + "' at " + Long.toHexString(entrypos);
+
+ byte[] entryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE];
+ readFully(entryBuffer, "reading central directory entry");
+ int versionMadeBy = getUnsignedShort(entryBuffer, 0);
+ int versionToExtract = getUnsignedShort(entryBuffer, 2);
+ int flags = getUnsignedShort(entryBuffer, 4);
+ int method = getUnsignedShort(entryBuffer, 6);
+ int dosTime = (int) getUnsignedInt(entryBuffer, 8);
+ int crc32 = (int) getUnsignedInt(entryBuffer, 12);
+ long compressedSize = getUnsignedInt(entryBuffer, 16);
+ long uncompressedSize = getUnsignedInt(entryBuffer, 20);
+ int filenameLength = getUnsignedShort(entryBuffer, 24);
+ int extraLength = getUnsignedShort(entryBuffer, 26);
+ int commentLength = getUnsignedShort(entryBuffer, 28);
+ int diskNumberStart = getUnsignedShort(entryBuffer, 30);
+ int internalAttributes = getUnsignedShort(entryBuffer, 32);
+ int externalAttributes = (int) getUnsignedInt(entryBuffer, 34);
+ long offset = getUnsignedInt(entryBuffer, 38);
+
+ byte[] filename = new byte[filenameLength];
+ readFully(filename, "reading file name");
+ skip(extraLength, "skipping extra data");
+ String name = new String(filename, "UTF-8");
+
+ if (!name.equals(entry.name)) {
+ throw new IOException(entryDesc + ": file name in central directory does not match original "
+ + "name");
+ }
+ if (offset != entry.pos) {
+ throw new IOException(entryDesc);
+ }
+ if (flags != entry.flags) {
+ throw new IOException(entryDesc);
+ }
+ if (method != entry.method) {
+ throw new IOException(entryDesc);
+ }
+ if (dosTime != entry.dosTime) {
+ throw new IOException(entryDesc);
+ }
+ }
+
+ private void validateCentralDirectory() throws IOException {
+ boolean first = true;
+ for (Entry entry : entries) {
+ if (first) {
+ first = false;
+ } else {
+ long id = getUnsignedInt("reading marker");
+ if (id != CENTRAL_DIRECTORY_MARKER) {
+ throw new IOException();
+ }
+ }
+ validateCentralDirectoryEntry(entry);
+ }
+ }
+
+ @SuppressWarnings("unused") // A couple of unused local variables.
+ private void validateEndOfCentralDirectory() throws IOException {
+ long id = getUnsignedInt("expecting end of central directory");
+ byte[] entryBuffer = new byte[END_OF_CENTRAL_DIRECTORY_BUFFER_SIZE];
+ readFully(entryBuffer, "reading end of central directory");
+ int diskNumber = getUnsignedShort(entryBuffer, 0);
+ int startDiskNumber = getUnsignedShort(entryBuffer, 2);
+ int numEntries = getUnsignedShort(entryBuffer, 4);
+ int numTotalEntries = getUnsignedShort(entryBuffer, 6);
+ long centralDirectorySize = getUnsignedInt(entryBuffer, 8);
+ long centralDirectoryOffset = getUnsignedInt(entryBuffer, 12);
+ int commentLength = getUnsignedShort(entryBuffer, 16);
+ if (diskNumber != 0) {
+ throw new IOException(String.format("diskNumber=%d", diskNumber));
+ }
+ if (startDiskNumber != 0) {
+ throw new IOException(String.format("startDiskNumber=%d", diskNumber));
+ }
+ if (numEntries != numTotalEntries) {
+ throw new IOException(String.format("numEntries=%d numTotalEntries=%d",
+ numEntries, numTotalEntries));
+ }
+ if (numEntries != (entries.size() % 0x10000)) {
+ throw new IOException("bad number of entries in central directory footer");
+ }
+ if (numTotalEntries != (entries.size() % 0x10000)) {
+ throw new IOException("bad number of entries in central directory footer");
+ }
+ if (commentLength != 0) {
+ throw new IOException("Zip file comment is unexpected");
+ }
+ if (id != END_OF_CENTRAL_DIRECTORY_MARKER) {
+ throw new IOException("Expected end of central directory marker");
+ }
+ }
+
+ public void validate() throws IOException {
+ while (true) {
+ long id = getUnsignedInt("reading marker");
+ if (id == LOCAL_FILE_HEADER_MARKER) {
+ readEntry();
+ } else if (id == CENTRAL_DIRECTORY_MARKER) {
+ validateCentralDirectory();
+ validateEndOfCentralDirectory();
+ return;
+ } else {
+ throw new IOException("unexpected result for marker: "
+ + Long.toHexString(id) + " at position " + Long.toHexString(pos - 4));
+ }
+ }
+ }
+}
diff --git a/src/main/cpp/BUILD b/src/main/cpp/BUILD
new file mode 100644
index 0000000..2681792
--- /dev/null
+++ b/src/main/cpp/BUILD
@@ -0,0 +1,71 @@
+cc_library(
+ name = "util",
+ srcs = [
+ "util/numbers.cc",
+ "util/port.cc",
+ "util/strings.cc",
+ ],
+ hdrs = [
+ "util/numbers.h",
+ "util/port.h",
+ "util/strings.h",
+ ],
+ copts = [
+ "-DBLAZE_OPENSOURCE=1",
+ ],
+ includes = ["."],
+)
+
+cc_library(
+ name = "md5",
+ srcs = ["util/md5.cc"],
+ hdrs = ["util/md5.h"],
+ includes = ["."],
+ visibility = ["//visibility:public"],
+)
+
+filegroup(
+ name = "blaze_util_os",
+ srcs = select({
+ "//src:darwin": ["blaze_util_darwin.cc"],
+ "//conditions:default": ["blaze_util_linux.cc"],
+ }),
+)
+
+cc_binary(
+ name = "client",
+ srcs = [
+ "blaze.cc",
+ "blaze_startup_options.cc",
+ "blaze_startup_options_common.cc",
+ "blaze_util.cc",
+ "option_processor.cc",
+ "util/file.cc",
+ ":blaze_util_os",
+ ],
+ copts = [
+ "-DBLAZE_JAVA_CPU=\\\"k8\\\"",
+ "-DBLAZE_OPENSOURCE=1",
+ ],
+ includes = ["."],
+ linkopts = select({
+ "//src:darwin": [
+ ],
+ "//conditions:default": [
+ "-larchive",
+ "-lrt",
+ ],
+ }),
+ visibility = ["//src:__pkg__"],
+ deps = select({
+ "//src:darwin": [
+ ":md5",
+ ":util",
+ "//fromhost:libarchive",
+ ],
+ "//conditions:default": [
+ ":md5",
+ ":util",
+ ],
+ }),
+)
diff --git a/src/main/cpp/blaze.cc b/src/main/cpp/blaze.cc
new file mode 100644
index 0000000..d51ab78
--- /dev/null
+++ b/src/main/cpp/blaze.cc
@@ -0,0 +1,1679 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// blaze.cc: bootstrap and client code for Blaze server.
+//
+// Responsible for:
+// - extracting the Python, C++ and Java components.
+// - starting the server or finding the existing one.
+// - client options parsing.
+// - passing the argv array, and printing the out/err streams.
+// - signal handling.
+// - exiting with the right error/WTERMSIG code.
+// - debugger + profiler support.
+// - mutual exclusion between batch invocations.
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <poll.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/un.h>
+#include <time.h>
+#include <unistd.h>
+#include <utime.h>
+#include <algorithm>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "blaze_exit_code.h"
+#include "blaze_startup_options.h"
+#include "blaze_util.h"
+#include "blaze_util_platform.h"
+#include "option_processor.h"
+#include "util/file.h"
+#include "util/md5.h"
+#include "util/numbers.h"
+#include "util/port.h"
+#include "util/strings.h"
+#include "archive.h"
+#include "archive_entry.h"
+
+using std::set;
+using std::vector;
+
+// This should already be defined in sched.h, but it's not.
+#ifndef SCHED_BATCH
+#define SCHED_BATCH 3
+#endif
+
+namespace blaze {
+
+// Enable messages mostly of interest to developers.
+static const bool SPAM = getenv("VERBOSE_BLAZE_CLIENT") != NULL;
+
+// Blaze is being run by a test.
+static const bool TESTING = getenv("TEST_TMPDIR") != NULL;
+
+extern char **environ;
+
+////////////////////////////////////////////////////////////////////////
+// Global Variables
+
+// The reason for a blaze server restart.
+// Keep in sync with logging.proto
+enum RestartReason {
+ NO_RESTART = 0,
+ NO_DAEMON,
+ NEW_VERSION,
+ NEW_OPTIONS
+};
+
+struct GlobalVariables {
+ // Used to make concurrent invocations of this program safe.
+ string lockfile; // = <output_base>/lock
+ int lockfd;
+
+ string jvm_log_file; // = <output_base>/server/jvm.out
+
+ string cwd;
+
+ // The nearest enclosing workspace directory, starting from cwd.
+ // If not under a workspace directory, this is equal to cwd.
+ string workspace;
+
+ // Option processor responsible for parsing RC files and converting them into
+ // the argument list passed on to the server.
+ OptionProcessor option_processor;
+
+ pid_t server_pid;
+
+ volatile sig_atomic_t sigint_count;
+
+ // The number of the last received signal that should cause the client
+ // to shutdown. This is saved so that the client's WTERMSIG can be set
+ // correctly. (Currently only SIGPIPE uses this mechanism.)
+ volatile sig_atomic_t received_signal;
+
+ // Contains the relative paths of all the files in the attached zip, and is
+ // populated during GetInstallDir().
+ vector<string> extracted_binaries;
+
+ // Parsed startup options
+ BlazeStartupOptions options;
+
+ // The time in ms the launcher spends before sending the request to the Blaze
+ uint64 startup_time;
+
+ // The time spent on extracting the new blaze version
+ // This is part of startup_time
+ uint64 extract_data_time;
+
+ // The time in ms if a command had to wait on a busy Blaze server process
+ // This is part of startup_time
+ uint64 command_wait_time;
+
+ RestartReason restart_reason;
+
+ // Absolute path of the blaze binary
+ string binary_path;
+};
+
+static GlobalVariables *globals;
+
+void InitGlobals() {
+ globals = new GlobalVariables;
+ globals->sigint_count = 0;
+ globals->startup_time = 0;
+ globals->extract_data_time = 0;
+ globals->command_wait_time = 0;
+ globals->restart_reason = NO_RESTART;
+}
+
+////////////////////////////////////////////////////////////////////////
+// Logic
+
+
+// Returns the canonical form of the base dir given a root and a hashable
+// string. The resulting dir is composed of the root + md5(hashable)
+static string GetHashedBaseDir(const string &root,
+ const string &hashable) {
+ unsigned char buf[17];
+ blaze_util::Md5Digest digest;
+ digest.Update(hashable.data(), hashable.size());
+ digest.Finish(buf);
+ return root + "/" + digest.String();
+}
+
+// Returns the install base (the root concatenated with the contents of the file
+// 'install_base_key' contained as a ZIP entry in the Blaze binary); as a side
+// effect, it also populates the extracted_binaries global variable.
+static string GetInstallBase(const string &root, const string &self_path) {
+ string key_file = "install_base_key";
+ struct archive *blaze_zip = archive_read_new();
+ archive_read_support_format_zip(blaze_zip);
+ int retval = archive_read_open_filename(blaze_zip, self_path.c_str(), 10240);
+ if (retval != ARCHIVE_OK) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to open blaze as a zip file: (%d) %s",
+ archive_errno(blaze_zip), archive_error_string(blaze_zip));
+ }
+
+ struct archive_entry *entry;
+ string install_base_key;
+ while (archive_read_next_header(blaze_zip, &entry) == ARCHIVE_OK) {
+ string pathname = archive_entry_pathname(entry);
+ globals->extracted_binaries.push_back(pathname);
+
+ if (key_file == pathname) {
+ const int size = 32;
+ char buf[size];
+ int bytesRead = archive_read_data(blaze_zip, &buf, size);
+ if (bytesRead < 0) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to extract install_base_key: (%d) %s",
+ archive_errno(blaze_zip), archive_error_string(blaze_zip));
+ }
+ if (bytesRead < 32) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to extract install_base_key: file too short");
+ }
+ install_base_key = string(buf, bytesRead);
+ }
+ }
+ retval = archive_read_free(blaze_zip);
+ if (retval != ARCHIVE_OK) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to close install_base_key's containing zip file");
+ }
+
+ return root + "/" + install_base_key;
+}
+
+// Escapes colons by replacing them with '_C' and underscores by replacing them
+// with '_U'. E.g. "name:foo_bar" becomes "name_Cfoo_Ubar"
+static string EscapeForOptionSource(const string& input) {
+ string result = input;
+ blaze_util::Replace("_", "_U", &result);
+ blaze_util::Replace(":", "_C", &result);
+ return result;
+}
+
+// Returns the JVM command argument array.
+static vector<string> GetArgumentArray() {
+ vector<string> result;
+
+ // e.g. A Blaze server process running in ~/src/build_root (where there's a
+ // ~/src/build_root/WORKSPACE file) will appear in ps(1) as "blaze(src)".
+ string workspace =
+ blaze_util::Basename(blaze_util::Dirname(globals->workspace));
+ result.push_back("blaze(" + workspace + ")");
+ if (globals->options.batch) {
+ result.push_back("-client");
+ result.push_back("-Xms256m");
+ result.push_back("-XX:NewRatio=4");
+ } else {
+ result.push_back("-server");
+ }
+
+ result.push_back("-XX:+HeapDumpOnOutOfMemoryError");
+ string heap_crash_path = globals->options.output_base;
+ result.push_back("-XX:HeapDumpPath=" + heap_crash_path);
+
+ result.push_back("-Xverify:none");
+
+ // Add JVM arguments particular to building blaze64 and particular JVM
+ // versions.
+ string error;
+ blaze_exit_code::ExitCode jvm_args_exit_code =
+ globals->options.AddJVMArguments(globals->options.GetHostJavabase(),
+ &result, &error);
+ if (jvm_args_exit_code != blaze_exit_code::SUCCESS) {
+ die(jvm_args_exit_code, "%s", error.c_str());
+ }
+
+ // We put all directories on the java.library.path that contain .so files.
+ string java_library_path = "-Djava.library.path=";
+ string real_install_dir = blaze_util::JoinPath(globals->options.install_base,
+ "_embedded_binaries");
+ bool first = true;
+ for (const auto& it : globals->extracted_binaries) {
+ if (blaze::IsSharedLibrary(it)) {
+ if (!first) {
+ java_library_path += ":";
+ }
+ first = false;
+ java_library_path += blaze_util::JoinPath(real_install_dir,
+ blaze_util::Dirname(it));
+ }
+ }
+ result.push_back(java_library_path);
+
+ // Force use of latin1 for file names.
+ result.push_back("-Dfile.encoding=ISO-8859-1");
+
+ if (globals->options.host_jvm_debug) {
+ fprintf(stderr,
+ "Running host JVM under debugger (listening on TCP port 5005).\n");
+ // Start JVM so that it listens for a connection from a
+ // JDWP-compliant debugger:
+ result.push_back("-Xdebug");
+ result.push_back("-Xrunjdwp:transport=dt_socket,server=y,address=5005");
+ }
+
+ blaze_util::SplitQuotedStringUsing(globals->options.host_jvm_args, ' ',
+ &result);
+
+ result.push_back("-jar");
+ result.push_back(blaze_util::JoinPath(real_install_dir,
+ globals->extracted_binaries[0]));
+
+ if (!globals->options.batch) {
+ result.push_back("--max_idle_secs");
+ result.push_back(std::to_string(globals->options.max_idle_secs));
+ } else {
+ result.push_back("--batch");
+ }
+ result.push_back("--install_base=" + globals->options.install_base);
+ result.push_back("--output_base=" + globals->options.output_base);
+ result.push_back("--workspace_directory=" + globals->workspace);
+ if (!globals->options.skyframe.empty()) {
+ result.push_back("--skyframe=" + globals->options.skyframe);
+ }
+ if (globals->options.allow_configurable_attributes) {
+ result.push_back("--allow_configurable_attributes");
+ }
+ if (globals->options.watchfs) {
+ result.push_back("--watchfs");
+ }
+ if (globals->options.fatal_event_bus_exceptions) {
+ result.push_back("--fatal_event_bus_exceptions");
+ } else {
+ result.push_back("--nofatal_event_bus_exceptions");
+ }
+ if (globals->options.webstatus_port) {
+ result.push_back("--use_webstatusserver=" + \
+ std::to_string(globals->options.webstatus_port));
+ }
+
+ // This is only for Blaze reporting purposes; the real interpretation of the
+ // jvm flags occurs when we set up the java command line.
+ if (globals->options.host_jvm_debug) {
+ result.push_back("--host_jvm_debug");
+ }
+ if (!globals->options.host_jvm_profile.empty()) {
+ result.push_back("--host_jvm_profile=" + globals->options.host_jvm_profile);
+ }
+ if (!globals->options.host_jvm_args.empty()) {
+ result.push_back("--host_jvm_args=" + globals->options.host_jvm_args);
+ }
+ globals->options.AddExtraOptions(&result);
+
+ // The option sources are transmitted in the following format:
+ // --option_sources=option1:source1:option2:source2:...
+ string option_sources = "--option_sources=";
+ first = true;
+ for (const auto& it : globals->options.option_sources) {
+ if (!first) {
+ option_sources += ":";
+ }
+
+ first = false;
+ option_sources += EscapeForOptionSource(it.first) + ":" +
+ EscapeForOptionSource(it.second);
+ }
+
+ result.push_back(option_sources);
+ return result;
+}
+
+// Add commom command options for logging to the given argument array.
+static void AddLoggingArgs(vector<string>* args) {
+ args->push_back("--startup_time=" + std::to_string(globals->startup_time));
+ if (globals->command_wait_time != 0) {
+ args->push_back("--command_wait_time=" +
+ std::to_string(globals->command_wait_time));
+ }
+ if (globals->extract_data_time != 0) {
+ args->push_back("--extract_data_time=" +
+ std::to_string(globals->extract_data_time));
+ }
+ if (globals->restart_reason != NO_RESTART) {
+ const char *reasons[] = {
+ "no_restart", "no_daemon", "new_version", "new_options"
+ };
+ args->push_back(
+ string("--restart_reason=") + reasons[globals->restart_reason]);
+ }
+ args->push_back(
+ string("--binary_path=") + globals->binary_path);
+}
+
+
+// Join the elements of the specified array with NUL's (\0's), akin to the
+// format of /proc/$PID/cmdline.
+string GetArgumentString(const vector<string>& argument_array) {
+ string result;
+ blaze_util::JoinStrings(argument_array, '\0', &result);
+ return result;
+}
+
+// Causes the current process to become a daemon (i.e. a child of
+// init, detached from the terminal, in its own session.) We don't
+// change cwd, though.
+static void Daemonize(int socket) {
+ // Don't call die() or exit() in this function; we're already in a
+ // child process so it won't work as expected. Just don't do
+ // anything that can possibly fail. :)
+
+ signal(SIGHUP, SIG_IGN);
+ if (fork() > 0) {
+ // This second fork is required iff there's any chance cmd will
+ // open an specific tty explicitly, e.g., open("/dev/tty23"). If
+ // not, this fork can be removed.
+ _exit(blaze_exit_code::SUCCESS);
+ }
+
+ setsid();
+
+ close(0);
+ close(1);
+ close(2);
+ close(socket);
+
+ open("/dev/null", O_RDONLY); // stdin
+ // stdout:
+ if (open(globals->jvm_log_file.c_str(),
+ O_WRONLY | O_CREAT | O_TRUNC, 0666) == -1) {
+ // In a daemon, no-one can hear you scream.
+ open("/dev/null", O_WRONLY);
+ }
+ dup(STDOUT_FILENO); // stderr (2>&1)
+
+ // Keep server from inheriting a useless fd.
+ // The file lock was already lost at fork().
+ close(globals->lockfd);
+}
+
+// Do a chdir into the workspace, and die if it fails.
+static void GoToWorkspace() {
+ if (BlazeStartupOptions::InWorkspace(globals->workspace) &&
+ chdir(globals->workspace.c_str()) != 0) {
+ pdie(blaze_exit_code::INTERNAL_ERROR,
+ "chdir() into %s failed", globals->workspace.c_str());
+ }
+}
+
+// Check the java version if a java version specification is bundled. On
+// success,
+// return the executable path of the java command.
+static string VerifyJavaVersionAndGetJvm() {
+ string exe = globals->options.GetJvm();
+
+ string version_spec_file = blaze_util::JoinPath(
+ blaze_util::JoinPath(globals->options.install_base, "_embedded_binaries"),
+ "java.version");
+ string version_spec = "";
+ if (ReadFile(version_spec_file, &version_spec)) {
+ blaze_util::StripWhitespace(&version_spec);
+ // A version specification is given, get version of java.
+ string jvm_version = GetJvmVersion(exe);
+
+ // Compare that jvm_version is found and at least the one specified.
+ if (jvm_version.size() == 0) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Java version not detected while at least %s is needed.\n"
+ "Please set JAVA_HOME.", version_spec.c_str());
+ } else if (!CheckJavaVersionIsAtLeast(jvm_version, version_spec)) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Java version is %s while at least %s is needed.\n"
+ "Please set JAVA_HOME.",
+ jvm_version.c_str(), version_spec.c_str());
+ }
+ }
+
+ return exe;
+}
+
+// Starts the Blaze server. Returns a readable fd connected to the server.
+// This is currently used only to detect liveness.
+static int StartServer(int socket) {
+ vector<string> jvm_args_vector = GetArgumentArray();
+ string argument_string = GetArgumentString(jvm_args_vector);
+
+ // Write the cmdline argument string to the server dir. If we get to this
+ // point, there is no server running, so we don't overwrite the cmdline file
+ // for the existing server. If might be that the server dies and the cmdline
+ // file stays there, but that is not a problem, since we always check the
+ // server, too.
+ WriteFile(argument_string, globals->options.output_base + "/server/cmdline");
+
+ // unless we restarted for a new-version, mark this as initial start
+ if (globals->restart_reason == NO_RESTART) {
+ globals->restart_reason = NO_DAEMON;
+ }
+
+ // Computing this path may report a fatal error, so do it before forking.
+ string exe = VerifyJavaVersionAndGetJvm();
+
+ // Go to the workspace before we daemonize, so
+ // we can still print errors to the terminal.
+ GoToWorkspace();
+
+ int fds[2];
+ if (pipe(fds)) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "pipe creation failed");
+ }
+ int child = fork();
+ if (child == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "fork() failed");
+ } else if (child > 0) { // we're the parent
+ close(fds[1]); // parent keeps only the reading side
+ return fds[0];
+ } else {
+ close(fds[0]); // child keeps only the writing side
+ }
+
+ Daemonize(socket);
+ ExecuteProgram(exe, jvm_args_vector);
+ pdie(blaze_exit_code::INTERNAL_ERROR, "execv of '%s' failed", exe.c_str());
+}
+
+static bool KillRunningServerIfAny();
+
+// Replace this process with blaze in standalone/batch mode.
+// The batch mode blaze process handles the command and exits.
+//
+// This function passes the commands array to the blaze process.
+// This array should start with a command ("build", "info", etc.).
+static void StartStandalone() {
+ KillRunningServerIfAny();
+
+ // Wall clock time since process startup.
+ globals->startup_time = ProcessClock() / 1000000LL;
+
+ if (VerboseLogging()) {
+ fprintf(stderr, "Starting blaze in batch mode.\n");
+ }
+ string command = globals->option_processor.GetCommand();
+ vector<string> command_arguments;
+ globals->option_processor.GetCommandArguments(&command_arguments);
+
+ if (!command_arguments.empty() && command == "shutdown") {
+ fprintf(stderr,
+ "WARNING: Running command \"shutdown\" in batch mode. Batch mode "
+ "is triggered\nwhen not running blaze within a workspace. If you "
+ "intend to shutdown an\nexisting blaze server, run \"blaze "
+ "shutdown\" from the directory where\nit was started.\n");
+ }
+ vector<string> jvm_args_vector = GetArgumentArray();
+ if (command != "") {
+ jvm_args_vector.push_back(command);
+ AddLoggingArgs(&jvm_args_vector);
+ }
+
+ jvm_args_vector.insert(jvm_args_vector.end(),
+ command_arguments.begin(),
+ command_arguments.end());
+
+ GoToWorkspace();
+
+ string exe = VerifyJavaVersionAndGetJvm();
+ ExecuteProgram(exe, jvm_args_vector);
+ pdie(blaze_exit_code::INTERNAL_ERROR, "execv of '%s' failed", exe.c_str());
+}
+
+// Like connect(2), but uses the AF_UNIX address denoted by socket_file,
+// resolving symbolic links. (The server may make "socket_file" a
+// symlink, to avoid ENAMETOOLONG, in which case the client must
+// resolve it in userspace before connecting.)
+static int Connect(int socket, const string &socket_file) {
+ struct sockaddr_un addr;
+ addr.sun_family = AF_UNIX;
+
+ char *resolved_path = realpath(socket_file.c_str(), NULL);
+ if (resolved_path != NULL) {
+ strncpy(addr.sun_path, resolved_path, sizeof addr.sun_path);
+ addr.sun_path[sizeof addr.sun_path - 1] = '\0';
+ free(resolved_path);
+ sockaddr *paddr = reinterpret_cast<sockaddr *>(&addr);
+ return connect(socket, paddr, sizeof addr);
+ } else if (errno == ENOENT) { // No socket means no server to connect to
+ errno = ECONNREFUSED;
+ return -1;
+ } else {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "realpath('%s') failed", socket_file.c_str());
+ }
+}
+
+// Write the contents of file_name to stream.
+static void WriteFileToStreamOrDie(FILE *stream, const char *file_name) {
+ FILE *fp = fopen(file_name, "r");
+ if (fp == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "opening %s failed", file_name);
+ }
+ char buffer[255];
+ int num_read;
+ while ((num_read = fread(buffer, 1, sizeof buffer, fp)) > 0) {
+ if (ferror(fp)) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "failed to read from '%s'", file_name);
+ }
+ fwrite(buffer, 1, num_read, stream);
+ }
+ fclose(fp);
+}
+
+// Connects to the Blaze server, returning the socket, or -1 if no
+// server is running and !start. If start, attempts to start a new
+// server, and exits on failure.
+static int ConnectToServer(bool start) {
+ int s = socket(PF_UNIX, SOCK_STREAM, 0);
+ if (s == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "can't create AF_UNIX socket");
+ }
+
+ string server_dir = globals->options.output_base + "/server";
+
+ // The server dir has the socket, so we don't allow access by other
+ // users.
+ if (MakeDirectories(server_dir, 0700) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "server directory '%s' could not be created", server_dir.c_str());
+ }
+
+ string socket_file = server_dir + "/server.socket";
+
+ if (Connect(s, socket_file) == 0) {
+ return s;
+ }
+ if (!start) {
+ return -1;
+ } else {
+ SetScheduling(
+ globals->options.batch_cpu_scheduling,
+ globals->options.io_nice_level);
+
+ int fd = StartServer(s);
+ if (fcntl(fd, F_SETFL, O_NONBLOCK | fcntl(fd, F_GETFL))) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Failed: fcntl to enable O_NONBLOCK on pipe");
+ }
+ // Give the server one minute to start up.
+ for (int ii = 0; ii < 600; ++ii) { // 60s; enough time to connect
+ // with debugger
+ if (Connect(s, socket_file) == 0) {
+ if (ii) {
+ fputc('\n', stderr);
+ fflush(stderr);
+ }
+ return s;
+ }
+ fputc('.', stderr);
+ fflush(stderr);
+ poll(NULL, 0, 100); // sleep 100ms. (usleep(3) is obsolete.)
+ char c;
+ if (read(fd, &c, 1) != -1 || errno != EAGAIN) {
+ fprintf(stderr, "\nunexpected pipe read status: %s\n"
+ "Server presumed dead. Now printing '%s':\n",
+ strerror(errno), globals->jvm_log_file.c_str());
+ WriteFileToStreamOrDie(stderr, globals->jvm_log_file.c_str());
+ exit(blaze_exit_code::INTERNAL_ERROR);
+ }
+ }
+ die(blaze_exit_code::INTERNAL_ERROR,
+ "\nError: couldn't connect to server at '%s' after 60 seconds.",
+ socket_file.c_str());
+ }
+}
+
+
+// Kills the specified running Blaze server.
+static void KillRunningServer(pid_t server_pid) {
+ fprintf(stderr, "Sending SIGTERM to previous Blaze server (pid=%d)... ",
+ server_pid);
+ fflush(stderr);
+ for (int ii = 0; ii < 100; ++ii) { // wait up to 10s
+ if (kill(server_pid, SIGTERM) == -1) {
+ fprintf(stderr, "done.\n");
+ return; // Ding! Dong! The witch is dead!
+ }
+ poll(NULL, 0, 100); // sleep 100ms. (usleep(3) is obsolete.)
+ }
+
+ // If the previous attempt did not suceeded, kill the whole group.
+ fprintf(stderr,
+ "Sending SIGKILL to previous Blaze server process group (pid=%d)... ",
+ server_pid);
+ fflush(stderr);
+ killpg(server_pid, SIGKILL);
+ if (kill(server_pid, 0) == -1) { // (probe)
+ fprintf(stderr, "could not be killed.\n"); // task state 'Z' or 'D'?
+ exit(1); // TODO(bazel-team): confirm whether this is an internal error.
+ } else {
+ fprintf(stderr, "killed.\n");
+ }
+}
+
+
+// Kills the running Blaze server, if any. Finds the pid from the socket.
+static bool KillRunningServerIfAny() {
+ int socket = ConnectToServer(false);
+ if (socket != -1) {
+ KillRunningServer(GetPeerProcessId(socket));
+ return true;
+ }
+ return false;
+}
+
+
+// Calls fsync() on the file (or directory) specified in 'file_path'.
+// pdie()'s if syncing fails.
+static void SyncFile(const char *file_path) {
+ // fsync always fails on Cygwin with "Permission denied" for some reason.
+#ifndef __CYGWIN__
+ int fd = open(file_path, O_RDONLY);
+ if (fd < 0) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "failed to open '%s' for syncing", file_path);
+ }
+ if (fsync(fd) < 0) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "failed to sync '%s'", file_path);
+ }
+ close(fd);
+#endif
+}
+
+// Walks the temporary directory recursively and collects full file paths.
+static void CollectExtractedFiles(const string &dir_path, vector<string> &files) {
+ DIR *dir;
+ struct dirent *ent;
+
+ if ((dir = opendir(dir_path.c_str())) == NULL) {
+ die(blaze_exit_code::INTERNAL_ERROR, "opendir failed");
+ }
+
+ while ((ent = readdir(dir)) != NULL) {
+ if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
+ continue;
+ }
+
+ string filename(blaze_util::JoinPath(dir_path, ent->d_name));
+ bool is_directory;
+ if (ent->d_type == DT_UNKNOWN) {
+ struct stat buf;
+ if (lstat(filename.c_str(), &buf) == -1) {
+ die(blaze_exit_code::INTERNAL_ERROR, "stat failed");
+ }
+ is_directory = S_ISDIR(buf.st_mode);
+ } else {
+ is_directory = (ent->d_type == DT_DIR);
+ }
+
+ if (is_directory) {
+ CollectExtractedFiles(filename, files);
+ } else {
+ files.push_back(filename);
+ }
+ }
+
+ closedir(dir);
+}
+
+// Actually extracts the embedded data files into the tree whose root
+// is 'embedded_binaries'.
+static void ActuallyExtractData(const string &argv0,
+ const string &embedded_binaries) {
+ if (MakeDirectories(embedded_binaries, 0777) == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR,
+ "couldn't create '%s'", embedded_binaries.c_str());
+ }
+
+ fprintf(stderr, "Extracting Blaze installation...\n");
+
+ struct archive *blaze_zip = archive_read_new();
+ archive_read_support_format_zip(blaze_zip);
+ int retval = archive_read_open_filename(blaze_zip, argv0.c_str(), 10240);
+ if (retval != ARCHIVE_OK) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to open blaze as a zip file");
+ }
+
+ struct archive_entry *entry;
+ string install_base_key;
+ while (archive_read_next_header(blaze_zip, &entry) == ARCHIVE_OK) {
+ string path = blaze_util::JoinPath(
+ embedded_binaries, archive_entry_pathname(entry));
+ if (MakeDirectories(blaze_util::Dirname(path), 0777) == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR,
+ "couldn't create '%s'", path.c_str());
+ }
+ int fd = open(path.c_str(), O_CREAT | O_WRONLY, archive_entry_perm(entry));
+ if (fd < 0) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to open extraction file: %s", strerror(errno));
+ }
+
+ const void *buf;
+ size_t size;
+ off_t offset;
+ while (true) {
+ retval = archive_read_data_block(blaze_zip, &buf, &size, &offset);
+ if (retval == ARCHIVE_EOF) {
+ break;
+ } else if (retval != ARCHIVE_OK) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to extract data from blaze zip: (%d) %s",
+ archive_errno(blaze_zip), archive_error_string(blaze_zip));
+ }
+ if (write(fd, buf, size) != size) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nError writing zipped file to %s", path.c_str());
+ }
+ }
+ if (close(fd) != 0) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nCould not close file %s", path.c_str());
+ }
+ }
+ retval = archive_read_free(blaze_zip);
+ if (retval != ARCHIVE_OK) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "\nFailed to close blaze zip");
+ }
+
+ const time_t TEN_YEARS_IN_SEC = 3600 * 24 * 365 * 10;
+ time_t future_time = time(NULL) + TEN_YEARS_IN_SEC;
+
+ // Set the timestamps of the extracted files to the future and make sure (or
+ // at least as sure as we can...) that the files we have written are actually
+ // on the disk.
+
+ vector<string> extracted_files;
+ CollectExtractedFiles(embedded_binaries, extracted_files);
+
+ set<string> synced_directories;
+ for (vector<string>::iterator it = extracted_files.begin(); it != extracted_files.end(); it++) {
+
+ const char *extracted_path = it->c_str();
+
+ // Set the time to a distantly futuristic value so we can observe tampering.
+ // Note that keeping the default timestamp set by unzip (1970-01-01) and using
+ // that to detect tampering is not enough, because we also need the timestamp
+ // to change between Blaze releases so that the metadata cache knows that
+ // the files may have changed. This is important for actions that use
+ // embedded binaries as artifacts.
+ struct utimbuf times = { future_time, future_time };
+ if (utime(extracted_path, ×) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "failed to set timestamp on '%s'", extracted_path);
+ }
+
+ SyncFile(extracted_path);
+
+ string directory = blaze_util::Dirname(extracted_path);
+
+ // Now walk up until embedded_binaries and sync every directory in between.
+ // synced_directories is used to avoid syncing the same directory twice.
+ // The !directory.empty() and directory != "/" conditions are not strictly
+ // needed, but it makes this loop more robust, because otherwise, if due to
+ // some glitch, directory was not under embedded_binaries, it would get
+ // into an infinite loop.
+ while (directory != embedded_binaries &&
+ synced_directories.count(directory) == 0 &&
+ !directory.empty() &&
+ directory != "/") {
+ SyncFile(directory.c_str());
+ synced_directories.insert(directory);
+ directory = blaze_util::Dirname(directory);
+ }
+ }
+
+ SyncFile(embedded_binaries.c_str());
+}
+
+// Installs Blaze by extracting the embedded data files, iff necessary.
+// The MD5-named install_base directory on disk is trusted; we assume
+// no-one has modified the extracted files beneath this directory once
+// it is in place. Concurrency during extraction is handled by
+// extracting in a tmp dir and then renaming it into place where it
+// becomes visible automically at the new path.
+// Populates globals->extracted_binaries with their extracted locations.
+static void ExtractData(const string &self_path) {
+ // If the install dir doesn't exist, create it, if it does, we know it's good.
+ struct stat buf;
+ if (stat(globals->options.install_base.c_str(), &buf) == -1) {
+ uint64 st = MonotonicClock();
+ // Work in a temp dir to avoid races.
+ string tmp_install = globals->options.install_base + ".tmp." +
+ std::to_string(getpid());
+ string tmp_binaries = tmp_install + "/_embedded_binaries";
+ ActuallyExtractData(self_path, tmp_binaries);
+
+ uint64 et = MonotonicClock();
+ globals->extract_data_time = (et - st) / 1000000LL;
+
+ // Now rename the completed installation to its final name. If this
+ // fails due to an ENOTEMPTY then we assume another good
+ // installation snuck in before us.
+ if (rename(tmp_install.c_str(), globals->options.install_base.c_str()) == -1
+ && errno != ENOTEMPTY) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "install base directory '%s' could not be renamed into place",
+ tmp_install.c_str());
+ }
+ } else {
+ if (!S_ISDIR(buf.st_mode)) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Error: Install base directory '%s' could not be created. "
+ "It exists but is not a directory.",
+ globals->options.install_base.c_str());
+ }
+
+ const time_t time_now = time(NULL);
+ string real_install_dir = blaze_util::JoinPath(
+ globals->options.install_base,
+ "_embedded_binaries");
+ for (const auto& it : globals->extracted_binaries) {
+ string path = blaze_util::JoinPath(real_install_dir, it);
+ // Check that the file exists and is readable.
+ if (stat(path.c_str(), &buf) == -1) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Error: corrupt installation: file '%s' missing."
+ " Please remove '%s' and try again.",
+ path.c_str(), globals->options.install_base.c_str());
+ }
+ // Check that the timestamp is in the future. A past timestamp would indicate
+ // that the file has been tampered with. See ActuallyExtractData().
+ if (buf.st_mtime <= time_now) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Error: corrupt installation: file '%s' "
+ "modified. Please remove '%s' and try again.",
+ path.c_str(), globals->options.install_base.c_str());
+ }
+ }
+ }
+}
+
+// Returns true if the server needs to be restarted to accommodate changes
+// between the two argument lists.
+static bool ServerNeedsToBeKilled(const vector<string>& args1,
+ const vector<string>& args2) {
+ // We need not worry about one side missing an argument and the other side
+ // having the default value, since this command line is already the
+ // canonicalized one that always contains every switch (with default values
+ // if it was not present on the real command line). Same applies for argument
+ // ordering.
+ if (args1.size() != args2.size()) {
+ return true;
+ }
+
+ for (int i = 0; i < args1.size(); i++) {
+ string option_sources = "--option_sources=";
+ if (args1[i].substr(0, option_sources.size()) == option_sources &&
+ args2[i].substr(0, option_sources.size()) == option_sources) {
+ continue;
+ }
+
+ if (args1[i] !=args2[i]) {
+ return true;
+ }
+
+ if (args1[i] == "--max_idle_secs") {
+ // Skip the argument of --max_idle_secs.
+ i++;
+ }
+ }
+
+ return false;
+}
+
+// Kills the running Blaze server, if any, if the startup options do not match.
+static void KillRunningServerIfDifferentStartupOptions() {
+ int socket = ConnectToServer(false);
+
+ if (socket == -1) {
+ return;
+ }
+
+ pid_t server_pid = GetPeerProcessId(socket);
+ close(socket);
+ string cmdline_path = globals->options.output_base + "/server/cmdline";
+ string joined_arguments;
+
+ // No, /proc/$PID/cmdline does not work, because it is limited to 4K. Even
+ // worse, its behavior differs slightly between kernels (in some, when longer
+ // command lines are truncated, the last 4 bytes are replaced with
+ // "..." + NUL.
+ ReadFile(cmdline_path, &joined_arguments);
+ vector<string> arguments = blaze_util::Split(joined_arguments, '\0');
+
+ // These strings contain null-separated command line arguments. If they are
+ // the same, the server can stay alive, otherwise, it needs shuffle off this
+ // mortal coil.
+ if (ServerNeedsToBeKilled(arguments, GetArgumentArray())) {
+ globals->restart_reason = NEW_OPTIONS;
+ fprintf(stderr,
+ "WARNING: Running Blaze server needs to be killed, because the "
+ "startup options are different.\n");
+ KillRunningServer(server_pid);
+ }
+}
+
+
+// Kills the old running server if it is not the same version as us,
+// dealing with various combinations of installation scheme
+// (installation symlink and older MD5_MANIFEST contents).
+// This function requires that the installation be complete, and the
+// server lock acquired.
+static void EnsureCorrectRunningVersion() {
+ // Read the previous installation's semaphore symlink in output_base. If the
+ // target dirs don't match, or if the symlink was not present, then kill any
+ // running servers. Lastly, symlink to our installation so others know which
+ // installation is running.
+ string installation_path = globals->options.output_base + "/install";
+ char prev_installation[PATH_MAX + 1] = ""; // NULs the whole array
+ if (readlink(installation_path.c_str(),
+ prev_installation, PATH_MAX) == -1 ||
+ prev_installation != globals->options.install_base) {
+ if (KillRunningServerIfAny()) {
+ globals->restart_reason = NEW_VERSION;
+ }
+ unlink(installation_path.c_str());
+ if (symlink(globals->options.install_base.c_str(),
+ installation_path.c_str())) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "failed to create installation symlink '%s'",
+ installation_path.c_str());
+ }
+ const time_t time_now = time(NULL);
+ struct utimbuf times = { time_now, time_now };
+ if (utime(globals->options.install_base.c_str(), ×) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "failed to set timestamp on '%s'",
+ globals->options.install_base.c_str());
+ }
+ }
+}
+
+
+// A signal-safe version of fprintf(stderr, ...).
+//
+// WARNING: any output from the blaze client may be interleaved
+// with output from the blaze server. In --curses mode,
+// the Blaze server often erases the previous line of output.
+// So, be sure to end each such message with TWO newlines,
+// otherwise it may be erased by the next message from the
+// Blaze server.
+// Also, it's a good idea to start each message with a newline,
+// in case the Blaze server has written a partial line.
+static void sigprintf(const char *format, ...) {
+ char buf[1024];
+ va_list ap;
+ va_start(ap, format);
+ int r = vsnprintf(buf, sizeof buf, format, ap);
+ va_end(ap);
+ write(STDERR_FILENO, buf, r);
+}
+
+
+// Signal handler.
+static void handler(int signum) {
+ // A defensive measure:
+ if (kill(globals->server_pid, 0) == -1 && errno == ESRCH) {
+ sigprintf("\nBlaze server has died; client exiting.\n\n");
+ _exit(1);
+ }
+
+ switch (signum) {
+ case SIGINT:
+ if (++globals->sigint_count >= 3) {
+ sigprintf("\nBlaze caught third interrupt signal; killed.\n\n");
+ kill(globals->server_pid, SIGKILL);
+ _exit(1);
+ }
+ sigprintf("\nBlaze caught interrupt signal; shutting down.\n\n");
+
+ kill(globals->server_pid, SIGINT);
+ break;
+ case SIGTERM:
+ sigprintf("\nBlaze caught terminate signal; shutting down.\n\n");
+ kill(globals->server_pid, SIGINT);
+ break;
+ case SIGPIPE:
+ // Don't bother the user with a message in this case; they're
+ // probably using head(1) or more(1).
+ kill(globals->server_pid, SIGINT);
+ signal(SIGPIPE, SIG_IGN); // ignore subsequent SIGPIPE signals
+ globals->received_signal = SIGPIPE;
+ break;
+ case SIGQUIT:
+ sigprintf("\nSending SIGQUIT to JVM process %d (see %s).\n\n",
+ globals->server_pid,
+ globals->jvm_log_file.c_str());
+ kill(globals->server_pid, SIGQUIT);
+ break;
+ }
+}
+
+
+// Reads a single char from the specified stream.
+static char read_server_char(FILE *fp) {
+ int c = getc(fp);
+ if (c == EOF) {
+ // e.g. external SIGKILL of server, misplaced System.exit() in the server,
+ // or a JVM crash. Print out the jvm.out file in case there's something
+ // useful.
+ fprintf(stderr, "Error: unexpected EOF from Blaze server.\n"
+ "Contents of '%s':\n", globals->jvm_log_file.c_str());
+ WriteFileToStreamOrDie(stderr, globals->jvm_log_file.c_str());
+ exit(blaze_exit_code::INTERNAL_ERROR);
+ }
+ return static_cast<char>(c);
+}
+
+// Constructs the command line for a server request,
+static string BuildServerRequest() {
+ vector<string> arg_vector;
+ string command = globals->option_processor.GetCommand();
+ if (command != "") {
+ arg_vector.push_back(command);
+ AddLoggingArgs(&arg_vector);
+ }
+
+ globals->option_processor.GetCommandArguments(&arg_vector);
+
+ string request("blaze");
+ for (vector<string>::iterator it = arg_vector.begin();
+ it != arg_vector.end(); it++) {
+ request.push_back('\0');
+ request.append(*it);
+ }
+ return request;
+}
+
+// Performs all I/O for a single client request to the server, and
+// shuts down the client (by exit or signal).
+static void SendServerRequest(void) ATTRIBUTE_NORETURN;
+static void SendServerRequest(void) {
+ int socket = -1;
+ while (true) {
+ socket = ConnectToServer(true);
+ globals->server_pid = GetPeerProcessId(socket);
+
+ // Check for deleted server cwd:
+ string server_cwd = GetProcessCWD(globals->server_pid);
+ if (server_cwd.empty() || // GetProcessCWD failed
+ server_cwd != globals->workspace || // changed
+ server_cwd.find(" (deleted)") != string::npos) { // deleted.
+ // There's a distant possibility that the two paths look the same yet are
+ // actually different because the two processes have different mount
+ // tables.
+ if (VerboseLogging()) {
+ fprintf(stderr, "Server's cwd moved or deleted (%s).\n",
+ server_cwd.c_str());
+ }
+ close(socket);
+ KillRunningServer(globals->server_pid);
+ } else {
+ break;
+ }
+ }
+
+ FILE *fp = fdopen(socket, "r"); // use buffering for reads--it's faster
+
+ if (VerboseLogging()) {
+ fprintf(stderr, "Connected (server pid=%d).\n", globals->server_pid);
+ }
+
+ // Wall clock time since process startup.
+ globals->startup_time = ProcessClock() / 1000000LL;
+ const string request = BuildServerRequest();
+
+ // Unblock all signals.
+ sigset_t sigset;
+ sigemptyset(&sigset);
+ sigprocmask(SIG_SETMASK, &sigset, NULL);
+
+ signal(SIGINT, handler);
+ signal(SIGTERM, handler);
+ signal(SIGPIPE, handler);
+ signal(SIGQUIT, handler);
+
+ // Send request and shutdown the write half of the connection:
+ // (Request is written in a single chunk.)
+ if (write(socket, request.data(), request.size()) != request.size()) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "write() to server failed");
+ }
+ // In this (totally bizarre) protocol, this is the
+ // client's way of saying "um, that's the end of the request".
+ if (shutdown(socket, SHUT_WR) == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "shutdown(WR) failed");
+ }
+
+ // Wait until we receive some response from the server.
+ // (We do this by calling select() with a timeout.)
+ // If we don't receive a response within 3 seconds, print a message,
+ // so that the user has some idea what is going on.
+ while (true) {
+ fd_set fdset;
+ FD_ZERO(&fdset);
+ FD_SET(socket, &fdset);
+ struct timeval timeout;
+ timeout.tv_sec = 3;
+ timeout.tv_usec = 0;
+ int result = select(socket + 1, &fdset, NULL, &fdset, &timeout);
+ if (result > 0) {
+ // Data is ready on socket. Go ahead and read it.
+ break;
+ } else if (result == 0) {
+ // Timeout. Print a message, then go ahead and read from
+ // the socket (the read will usually block).
+ fprintf(stderr,
+ "INFO: Waiting for response from blaze server (pid %d)...\n",
+ globals->server_pid);
+ break;
+ } else { // result < 0
+ // Error. For EINTR we try again, all other errors are fatal.
+ if (errno != EINTR) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "select() on server socket failed");
+ }
+ }
+ }
+
+ // Read and demux the response. This protocol is awful.
+ for (;;) {
+ // Read one line:
+ char at = read_server_char(fp);
+ assert(at == '@');
+ (void) at; // avoid warning about unused variable
+ char tag = read_server_char(fp);
+ assert(tag == '1' || tag == '2' || tag == '3');
+ char at_or_newline = read_server_char(fp);
+ bool second_at = at_or_newline == '@';
+ if (second_at) {
+ at_or_newline = read_server_char(fp);
+ }
+ assert(at_or_newline == '\n');
+
+ if (tag == '3') {
+ // In this (totally bizarre) protocol, this is the
+ // server's way of saying "um, that's the end of the response".
+ break;
+ }
+ FILE *stream = tag == '1' ? stdout : stderr;
+ for (;;) {
+ char c = read_server_char(fp);
+ if (c == '\n') {
+ if (!second_at) fputc(c, stream);
+ fflush(stream);
+ break;
+ } else {
+ fputc(c, stream);
+ }
+ }
+ }
+
+ char line[255];
+ if (fgets(line, sizeof line, fp) == NULL ||
+ !isdigit(line[0])) {
+ die(blaze_exit_code::INTERNAL_ERROR,
+ "Error: can't read exit code from server.");
+ }
+ int exit_code;
+ blaze_util::safe_strto32(line, &exit_code);
+
+ close(socket); // might fail EINTR, just ignore.
+
+ if (globals->received_signal) { // Kill ourselves with the same signal, so
+ // that callers see the right WTERMSIG value.
+ signal(globals->received_signal, SIG_DFL);
+ raise(globals->received_signal);
+ exit(1); // (in case raise didn't kill us for some reason)
+ }
+
+ exit(exit_code);
+}
+
+// Parse the options, storing parsed values in globals.
+// Returns the index of the first non-option argument.
+static void ParseOptions(int argc, const char *argv[]) {
+ string error;
+ blaze_exit_code::ExitCode parse_exit_code =
+ globals->option_processor.ParseOptions(argc, argv, globals->workspace,
+ globals->cwd, &error);
+ if (parse_exit_code != blaze_exit_code::SUCCESS) {
+ die(parse_exit_code, "%s", error.c_str());
+ }
+ globals->options = globals->option_processor.GetParsedStartupOptions();
+}
+
+// Returns the canonical form of a path.
+static string MakeCanonical(const char *path) {
+ char *resolved_path = realpath(path, NULL);
+ if (resolved_path == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "realpath('%s') failed", path);
+ }
+
+ string ret = resolved_path;
+ free(resolved_path);
+ return ret;
+}
+
+// Compute the globals globals->cwd and globals->workspace.
+static void ComputeWorkspace() {
+ char cwdbuf[PATH_MAX];
+ if (getcwd(cwdbuf, sizeof cwdbuf) == NULL) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "getcwd() failed");
+ }
+ globals->cwd = MakeCanonical(cwdbuf);
+ globals->workspace = BlazeStartupOptions::GetWorkspace(globals->cwd);
+}
+
+// Figure out the base directories based on embedded data, username, cwd, etc.
+// Sets globals->options.install_base, globals->options.output_base,
+// globals->lock_file, globals->jvm_log_file.
+static void ComputeBaseDirectories(const string self_path) {
+ // Only start a server when in a workspace because otherwise we won't do more
+ // than emit a help message.
+ if (!BlazeStartupOptions::InWorkspace(globals->workspace)) {
+ globals->options.batch = true;
+ }
+
+ // The default install_base is <output_user_root>/install/<md5(blaze)>
+ // but if an install_base is specified on the command line, we use that as
+ // the base instead.
+ if (globals->options.install_base.empty()) {
+ string install_user_root = globals->options.output_user_root + "/install";
+ globals->options.install_base =
+ GetInstallBase(install_user_root, self_path);
+ } else {
+ // We call GetInstallBase anyway to populate extracted_binaries.
+ GetInstallBase("", self_path);
+ }
+
+ if (globals->options.output_base.empty()) {
+ globals->options.output_base = GetHashedBaseDir(
+ globals->options.output_user_root, globals->workspace);
+ }
+
+ struct stat buf;
+ if (stat(globals->options.output_base.c_str(), &buf) == -1) {
+ if (MakeDirectories(globals->options.output_base, 0777) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Output base directory '%s' could not be created",
+ globals->options.output_base.c_str());
+ }
+ } else {
+ if (!S_ISDIR(buf.st_mode)) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Error: Output base directory '%s' could not be created. "
+ "It exists but is not a directory.",
+ globals->options.output_base.c_str());
+ }
+ }
+ if (access(globals->options.output_base.c_str(), R_OK | W_OK | X_OK) != 0) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Error: Output base directory '%s' must be readable and writable.",
+ globals->options.output_base.c_str());
+ }
+
+ globals->options.output_base =
+ MakeCanonical(globals->options.output_base.c_str());
+ globals->lockfile = globals->options.output_base + "/lock";
+ globals->jvm_log_file = globals->options.output_base + "/server/jvm.out";
+}
+
+static void CheckEnvironment() {
+ char pthread_impl[512];
+#ifndef _CS_GNU_LIBPTHREAD_VERSION
+#define _CS_GNU_LIBPTHREAD_VERSION 3
+#endif
+ if (confstr(_CS_GNU_LIBPTHREAD_VERSION, pthread_impl, sizeof pthread_impl) &&
+ strprefix(pthread_impl, "linuxthreads")) {
+ fprintf(stderr, "Warning: LinuxThreads detected. NPTL is preferred.\n"
+ " (Perhaps unset LD_ASSUME_KERNEL or LD_LIBRARY_PATH.)\n");
+ }
+
+ if (getenv("LD_ASSUME_KERNEL") != NULL) {
+ // Fix for bug: if ulimit -s and LD_ASSUME_KERNEL are both
+ // specified, the JVM fails to create threads. See thread_stack_regtest.
+ // This is also provoked by LD_LIBRARY_PATH=/usr/lib/debug,
+ // or anything else that causes the JVM to use LinuxThreads.
+ fprintf(stderr, "Warning: ignoring LD_ASSUME_KERNEL in environment.\n");
+ unsetenv("LD_ASSUME_KERNEL");
+ }
+
+ if (getenv("LD_PRELOAD") != NULL) {
+ fprintf(stderr, "Warning: ignoring LD_PRELOAD in environment.\n");
+ unsetenv("LD_PRELOAD");
+ }
+
+ if (getenv("_JAVA_OPTIONS") != NULL) {
+ // This would override --host_jvm_args
+ fprintf(stderr, "Warning: ignoring _JAVA_OPTIONS in environment.\n");
+ unsetenv("_JAVA_OPTIONS");
+ }
+
+ if (TESTING) {
+ fprintf(stderr, "INFO: $TEST_TMPDIR defined: output root default is "
+ "'%s'.\n", globals->options.output_root.c_str());
+ }
+
+ // TODO(bazel-team): We've also seen a failure during loading (creating
+ // threads?) when ulimit -Hs 8192. Characterize that and check for it here.
+
+ // Make the JVM use ISO-8859-1 for parsing its command line because "blaze
+ // run" doesn't handle non-ASCII command line arguments. This is apparently
+ // the most reliable way to select the platform default encoding.
+ setenv("LANG", "en_US.ISO-8859-1", 1);
+ setenv("LANGUAGE", "en_US.ISO-8859-1", 1);
+ setenv("LC_ALL", "en_US.ISO-8859-1", 1);
+ setenv("LC_CTYPE", "en_US.ISO-8859-1", 1);
+}
+
+// Create the lockfile and take an exclusive lock on a region within it. This
+// lock is inherited with the file descriptor across execve(), but not fork().
+// So in the batch case, the JVM holds the lock until exit; otherwise, this
+// program holds it until exit.
+static void AcquireLock() {
+ globals->lockfd = open(globals->lockfile.c_str(), O_CREAT|O_RDWR, 0644);
+ if (globals->lockfd < 0) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "cannot open lockfile '%s' for writing", globals->lockfile.c_str());
+ }
+
+ struct flock lock;
+ lock.l_type = F_WRLCK;
+ lock.l_whence = SEEK_SET;
+ lock.l_start = 0;
+ // This doesn't really matter now, but allows us to subdivide the lock
+ // later if that becomes meaningful. (Ranges beyond EOF can be locked.)
+ lock.l_len = 4096;
+
+ // Try to take the lock, without blocking.
+ if (fcntl(globals->lockfd, F_SETLK, &lock) == -1) {
+ if (errno != EACCES && errno != EAGAIN) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "unexpected result from F_SETLK");
+ }
+
+ // We didn't get the lock. Find out who has it.
+ struct flock probe = lock;
+ probe.l_pid = 0;
+ if (fcntl(globals->lockfd, F_GETLK, &probe) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "unexpected result from F_GETLK");
+ }
+ if (!globals->options.block_for_lock) {
+ die(blaze_exit_code::BAD_ARGV,
+ "Another Blaze command is running (pid=%d). Exiting immediately.",
+ probe.l_pid);
+ }
+ fprintf(stderr, "Another Blaze command is running (pid = %d). "
+ "Waiting for it to complete...", probe.l_pid);
+ fflush(stderr);
+
+ // Take a clock sample for that start of the waiting time
+ uint64 st = MonotonicClock();
+ // Try to take the lock again (blocking).
+ int r;
+ do {
+ r = fcntl(globals->lockfd, F_SETLKW, &lock);
+ } while (r == -1 && errno == EINTR);
+ fprintf(stderr, "\n");
+ if (r == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "couldn't acquire file lock");
+ }
+ // Take another clock sample, calculate elapsed
+ uint64 et = MonotonicClock();
+ globals->command_wait_time = (et - st) / 1000000LL;
+ }
+
+ // Identify ourselves in the lockfile.
+ ftruncate(globals->lockfd, 0);
+ const char *tty = ttyname(STDIN_FILENO); // NOLINT (single-threaded)
+ string msg = "owner=blaze launcher\npid=" + std::to_string(getpid()) +
+ "\ntty=" + (tty ? tty : "") + "\n";
+ // Don't bother checking for error, since it's unlikely and unimportant.
+ // The contents are currently meant only for debugging.
+ write(globals->lockfd, msg.data(), msg.size());
+}
+
+// Returns the mountpoint containing the specified directory, which
+// must exist. Fails if any parent path could not be statted or
+// canonicalised.
+static string GetMountpoint(string dir) {
+ dev_t initial_device = -1;
+ ino_t prev_inode = -1;
+ string prev_dir = dir;
+ for (;;) {
+ struct stat buf;
+ if (stat(dir.c_str(), &buf) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "stat('%s') failed", dir.c_str());
+ } else if (initial_device == -1 && prev_inode == -1) { // first time
+ initial_device = buf.st_dev;
+ } else if (initial_device != buf.st_dev) { // we crossed file systems
+ char *resolved_path = realpath(prev_dir.c_str(), NULL);
+ if (resolved_path == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "realpath('%s') failed", prev_dir.c_str());
+ }
+ dir = resolved_path;
+ free(resolved_path);
+ return dir;
+ } else if (prev_inode == buf.st_ino) { // ".." had no effect => root.
+ return "/";
+ }
+
+ prev_inode = buf.st_ino;
+ prev_dir = dir;
+ dir += "/..";
+ }
+
+ return "/";
+}
+
+// Issue a warning if disk has less than 10% free blocks or inodes.
+static void WarnIfFullDisk() {
+ struct statvfs buf;
+ if (statvfs(globals->options.output_base.c_str(), &buf) < 0) {
+ fprintf(stderr, "WARNING: couldn't get file system information for '%s': "
+ "%s\n", globals->options.output_base.c_str(), strerror(errno));
+ return;
+ }
+
+ if (10LL * buf.f_favail < buf.f_files) {
+ fprintf(stderr,
+ "WARNING: build volume %s is nearly full "
+ "(%llu inodes remain).\n",
+ GetMountpoint(globals->options.output_base).c_str(),
+ static_cast<int64>(buf.f_favail));
+ }
+ if (10LL * buf.f_bavail < buf.f_blocks) {
+ fprintf(stderr,
+ "WARNING: build volume %s is nearly full "
+ "(%.1fGB remain).\n",
+ GetMountpoint(globals->options.output_base).c_str(),
+ (1.0 * buf.f_bavail) * buf.f_frsize / 1E9);
+ }
+}
+
+void SetupStreams() {
+ // Line-buffer stderr, since we always flush at the end of a server
+ // message. This saves lots of single-char calls to write(2).
+ // This doesn't work if any writes to stderr have already occurred!
+ setlinebuf(stderr);
+
+ // Ensure we have three open fds. Otherwise we can end up with
+ // bizarre things like stdout going to the lock file, etc.
+ if (fcntl(0, F_GETFL) == -1) open("/dev/null", O_RDONLY);
+ if (fcntl(1, F_GETFL) == -1) open("/dev/null", O_WRONLY);
+ if (fcntl(2, F_GETFL) == -1) open("/dev/null", O_WRONLY);
+}
+
+// Set an 8MB stack for Blaze. When the stack max is unbounded, it changes the
+// layout in the JVM's address space, and we are unable to instantiate the
+// default 3000MB heap.
+static void EnsureFiniteStackLimit() {
+ struct rlimit limit;
+ const int default_stack = 8 * 1024 * 1024; // 8MB.
+ if (getrlimit(RLIMIT_STACK, &limit)) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "getrlimit() failed");
+ }
+
+ if (default_stack < limit.rlim_cur) {
+ limit.rlim_cur = default_stack;
+ if (setrlimit(RLIMIT_STACK, &limit)) {
+ perror("setrlimit() failed: If the stack limit is too high, "
+ "this can cause the JVM to be unable to allocate enough "
+ "contiguous address space for its heap");
+ }
+ }
+}
+
+static void CheckBinaryPath(const string& argv0) {
+ if (argv0[0] == '/') {
+ globals->binary_path = argv0;
+ } else {
+ string abs_path = globals->cwd + '/' + argv0;
+ char *resolved_path = realpath(abs_path.c_str(), NULL);
+ if (resolved_path) {
+ globals->binary_path = resolved_path;
+ free(resolved_path);
+ } else {
+ // This happens during our integration tests, but thats okay, as we won't
+ // log the invocation anyway.
+ globals->binary_path = abs_path;
+ }
+ }
+}
+
+// Create the user's directory where we keep state, installations etc.
+// Typically, this happens inside a temp directory, so we have to be
+// careful about symlink attacks.
+static void CreateSecureOutputRoot() {
+ const char* root = globals->options.output_user_root.c_str();
+ struct stat fileinfo = {};
+
+ if (mkdir(root, 0755) == 0) {
+ return; // mkdir succeeded, no need to verify ownership/mode.
+ }
+ if (errno != EEXIST) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "mkdir('%s')", root);
+ }
+
+ // The path already exists.
+ // Check ownership and mode, and verify that it is a directory.
+
+ if (lstat(root, &fileinfo) < 0) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "lstat('%s')", root);
+ }
+
+ if (fileinfo.st_uid != geteuid()) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "'%s' is not owned by me",
+ root);
+ }
+
+ if ((fileinfo.st_mode & 022) != 0) {
+ int new_mode = fileinfo.st_mode & (~022);
+ if (chmod(root, new_mode) < 0) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "'%s' has mode %o, chmod to %o failed", root,
+ fileinfo.st_mode & 07777, new_mode);
+ }
+ }
+
+ if (stat(root, &fileinfo) < 0) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "stat('%s')", root);
+ }
+
+ if (!S_ISDIR(fileinfo.st_mode)) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "'%s' is not a directory",
+ root);
+ }
+}
+
+// TODO(bazel-team): Execute the server as a child process and write its exit
+// code to a file. In case the server becomes unresonsive or terminates
+// unexpectedly (in a way that isn't already handled), we can observe the file,
+// if it exists. (If it doesn't, then we know something went horribly wrong.)
+int main(int argc, const char *argv[]) {
+ InitGlobals();
+ SetupStreams();
+
+ // Must be done before command line parsing.
+ ComputeWorkspace();
+ CheckBinaryPath(argv[0]);
+ ParseOptions(argc, argv);
+ string error;
+ blaze_exit_code::ExitCode reexec_options_exit_code =
+ globals->options.CheckForReExecuteOptions(argc, argv, &error);
+ if (reexec_options_exit_code != blaze_exit_code::SUCCESS) {
+ die(reexec_options_exit_code, "%s", error.c_str());
+ }
+ CheckEnvironment();
+ CreateSecureOutputRoot();
+
+ const string self_path = GetSelfPath();
+ ComputeBaseDirectories(self_path);
+
+ AcquireLock();
+
+ WarnIfFullDisk();
+ WarnFilesystemType(globals->options.output_base);
+ EnsureFiniteStackLimit();
+
+ ExtractData(self_path);
+ EnsureCorrectRunningVersion();
+ KillRunningServerIfDifferentStartupOptions();
+
+ if (globals->options.batch) {
+ SetScheduling(globals->options.batch_cpu_scheduling,
+ globals->options.io_nice_level);
+ StartStandalone();
+ } else {
+ SendServerRequest();
+ }
+ return 0;
+}
+} // namespace blaze
+
+int main(int argc, const char *argv[]) {
+ return blaze::main(argc, argv);
+}
diff --git a/src/main/cpp/blaze_exit_code.h b/src/main/cpp/blaze_exit_code.h
new file mode 100644
index 0000000..50cad74
--- /dev/null
+++ b/src/main/cpp/blaze_exit_code.h
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// blaze_exit_code.h: Exit codes for Blaze.
+// Must be kept in sync with the Java counterpart under
+// com/google/devtools/build/lib/util/ExitCode.java
+
+#ifndef DEVTOOLS_BLAZE_MAIN_BLAZE_EXIT_CODE_H_
+#define DEVTOOLS_BLAZE_MAIN_BLAZE_EXIT_CODE_H_
+
+namespace blaze_exit_code {
+
+enum ExitCode {
+ // Success.
+ SUCCESS = 0,
+
+ // Command Line Problem, Bad or Illegal flags or command combination, or
+ // Bad environment variables. The user must modify their command line.
+ BAD_ARGV = 2,
+
+ LOCAL_ENVIRONMENTAL_ERROR = 36,
+
+ // Unexpected server termination, due to e.g. external SIGKILL, misplaced
+ // System.exit(), or a JVM crash.
+ // This exit code should be a last resort.
+ INTERNAL_ERROR = 37,
+};
+
+} // namespace blaze_exit_code
+
+#endif // DEVTOOLS_BLAZE_MAIN_BLAZE_EXIT_CODE_H_
diff --git a/src/main/cpp/blaze_startup_options.cc b/src/main/cpp/blaze_startup_options.cc
new file mode 100644
index 0000000..569c561
--- /dev/null
+++ b/src/main/cpp/blaze_startup_options.cc
@@ -0,0 +1,160 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "blaze_startup_options.h"
+
+#include <assert.h>
+#include <errno.h> // errno, ENOENT
+#include <stdlib.h> // getenv, exit
+#include <unistd.h> // access
+
+#include <cstdio>
+
+#include "blaze_exit_code.h"
+#include "blaze_util_platform.h"
+#include "blaze_util.h"
+#include "util/file.h"
+#include "util/strings.h"
+
+namespace blaze {
+
+using std::vector;
+
+struct StartupOptions {};
+
+BlazeStartupOptions::BlazeStartupOptions() {
+ Init();
+}
+
+BlazeStartupOptions::BlazeStartupOptions(const BlazeStartupOptions &rhs)
+ : output_base(rhs.output_base),
+ install_base(rhs.install_base),
+ output_root(rhs.output_root),
+ output_user_root(rhs.output_user_root),
+ block_for_lock(rhs.block_for_lock),
+ host_jvm_debug(rhs.host_jvm_debug),
+ host_jvm_profile(rhs.host_jvm_profile),
+ host_jvm_args(rhs.host_jvm_args),
+ batch(rhs.batch),
+ batch_cpu_scheduling(rhs.batch_cpu_scheduling),
+ io_nice_level(rhs.io_nice_level),
+ max_idle_secs(rhs.max_idle_secs),
+ skyframe(rhs.skyframe),
+ watchfs(rhs.watchfs),
+ allow_configurable_attributes(rhs.allow_configurable_attributes),
+ option_sources(rhs.option_sources),
+ webstatus_port(rhs.webstatus_port),
+ host_javabase(rhs.host_javabase) {}
+
+BlazeStartupOptions::~BlazeStartupOptions() {
+}
+
+BlazeStartupOptions& BlazeStartupOptions::operator=(
+ const BlazeStartupOptions &rhs) {
+ Copy(rhs, this);
+ return *this;
+}
+
+string BlazeStartupOptions::GetOutputRoot() {
+ return "/var/tmp";
+}
+
+void BlazeStartupOptions::AddExtraOptions(vector<string> *result) const {}
+
+static const char kWorkspaceMarker[] = "WORKSPACE";
+
+// static
+bool BlazeStartupOptions::InWorkspace(const string &workspace) {
+ return access(
+ blaze_util::JoinPath(workspace, kWorkspaceMarker).c_str(), F_OK) == 0;
+}
+
+// static
+string BlazeStartupOptions::GetWorkspace(const string &cwd) {
+ assert(!cwd.empty());
+ string workspace = cwd;
+
+ do {
+ if (access(blaze_util::JoinPath(
+ workspace, kWorkspaceMarker).c_str(), F_OK) != -1) {
+ return workspace;
+ }
+ workspace = blaze_util::Dirname(workspace);
+ } while (!workspace.empty() && workspace != "/");
+ return "";
+}
+
+blaze_exit_code::ExitCode BlazeStartupOptions::ProcessArgExtra(
+ const char *arg, const char *next_arg, const string &rcfile,
+ const char **value, bool *is_processed, string *error) {
+ *is_processed = false;
+ return blaze_exit_code::SUCCESS;
+}
+
+blaze_exit_code::ExitCode BlazeStartupOptions::CheckForReExecuteOptions(
+ int argc, const char *argv[], string *error) {
+ return blaze_exit_code::SUCCESS;
+}
+
+string BlazeStartupOptions::GetDefaultHostJavabase() const {
+ return blaze::GetDefaultHostJavabase();
+}
+
+string BlazeStartupOptions::GetJvm() {
+ string java_program = GetHostJavabase() + "/bin/java";
+ if (access(java_program.c_str(), X_OK) == -1) {
+ if (errno == ENOENT) {
+ fprintf(stderr, "Couldn't find java at '%s'.\n", java_program.c_str());
+ } else {
+ fprintf(stderr, "Couldn't access %s: %s\n", java_program.c_str(),
+ strerror(errno));
+ }
+ exit(1);
+ }
+ for (string rt_jar : {
+ // If the full JDK is installed
+ GetHostJavabase() + "/jre/lib/rt.jar",
+ // If just the JRE is installed
+ GetHostJavabase() + "/lib/rt.jar"
+ }) {
+ if (access(rt_jar.c_str(), R_OK) == 0) {
+ return java_program;
+ }
+ }
+ fprintf(stderr, "Problem with java installation: "
+ "couldn't find/access rt.jar in %s\n", GetHostJavabase().c_str());
+ exit(1);
+}
+
+BlazeStartupOptions::Architecture BlazeStartupOptions::GetBlazeArchitecture()
+ const {
+ return strcmp(BLAZE_JAVA_CPU, "64") == 0 ? k64Bit : k32Bit;
+}
+
+blaze_exit_code::ExitCode BlazeStartupOptions::AddJVMArguments(
+ const string &host_javabase, vector<string> *result, string *error) const {
+ // TODO(bazel-team): see what tuning options make sense in the
+ // open-source world.
+ return blaze_exit_code::SUCCESS;
+}
+
+string BlazeStartupOptions::RcBasename() {
+ return ".bazelrc";
+}
+
+void BlazeStartupOptions::WorkspaceRcFileSearchPath(
+ vector<string>* candidates) {
+ candidates->push_back("tools/bazel.rc");
+}
+
+} // namespace blaze
diff --git a/src/main/cpp/blaze_startup_options.h b/src/main/cpp/blaze_startup_options.h
new file mode 100644
index 0000000..c8d902d
--- /dev/null
+++ b/src/main/cpp/blaze_startup_options.h
@@ -0,0 +1,210 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef DEVTOOLS_BLAZE_MAIN_BLAZE_STARTUP_OPTIONS_H_
+#define DEVTOOLS_BLAZE_MAIN_BLAZE_STARTUP_OPTIONS_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "blaze_exit_code.h"
+
+namespace blaze {
+
+using std::string;
+
+struct StartupOptions;
+
+// This class holds the parsed startup options for Blaze.
+// These options and their defaults must be kept in sync with those
+// in java/com/google/devtools/build/lib/blaze/BlazeServerStartupOptions.
+// The latter are purely decorative (they affect the help message,
+// which displays the defaults). The actual defaults are defined
+// in the constructor.
+//
+// TODO(bazel-team): The encapsulation is not quite right -- there are some
+// places in blaze.cc where some of these fields are explicitly modified. Their
+// names also don't conform to the style guide.
+class BlazeStartupOptions {
+ public:
+ enum Architecture { k32Bit, k64Bit };
+
+ BlazeStartupOptions();
+ BlazeStartupOptions(const BlazeStartupOptions &rhs);
+ ~BlazeStartupOptions();
+ BlazeStartupOptions& operator=(const BlazeStartupOptions &rhs);
+
+ // Parses a single argument, either from the command line or from the .blazerc
+ // "startup" options.
+ //
+ // rcfile should be an empty string if the option being parsed does not come
+ // from a blazerc.
+ //
+ // Sets "is_space_seperated" true if arg is unary and uses the "--foo bar"
+ // style, so its value is in next_arg.
+ //
+ // Sets "is_space_seperated" false if arg is either nullary
+ // (e.g. "--[no]batch") or is unary but uses the "--foo=bar" style.
+ //
+ // Returns the exit code after processing the argument. "error" will contain
+ // a descriptive string for any return value other than
+ // blaze_exit_code::SUCCESS.
+ blaze_exit_code::ExitCode ProcessArg(
+ const string &arg, const string &next_arg, const string &rcfile,
+ bool *is_space_seperated, string *error);
+
+ // Adds any other options needed to result.
+ void AddExtraOptions(std::vector<string> *result) const;
+
+ // Checks if Blaze needs to be re-executed. Does not return, if so.
+ //
+ // Returns the exit code after the check. "error" will contain a descriptive
+ // string for any return value other than blaze_exit_code::SUCCESS.
+ blaze_exit_code::ExitCode CheckForReExecuteOptions(
+ int argc, const char *argv[], string *error);
+
+ // Checks extra fields when processing arg.
+ //
+ // Returns the exit code after processing the argument. "error" will contain
+ // a descriptive string for any return value other than
+ // blaze_exit_code::SUCCESS.
+ blaze_exit_code::ExitCode ProcessArgExtra(
+ const char *arg, const char *next_arg, const string &rcfile,
+ const char **value, bool *is_processed, string *error);
+
+ // Return the default path to the JDK used to run Blaze itself
+ // (must be an absolute directory).
+ string GetDefaultHostJavabase() const;
+
+ Architecture GetBlazeArchitecture() const;
+
+ // Returns the path to the JVM. This should be called after parsing
+ // the startup options.
+ string GetJvm();
+
+ // Adds JVM tuning flags for Blaze.
+ //
+ // Returns the exit code after this operation. "error" will be set to a
+ // descriptive string for any value other than blaze_exit_code::SUCCESS.
+ blaze_exit_code::ExitCode AddJVMArguments(const string &host_javabase,
+ std::vector<string> *result,
+ string *error) const;
+
+ // Blaze's output base. Everything is relative to this. See
+ // the BlazeDirectories Java class for details.
+ string output_base;
+
+ // Installation base for a specific release installation.
+ string install_base;
+
+ // The toplevel directory containing Blaze's output. When Blaze is
+ // run by a test, we use TEST_TMPDIR, simplifying the correct
+ // hermetic invocation of Blaze from tests.
+ string output_root;
+
+ // Blaze's output_user_root. Used only for computing install_base and
+ // output_base.
+ string output_user_root;
+
+ // Block for the Blaze server lock. Otherwise,
+ // quit with non-0 exit code if lock can't
+ // be acquired immediately.
+ bool block_for_lock;
+
+ bool host_jvm_debug;
+
+ string host_jvm_profile;
+
+ string host_jvm_args;
+
+ bool batch;
+
+ // From the man page: "This policy is useful for workloads that are
+ // non-interactive, but do not want to lower their nice value, and for
+ // workloads that want a deterministic scheduling policy without
+ // interactivity causing extra preemptions (between the workload's tasks)."
+ bool batch_cpu_scheduling;
+
+ // If negative, don't mess with ionice. Otherwise, set a level from 0-7
+ // for best-effort scheduling. 0 is highest priority, 7 is lowest.
+ int io_nice_level;
+
+ int max_idle_secs;
+
+ string skyframe;
+
+ // If true, Blaze will listen to OS-level file change notifications.
+ bool watchfs;
+
+ // Temporary experimental flag that permits configurable attribute syntax
+ // in BUILD files. This will be removed when configurable attributes is
+ // a more stable feature.
+ bool allow_configurable_attributes;
+
+ // Temporary flag for enabling EventBus exceptions to be fatal.
+ bool fatal_event_bus_exceptions;
+
+ // A string to string map specifying where each option comes from. If the
+ // value is empty, it was on the command line, if it is a string, it comes
+ // from a blazerc file, if a key is not present, it is the default.
+ std::map<string, string> option_sources;
+
+ std::unique_ptr<StartupOptions> extra_options;
+
+ // Given the working directory, returns the nearest enclosing directory with a
+ // WORKSPACE file in it. If there is no such enclosing directory, returns "".
+ //
+ // E.g., if there was a WORKSPACE file in foo/bar/build_root:
+ // GetWorkspace('foo/bar') --> ''
+ // GetWorkspace('foo/bar/build_root') --> 'foo/bar/build_root'
+ // GetWorkspace('foo/bar/build_root/biz') --> 'foo/bar/build_root'
+ //
+ // The returned path is relative or absolute depending on whether cwd was
+ // relative or absolute.
+ static string GetWorkspace(const string &cwd);
+
+ // Returns if workspace is a valid build workspace.
+ static bool InWorkspace(const string &workspace);
+
+ // Returns the basename for the rc file.
+ static string RcBasename();
+
+ // Returns the search paths for the RC file in the workspace.
+ static void WorkspaceRcFileSearchPath(std::vector<string>* candidates);
+
+ // Returns the GetHostJavabase. This should be called after parsing
+ // the --host_javabase option.
+ string GetHostJavabase();
+
+ // Port for web status server, 0 to disable
+ int webstatus_port;
+
+ private:
+ string host_javabase;
+
+ // Sets default values for members.
+ void Init();
+
+ // Copies member variables from rhs to lhs. This cannot use the compiler-
+ // generated copy constructor because extra_options is a unique_ptr and
+ // unique_ptr deletes its copy constructor.
+ void Copy(const BlazeStartupOptions &rhs, BlazeStartupOptions *lhs);
+
+ // Returns the directory to use for storing outputs.
+ string GetOutputRoot();
+};
+
+} // namespace blaze
+#endif // DEVTOOLS_BLAZE_MAIN_BLAZE_STARTUP_OPTIONS_H_
diff --git a/src/main/cpp/blaze_startup_options_common.cc b/src/main/cpp/blaze_startup_options_common.cc
new file mode 100644
index 0000000..957546c
--- /dev/null
+++ b/src/main/cpp/blaze_startup_options_common.cc
@@ -0,0 +1,226 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "blaze_startup_options.h"
+
+#include <cassert>
+#include <cstdlib>
+
+#include "blaze_exit_code.h"
+#include "blaze_util_platform.h"
+#include "blaze_util.h"
+#include "util/numbers.h"
+#include "util/strings.h"
+
+namespace blaze {
+
+void BlazeStartupOptions::Init() {
+ bool testing = getenv("TEST_TMPDIR") != NULL;
+ if (testing) {
+ output_root = MakeAbsolute(getenv("TEST_TMPDIR"));
+ } else {
+ output_root = GetOutputRoot();
+ }
+
+ output_user_root = output_root + "/_blaze_" + GetUserName();
+ block_for_lock = true;
+ host_jvm_debug = false;
+ host_javabase = "";
+ batch = false;
+ batch_cpu_scheduling = false;
+ allow_configurable_attributes = false;
+ fatal_event_bus_exceptions = false;
+ io_nice_level = -1;
+ // 3 hours (but only 5 seconds if used within a test)
+ max_idle_secs = testing ? 5 : (3 * 3600);
+ webstatus_port = 0;
+ watchfs = false;
+}
+
+string BlazeStartupOptions::GetHostJavabase() {
+ if (host_javabase.empty()) {
+ host_javabase = GetDefaultHostJavabase();
+ }
+ return host_javabase;
+}
+
+void BlazeStartupOptions::Copy(
+ const BlazeStartupOptions &rhs, BlazeStartupOptions *lhs) {
+ assert(lhs);
+
+ lhs->output_base = rhs.output_base;
+ lhs->install_base = rhs.install_base;
+ lhs->output_root = rhs.output_root;
+ lhs->output_user_root = rhs.output_user_root;
+ lhs->block_for_lock = rhs.block_for_lock;
+ lhs->host_jvm_debug = rhs.host_jvm_debug;
+ lhs->host_jvm_profile = rhs.host_jvm_profile;
+ lhs->host_javabase = rhs.host_javabase;
+ lhs->host_jvm_args = rhs.host_jvm_args;
+ lhs->batch = rhs.batch;
+ lhs->batch_cpu_scheduling = rhs.batch_cpu_scheduling;
+ lhs->io_nice_level = rhs.io_nice_level;
+ lhs->max_idle_secs = rhs.max_idle_secs;
+ lhs->skyframe = rhs.skyframe;
+ lhs->webstatus_port = rhs.webstatus_port;
+ lhs->watchfs = rhs.watchfs;
+ lhs->allow_configurable_attributes = rhs.allow_configurable_attributes;
+ lhs->fatal_event_bus_exceptions = rhs.fatal_event_bus_exceptions;
+ lhs->option_sources = rhs.option_sources;
+}
+
+blaze_exit_code::ExitCode BlazeStartupOptions::ProcessArg(
+ const string &argstr, const string &next_argstr, const string &rcfile,
+ bool *is_space_seperated, string *error) {
+ // We have to parse a specific option syntax, so GNU getopts won't do. All
+ // options begin with "--" or "-". Values are given together with the option
+ // delimited by '=' or in the next option.
+ const char* arg = argstr.c_str();
+ const char* next_arg = next_argstr.empty() ? NULL : next_argstr.c_str();
+ const char* value = NULL;
+
+ if ((value = GetUnaryOption(arg, next_arg, "--output_base")) != NULL) {
+ output_base = MakeAbsolute(value);
+ option_sources["output_base"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--install_base")) != NULL) {
+ install_base = MakeAbsolute(value);
+ option_sources["install_base"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--output_user_root")) != NULL) {
+ output_user_root = MakeAbsolute(value);
+ option_sources["output_user_root"] = rcfile;
+ } else if (GetNullaryOption(arg, "--noblock_for_lock")) {
+ block_for_lock = false;
+ option_sources["block_for_lock"] = rcfile;
+ } else if (GetNullaryOption(arg, "--host_jvm_debug")) {
+ host_jvm_debug = true;
+ option_sources["host_jvm_debug"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--host_jvm_profile")) != NULL) {
+ host_jvm_profile = value;
+ option_sources["host_jvm_profile"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--host_javabase")) != NULL) {
+ // TODO(bazel-team): Consider examining the javabase, and in case of
+ // architecture mismatch, treating this option like --blaze_cpu
+ // and re-execing.
+ host_javabase = MakeAbsolute(value);
+ option_sources["host_javabase"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--host_jvm_args")) != NULL) {
+ if (host_jvm_args.empty()) {
+ host_jvm_args = value;
+ } else {
+ host_jvm_args = host_jvm_args + " " + value;
+ }
+ option_sources["host_jvm_args"] = rcfile; // NB: This is incorrect
+ } else if ((value = GetUnaryOption(arg, next_arg, "--blaze_cpu")) != NULL) {
+ fprintf(stderr, "WARNING: The --blaze_cpu startup option is now ignored "
+ "and will be removed in a future release\n");
+ } else if ((value = GetUnaryOption(arg, next_arg, "--blazerc")) != NULL) {
+ if (rcfile != "") {
+ *error = "Can't specify --blazerc in the .blazerc file.";
+ return blaze_exit_code::BAD_ARGV;
+ }
+ } else if (GetNullaryOption(arg, "--nomaster_blazerc") ||
+ GetNullaryOption(arg, "--master_blazerc")) {
+ if (rcfile != "") {
+ *error = "Can't specify --[no]master_blazerc in .blazerc file.";
+ return blaze_exit_code::BAD_ARGV;
+ }
+ option_sources["blazerc"] = rcfile;
+ } else if (GetNullaryOption(arg, "--batch")) {
+ batch = true;
+ option_sources["batch"] = rcfile;
+ } else if (GetNullaryOption(arg, "--nobatch")) {
+ batch = false;
+ option_sources["batch"] = rcfile;
+ } else if (GetNullaryOption(arg, "--batch_cpu_scheduling")) {
+ batch_cpu_scheduling = true;
+ option_sources["batch_cpu_scheduling"] = rcfile;
+ } else if (GetNullaryOption(arg, "--nobatch_cpu_scheduling")) {
+ batch_cpu_scheduling = false;
+ option_sources["batch_cpu_scheduling"] = rcfile;
+ } else if (GetNullaryOption(arg, "--allow_configurable_attributes")) {
+ allow_configurable_attributes = true;
+ option_sources["allow_configurable_attributes"] = rcfile;
+ } else if (GetNullaryOption(arg, "--noallow_configurable_attributes")) {
+ allow_configurable_attributes = false;
+ option_sources["allow_configurable_attributes"] = rcfile;
+ } else if (GetNullaryOption(arg, "--fatal_event_bus_exceptions")) {
+ fatal_event_bus_exceptions = true;
+ option_sources["fatal_event_bus_exceptions"] = rcfile;
+ } else if (GetNullaryOption(arg, "--nofatal_event_bus_exceptions")) {
+ fatal_event_bus_exceptions = false;
+ option_sources["fatal_event_bus_exceptions"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--io_nice_level")) != NULL) {
+ if (!blaze_util::safe_strto32(value, &io_nice_level) ||
+ io_nice_level > 7) {
+ blaze_util::StringPrintf(error,
+ "Invalid argument to --io_nice_level: '%s'. Must not exceed 7.",
+ value);
+ return blaze_exit_code::BAD_ARGV;
+ }
+ option_sources["io_nice_level"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--max_idle_secs")) != NULL) {
+ if (!blaze_util::safe_strto32(value, &max_idle_secs) ||
+ max_idle_secs < 0) {
+ blaze_util::StringPrintf(error,
+ "Invalid argument to --max_idle_secs: '%s'.", value);
+ return blaze_exit_code::BAD_ARGV;
+ }
+ option_sources["max_idle_secs"] = rcfile;
+ } else if ((value = GetUnaryOption(arg, next_arg,
+ "--skyframe")) != NULL) {
+ fprintf(stderr, "WARNING: The --skyframe startup option is now ignored "
+ "and will be removed in a future release\n");
+ } else if (GetNullaryOption(arg, "-x")) {
+ fprintf(stderr, "WARNING: The -x startup option is now ignored "
+ "and will be removed in a future release\n");
+ } else if (GetNullaryOption(arg, "--watchfs")) {
+ watchfs = true;
+ option_sources["watchfs"] = rcfile;
+ } else if ((value = GetUnaryOption(
+ arg, next_arg, "--use_webstatusserver")) != NULL) {
+ if (!blaze_util::safe_strto32(value, &webstatus_port) ||
+ webstatus_port < 0 || webstatus_port > 65535) {
+ blaze_util::StringPrintf(error,
+ "Invalid argument to --use_webstatusserver: '%s'. "
+ "Must be a valid port number or 0 if server disabled.\n", value);
+ return blaze_exit_code::BAD_ARGV;
+ }
+ option_sources["webstatusserver"] = rcfile;
+ } else {
+ bool extra_argument_processed;
+ blaze_exit_code::ExitCode process_extra_arg_exit_code = ProcessArgExtra(
+ arg, next_arg, rcfile, &value, &extra_argument_processed, error);
+ if (process_extra_arg_exit_code != blaze_exit_code::SUCCESS) {
+ return process_extra_arg_exit_code;
+ }
+ if (!extra_argument_processed) {
+ blaze_util::StringPrintf(error,
+ "Unknown Blaze startup option: '%s'.\n"
+ " For more info, run 'blaze help startup_options'.",
+ arg);
+ return blaze_exit_code::BAD_ARGV;
+ }
+ }
+
+ *is_space_seperated = ((value == next_arg) && (value != NULL));
+ return blaze_exit_code::SUCCESS;
+}
+
+} // namespace blaze
diff --git a/src/main/cpp/blaze_util.cc b/src/main/cpp/blaze_util.cc
new file mode 100644
index 0000000..287cea0
--- /dev/null
+++ b/src/main/cpp/blaze_util.cc
@@ -0,0 +1,336 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "blaze_util.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <pwd.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/xattr.h>
+#include <unistd.h>
+#include <sstream>
+
+#include "util/numbers.h"
+#include "util/strings.h"
+
+using std::vector;
+
+namespace blaze {
+
+void die(const int exit_status, const char *format, ...) {
+ va_list ap;
+ va_start(ap, format);
+ vfprintf(stderr, format, ap);
+ va_end(ap);
+ fputc('\n', stderr);
+ exit(exit_status);
+}
+
+void pdie(const int exit_status, const char *format, ...) {
+ fprintf(stderr, "Error: ");
+ va_list ap;
+ va_start(ap, format);
+ vfprintf(stderr, format, ap);
+ va_end(ap);
+ fprintf(stderr, ": %s\n", strerror(errno));
+ exit(exit_status);
+}
+
+string GetUserName() {
+ const char *user = getenv("USER");
+ if (user && user[0] != '\0') return user;
+ errno = 0;
+ passwd *pwent = getpwuid(getuid()); // NOLINT (single-threaded)
+ if (pwent == NULL || pwent->pw_name == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "$USER is not set, and unable to look up name of current user");
+ }
+ return pwent->pw_name;
+}
+
+// Returns the given path in absolute form. Does not change paths that are
+// already absolute.
+//
+// If called from working directory "/bar":
+// MakeAbsolute("foo") --> "/bar/foo"
+// MakeAbsolute("/foo") ---> "/foo"
+string MakeAbsolute(string path) {
+ // Check if path is already absolute.
+ if (path.empty() || path[0] == '/') {
+ return path;
+ }
+
+ char cwdbuf[PATH_MAX];
+ if (getcwd(cwdbuf, sizeof cwdbuf) == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "getcwd() failed");
+ }
+
+ // Determine whether the cwd ends with "/" or not.
+ string separator = (cwdbuf[strlen(cwdbuf) - 1] == '/') ? "" : "/";
+ return cwdbuf + separator + path;
+}
+
+// mkdir -p path. Returns -1 on failure, sets errno.
+int MakeDirectories(string path, int mode) {
+ path.push_back('\0');
+ char *buf = &path[0];
+ for (char *slash = strchr(buf + 1, '/'); slash != NULL;
+ slash = strchr(slash + 1, '/')) {
+ *slash = '\0';
+ if (mkdir(buf, mode) == -1 && errno != EEXIST) {
+ return -1;
+ }
+ *slash = '/';
+ }
+ // TODO(bazel-team): EEXIST does not prove that it's a directory!
+ if (mkdir(buf, mode) == -1 && errno != EEXIST) {
+ return -1;
+ }
+ return 0;
+}
+
+// Replaces 'content' with contents of file 'filename'.
+// Returns false on error.
+bool ReadFile(const string &filename, string *content) {
+ content->clear();
+ int fd = open(filename.c_str(), O_RDONLY);
+ if (fd == -1) return false;
+ char buf[4096];
+ // OPT: This loop generates one spurious read on regular files.
+ while (int r = read(fd, buf, sizeof buf)) {
+ if (r == -1) {
+ if (errno == EINTR) continue;
+ return false;
+ }
+ content->append(buf, r);
+ }
+ close(fd);
+ return true;
+}
+
+// Writes 'content' into file 'filename', and makes it executable.
+// Returns false on failure, sets errno.
+bool WriteFile(const string &content, const string &filename) {
+ unlink(filename.c_str());
+ int fd = open(filename.c_str(), O_CREAT|O_WRONLY|O_TRUNC, 0755); // chmod +x
+ if (fd == -1) return false;
+ int r = write(fd, content.data(), content.size());
+ int saved_errno = errno;
+ if (close(fd)) return false; // Can fail on NFS.
+ errno = saved_errno; // Caller should see errno from write().
+ return r == content.size();
+}
+
+// Returns true iff both stdout and stderr are connected to a
+// terminal, and it can support color and cursor movement
+// (this is computed heuristically based on the values of
+// environment variables).
+bool IsStandardTerminal() {
+ string term = getenv("TERM") == nullptr ? "" : getenv("TERM");
+ string emacs = getenv("EMACS") == nullptr ? "" : getenv("EMACS");
+ if (term == "" || term == "dumb" || term == "emacs" || term == "xterm-mono" ||
+ term == "symbolics" || term == "9term" || emacs == "t") {
+ return false;
+ }
+ return isatty(STDOUT_FILENO) && isatty(STDERR_FILENO);
+}
+
+// Returns the number of columns of the terminal to which stdout is
+// connected, or $COLUMNS (default 80) if there is no such terminal.
+int GetTerminalColumns() {
+ struct winsize ws;
+ if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) != -1) {
+ return ws.ws_col;
+ }
+ const char* columns_env = getenv("COLUMNS");
+ if (columns_env != NULL && columns_env[0] != '\0') {
+ char* endptr;
+ int columns = blaze_util::strto32(columns_env, &endptr, 10);
+ if (*endptr == '\0') { // $COLUMNS is a valid number
+ return columns;
+ }
+ }
+ return 80; // default if not a terminal.
+}
+
+// Replace the current process with the given program in the given working
+// directory, using the given argument vector.
+// This function does not return on success.
+void ExecuteProgram(string exe, const vector<string>& args_vector) {
+ if (VerboseLogging()) {
+ string dbg;
+ for (const auto& s : args_vector) {
+ dbg.append(s);
+ dbg.append(" ");
+ }
+
+ char cwd[PATH_MAX] = {};
+ getcwd(cwd, sizeof(cwd));
+
+ fprintf(stderr, "Invoking binary %s in %s:\n %s\n",
+ exe.c_str(), cwd, dbg.c_str());
+ }
+
+ // Copy to a char* array for execv:
+ int n = args_vector.size();
+ const char **argv = new const char *[n + 1];
+ for (int i = 0; i < n; ++i) {
+ argv[i] = args_vector[i].c_str();
+ }
+ argv[n] = NULL;
+
+ execv(exe.c_str(), const_cast<char**>(argv));
+}
+
+// Re-execute the blaze command line with a different binary as argv[0].
+// This function does not return on success.
+void ReExecute(const string &executable, int argc, const char *argv[]) {
+ vector<string> args;
+ args.push_back(executable);
+ for (int i = 1; i < argc; i++) {
+ args.push_back(argv[i]);
+ }
+ ExecuteProgram(args[0], args);
+}
+
+const char* GetUnaryOption(const char *arg, const char *next_arg,
+ const char *key) {
+ const char *value = blaze_util::var_strprefix(arg, key);
+ if (value == NULL) {
+ return NULL;
+ } else if (value[0] == '=') {
+ return value + 1;
+ } else if (value[0]) {
+ return NULL; // trailing garbage in key name
+ }
+
+ return next_arg;
+}
+
+bool GetNullaryOption(const char *arg, const char *key) {
+ const char *value = blaze_util::var_strprefix(arg, key);
+ if (value == NULL) {
+ return false;
+ } else if (value[0] == '=') {
+ die(blaze_exit_code::BAD_ARGV,
+ "In argument '%s': option '%s' does not take a value.", arg, key);
+ } else if (value[0]) {
+ return false; // trailing garbage in key name
+ }
+
+ return true;
+}
+
+blaze_exit_code::ExitCode CheckValidPort(
+ const string &str, const string &option, string *error) {
+ int number;
+ if (blaze_util::safe_strto32(str, &number) && number > 0 && number < 65536) {
+ return blaze_exit_code::SUCCESS;
+ }
+
+ blaze_util::StringPrintf(error,
+ "Invalid argument to %s: '%s' (must be a valid port number).",
+ option.c_str(), str.c_str());
+ return blaze_exit_code::BAD_ARGV;
+}
+
+bool VerboseLogging() {
+ return getenv("VERBOSE_BLAZE_CLIENT") != NULL;
+}
+
+// Read the Jvm version from a file descriptor. The read fd
+// should contains a similar output as the java -version output.
+string ReadJvmVersion(int fd) {
+ static const int bytes_to_read = 255;
+ char buf[bytes_to_read + 1]; // leave one extra space for null
+ ssize_t size = read(fd, buf, bytes_to_read);
+ close(fd);
+ if (size > 0) {
+ buf[size] = 0;
+ // try to look out for 'version "'
+ static const char version_pattern[] = "version \"";
+ char *ptr = strstr(buf, version_pattern);
+ if (ptr != NULL) {
+ ptr += sizeof(version_pattern)-1;
+ // If we found "version \"", process until next '"'
+ char *endptr = strchr(ptr, '"');
+ if (endptr != NULL) {
+ *endptr = 0;
+ }
+ return string(ptr);
+ }
+ }
+ return "";
+}
+
+string GetJvmVersion(string java_exe) {
+ vector<string> args;
+ args.push_back("java");
+ args.push_back("-version");
+
+ int fds[2];
+ if (pipe(fds)) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "pipe creation failed");
+ }
+
+ int child = fork();
+ if (child == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "fork() failed");
+ } else if (child > 0) { // we're the parent
+ close(fds[1]); // parent keeps only the reading side
+ return ReadJvmVersion(fds[0]);
+ } else {
+ close(fds[0]); // child keeps only the writing side
+ // Redirect output to the writing side of the dup.
+ dup2(fds[1], STDOUT_FILENO);
+ dup2(fds[1], STDERR_FILENO);
+ // Execute java -version
+ ExecuteProgram(java_exe, args);
+ pdie(blaze_exit_code::INTERNAL_ERROR, "Failed to run java -version");
+ }
+}
+
+bool CheckJavaVersionIsAtLeast(string jvm_version, string version_spec) {
+ vector<string> jvm_version_vect = blaze_util::Split(jvm_version, '.');
+ vector<string> version_spec_vect = blaze_util::Split(version_spec, '.');
+ int i;
+ for (i = 0; i < jvm_version_vect.size() && i < version_spec_vect.size();
+ i++) {
+ int jvm = blaze_util::strto32(jvm_version_vect[i].c_str(), NULL, 10);
+ int spec = blaze_util::strto32(version_spec_vect[i].c_str(), NULL, 10);
+ if (jvm > spec) {
+ return true;
+ } else if (jvm < spec) {
+ return false;
+ }
+ }
+ if (i < version_spec_vect.size()) {
+ for (; i < version_spec_vect.size(); i++) {
+ if (version_spec_vect[i] != "0") {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+} // namespace blaze
diff --git a/src/main/cpp/blaze_util.h b/src/main/cpp/blaze_util.h
new file mode 100644
index 0000000..0c006a2
--- /dev/null
+++ b/src/main/cpp/blaze_util.h
@@ -0,0 +1,127 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// blaze_util.h: Miscellaneous utility functions used by the blaze.cc
+// Blaze client.
+//
+
+#ifndef DEVTOOLS_BLAZE_MAIN_BLAZE_UTIL_H__
+#define DEVTOOLS_BLAZE_MAIN_BLAZE_UTIL_H__
+
+#include <pwd.h>
+#include <stdarg.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <string>
+#include <vector>
+
+#include "blaze_exit_code.h"
+#include "util/numbers.h"
+#include "util/port.h"
+
+namespace blaze {
+
+using std::string;
+
+// Prints the specified error message and exits nonzero.
+void die(const int exit_status, const char *format, ...) ATTRIBUTE_NORETURN
+ PRINTF_ATTRIBUTE(2, 3);
+// Prints "Error: <formatted-message>: <strerror(errno)>\n", and exits nonzero.
+void pdie(const int exit_status, const char *format, ...) ATTRIBUTE_NORETURN
+ PRINTF_ATTRIBUTE(2, 3);
+
+string GetUserName();
+
+// Return the path to the JVM launcher.
+string GetJvm();
+
+// Returns the given path in absolute form. Does not change paths that are
+// already absolute.
+//
+// If called from working directory "/bar":
+// MakeAbsolute("foo") --> "/bar/foo"
+// MakeAbsolute("/foo") ---> "/foo"
+string MakeAbsolute(string path);
+
+// mkdir -p path. All newly created directories use the given mode.
+// Returns -1 on failure, sets errno.
+int MakeDirectories(string path, int mode);
+
+// Replaces 'content' with contents of file 'filename'.
+// Returns false on error.
+bool ReadFile(const string &filename, string *content);
+
+// Writes 'content' into file 'filename', and makes it executable.
+// Returns false on failure, sets errno.
+bool WriteFile(const string &content, const string &filename);
+
+// Returns true iff the current terminal can support color and cursor movement.
+bool IsStandardTerminal();
+
+// Returns the number of columns of the terminal to which stdout is
+// connected, or 80 if there is no such terminal.
+int GetTerminalColumns();
+
+// blaze's JVM arch is set at build time (--java_cpu), since the blaze java
+// process includes native code.
+bool Is64BitBlazeJavabase();
+
+// Adds JVM arguments particular to running blaze with JVM v3 or higher.
+void AddJVMSpecificArguments(const string &host_javabase,
+ std::vector<string> *result);
+
+void ExecuteProgram(string exe, const std::vector<string>& args_vector);
+
+void ReExecute(const string &executable, int argc, const char *argv[]);
+
+// If 'arg' matches 'key=value', returns address of 'value'.
+// If it matches 'key' alone, returns address of next_arg.
+// Returns NULL otherwise.
+const char* GetUnaryOption(const char *arg, const char *next_arg,
+ const char *key);
+
+// Returns true iff 'arg' equals 'key'.
+// Dies with a syntax error if arg starts with 'key='.
+// Returns NULL otherwise.
+bool GetNullaryOption(const char *arg, const char *key);
+
+blaze_exit_code::ExitCode CheckValidPort(
+ const string &str, const string &option, string *error);
+
+bool VerboseLogging();
+
+// Read the JVM version from a file descriptor. The fd should point
+// to the output of a "java -version" execution and is supposed to contains
+// a string of the form 'version "version-number"' in the first 255 bytes.
+// If the string is found, version-number is returned, else the empty string
+// is returned.
+string ReadJvmVersion(int fd);
+
+// Get the version string from the given java executable. The java executable
+// is supposed to output a string in the form '.*version ".*".*'. This method
+// will return the part in between the two quote or the empty string on failure
+// to match the good string.
+string GetJvmVersion(string java_exe);
+
+// Returns true iff jvm_version is at least the version specified by
+// version_spec.
+// jvm_version is supposed to be a string specifying a java runtime version
+// as specified by the JSR-56 appendix A. version_spec is supposed to be a
+// version is the format [0-9]+(.[1-9]+)*.
+bool CheckJavaVersionIsAtLeast(string jvm_version, string version_spec);
+
+} // namespace blaze
+#endif // DEVTOOLS_BLAZE_MAIN_BLAZE_UTIL_H__
diff --git a/src/main/cpp/blaze_util_darwin.cc b/src/main/cpp/blaze_util_darwin.cc
new file mode 100644
index 0000000..f6eb39c
--- /dev/null
+++ b/src/main/cpp/blaze_util_darwin.cc
@@ -0,0 +1,115 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <libproc.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <cstdio>
+
+#include "blaze_exit_code.h"
+#include "blaze_util.h"
+#include "blaze_util_platform.h"
+#include "util/strings.h"
+
+namespace blaze {
+
+using std::string;
+
+void WarnFilesystemType(const string& output_base) {
+ // TODO(bazel-team): Should check for NFS.
+ // TODO(bazel-team): Should check for case insensitive file systems?
+}
+
+pid_t GetPeerProcessId(int socket) {
+ pid_t pid = 0;
+ socklen_t len = sizeof(pid_t);
+ if (getsockopt(socket, SOL_LOCAL, LOCAL_PEERPID, &pid, &len) < 0) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "can't get server pid from connection");
+ }
+ return pid;
+}
+
+string GetSelfPath() {
+ char pathbuf[PROC_PIDPATHINFO_MAXSIZE] = {};
+ int len = proc_pidpath(getpid(), pathbuf, sizeof(pathbuf));
+ if (len == 0) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "error calling proc_pidpath");
+ }
+ return string(pathbuf, len);
+}
+
+uint64 MonotonicClock() {
+ struct timeval ts = {};
+ if (gettimeofday(&ts, NULL) < 0) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "error calling gettimeofday");
+ }
+ return ts.tv_sec * 1000000000LL + ts.tv_usec * 1000;
+}
+
+uint64 ProcessClock() {
+ return clock() * (1000000000LL / CLOCKS_PER_SEC);
+}
+
+void SetScheduling(bool batch_cpu_scheduling, int io_nice_level) {
+ // stubbed out so we can compile for Darwin.
+}
+
+string GetProcessCWD(int pid) {
+ struct proc_vnodepathinfo info = {};
+ if (proc_pidinfo(
+ pid, PROC_PIDVNODEPATHINFO, 0, &info, sizeof(info)) != sizeof(info)) {
+ return "";
+ }
+ return string(info.pvi_cdir.vip_path);
+}
+
+bool IsSharedLibrary(string filename) {
+ return blaze_util::ends_with(filename, ".dylib");
+}
+
+string GetDefaultHostJavabase() {
+ const char *java_home = getenv("JAVA_HOME");
+ if (java_home) {
+ return std::string(java_home);
+ }
+
+ FILE *output = popen("/usr/libexec/java_home -v 1.7+", "r");
+ if (output == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Could not run /usr/libexec/java_home");
+ }
+
+ char buf[512];
+ char *result = fgets(buf, sizeof(buf), output);
+ pclose(output);
+ if (result == NULL) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "No output from /usr/libexec/java_home");
+ }
+
+ string javabase = buf;
+ if (javabase.empty()) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Empty output from /usr/libexec/java_home - "
+ "install a JDK, or install a JRE and point your JAVA_HOME to it");
+ }
+
+ // The output ends with a \n, trim it off.
+ return javabase.substr(0, javabase.length()-1);
+}
+
+} // namespace blaze.
diff --git a/src/main/cpp/blaze_util_linux.cc b/src/main/cpp/blaze_util_linux.cc
new file mode 100644
index 0000000..3c1bceb
--- /dev/null
+++ b/src/main/cpp/blaze_util_linux.cc
@@ -0,0 +1,152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <limits.h>
+#include <string.h> // strerror
+#include <sys/statfs.h>
+#include <unistd.h>
+
+#include "blaze_exit_code.h"
+#include "blaze_util_platform.h"
+#include "blaze_util.h"
+#include "util/file.h"
+#include "util/strings.h"
+
+namespace blaze {
+
+using std::string;
+
+void WarnFilesystemType(const string& output_base) {
+ struct statfs buf = {};
+ if (statfs(output_base.c_str(), &buf) < 0) {
+ fprintf(stderr,
+ "WARNING: couldn't get file system type information for '%s': %s\n",
+ output_base.c_str(), strerror(errno));
+ return;
+ }
+
+ if (buf.f_type == 0x00006969) { // NFS_SUPER_MAGIC
+ fprintf(stderr, "WARNING: Output base '%s' is on NFS. This may lead "
+ "to surprising failures and undetermined behavior.\n",
+ output_base.c_str());
+ }
+}
+
+string GetSelfPath() {
+ char buffer[PATH_MAX] = {};
+ ssize_t bytes = readlink("/proc/self/exe", buffer, sizeof(buffer));
+ if (bytes == sizeof(buffer)) {
+ // symlink contents truncated
+ bytes = -1;
+ errno = ENAMETOOLONG;
+ }
+ if (bytes == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "error reading /proc/self/exe");
+ }
+ buffer[bytes] = '\0'; // readlink does not NUL-terminate
+ return string(buffer);
+}
+
+pid_t GetPeerProcessId(int socket) {
+ struct ucred creds = {};
+ socklen_t len = sizeof creds;
+ if (getsockopt(socket, SOL_SOCKET, SO_PEERCRED, &creds, &len) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "can't get server pid from connection");
+ }
+ return creds.pid;
+}
+
+uint64 MonotonicClock() {
+ struct timespec ts = {};
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+uint64 ProcessClock() {
+ struct timespec ts = {};
+ clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
+ return ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+void SetScheduling(bool batch_cpu_scheduling, int io_nice_level) {
+ // Move ourself into a low priority CPU scheduling group if the
+ // machine is configured appropriately. Fail silently, because this
+ // isn't available on all kernels.
+ if (FILE *f = fopen("/dev/cgroup/cpu/batch/tasks", "w")) {
+ fprintf(f, "%d", getpid());
+ fclose(f);
+ }
+
+ if (batch_cpu_scheduling) {
+ sched_param param = {};
+ param.sched_priority = 0;
+ if (sched_setscheduler(0, SCHED_BATCH, ¶m)) {
+ pdie(blaze_exit_code::INTERNAL_ERROR,
+ "sched_setscheduler(SCHED_BATCH) failed");
+ }
+ }
+
+ if (io_nice_level >= 0) {
+ if (blaze_util::sys_ioprio_set(
+ IOPRIO_WHO_PROCESS, getpid(),
+ IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, io_nice_level)) < 0) {
+ pdie(blaze_exit_code::INTERNAL_ERROR,
+ "ioprio_set() with class %d and level %d failed",
+ IOPRIO_CLASS_BE, io_nice_level);
+ }
+ }
+}
+
+string GetProcessCWD(int pid) {
+ char server_cwd[PATH_MAX] = {};
+ if (readlink(
+ ("/proc/" + std::to_string(pid) + "/cwd").c_str(),
+ server_cwd, sizeof(server_cwd)) < 0) {
+ return "";
+ }
+
+ return string(server_cwd);
+}
+
+bool IsSharedLibrary(string filename) {
+ return blaze_util::ends_with(filename, ".so");
+}
+
+string GetDefaultHostJavabase() {
+ // if JAVA_HOME is defined, then use it as default.
+ const char *javahome = getenv("JAVA_HOME");
+ if (javahome != NULL) {
+ return string(javahome);
+ }
+
+ // which javac
+ string javac_dir = blaze_util::Which("javac");
+ if (javac_dir.empty()) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR, "Could not find javac");
+ }
+
+ // Resolve all symlinks.
+ char resolved_path[PATH_MAX];
+ if (realpath(javac_dir.c_str(), resolved_path) == NULL) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Could not resolve javac directory");
+ }
+ javac_dir = resolved_path;
+
+ // dirname dirname
+ return blaze_util::Dirname(blaze_util::Dirname(javac_dir));
+}
+
+} // namespace blaze
diff --git a/src/main/cpp/blaze_util_mingw.cc b/src/main/cpp/blaze_util_mingw.cc
new file mode 100644
index 0000000..07fabe6
--- /dev/null
+++ b/src/main/cpp/blaze_util_mingw.cc
@@ -0,0 +1,102 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <errno.h>
+#include <limits.h>
+#include <string.h> // strerror
+#include <sys/statfs.h>
+#include <unistd.h>
+
+#include <cstdlib>
+#include <cstdio>
+
+#include "blaze_exit_code.h"
+#include "blaze_util_platform.h"
+#include "blaze_util.h"
+#include "util/file.h"
+#include "util/strings.h"
+
+namespace blaze {
+
+using std::string;
+
+void WarnFilesystemType(const string& output_base) {
+}
+
+string GetSelfPath() {
+ char buffer[PATH_MAX] = {};
+ ssize_t bytes = readlink("/proc/self/exe", buffer, sizeof(buffer));
+ if (bytes == sizeof(buffer)) {
+ // symlink contents truncated
+ bytes = -1;
+ errno = ENAMETOOLONG;
+ }
+ if (bytes == -1) {
+ pdie(blaze_exit_code::INTERNAL_ERROR, "error reading /proc/self/exe");
+ }
+ buffer[bytes] = '\0'; // readlink does not NUL-terminate
+ return string(buffer);
+}
+
+pid_t GetPeerProcessId(int socket) {
+ struct ucred creds = {};
+ socklen_t len = sizeof creds;
+ if (getsockopt(socket, SOL_SOCKET, SO_PEERCRED, &creds, &len) == -1) {
+ pdie(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "can't get server pid from connection");
+ }
+ return creds.pid;
+}
+
+uint64 MonotonicClock() {
+ struct timespec ts = {};
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+uint64 ProcessClock() {
+ struct timespec ts = {};
+ clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
+ return ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+void SetScheduling(bool batch_cpu_scheduling, int io_nice_level) {
+ // TODO(bazel-team): There should be a similar function on Windows.
+}
+
+string GetProcessCWD(int pid) {
+ char server_cwd[PATH_MAX] = {};
+ if (readlink(
+ ("/proc/" + std::to_string(pid) + "/cwd").c_str(),
+ server_cwd, sizeof(server_cwd)) < 0) {
+ return "";
+ }
+
+ return string(server_cwd);
+}
+
+bool IsSharedLibrary(string filename) {
+ return blaze_util::ends_with(filename, ".dll");
+}
+
+string GetDefaultHostJavabase() {
+ const char *javahome = getenv("JAVA_HOME");
+ if (javahome == NULL) {
+ die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Error: JAVA_HOME not set.");
+ }
+ return javahome;
+}
+
+} // namespace blaze
diff --git a/src/main/cpp/blaze_util_platform.h b/src/main/cpp/blaze_util_platform.h
new file mode 100644
index 0000000..1de65b4
--- /dev/null
+++ b/src/main/cpp/blaze_util_platform.h
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DEVTOOLS_BLAZE_MAIN_BLAZE_UTIL_PLATFORM_H_
+#define DEVTOOLS_BLAZE_MAIN_BLAZE_UTIL_PLATFORM_H_
+
+#include <string>
+#include "util/numbers.h"
+
+namespace blaze {
+
+// Get the absolute path to the binary being executed.
+std::string GetSelfPath();
+
+// Returns the process id of the peer connected to this socket.
+pid_t GetPeerProcessId(int socket);
+
+// Warn about dubious filesystem types, such as NFS, case-insensitive (?).
+void WarnFilesystemType(const std::string& output_base);
+
+// Wrapper around clock_gettime(CLOCK_MONOTONIC) that returns the time
+// as a uint64 nanoseconds since epoch.
+uint64 MonotonicClock();
+
+// Wrapper around clock_gettime(CLOCK_PROCESS_CPUTIME_ID) that returns the
+// nanoseconds consumed by the current process since it started.
+uint64 ProcessClock();
+
+// Set cpu and IO scheduling properties. Note that this can take ~50ms
+// on Linux, so it should only be called when necessary.
+void SetScheduling(bool batch_cpu_scheduling, int io_nice_level);
+
+// Returns the cwd for a process.
+std::string GetProcessCWD(int pid);
+
+bool IsSharedLibrary(std::string filename);
+
+// Return the default path to the JDK used to run Blaze itself
+// (must be an absolute directory).
+std::string GetDefaultHostJavabase();
+
+} // namespace blaze
+
+#endif // DEVTOOLS_BLAZE_MAIN_BLAZE_UTIL_PLATFORM_H_
diff --git a/src/main/cpp/option_processor.cc b/src/main/cpp/option_processor.cc
new file mode 100644
index 0000000..ce4cd63
--- /dev/null
+++ b/src/main/cpp/option_processor.cc
@@ -0,0 +1,480 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "option_processor.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <algorithm>
+#include <cassert>
+#include <utility>
+
+#include "blaze_util.h"
+#include "blaze_util_platform.h"
+#include "util/file.h"
+#include "util/strings.h"
+
+using std::list;
+using std::map;
+using std::vector;
+
+// On OSX, there apparently is no header that defines this.
+extern char **environ;
+
+namespace blaze {
+
+OptionProcessor::RcOption::RcOption(int rcfile_index, const string& option) {
+ rcfile_index_ = rcfile_index;
+ option_ = option;
+}
+
+
+OptionProcessor::RcFile::RcFile(const string& filename, int index) {
+ filename_ = filename;
+ index_ = index;
+}
+
+blaze_exit_code::ExitCode OptionProcessor::RcFile::Parse(
+ vector<RcFile>* rcfiles,
+ map<string, vector<RcOption> >* rcoptions,
+ string* error) {
+ list<string> initial_import_stack;
+ initial_import_stack.push_back(filename_);
+ return Parse(
+ filename_, index_, rcfiles, rcoptions, &initial_import_stack, error);
+}
+
+blaze_exit_code::ExitCode OptionProcessor::RcFile::Parse(
+ string filename, const int index,
+ vector<RcFile>* rcfiles,
+ map<string, vector<RcOption> >* rcoptions,
+ list<string>* import_stack,
+ string* error) {
+ string contents;
+ if (!ReadFile(filename, &contents)) {
+ // We checked for file readability before, so this is unexpected.
+ blaze_util::StringPrintf(error,
+ "Unexpected error reading .blazerc file '%s'", filename.c_str());
+ return blaze_exit_code::INTERNAL_ERROR;
+ }
+
+ // A '\' at the end of a line continues the line.
+ blaze_util::Replace("\\\r\n", "", &contents);
+ blaze_util::Replace("\\\n", "", &contents);
+ vector<string> startup_options;
+
+ vector<string> lines = blaze_util::Split(contents, '\n');
+ for (int line = 0; line < lines.size(); ++line) {
+ blaze_util::StripWhitespace(&lines[line]);
+
+ // Check for an empty line.
+ if (lines[line].empty()) {
+ continue;
+ }
+
+ vector<string> words;
+
+ // This will treat "#" as a comment, and properly
+ // quote single and double quotes, and treat '\'
+ // as an escape character.
+ // TODO(bazel-team): This function silently ignores
+ // dangling backslash escapes and missing end-quotes.
+ blaze_util::Tokenize(lines[line], '#', &words);
+
+ if (words.empty()) {
+ // Could happen if line starts with "#"
+ continue;
+ }
+
+ string command = words[0];
+
+ if (command == "import") {
+ if (words.size() != 2) {
+ blaze_util::StringPrintf(error,
+ "Invalid import declaration in .blazerc file '%s': '%s'",
+ filename.c_str(), lines[line].c_str());
+ return blaze_exit_code::BAD_ARGV;
+ }
+
+ if (std::find(import_stack->begin(), import_stack->end(), words[1]) !=
+ import_stack->end()) {
+ string loop;
+ for (list<string>::const_iterator imported_rc = import_stack->begin();
+ imported_rc != import_stack->end(); ++imported_rc) {
+ loop += " " + *imported_rc + "\n";
+ }
+ blaze_util::StringPrintf(error,
+ "Import loop detected:\n%s", loop.c_str());
+ return blaze_exit_code::BAD_ARGV;
+ }
+
+ rcfiles->push_back(RcFile(words[1], rcfiles->size()));
+ import_stack->push_back(words[1]);
+ blaze_exit_code::ExitCode parse_exit_code = RcFile::Parse(
+ rcfiles->back().Filename(), rcfiles->back().Index(),
+ rcfiles, rcoptions, import_stack, error);
+ if (parse_exit_code != blaze_exit_code::SUCCESS) {
+ return parse_exit_code;
+ }
+ import_stack->pop_back();
+ } else {
+ for (int word = 1; word < words.size(); ++word) {
+ (*rcoptions)[command].push_back(RcOption(index, words[word]));
+ if (command == "startup") {
+ startup_options.push_back(words[word]);
+ }
+ }
+ }
+ }
+
+ if (!startup_options.empty()) {
+ string startup_args;
+ blaze_util::JoinStrings(startup_options, ' ', &startup_args);
+ fprintf(stderr, "INFO: Reading 'startup' options from %s: %s\n",
+ filename.c_str(), startup_args.c_str());
+ }
+ return blaze_exit_code::SUCCESS;
+}
+
+OptionProcessor::OptionProcessor()
+ : initialized_(false), parsed_startup_options_(new BlazeStartupOptions()) {
+}
+
+// Return the path of the depot .blazerc file.
+string OptionProcessor::FindDepotBlazerc(const string& workspace) {
+ // Package semantics are ignored here, but that's acceptable because
+ // blaze.blazerc is a configuration file.
+ vector<string> candidates;
+ BlazeStartupOptions::WorkspaceRcFileSearchPath(&candidates);
+ for (const auto& candidate : candidates) {
+ string blazerc = blaze_util::JoinPath(workspace, candidate);
+ if (!access(blazerc.c_str(), R_OK)) {
+ return blazerc;
+ }
+ }
+
+ return "";
+}
+
+// Return the path of the .blazerc file that sits alongside the binary.
+// This allows for canary or cross-platform Blazes operating on the same depot
+// to have customized behavior.
+string OptionProcessor::FindAlongsideBinaryBlazerc(const string& cwd,
+ const string& arg0) {
+ string path = arg0[0] == '/' ? arg0 : blaze_util::JoinPath(cwd, arg0);
+ string base = blaze_util::Basename(arg0);
+ string binary_blazerc_path = path + "." + base + "rc";
+ if (!access(binary_blazerc_path.c_str(), R_OK)) {
+ return binary_blazerc_path;
+ }
+ return "";
+}
+
+
+// Return the path the the user rc file. If cmdLineRcFile != NULL,
+// use it, dying if it is not readable. Otherwise, return the first
+// readable file called rc_basename from [workspace, $HOME]
+//
+// If no readable .blazerc file is found, return the empty string.
+blaze_exit_code::ExitCode OptionProcessor::FindUserBlazerc(
+ const char* cmdLineRcFile,
+ const string& rc_basename,
+ const string& workspace,
+ string* blaze_rc_file,
+ string* error) {
+ if (cmdLineRcFile != NULL) {
+ string rcFile = MakeAbsolute(cmdLineRcFile);
+ if (access(rcFile.c_str(), R_OK)) {
+ blaze_util::StringPrintf(error,
+ "Error: Unable to read .blazerc file '%s'.", rcFile.c_str());
+ return blaze_exit_code::BAD_ARGV;
+ }
+ *blaze_rc_file = rcFile;
+ return blaze_exit_code::SUCCESS;
+ }
+
+ string workspaceRcFile = blaze_util::JoinPath(workspace, rc_basename);
+ if (!access(workspaceRcFile.c_str(), R_OK)) {
+ *blaze_rc_file = workspaceRcFile;
+ return blaze_exit_code::SUCCESS;
+ }
+
+ const char* home = getenv("HOME");
+ if (home == NULL) {
+ *blaze_rc_file = "";
+ return blaze_exit_code::SUCCESS;
+ }
+
+ string userRcFile = blaze_util::JoinPath(home, rc_basename);
+ if (!access(userRcFile.c_str(), R_OK)) {
+ *blaze_rc_file = userRcFile;
+ return blaze_exit_code::SUCCESS;
+ }
+ *blaze_rc_file = "";
+ return blaze_exit_code::SUCCESS;
+}
+
+blaze_exit_code::ExitCode OptionProcessor::ParseOptions(
+ const vector<string>& args,
+ const string& workspace,
+ const string& cwd,
+ string* error) {
+ assert(!initialized_);
+ initialized_ = true;
+
+ const char* blazerc = NULL;
+ bool use_master_blazerc = true;
+
+ // Check if there is a blazerc related option given
+ args_ = args;
+ for (int i= 1; i < args.size(); i++) {
+ const char* arg_chr = args[i].c_str();
+ const char* next_arg_chr = (i + 1) < args.size()
+ ? args[i + 1].c_str()
+ : NULL;
+ if (blazerc == NULL) {
+ blazerc = GetUnaryOption(arg_chr, next_arg_chr, "--blazerc");
+ }
+ if (use_master_blazerc &&
+ GetNullaryOption(arg_chr, "--nomaster_blazerc")) {
+ use_master_blazerc = false;
+ }
+ }
+
+ // Parse depot and user blazerc files.
+ // This is not a little ineffective (copying a multimap around), but it is a
+ // small one and this way I don't have to care about memory management.
+ if (use_master_blazerc) {
+ string depot_blazerc_path = FindDepotBlazerc(workspace);
+ if (!depot_blazerc_path.empty()) {
+ blazercs_.push_back(RcFile(depot_blazerc_path, blazercs_.size()));
+ blaze_exit_code::ExitCode parse_exit_code =
+ blazercs_.back().Parse(&blazercs_, &rcoptions_, error);
+ if (parse_exit_code != blaze_exit_code::SUCCESS) {
+ return parse_exit_code;
+ }
+ }
+ string alongside_binary_blazerc = FindAlongsideBinaryBlazerc(cwd, args[0]);
+ if (!alongside_binary_blazerc.empty()) {
+ blazercs_.push_back(RcFile(alongside_binary_blazerc, blazercs_.size()));
+ blaze_exit_code::ExitCode parse_exit_code =
+ blazercs_.back().Parse(&blazercs_, &rcoptions_, error);
+ if (parse_exit_code != blaze_exit_code::SUCCESS) {
+ return parse_exit_code;
+ }
+ }
+ }
+
+ string user_blazerc_path;
+ blaze_exit_code::ExitCode find_blazerc_exit_code = FindUserBlazerc(
+ blazerc, BlazeStartupOptions::RcBasename(), workspace, &user_blazerc_path,
+ error);
+ if (find_blazerc_exit_code != blaze_exit_code::SUCCESS) {
+ return find_blazerc_exit_code;
+ }
+ if (!user_blazerc_path.empty()) {
+ blazercs_.push_back(RcFile(user_blazerc_path, blazercs_.size()));
+ blaze_exit_code::ExitCode parse_exit_code =
+ blazercs_.back().Parse(&blazercs_, &rcoptions_, error);
+ if (parse_exit_code != blaze_exit_code::SUCCESS) {
+ return parse_exit_code;
+ }
+ }
+
+ blaze_exit_code::ExitCode parse_startup_options_exit_code =
+ ParseStartupOptions(error);
+ if (parse_startup_options_exit_code != blaze_exit_code::SUCCESS) {
+ return parse_startup_options_exit_code;
+ }
+
+ // Determine command
+ if (startup_args_ + 1 >= args.size()) {
+ command_ = "";
+ return blaze_exit_code::SUCCESS;
+ }
+
+ command_ = args[startup_args_ + 1];
+
+#if __APPLE__
+ // This is a temporary hack until we work out how to actually reference the
+ // system JDK in a sound way.
+ if (command_ == "build" ||
+ command_ == "test" ||
+ command_ == "coverage" ||
+ command_ == "run" ||
+ command_ == "info" ||
+ command_ == "version") {
+ string javabase = blaze::GetDefaultHostJavabase();
+ command_arguments_.push_back("--javabase=" + javabase);
+ command_arguments_.push_back("--host_javabase=" + javabase);
+ }
+#endif
+
+ AddRcfileArgsAndOptions(parsed_startup_options_->batch, cwd);
+ for (unsigned int cmd_arg = startup_args_ + 2;
+ cmd_arg < args.size(); cmd_arg++) {
+ command_arguments_.push_back(args[cmd_arg]);
+ }
+ return blaze_exit_code::SUCCESS;
+}
+
+blaze_exit_code::ExitCode OptionProcessor::ParseOptions(
+ int argc,
+ const char* argv[],
+ const string& workspace,
+ const string& cwd,
+ string* error) {
+ vector<string> args(argc);
+ for (int arg = 0; arg < argc; arg++) {
+ args[arg] = argv[arg];
+ }
+
+ return ParseOptions(args, workspace, cwd, error);
+}
+
+static bool IsArg(const string& arg) {
+ return blaze_util::starts_with(arg, "-") && (arg != "--help")
+ && (arg != "-help") && (arg != "-h");
+}
+
+blaze_exit_code::ExitCode OptionProcessor::ParseStartupOptions(string *error) {
+ // Process rcfile startup options
+ map< string, vector<RcOption> >::const_iterator it =
+ rcoptions_.find("startup");
+ blaze_exit_code::ExitCode process_arg_exit_code;
+ bool is_space_separated;
+ if (it != rcoptions_.end()) {
+ const vector<RcOption>& startup_options = it->second;
+ int i = 0;
+ // Process all elements except the last one.
+ for (; i < startup_options.size() - 1; i++) {
+ const RcOption& option = startup_options[i];
+ const string& blazerc = blazercs_[option.rcfile_index()].Filename();
+ process_arg_exit_code = parsed_startup_options_->ProcessArg(
+ option.option(), startup_options[i + 1].option(), blazerc,
+ &is_space_separated, error);
+ if (process_arg_exit_code != blaze_exit_code::SUCCESS) {
+ return process_arg_exit_code;
+ }
+ if (is_space_separated) {
+ i++;
+ }
+ }
+ // Process last element, if any.
+ if (i < startup_options.size()) {
+ const RcOption& option = startup_options[i];
+ if (IsArg(option.option())) {
+ const string& blazerc = blazercs_[option.rcfile_index()].Filename();
+ process_arg_exit_code = parsed_startup_options_->ProcessArg(
+ option.option(), "", blazerc, &is_space_separated, error);
+ if (process_arg_exit_code != blaze_exit_code::SUCCESS) {
+ return process_arg_exit_code;
+ }
+ }
+ }
+ }
+
+ // Process command-line args next, so they override any of the same options
+ // from .blazerc. Stop on first non-arg, this includes --help
+ unsigned int i = 1;
+ if (!args_.empty()) {
+ for (; (i < args_.size() - 1) && IsArg(args_[i]); i++) {
+ process_arg_exit_code = parsed_startup_options_->ProcessArg(
+ args_[i], args_[i + 1], "", &is_space_separated, error);
+ if (process_arg_exit_code != blaze_exit_code::SUCCESS) {
+ return process_arg_exit_code;
+ }
+ if (is_space_separated) {
+ i++;
+ }
+ }
+ if (i < args_.size() && IsArg(args_[i])) {
+ process_arg_exit_code = parsed_startup_options_->ProcessArg(
+ args_[i], "", "", &is_space_separated, error);
+ if (process_arg_exit_code != blaze_exit_code::SUCCESS) {
+ return process_arg_exit_code;
+ }
+ i++;
+ }
+ }
+ startup_args_ = i -1;
+
+ return blaze_exit_code::SUCCESS;
+}
+
+// Appends the command and arguments from argc/argv to the end of arg_vector,
+// and also splices in some additional terminal and environment options between
+// the command and the arguments. NB: Keep the options added here in sync with
+// BlazeCommandDispatcher.INTERNAL_COMMAND_OPTIONS!
+void OptionProcessor::AddRcfileArgsAndOptions(bool batch, const string& cwd) {
+ // Push the options mapping .blazerc numbers to filenames.
+ for (int i_blazerc = 0; i_blazerc < blazercs_.size(); i_blazerc++) {
+ const RcFile& blazerc = blazercs_[i_blazerc];
+ command_arguments_.push_back("--rc_source=" + blazerc.Filename());
+ }
+
+ // Push the option defaults
+ for (map<string, vector<RcOption> >::const_iterator it = rcoptions_.begin();
+ it != rcoptions_.end(); ++it) {
+ if (it->first == "startup") {
+ // Skip startup options, they are parsed in the C++ wrapper
+ continue;
+ }
+
+ for (int ii = 0; ii < it->second.size(); ii++) {
+ const RcOption& rcoption = it->second[ii];
+ command_arguments_.push_back(
+ "--default_override=" + std::to_string(rcoption.rcfile_index()) + ":"
+ + it->first + "=" + rcoption.option());
+ }
+ }
+
+ // Splice the terminal options.
+ command_arguments_.push_back(
+ "--isatty=" + std::to_string(IsStandardTerminal()));
+ command_arguments_.push_back(
+ "--terminal_columns=" + std::to_string(GetTerminalColumns()));
+
+ // Pass the client environment to the server in server mode.
+ if (batch) {
+ command_arguments_.push_back("--ignore_client_env");
+ } else {
+ for (char** env = environ; *env != NULL; env++) {
+ command_arguments_.push_back("--client_env=" + string(*env));
+ }
+ }
+ command_arguments_.push_back("--client_cwd=" + cwd);
+
+ const char *emacs = getenv("EMACS");
+ if (emacs != NULL && strcmp(emacs, "t") == 0) {
+ command_arguments_.push_back("--emacs");
+ }
+}
+
+void OptionProcessor::GetCommandArguments(vector<string>* result) const {
+ result->insert(result->end(),
+ command_arguments_.begin(),
+ command_arguments_.end());
+}
+
+const string& OptionProcessor::GetCommand() const {
+ return command_;
+}
+
+const BlazeStartupOptions& OptionProcessor::GetParsedStartupOptions() const {
+ return *parsed_startup_options_.get();
+}
+} // namespace blaze
diff --git a/src/main/cpp/option_processor.h b/src/main/cpp/option_processor.h
new file mode 100644
index 0000000..8246216
--- /dev/null
+++ b/src/main/cpp/option_processor.h
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DEVTOOLS_BLAZE_MAIN_OPTION_PROCESSOR_H_
+#define DEVTOOLS_BLAZE_MAIN_OPTION_PROCESSOR_H_
+
+#include <list>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "blaze_exit_code.h"
+#include "blaze_startup_options.h"
+
+namespace blaze {
+
+using std::string;
+
+// This class is responsible for parsing the command line of the Blaze binary,
+// parsing blazerc files, and putting together the command that should be sent
+// to the server.
+class OptionProcessor {
+ public:
+ OptionProcessor();
+
+ virtual ~OptionProcessor() {}
+
+ // Parse a command line and the appropriate blazerc files. This should be
+ // invoked only once per OptionProcessor object.
+ blaze_exit_code::ExitCode ParseOptions(const std::vector<string>& args,
+ const string& workspace,
+ const string& cwd,
+ string* error);
+
+ blaze_exit_code::ExitCode ParseOptions(int argc, const char* argv[],
+ const string& workspace,
+ const string& cwd,
+ string* error);
+
+ // Get the Blaze command to be executed.
+ // Returns an empty string if no command was found on the command line.
+ const string& GetCommand() const;
+
+ // Gets the arguments to the command. This is put together from the default
+ // options specified in the blazerc file(s), the command line, and various
+ // bits and pieces of information about the environment the blaze binary is
+ // executed in.
+ void GetCommandArguments(std::vector<string>* result) const;
+
+ const BlazeStartupOptions& GetParsedStartupOptions() const;
+
+ virtual string FindDepotBlazerc(const string& workspace);
+ virtual string FindAlongsideBinaryBlazerc(const string& cwd,
+ const string& arg0);
+ virtual blaze_exit_code::ExitCode FindUserBlazerc(const char* cmdLineRcFile,
+ const string& rc_basename,
+ const string& workspace,
+ string* user_blazerc_file,
+ string* error);
+
+ private:
+ class RcOption {
+ public:
+ RcOption(int rcfile_index, const string& option);
+
+ const int rcfile_index() const { return rcfile_index_; }
+ const string& option() const { return option_; }
+
+ private:
+ int rcfile_index_;
+ string option_;
+ };
+
+ class RcFile {
+ public:
+ RcFile(const string& filename, int index);
+ blaze_exit_code::ExitCode Parse(
+ std::vector<RcFile>* rcfiles,
+ std::map<string, std::vector<RcOption> >* rcoptions,
+ string* error);
+ const string& Filename() const { return filename_; }
+ const int Index() const { return index_; }
+
+ private:
+ static blaze_exit_code::ExitCode Parse(string filename, const int index,
+ std::vector<RcFile>* rcfiles,
+ std::map<string,
+ std::vector<RcOption> >* rcoptions,
+ std::list<string>* import_stack,
+ string* error);
+
+ string filename_;
+ int index_;
+ };
+
+ void AddRcfileArgsAndOptions(bool batch, const string& cwd);
+ blaze_exit_code::ExitCode ParseStartupOptions(string *error);
+
+ std::vector<RcFile> blazercs_;
+ std::map<string, std::vector<RcOption> > rcoptions_;
+ std::vector<string> args_;
+ unsigned int startup_args_;
+ string command_;
+ std::vector<string> command_arguments_;
+ bool initialized_;
+ std::unique_ptr<BlazeStartupOptions> parsed_startup_options_;
+};
+
+} // namespace blaze
+#endif // DEVTOOLS_BLAZE_MAIN_OPTION_PROCESSOR_H_
diff --git a/src/main/cpp/util/file.cc b/src/main/cpp/util/file.cc
new file mode 100644
index 0000000..581b71a
--- /dev/null
+++ b/src/main/cpp/util/file.cc
@@ -0,0 +1,100 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "util/file.h"
+
+#include <errno.h> // EINVAL
+#include <limits.h> // PATH_MAX
+#include <sys/stat.h>
+#include <unistd.h> // access
+#include <cstdlib>
+#include <vector>
+
+#include "blaze_exit_code.h"
+#include "blaze_util.h"
+#include "util/strings.h"
+
+using std::pair;
+
+namespace blaze_util {
+
+pair<string, string> SplitPath(const string &path) {
+ size_t pos = path.rfind('/');
+
+ // Handle the case with no '/' in 'path'.
+ if (pos == string::npos) return std::make_pair("", path);
+
+ // Handle the case with a single leading '/' in 'path'.
+ if (pos == 0) return std::make_pair(string(path, 0, 1), string(path, 1));
+
+ return std::make_pair(string(path, 0, pos), string(path, pos + 1));
+}
+
+string Dirname(const string &path) {
+ return SplitPath(path).first;
+}
+
+string Basename(const string &path) {
+ return SplitPath(path).second;
+}
+
+string JoinPath(const string &path1, const string &path2) {
+ if (path1.empty()) {
+ // "" + "/bar"
+ return path2;
+ }
+
+ if (path1[path1.size() - 1] == '/') {
+ if (path2.find('/') == 0) {
+ // foo/ + /bar
+ return path1 + path2.substr(1);
+ } else {
+ // foo/ + bar
+ return path1 + path2;
+ }
+ } else {
+ if (path2.find('/') == 0) {
+ // foo + /bar
+ return path1 + path2;
+ } else {
+ // foo + bar
+ return path1 + "/" + path2;
+ }
+ }
+}
+
+string Which(const string &executable) {
+ string path(getenv("PATH"));
+ if (path.empty()) {
+ blaze::die(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR,
+ "Could not get PATH to find %s", executable.c_str());
+ }
+
+ std::vector<std::string> pieces = blaze_util::Split(path, ':');
+ for (auto piece : pieces) {
+ if (piece.empty()) {
+ piece = ".";
+ }
+
+ struct stat file_stat;
+ string candidate = blaze_util::JoinPath(piece, executable);
+ if (access(candidate.c_str(), X_OK) == 0 &&
+ stat(candidate.c_str(), &file_stat) == 0 &&
+ S_ISREG(file_stat.st_mode)) {
+ return candidate;
+ }
+ }
+ return "";
+}
+
+} // namespace blaze_util
diff --git a/src/main/cpp/util/file.h b/src/main/cpp/util/file.h
new file mode 100644
index 0000000..1087153
--- /dev/null
+++ b/src/main/cpp/util/file.h
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef DEVTOOLS_BLAZE_MAIN_UTIL_FILE_H_
+#define DEVTOOLS_BLAZE_MAIN_UTIL_FILE_H_
+
+#include <string>
+
+namespace blaze_util {
+
+using std::string;
+
+// Returns the part of the path before the final "/". If there is a single
+// leading "/" in the path, the result will be the leading "/". If there is
+// no "/" in the path, the result is the empty prefix of the input (i.e., "").
+string Dirname(const string &path);
+
+// Returns the part of the path after the final "/". If there is no
+// "/" in the path, the result is the same as the input.
+string Basename(const string &path);
+
+string JoinPath(const string &path1, const string &path2);
+
+// Checks each element of the PATH variable for executable. If none is found, ""
+// is returned. Otherwise, the full path to executable is returned. Can die if
+// looking up PATH fails.
+string Which(const string &executable);
+
+} // namespace blaze_util
+
+#endif // DEVTOOLS_BLAZE_MAIN_UTIL_FILE_H_
diff --git a/src/main/cpp/util/md5.cc b/src/main/cpp/util/md5.cc
new file mode 100644
index 0000000..b986986
--- /dev/null
+++ b/src/main/cpp/util/md5.cc
@@ -0,0 +1,345 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+/* MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm
+ */
+
+/*
+ Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
+ rights reserved.
+
+ License to copy and use this software is granted provided that it
+ is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ Algorithm" in all material mentioning or referencing this software
+ or this function.
+
+ License is also granted to make and use derivative works provided
+ that such works are identified as "derived from the RSA Data
+ Security, Inc. MD5 Message-Digest Algorithm" in all material
+ mentioning or referencing the derived work.
+
+ RSA Data Security, Inc. makes no representations concerning either
+ the merchantability of this software or the suitability of this
+ software for any particular purpose. It is provided "as is"
+ without express or implied warranty of any kind.
+
+ These notices must be retained in any copies of any part of this
+ documentation and/or software.
+*/
+
+
+#include "util/md5.h"
+
+#include <string.h> // for memcpy
+#include <stddef.h> // for ofsetof
+
+#include "util/numbers.h"
+
+#if !_STRING_ARCH_unaligned
+# ifdef _LP64
+# define UNALIGNED_P(p) (reinterpret_cast<uint64>(p) % \
+ __alignof__(uint32) != 0) // NOLINT
+# else
+# define UNALIGNED_P(p) (reinterpret_cast<uint32>(p) % \
+ __alignof__(uint32) != 0) // NOLINT
+# endif
+#else
+# define UNALIGNED_P(p) (0)
+#endif
+
+namespace blaze_util {
+
+static const unsigned int k8Bytes = 64;
+static const unsigned int k8ByteMask = 63;
+
+static const unsigned char kPadding[64] = {
+ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+// Digit conversion.
+static char hex_char[] = "0123456789abcdef";
+
+// This is a templated function so that T can be either a char*
+// or a string. This works because we use the [] operator to access
+// individual characters at a time.
+template <typename T>
+static void b2a_hex_t(const unsigned char* b, T a, int num) {
+ for (int i = 0; i < num; i++) {
+ a[i * 2 + 0] = hex_char[b[i] >> 4];
+ a[i * 2 + 1] = hex_char[b[i] & 0xf];
+ }
+}
+
+// ----------------------------------------------------------------------
+// b2a_hex()
+// Description: Binary-to-Ascii hex conversion. This converts
+// 'num' bytes of binary to a 2*'num'-character hexadecimal representation
+// Return value: 2*'num' characters of ascii text (via the 'to' argument)
+// ----------------------------------------------------------------------
+static void b2a_hex(const unsigned char* from, string* to, int num) {
+ to->resize(num << 1);
+ b2a_hex_t<string&>(from, *to, num);
+}
+
+Md5Digest::Md5Digest() {
+ Reset();
+}
+
+Md5Digest::Md5Digest(const Md5Digest& original) {
+ memcpy(state, original.state, sizeof(original.state));
+ memcpy(count, original.count, sizeof(original.count));
+ memcpy(ctx_buffer, original.ctx_buffer, original.ctx_buffer_len);
+ ctx_buffer_len = original.ctx_buffer_len;
+}
+
+void Md5Digest::Reset() {
+ count[0] = count[1] = 0;
+ ctx_buffer_len = 0;
+ // Load magic initialization constants.
+ state[0] = 0x67452301;
+ state[1] = 0xefcdab89;
+ state[2] = 0x98badcfe;
+ state[3] = 0x10325476;
+}
+
+void Md5Digest::Update(const void *buf, unsigned int length) {
+ const unsigned char *input = reinterpret_cast<const unsigned char*>(buf);
+ unsigned int buffer_space_len;
+
+ buffer_space_len = k8Bytes - ctx_buffer_len;
+
+ // Transform as many times as possible.
+ if (length >= buffer_space_len) {
+ if (buffer_space_len != 0 && ctx_buffer_len != 0) {
+ // Copy more bytes to fill the complete buffer
+ memcpy(ctx_buffer + ctx_buffer_len, input, buffer_space_len);
+ Transform(ctx_buffer, k8Bytes);
+ input += buffer_space_len;
+ length -= buffer_space_len;
+ ctx_buffer_len = 0;
+ }
+
+ if (UNALIGNED_P(input)) {
+ while (length >= k8Bytes) {
+ memcpy(ctx_buffer, input, k8Bytes);
+ Transform(ctx_buffer, k8Bytes);
+ input += k8Bytes;
+ length -= k8Bytes;
+ }
+ } else if (length >= k8Bytes) {
+ Transform(input, length & ~k8ByteMask);
+ input += length & ~k8ByteMask;
+ length &= k8ByteMask;
+ }
+ }
+
+ // Buffer remaining input
+ memcpy(ctx_buffer + ctx_buffer_len, input, length);
+ ctx_buffer_len += length;
+}
+
+void Md5Digest::Finish(unsigned char digest[16]) {
+ count[0] += ctx_buffer_len;
+ if (count[0] < ctx_buffer_len) {
+ ++count[1];
+ }
+
+ /* Put the 64-bit file length in *bits* at the end of the buffer. */
+ unsigned int size = (ctx_buffer_len < 56 ? 64 : 128);
+ *(reinterpret_cast<uint32*>(ctx_buffer + size - 8)) = count[0] << 3;
+ *(reinterpret_cast<uint32*>(ctx_buffer + size - 4)) =
+ (count[1] << 3) | (count[0] >> 29);
+
+ memcpy(ctx_buffer + ctx_buffer_len, kPadding, size - 8 - ctx_buffer_len);
+
+ Transform(ctx_buffer, size);
+
+ uint32* r = reinterpret_cast<uint32*>(digest);
+ r[0] = state[0];
+ r[1] = state[1];
+ r[2] = state[2];
+ r[3] = state[3];
+}
+
+void Md5Digest::Transform(
+ const unsigned char* buffer, unsigned int len) {
+ // Constants for transform routine.
+#define S11 7
+#define S12 12
+#define S13 17
+#define S14 22
+#define S21 5
+#define S22 9
+#define S23 14
+#define S24 20
+#define S31 4
+#define S32 11
+#define S33 16
+#define S34 23
+#define S41 6
+#define S42 10
+#define S43 15
+#define S44 21
+
+ // F, G, H and I are basic MD5 functions.
+/* These are the four functions used in the four steps of the MD5 algorithm
+ and defined in the RFC 1321. The first function is a little bit optimized
+ (as found in Colin Plumbs public domain implementation). */
+/* #define F(b, c, d) ((b & c) | (~b & d)) */
+#define F(x, y, z) (z ^ (x & (y ^ z)))
+#define G(x, y, z) F (z, x, y)
+#define H(x, y, z) ((x) ^ (y) ^ (z))
+#define I(x, y, z) ((y) ^ ((x) | (~z)))
+
+ // ROTATE_LEFT rotates x left n bits.
+#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
+
+ // FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
+ // Rotation is separate from addition to prevent recomputation.
+#define FF(a, b, c, d, s, ac) { \
+ (a) += F((b), (c), (d)) + ((*x_pos++ = *cur_word++)) + \
+ static_cast<uint32>(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+
+#define GG(a, b, c, d, x, s, ac) { \
+ (a) += G((b), (c), (d)) + (x) + static_cast<uint32>(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+#define HH(a, b, c, d, x, s, ac) { \
+ (a) += H((b), (c), (d)) + (x) + static_cast<uint32>(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+#define II(a, b, c, d, x, s, ac) { \
+ (a) += I((b), (c), (d)) + (x) + static_cast<uint32>(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+
+ count[0] += len;
+ if (count[0] < len) {
+ ++count[1];
+ }
+
+ uint32 a = state[0];
+ uint32 b = state[1];
+ uint32 c = state[2];
+ uint32 d = state[3];
+ uint32 x[16];
+
+ const uint32 *cur_word = reinterpret_cast<const uint32*>(buffer);
+ const uint32 *end_word = cur_word + (len / sizeof(uint32));
+
+ while (cur_word < end_word) {
+ uint32 *x_pos = x;
+ uint32 prev_a = a;
+ uint32 prev_b = b;
+ uint32 prev_c = c;
+ uint32 prev_d = d;
+
+ // Round 1
+ FF(a, b, c, d, S11, 0xd76aa478); // 1
+ FF(d, a, b, c, S12, 0xe8c7b756); // 2
+ FF(c, d, a, b, S13, 0x242070db); // 3
+ FF(b, c, d, a, S14, 0xc1bdceee); // 4
+ FF(a, b, c, d, S11, 0xf57c0faf); // 5
+ FF(d, a, b, c, S12, 0x4787c62a); // 6
+ FF(c, d, a, b, S13, 0xa8304613); // 7
+ FF(b, c, d, a, S14, 0xfd469501); // 8
+ FF(a, b, c, d, S11, 0x698098d8); // 9
+ FF(d, a, b, c, S12, 0x8b44f7af); // 10
+ FF(c, d, a, b, S13, 0xffff5bb1); // 11
+ FF(b, c, d, a, S14, 0x895cd7be); // 12
+ FF(a, b, c, d, S11, 0x6b901122); // 13
+ FF(d, a, b, c, S12, 0xfd987193); // 14
+ FF(c, d, a, b, S13, 0xa679438e); // 15
+ FF(b, c, d, a, S14, 0x49b40821); // 16
+
+ // Round 2
+ GG(a, b, c, d, x[ 1], S21, 0xf61e2562); // 17
+ GG(d, a, b, c, x[ 6], S22, 0xc040b340); // 18
+ GG(c, d, a, b, x[11], S23, 0x265e5a51); // 19
+ GG(b, c, d, a, x[ 0], S24, 0xe9b6c7aa); // 20
+ GG(a, b, c, d, x[ 5], S21, 0xd62f105d); // 21
+ GG(d, a, b, c, x[10], S22, 0x2441453); // 22
+ GG(c, d, a, b, x[15], S23, 0xd8a1e681); // 23
+ GG(b, c, d, a, x[ 4], S24, 0xe7d3fbc8); // 24
+ GG(a, b, c, d, x[ 9], S21, 0x21e1cde6); // 25
+ GG(d, a, b, c, x[14], S22, 0xc33707d6); // 26
+ GG(c, d, a, b, x[ 3], S23, 0xf4d50d87); // 27
+ GG(b, c, d, a, x[ 8], S24, 0x455a14ed); // 28
+ GG(a, b, c, d, x[13], S21, 0xa9e3e905); // 29
+ GG(d, a, b, c, x[ 2], S22, 0xfcefa3f8); // 30
+ GG(c, d, a, b, x[ 7], S23, 0x676f02d9); // 31
+ GG(b, c, d, a, x[12], S24, 0x8d2a4c8a); // 32
+
+ // Round 3
+ HH(a, b, c, d, x[ 5], S31, 0xfffa3942); // 33
+ HH(d, a, b, c, x[ 8], S32, 0x8771f681); // 34
+ HH(c, d, a, b, x[11], S33, 0x6d9d6122); // 35
+ HH(b, c, d, a, x[14], S34, 0xfde5380c); // 36
+ HH(a, b, c, d, x[ 1], S31, 0xa4beea44); // 37
+ HH(d, a, b, c, x[ 4], S32, 0x4bdecfa9); // 38
+ HH(c, d, a, b, x[ 7], S33, 0xf6bb4b60); // 39
+ HH(b, c, d, a, x[10], S34, 0xbebfbc70); // 40
+ HH(a, b, c, d, x[13], S31, 0x289b7ec6); // 41
+ HH(d, a, b, c, x[ 0], S32, 0xeaa127fa); // 42
+ HH(c, d, a, b, x[ 3], S33, 0xd4ef3085); // 43
+ HH(b, c, d, a, x[ 6], S34, 0x4881d05); // 44
+ HH(a, b, c, d, x[ 9], S31, 0xd9d4d039); // 45
+ HH(d, a, b, c, x[12], S32, 0xe6db99e5); // 46
+ HH(c, d, a, b, x[15], S33, 0x1fa27cf8); // 47
+ HH(b, c, d, a, x[ 2], S34, 0xc4ac5665); // 48
+
+ // Round 4
+ II(a, b, c, d, x[ 0], S41, 0xf4292244); // 49
+ II(d, a, b, c, x[ 7], S42, 0x432aff97); // 50
+ II(c, d, a, b, x[14], S43, 0xab9423a7); // 51
+ II(b, c, d, a, x[ 5], S44, 0xfc93a039); // 52
+ II(a, b, c, d, x[12], S41, 0x655b59c3); // 53
+ II(d, a, b, c, x[ 3], S42, 0x8f0ccc92); // 54
+ II(c, d, a, b, x[10], S43, 0xffeff47d); // 55
+ II(b, c, d, a, x[ 1], S44, 0x85845dd1); // 56
+ II(a, b, c, d, x[ 8], S41, 0x6fa87e4f); // 57
+ II(d, a, b, c, x[15], S42, 0xfe2ce6e0); // 58
+ II(c, d, a, b, x[ 6], S43, 0xa3014314); // 59
+ II(b, c, d, a, x[13], S44, 0x4e0811a1); // 60
+ II(a, b, c, d, x[ 4], S41, 0xf7537e82); // 61
+ II(d, a, b, c, x[11], S42, 0xbd3af235); // 62
+ II(c, d, a, b, x[ 2], S43, 0x2ad7d2bb); // 63
+ II(b, c, d, a, x[ 9], S44, 0xeb86d391); // 64
+
+ a += prev_a;
+ b += prev_b;
+ c += prev_c;
+ d += prev_d;
+ }
+
+ state[0] = a;
+ state[1] = b;
+ state[2] = c;
+ state[3] = d;
+}
+
+string Md5Digest::String() const {
+ string result;
+ b2a_hex(reinterpret_cast<const uint8*>(state), &result, 16);
+ return result;
+}
+
+} // namespace blaze_util
diff --git a/src/main/cpp/util/md5.h b/src/main/cpp/util/md5.h
new file mode 100644
index 0000000..fbab188
--- /dev/null
+++ b/src/main/cpp/util/md5.h
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// Provides a fast MD5 implementation.
+//
+// This implementation saves us from linking huge OpenSSL library.
+
+#ifndef DEVTOOLS_BLAZE_MAIN_UTIL_MD5_H_
+#define DEVTOOLS_BLAZE_MAIN_UTIL_MD5_H_
+
+#include <string>
+
+#include "util/port.h"
+
+namespace blaze_util {
+
+using std::string;
+
+// The <code>Context</code> class performs the actual MD5
+// computation. It works incrementally and can be fed a single byte at
+// a time if desired.
+class Md5Digest {
+ public:
+ Md5Digest();
+
+ Md5Digest(const Md5Digest& original);
+
+ // the MD5 digest is always 128 bits = 16 bytes
+ static const int kDigestLength = 16;
+
+ // Resets the context so that it can be used to calculate another
+ // MD5 digest. The context is in the same state as if it had just
+ // been constructed. It is unnecessary to call <code>Reset</code> on
+ // a newly created context.
+ void Reset();
+
+ // Add <code>length</code> bytes of <code>buf</code> to the MD5
+ // digest.
+ void Update(const void *buf, unsigned int length);
+
+ // Retrieve the computed MD5 digest as a 16 byte array.
+ void Finish(unsigned char* digest);
+
+ // Produces a hexadecimal string representation of this digest in the form:
+ // [0-9a-f]{32}
+ string String() const;
+
+ private:
+ void Transform(const unsigned char* buffer, unsigned int len);
+
+ private:
+ unsigned int state[4]; // state (ABCD)
+ unsigned int count[2]; // number of bits, modulo 2^64 (lsb first)
+ unsigned char ctx_buffer[128]; // input buffer
+ unsigned int ctx_buffer_len;
+};
+
+} // namespace blaze_util
+
+
+#endif // DEVTOOLS_BLAZE_MAIN_UTIL_MD5_H_
diff --git a/src/main/cpp/util/numbers.cc b/src/main/cpp/util/numbers.cc
new file mode 100644
index 0000000..1a758a8
--- /dev/null
+++ b/src/main/cpp/util/numbers.cc
@@ -0,0 +1,209 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "util/numbers.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <cassert>
+#include <cstdlib>
+#include <limits>
+
+#include "util/strings.h"
+
+namespace blaze_util {
+
+static const int32 kint32min = static_cast<int32>(~0x7FFFFFFF);
+static const int32 kint32max = static_cast<int32>(0x7FFFFFFF);
+
+// Represents integer values of digits.
+// Uses 36 to indicate an invalid character since we support
+// bases up to 36.
+static const int8 kAsciiToInt[256] = {
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, // 16 36s.
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+ 36, 36, 36, 36, 36, 36, 36,
+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
+ 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+ 36, 36, 36, 36, 36, 36,
+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
+ 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+ 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36,
+ 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36 };
+
+// Parse the sign.
+inline bool safe_parse_sign(const string &text /*inout*/,
+ bool* negative_ptr /*output*/) {
+ const char* start = text.data();
+ const char* end = start + text.size();
+
+ // Consume whitespace.
+ while (start < end && ascii_isspace(start[0])) {
+ ++start;
+ }
+ while (start < end && ascii_isspace(end[-1])) {
+ --end;
+ }
+ if (start >= end) {
+ return false;
+ }
+
+ // Consume sign.
+ *negative_ptr = (start[0] == '-');
+ if (*negative_ptr || start[0] == '+') {
+ ++start;
+ if (start >= end) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Consume digits.
+//
+// The classic loop:
+//
+// for each digit
+// value = value * base + digit
+// value *= sign
+//
+// The classic loop needs overflow checking. It also fails on the most
+// negative integer, -2147483648 in 32-bit two's complement representation.
+//
+// My improved loop:
+//
+// if (!negative)
+// for each digit
+// value = value * base
+// value = value + digit
+// else
+// for each digit
+// value = value * base
+// value = value - digit
+//
+// Overflow checking becomes simple.
+
+inline bool safe_parse_positive_int(const string &text, int* value_p) {
+ int value = 0;
+ const int vmax = std::numeric_limits<int>::max();
+ assert(vmax > 0);
+ const int vmax_over_base = vmax / 10;
+ const char* start = text.data();
+ const char* end = start + text.size();
+ // loop over digits
+ for (; start < end; ++start) {
+ unsigned char c = static_cast<unsigned char>(start[0]);
+ int digit = kAsciiToInt[c];
+ if (digit >= 10) {
+ *value_p = value;
+ return false;
+ }
+ if (value > vmax_over_base) {
+ *value_p = vmax;
+ return false;
+ }
+ value *= 10;
+ if (value > vmax - digit) {
+ *value_p = vmax;
+ return false;
+ }
+ value += digit;
+ }
+ *value_p = value;
+ return true;
+}
+
+inline bool safe_parse_negative_int(const string &text, int* value_p) {
+ int value = 0;
+ const int vmin = std::numeric_limits<int>::min();
+ assert(vmin < 0);
+ int vmin_over_base = vmin / 10;
+ // 2003 c++ standard [expr.mul]
+ // "... the sign of the remainder is implementation-defined."
+ // Although (vmin/base)*base + vmin%base is always vmin.
+ // 2011 c++ standard tightens the spec but we cannot rely on it.
+ if (vmin % 10 > 0) {
+ vmin_over_base += 1;
+ }
+ const char* start = text.data();
+ const char* end = start + text.size();
+ // loop over digits
+ for (; start < end; ++start) {
+ unsigned char c = static_cast<unsigned char>(start[0]);
+ int digit = kAsciiToInt[c];
+ if (digit >= 10) {
+ *value_p = value;
+ return false;
+ }
+ if (value < vmin_over_base) {
+ *value_p = vmin;
+ return false;
+ }
+ value *= 10;
+ if (value < vmin + digit) {
+ *value_p = vmin;
+ return false;
+ }
+ value -= digit;
+ }
+ *value_p = value;
+ return true;
+}
+
+bool safe_strto32(const string &text, int *value_p) {
+ *value_p = 0;
+ bool negative;
+ if (!safe_parse_sign(text, &negative)) {
+ return false;
+ }
+ if (!negative) {
+ return safe_parse_positive_int(text, value_p);
+ } else {
+ return safe_parse_negative_int(text, value_p);
+ }
+}
+
+int32 strto32(const char *str, char **endptr, int base) {
+ if (sizeof(int32) == sizeof(long)) { // NOLINT
+ return static_cast<int32>(strtol(str, endptr, base)); // NOLINT
+ }
+ const int saved_errno = errno;
+ errno = 0;
+ const long result = strtol(str, endptr, base); // NOLINT
+ if (errno == ERANGE && result == LONG_MIN) {
+ return kint32min;
+ } else if (errno == ERANGE && result == LONG_MAX) {
+ return kint32max;
+ } else if (errno == 0 && result < kint32min) {
+ errno = ERANGE;
+ return kint32min;
+ } else if (errno == 0 && result > kint32max) {
+ errno = ERANGE;
+ return kint32max;
+ }
+ if (errno == 0)
+ errno = saved_errno;
+ return static_cast<int32>(result);
+}
+
+} // namespace blaze_util
diff --git a/src/main/cpp/util/numbers.h b/src/main/cpp/util/numbers.h
new file mode 100644
index 0000000..3285a61
--- /dev/null
+++ b/src/main/cpp/util/numbers.h
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef DEVTOOLS_BLAZE_MAIN_UTIL_NUMBERS_H_
+#define DEVTOOLS_BLAZE_MAIN_UTIL_NUMBERS_H_
+
+#include <string>
+
+typedef signed char int8;
+typedef int int32;
+typedef long long int64; // NOLINT
+
+typedef unsigned char uint8;
+typedef unsigned int uint32;
+typedef unsigned long long uint64; // NOLINT
+
+namespace blaze_util {
+
+using std::string;
+
+bool safe_strto32(const string &text, int *value);
+
+int32 strto32(const char *str, char **endptr, int base);
+
+} // namespace blaze_util
+
+#endif // DEVTOOLS_BLAZE_MAIN_UTIL_NUMBERS_H_
diff --git a/src/main/cpp/util/port.cc b/src/main/cpp/util/port.cc
new file mode 100644
index 0000000..0f4bcf2
--- /dev/null
+++ b/src/main/cpp/util/port.cc
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __linux
+#include <sys/syscall.h>
+#endif // __linux
+
+#include <unistd.h>
+
+namespace blaze_util {
+
+#ifdef __linux
+
+int sys_ioprio_set(int which, int who, int ioprio) {
+ return syscall(SYS_ioprio_set, which, who, ioprio);
+}
+
+#else // Not Linux.
+
+int sys_ioprio_set(int which, int who, int ioprio) {
+ return 0;
+}
+
+#endif // __linux
+
+} // namespace blaze_util
diff --git a/src/main/cpp/util/port.h b/src/main/cpp/util/port.h
new file mode 100644
index 0000000..1f64f05
--- /dev/null
+++ b/src/main/cpp/util/port.h
@@ -0,0 +1,119 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef DEVTOOLS_BLAZE_MAIN_UTIL_PORT_H_
+#define DEVTOOLS_BLAZE_MAIN_UTIL_PORT_H_
+
+#include <stddef.h> // For size_t
+
+// GCC-specific features
+#if (defined(COMPILER_GCC3) || defined(__APPLE__)) && !defined(SWIG)
+
+//
+// Tell the compiler to do printf format string checking if the
+// compiler supports it; see the 'format' attribute in
+// <http://gcc.gnu.org/onlinedocs/gcc-4.3.0/gcc/Function-Attributes.html>.
+//
+// N.B.: As the GCC manual states, "[s]ince non-static C++ methods
+// have an implicit 'this' argument, the arguments of such methods
+// should be counted from two, not one."
+//
+#define PRINTF_ATTRIBUTE(string_index, first_to_check) \
+ __attribute__((__format__ \
+ (__printf__, string_index, first_to_check)))
+
+//
+// Tell the compiler that a given function never returns
+//
+#define ATTRIBUTE_NORETURN __attribute__((noreturn))
+#define ATTRIBUTE_UNUSED __attribute__ ((unused))
+
+#else // Not GCC
+
+#define PRINTF_ATTRIBUTE(string_index, first_to_check)
+#define ATTRIBUTE_NORETURN
+#define ATTRIBUTE_UNUSED
+
+#endif // GCC
+
+// Linux I/O priorities support is available only in later versions of glibc.
+// Therefore, we include some of the needed definitions here. May need to
+// be removed once we switch to a new version of glibc
+// (As of 10/24/08 it is unclear when glibc support will become available.)
+enum IOPriorityClass {
+ // No I/O priority value has yet been set. The kernel may assign I/O
+ // priority based on the process nice value.
+ IOPRIO_CLASS_NONE,
+
+ // Real-time, highest priority. Given first access to the disk at
+ // every opportunity. Use with care: one such process can STARVE
+ // THE ENTIRE SYSTEM. Has 8 priority levels (0-7).
+ IOPRIO_CLASS_RT,
+
+ // Best-effort, default for any process. Has 8 priority levels (0-7).
+ IOPRIO_CLASS_BE,
+
+ // Idle, lowest priority. Processes running at this level only get
+ // I/O time when no one else needs the disk, and MAY BECOME
+ // STARVED if higher priority processes are constantly accessing
+ // the disk. With the "anticipatory" I/O scheduler, mapped to
+ // IOPRIO_CLASS_BE, level 3.
+ IOPRIO_CLASS_IDLE,
+};
+
+enum {
+ IOPRIO_WHO_PROCESS = 1,
+ IOPRIO_WHO_PGRP,
+ IOPRIO_WHO_USER,
+};
+
+#ifndef IOPRIO_CLASS_SHIFT
+#define IOPRIO_CLASS_SHIFT 13
+#endif
+
+#ifndef IOPRIO_PRIO_VALUE
+#define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data)
+#endif
+
+namespace blaze_util {
+
+int sys_ioprio_set(int which, int who, int ioprio);
+
+} // namespace blaze_util
+
+// The arraysize(arr) macro returns the # of elements in an array arr.
+// The expression is a compile-time constant, and therefore can be
+// used in defining new arrays, for example. If you use arraysize on
+// a pointer by mistake, you will get a compile-time error.
+//
+// One caveat is that, for C++03, arraysize() doesn't accept any array of
+// an anonymous type or a type defined inside a function. In these rare
+// cases, you have to use the unsafe ARRAYSIZE() macro below. This is
+// due to a limitation in C++03's template system. The limitation has
+// been removed in C++11.
+
+// This template function declaration is used in defining arraysize.
+// Note that the function doesn't need an implementation, as we only
+// use its type.
+template <typename T, size_t N>
+char (&ArraySizeHelper(T (&array)[N]))[N];
+
+// That gcc wants both of these prototypes seems mysterious. VC, for
+// its part, can't decide which to use (another mystery). Matching of
+// template overloads: the final frontier.
+template <typename T, size_t N>
+char (&ArraySizeHelper(const T (&array)[N]))[N];
+
+#define arraysize(array) (sizeof(ArraySizeHelper(array)))
+
+#endif // DEVTOOLS_BLAZE_MAIN_UTIL_PORT_H_
diff --git a/src/main/cpp/util/strings.cc b/src/main/cpp/util/strings.cc
new file mode 100644
index 0000000..192758b
--- /dev/null
+++ b/src/main/cpp/util/strings.cc
@@ -0,0 +1,298 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "util/strings.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <cassert>
+
+#include "blaze_exit_code.h"
+
+using std::vector;
+
+namespace blaze_util {
+
+static const char kSeparator[] = " \n\t\r";
+
+// # Table generated by this Python code (bit 0x02 is currently unused):
+// def Hex2(n):
+// return '0x' + hex(n/16)[2:] + hex(n%16)[2:]
+// def IsPunct(ch):
+// return (ord(ch) >= 32 and ord(ch) < 127 and
+// not ch.isspace() and not ch.isalnum())
+// def IsBlank(ch):
+// return ch in ' \t'
+// def IsCntrl(ch):
+// return ord(ch) < 32 or ord(ch) == 127
+// def IsXDigit(ch):
+// return ch.isdigit() or ch.lower() in 'abcdef'
+// for i in range(128):
+// ch = chr(i)
+// mask = ((ch.isalpha() and 0x01 or 0) |
+// (ch.isalnum() and 0x04 or 0) |
+// (ch.isspace() and 0x08 or 0) |
+// (IsPunct(ch) and 0x10 or 0) |
+// (IsBlank(ch) and 0x20 or 0) |
+// (IsCntrl(ch) and 0x40 or 0) |
+// (IsXDigit(ch) and 0x80 or 0))
+// print Hex2(mask) + ',',
+// if i % 16 == 7:
+// print ' //', Hex2(i & 0x78)
+// elif i % 16 == 15:
+// print
+const unsigned char kAsciiPropertyBits[256] = {
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x00
+ 0x40, 0x68, 0x48, 0x48, 0x48, 0x48, 0x40, 0x40,
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x10
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+ 0x28, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x20
+ 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
+ 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, // 0x30
+ 0x84, 0x84, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
+ 0x10, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x05, // 0x40
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, // 0x50
+ 0x05, 0x05, 0x05, 0x10, 0x10, 0x10, 0x10, 0x10,
+ 0x10, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x05, // 0x60
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, // 0x70
+ 0x05, 0x05, 0x05, 0x10, 0x10, 0x10, 0x10, 0x40,
+};
+
+
+bool starts_with(const string &haystack, const string &needle) {
+ return (haystack.length() >= needle.length()) &&
+ (memcmp(haystack.c_str(), needle.c_str(), needle.length()) == 0);
+}
+
+bool ends_with(const string &haystack, const string &needle) {
+ return ((haystack.length() >= needle.length()) &&
+ (memcmp(haystack.c_str() + (haystack.length()-needle.length()),
+ needle.c_str(), needle.length()) == 0));
+}
+
+void JoinStrings(
+ const vector<string> &pieces, const char delimeter, string *output) {
+ bool first = true;
+ for (const auto &piece : pieces) {
+ if (first) {
+ *output = piece;
+ first = false;
+ } else {
+ *output += delimeter + piece;
+ }
+ }
+}
+
+vector<string> Split(const string &contents, const char delimeter) {
+ vector<string> result;
+ SplitStringUsing(contents, delimeter, &result);
+ return result;
+}
+
+void SplitStringUsing(
+ const string &contents, const char delimeter, vector<string> *result) {
+ assert(result);
+
+ size_t start = 0;
+ while (start < contents.length() && contents[start] == delimeter) {
+ ++start;
+ }
+
+ size_t newline = contents.find(delimeter, start);
+ while (newline != string::npos) {
+ result->push_back(string(contents, start, newline - start));
+ start = newline;
+ while (start < contents.length() && contents[start] == delimeter) {
+ ++start;
+ }
+ newline = contents.find(delimeter, start);
+ }
+
+ // If there is a trailing line, add that.
+ if (start != newline && start != contents.size()) {
+ result->push_back(string(contents, start));
+ }
+}
+
+vector<string> SplitQuoted(const string &contents, const char delimeter) {
+ vector<string> result;
+ SplitQuotedStringUsing(contents, delimeter, &result);
+ return result;
+}
+
+void SplitQuotedStringUsing(const string &contents, const char delimeter,
+ std::vector<string> *output) {
+ size_t len = contents.length();
+ size_t start = 0;
+ size_t quote = string::npos; // quote position
+
+ for (size_t pos = 0; pos < len; ++pos) {
+ if (start == pos && contents[start] == delimeter) {
+ ++start;
+ } else if (contents[pos] == '\\') {
+ ++pos;
+ } else if (quote != string::npos && contents[pos] == contents[quote]) {
+ quote = string::npos;
+ } else if (quote == string::npos &&
+ (contents[pos] == '"' || contents[pos] == '\'')) {
+ quote = pos;
+ } else if (quote == string::npos && contents[pos] == delimeter) {
+ output->push_back(string(contents, start, pos - start));
+ start = pos + 1;
+ }
+ }
+
+ // A trailing element
+ if (start < len) {
+ output->push_back(string(contents, start));
+ }
+}
+
+void Replace(const string &oldsub, const string &newsub, string *str) {
+ size_t start = 0;
+ // This is O(n^2) (the complexity of erase() is actually unspecified, but
+ // usually linear).
+ while ((start = str->find(oldsub, start)) != string::npos) {
+ str->erase(start, oldsub.length());
+ str->insert(start, newsub);
+ start += newsub.length();
+ }
+}
+
+void StripWhitespace(string *str) {
+ int str_length = str->length();
+
+ // Strip off leading whitespace.
+ int first = 0;
+ while (first < str_length && ascii_isspace(str->at(first))) {
+ ++first;
+ }
+ // If entire string is white space.
+ if (first == str_length) {
+ str->clear();
+ return;
+ }
+ if (first > 0) {
+ str->erase(0, first);
+ str_length -= first;
+ }
+
+ // Strip off trailing whitespace.
+ int last = str_length - 1;
+ while (last >= 0 && ascii_isspace(str->at(last))) {
+ --last;
+ }
+ if (last != (str_length - 1) && last >= 0) {
+ str->erase(last + 1, string::npos);
+ }
+}
+
+static void GetNextToken(const string &str, const char &comment,
+ string::const_iterator *iter, vector<string> *words) {
+ string output;
+ auto last = *iter;
+ char quote = '\0';
+ // While not a delimiter.
+ while (last != str.end() && (quote || strchr(kSeparator, *last) == nullptr)) {
+ // Absorb escapes.
+ if (*last == '\\') {
+ ++last;
+ if (last == str.end()) {
+ break;
+ }
+ output += *last++;
+ continue;
+ }
+
+ if (quote) {
+ if (*last == quote) {
+ // Absorb closing quote.
+ quote = '\0';
+ ++last;
+ } else {
+ output += *last++;
+ }
+ } else {
+ if (*last == comment) {
+ last = str.end();
+ break;
+ }
+ if (*last == '\'' || *last == '"') {
+ // Absorb opening quote.
+ quote = *last++;
+ } else {
+ output += *last++;
+ }
+ }
+ }
+
+ if (!output.empty()) {
+ words->push_back(output);
+ }
+
+ *iter = last;
+}
+
+void Tokenize(const string &str, const char &comment, vector<string> *words) {
+ assert(words);
+ words->clear();
+
+ string::const_iterator i = str.begin();
+ while (i != str.end()) {
+ // Skip whitespace.
+ while (i != str.end() && strchr(kSeparator, *i) != nullptr) {
+ i++;
+ }
+ if (i != str.end() && *i == comment) {
+ break;
+ }
+ GetNextToken(str, comment, &i, words);
+ }
+}
+
+
+// Evaluate a format string and store the result in 'str'.
+void StringPrintf(string *str, const char *format, ...) {
+ assert(str);
+
+ // Determine the required buffer size. vsnpritnf won't account for the
+ // terminating '\0'.
+ va_list args;
+ va_start(args, format);
+ int output_size = vsnprintf(nullptr, 0, format, args);
+ if (output_size < 0) {
+ fprintf(stderr, "Fatal error formatting string: %d", output_size);
+ exit(blaze_exit_code::INTERNAL_ERROR);
+ }
+ va_end(args);
+
+ // Allocate a buffer and format the input.
+ int buffer_size = output_size + sizeof '\0';
+ char *buf = new char[buffer_size];
+ va_start(args, format);
+ int print_result = vsnprintf(buf, buffer_size, format, args);
+ if (print_result < 0) {
+ fprintf(stderr, "Fatal error formatting string: %d", print_result);
+ exit(blaze_exit_code::INTERNAL_ERROR);
+ }
+ va_end(args);
+
+ *str = buf;
+ delete[] buf;
+}
+
+} // namespace blaze_util
diff --git a/src/main/cpp/util/strings.h b/src/main/cpp/util/strings.h
new file mode 100644
index 0000000..5d2dc46
--- /dev/null
+++ b/src/main/cpp/util/strings.h
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef DEVTOOLS_BLAZE_MAIN_UTIL_STRINGS_H_
+#define DEVTOOLS_BLAZE_MAIN_UTIL_STRINGS_H_
+
+#include <string>
+#include <vector>
+
+#ifdef BLAZE_OPENSOURCE
+#include <string.h>
+#endif
+
+namespace blaze_util {
+
+using std::string;
+
+extern const unsigned char kAsciiPropertyBits[256];
+#define kApb kAsciiPropertyBits
+
+static inline bool ascii_isspace(unsigned char c) { return kApb[c] & 0x08; }
+
+bool starts_with(const string &haystack, const string &needle);
+
+bool ends_with(const string &haystack, const string &needle);
+
+// Matches a prefix (which must be a char* literal!) against the beginning of
+// str. Returns a pointer past the prefix, or NULL if the prefix wasn't matched.
+// (Like the standard strcasecmp(), but for efficiency doesn't call strlen() on
+// prefix, and returns a pointer rather than an int.)
+//
+// The ""'s catch people who don't pass in a literal for "prefix"
+#ifndef strprefix
+#define strprefix(str, prefix) \
+ (strncmp(str, prefix, sizeof("" prefix "")-1) == 0 ? \
+ str + sizeof(prefix)-1 : NULL)
+#endif
+
+// Matches a prefix; returns a pointer past the prefix, or NULL if not found.
+// (Like strprefix() and strcaseprefix() but not restricted to searching for
+// char* literals). Templated so searching a const char* returns a const char*,
+// and searching a non-const char* returns a non-const char*.
+// Matches a prefix; returns a pointer past the prefix, or NULL if not found.
+// (Like strprefix() and strcaseprefix() but not restricted to searching for
+// char* literals). Templated so searching a const char* returns a const char*,
+// and searching a non-const char* returns a non-const char*.
+template<class CharStar>
+inline CharStar var_strprefix(CharStar str, const char* prefix) {
+ const int len = strlen(prefix);
+ return strncmp(str, prefix, len) == 0 ? str + len : NULL;
+}
+
+// Returns a mutable char* pointing to a string's internal buffer, which may not
+// be null-terminated. Returns NULL for an empty string. If not non-null,
+// writing through this pointer will modify the string.
+inline char* string_as_array(string* str) {
+ // DO NOT USE const_cast<char*>(str->data())! See the unittest for why.
+ return str->empty() ? NULL : &*str->begin();
+}
+
+// Join the elements of pieces separated by delimeter. Returns the joined
+// string in output.
+void JoinStrings(
+ const std::vector<string> &pieces, const char delimeter, string *output);
+
+// Splits contents by delimeter. Skips empty subsections.
+std::vector<string> Split(const string &contents, const char delimeter);
+
+// Same as above, but adds results to output.
+void SplitStringUsing(
+ const string &contents, const char delimeter, std::vector<string> *output);
+
+// Splits contents by delimeter with possible elements quoted by ' or ".
+// backslashes (\) can be used to escape the quotes or delimeter. Skips
+// empty subsections.
+std::vector<string> SplitQuoted(const string &contents, const char delimeter);
+
+// Same as above, but adds results to output.
+void SplitQuotedStringUsing(const string &contents, const char delimeter,
+ std::vector<string> *output);
+
+// Global replace of oldsub with newsub.
+void Replace(const string &oldsub, const string &newsub, string *str);
+
+// Removes whitespace from both ends of a string.
+void StripWhitespace(string *str);
+
+// Tokenizes str on whitespace and places the tokens in words. Splits on spaces,
+// newlines, carriage returns, and tabs. Respects single and double quotes (that
+// is, "a string of 'some stuff'" would be 4 tokens). If the comment character
+// is found (outside of quotes), the rest of the string will be ignored. Any
+// token can be escaped with \, e.g., "this\\ is\\ one\\ token".
+void Tokenize(
+ const string &str, const char &comment, std::vector<string> *words);
+
+// Evaluate a format string and store the result in 'str'.
+void StringPrintf(string *str, const char *format, ...);
+
+} // namespace blaze_util
+
+#endif // DEVTOOLS_BLAZE_MAIN_UTIL_STRINGS_H_
diff --git a/src/main/java/BUILD b/src/main/java/BUILD
new file mode 100644
index 0000000..503426f
--- /dev/null
+++ b/src/main/java/BUILD
@@ -0,0 +1,87 @@
+java_library(
+ name = "shell",
+ srcs = glob(["com/google/devtools/build/lib/shell/*.java"]),
+ visibility = ["//src:__subpackages__"],
+ deps = ["//third_party:guava"],
+)
+
+java_library(
+ name = "bazel-core",
+ srcs = glob(
+ ["**/*.java"],
+ exclude = ["com/google/devtools/build/lib/shell/*.java"],
+ ),
+ resources = glob([
+ "**/*.txt",
+ "**/*.html",
+ "**/*.css",
+ "**/*.js",
+ ]),
+ visibility = ["//src/test/java:__subpackages__"],
+ runtime_deps = [
+ "//third_party:aether",
+ "//third_party:apache_commons_logging",
+ "//third_party:apache_httpclient",
+ "//third_party:apache_httpcore",
+ "//third_party:maven_model",
+ "//third_party:plexus_interpolation",
+ "//third_party:plexus_utils",
+ ],
+ deps = [
+ ":shell",
+ "//src/main/protobuf:proto_build",
+ "//src/main/protobuf:proto_bundlemerge",
+ "//src/main/protobuf:proto_crosstool_config",
+ "//src/main/protobuf:proto_extra_actions_base",
+ "//src/main/protobuf:proto_test_status",
+ "//src/main/protobuf:proto_xcodegen",
+ "//src/tools/xcode-common",
+ "//third_party:aether",
+ "//third_party:apache_commons_compress",
+ "//third_party:gson",
+ "//third_party:guava",
+ "//third_party:joda-time",
+ "//third_party:jsr305",
+ "//third_party:maven_model",
+ "//third_party:protobuf",
+ ],
+)
+
+java_binary(
+ name = "bazel-main",
+ main_class = "com.google.devtools.build.lib.bazel.BazelMain",
+ visibility = ["//src:__pkg__"],
+ runtime_deps = [
+ ":bazel-core",
+ ],
+)
+
+# Build encyclopedia generation.
+filegroup(
+ name = "gen_be_sources",
+ srcs = glob(["com/google/devtools/build/lib/**/*.java"]),
+)
+
+java_binary(
+ name = "docgen_bin",
+ srcs = glob(["com/google/devtools/build/docgen/*.java"]),
+ data = [":gen_be_sources"],
+ main_class = "com.google.devtools.build.docgen.BuildEncyclopediaGenerator",
+ resources = glob(
+ ["com/google/devtools/build/docgen/templates/*.html"],
+ ),
+ deps = [
+ ":bazel-core",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
+
+genrule(
+ name = "gen_buildencyclopedia",
+ srcs = [":gen_be_sources"],
+ outs = ["build-encyclopedia.html"],
+ cmd = " docgen_bin $$PWD/src/main/java/com/google/devtools/build/lib $$PWD;" +
+ "cp $$PWD/build-encyclopedia.html $@",
+ tools = [":docgen_bin"],
+)
diff --git a/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
new file mode 100644
index 0000000..12687db
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A helper class to load and store printable build rule documentation. The doc
+ * printed here doesn't contain attribute and implicit output definitions, just
+ * the general rule documentation and examples.
+ */
+public class BlazeRuleHelpPrinter {
+
+ private static Map<String, RuleDocumentation> ruleDocMap = null;
+
+ /**
+ * Returns the documentation of the given rule to be printed on the console.
+ */
+ public static String getRuleDoc(String ruleName, ConfiguredRuleClassProvider provider) {
+ if (ruleDocMap == null) {
+ try {
+ BuildEncyclopediaProcessor processor = new BuildEncyclopediaProcessor(provider);
+ Set<RuleDocumentation> ruleDocs = processor.collectAndProcessRuleDocs(
+ new String[] {"java/com/google/devtools/build/lib/view",
+ "java/com/google/devtools/build/lib/rules"}, false);
+ ruleDocMap = new HashMap<>();
+ for (RuleDocumentation ruleDoc : ruleDocs) {
+ ruleDocMap.put(ruleDoc.getRuleName(), ruleDoc);
+ }
+ } catch (BuildEncyclopediaDocException e) {
+ return e.getErrorMsg();
+ } catch (IOException e) {
+ return e.getMessage();
+ }
+ }
+ // Every rule should be documented and this method should be called only
+ // for existing rules (a check is performed in HelpCommand).
+ Preconditions.checkState(ruleDocMap.containsKey(ruleName), String.format(
+ "ERROR: Documentation of rule %s does not exist.", ruleName));
+ return "Rule " + ruleName + ":"
+ + ruleDocMap.get(ruleName).getCommandLineDocumentation() + "\n";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaDocException.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaDocException.java
new file mode 100644
index 0000000..d3b12c4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaDocException.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+/**
+ * An exception for Build Encyclopedia generation implementing the common BLAZE
+ * error formatting, i.e. displaying file name and line number.
+ */
+public class BuildEncyclopediaDocException extends Exception {
+
+ private String fileName;
+ private int lineNumber;
+ private String errorMsg;
+
+ public BuildEncyclopediaDocException(String fileName, int lineNumber, String errorMsg) {
+ this.fileName = fileName;
+ this.lineNumber = lineNumber;
+ this.errorMsg = errorMsg;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public int getLineNumber() {
+ return lineNumber;
+ }
+
+ public String getErrorMsg() {
+ return errorMsg;
+ }
+
+ @Override
+ public String getMessage() {
+ return "Error in " + fileName + ":" + lineNumber + ": " + errorMsg;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java
new file mode 100644
index 0000000..e2bb844
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+/**
+ * The main class for the docgen project. The class checks the input arguments
+ * and uses the BuildEncyclopediaProcessor for the actual documentation generation.
+ */
+public class BuildEncyclopediaGenerator {
+
+ private static boolean checkArgs(String[] args) {
+ if (args.length < 1) {
+ System.err.println("There has to be one or two input parameters\n"
+ + " - a comma separated list for input directories\n"
+ + " - an output directory (optional).");
+ return false;
+ }
+ return true;
+ }
+
+ private static void fail(Throwable e, boolean printStackTrace) {
+ System.err.println("ERROR: " + e.getMessage());
+ if (printStackTrace) {
+ e.printStackTrace();
+ }
+ Runtime.getRuntime().exit(1);
+ }
+
+ private static ConfiguredRuleClassProvider createRuleClassProvider()
+ throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
+ IllegalAccessException {
+ Class<?> providerClass = Class.forName(Constants.MAIN_RULE_CLASS_PROVIDER);
+ Method createMethod = providerClass.getMethod("create");
+ return (ConfiguredRuleClassProvider) createMethod.invoke(null);
+ }
+
+ public static void main(String[] args) {
+ if (checkArgs(args)) {
+ // TODO(bazel-team): use flags
+ try {
+ BuildEncyclopediaProcessor processor = new BuildEncyclopediaProcessor(
+ createRuleClassProvider());
+ processor.generateDocumentation(args[0].split(","), args.length > 1 ? args[1] : null);
+ } catch (BuildEncyclopediaDocException e) {
+ fail(e, false);
+ } catch (Throwable e) {
+ fail(e, true);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
new file mode 100644
index 0000000..fb52a4c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
@@ -0,0 +1,403 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.docgen.DocgenConsts.RuleType;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RuleClass;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/**
+ * A class to assemble documentation for the Build Encyclopedia. The
+ * program parses the documentation fragments of rule-classes and
+ * generates the html format documentation.
+ */
+public class BuildEncyclopediaProcessor {
+
+ private ConfiguredRuleClassProvider ruleClassProvider;
+
+ /**
+ * Creates the BuildEncyclopediaProcessor instance. The ruleClassProvider parameter
+ * is used for rule class hierarchy and attribute checking.
+ *
+ */
+ public BuildEncyclopediaProcessor(ConfiguredRuleClassProvider ruleClassProvider) {
+ this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider);
+ }
+
+ /**
+ * Collects and processes all the rule and attribute documentation in inputDirs and
+ * generates the Build Encyclopedia into the outputRootDir.
+ */
+ public void generateDocumentation(String[] inputDirs, String outputRootDir)
+ throws BuildEncyclopediaDocException, IOException {
+ BufferedWriter bw = null;
+ File buildEncyclopediaPath = setupDirectories(outputRootDir);
+ try {
+ bw = new BufferedWriter(new FileWriter(buildEncyclopediaPath));
+ bw.write(DocgenConsts.HEADER_COMMENT);
+
+ Set<RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false);
+ writeRuleClassDocs(ruleDocEntries, bw);
+
+ bw.write(SourceFileReader.readTemplateContents(DocgenConsts.FOOTER_TEMPLATE));
+
+ } finally {
+ if (bw != null) {
+ bw.close();
+ }
+ }
+ }
+
+ /**
+ * Collects all the rule and attribute documentation present in inputDirs, integrates the
+ * attribute documentation in the rule documentation and returns the rule documentation.
+ */
+ public Set<RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs,
+ boolean printMessages) throws BuildEncyclopediaDocException, IOException {
+ // RuleDocumentations are generated in order (based on rule type then alphabetically).
+ // The ordering is also used to determine in which rule doc the common attribute docs are
+ // generated (they are generated at the first appearance).
+ Set<RuleDocumentation> ruleDocEntries = new TreeSet<>();
+ // RuleDocumentationAttribute objects equal based on attributeName so they have to be
+ // collected in a List instead of a Set.
+ ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries =
+ LinkedListMultimap.create();
+ for (String inputDir : inputDirs) {
+ if (printMessages) {
+ System.out.println(" Processing input directory: " + inputDir);
+ }
+ int ruleNum = ruleDocEntries.size();
+ collectDocs(ruleDocEntries, attributeDocEntries, new File(inputDir));
+ if (printMessages) {
+ System.out.println(
+ " " + (ruleDocEntries.size() - ruleNum) + " rule documentations found.");
+ }
+ }
+
+ processAttributeDocs(ruleDocEntries, attributeDocEntries);
+ return ruleDocEntries;
+ }
+
+ /**
+ * Go through all attributes of all documented rules and search the best attribute documentation
+ * if exists. The best documentation is the closest documentation in the ancestor graph. E.g. if
+ * java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to
+ * apply since it's a closer ancestor of java_library.
+ */
+ private void processAttributeDocs(Set<RuleDocumentation> ruleDocEntries,
+ ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries)
+ throws BuildEncyclopediaDocException {
+ for (RuleDocumentation ruleDoc : ruleDocEntries) {
+ RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
+ if (ruleClass != null) {
+ if (ruleClass.isDocumented()) {
+ Class<? extends RuleDefinition> ruleDefinition =
+ ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName());
+ for (Attribute attribute : ruleClass.getAttributes()) {
+ String attrName = attribute.getName();
+ List<RuleDocumentationAttribute> attributeDocList =
+ attributeDocEntries.get(attrName);
+ if (attributeDocList != null) {
+ // There are attribute docs for this attribute.
+ // Search the closest one in the ancestor graph.
+ // Note that there can be only one 'closest' attribute since we forbid multiple
+ // inheritance of the same attribute in RuleClass.
+ int minLevel = Integer.MAX_VALUE;
+ RuleDocumentationAttribute bestAttributeDoc = null;
+ for (RuleDocumentationAttribute attributeDoc : attributeDocList) {
+ int level = attributeDoc.getDefinitionClassAncestryLevel(ruleDefinition);
+ if (level >= 0 && level < minLevel) {
+ bestAttributeDoc = attributeDoc;
+ minLevel = level;
+ }
+ }
+ if (bestAttributeDoc != null) {
+ ruleDoc.addAttribute(bestAttributeDoc);
+ // If there is no matching attribute doc try to add the common.
+ } else if (ruleDoc.getRuleType().equals(RuleType.BINARY)
+ && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) {
+ ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName));
+ } else if (ruleDoc.getRuleType().equals(RuleType.TEST)
+ && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) {
+ ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName));
+ } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) {
+ ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName));
+ }
+ }
+ }
+ }
+ } else {
+ throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
+ }
+ }
+ }
+
+ /**
+ * Categorizes, checks and prints all the rule-class documentations.
+ */
+ private void writeRuleClassDocs(Set<RuleDocumentation> docEntries, BufferedWriter bw)
+ throws BuildEncyclopediaDocException, IOException {
+ Set<RuleDocumentation> binaryDocs = new TreeSet<>();
+ Set<RuleDocumentation> libraryDocs = new TreeSet<>();
+ Set<RuleDocumentation> testDocs = new TreeSet<>();
+ Set<RuleDocumentation> generateDocs = new TreeSet<>();
+ Set<RuleDocumentation> otherDocs = new TreeSet<>();
+
+ for (RuleDocumentation doc : docEntries) {
+ RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(doc.getRuleName());
+ if (ruleClass.isDocumented()) {
+ if (doc.isLanguageSpecific()) {
+ switch(doc.getRuleType()) {
+ case BINARY:
+ binaryDocs.add(doc);
+ break;
+ case LIBRARY:
+ libraryDocs.add(doc);
+ break;
+ case TEST:
+ testDocs.add(doc);
+ break;
+ case OTHER:
+ otherDocs.add(doc);
+ break;
+ }
+ } else {
+ otherDocs.add(doc);
+ }
+ }
+ }
+
+ bw.write(SourceFileReader.readTemplateContents(DocgenConsts.HEADER_TEMPLATE,
+ generateBEHeaderMapping(docEntries)));
+
+ Map<String, String> sectionMapping = ImmutableMap.of(
+ DocgenConsts.VAR_SECTION_BINARY, getRuleDocs(binaryDocs),
+ DocgenConsts.VAR_SECTION_LIBRARY, getRuleDocs(libraryDocs),
+ DocgenConsts.VAR_SECTION_TEST, getRuleDocs(testDocs),
+ DocgenConsts.VAR_SECTION_GENERATE, getRuleDocs(generateDocs),
+ DocgenConsts.VAR_SECTION_OTHER, getRuleDocs(otherDocs));
+ bw.write(SourceFileReader.readTemplateContents(DocgenConsts.BODY_TEMPLATE, sectionMapping));
+ }
+
+ private Map<String, String> generateBEHeaderMapping(Set<RuleDocumentation> docEntries)
+ throws BuildEncyclopediaDocException {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("<table id=\"rules\" summary=\"Table of rules sorted by language\">\n")
+ .append("<colgroup span=\"5\" width=\"20%\"></colgroup>\n")
+ .append("<tr><th>Language</th><th>Binary rules</th><th>Library rules</th>"
+ + "<th>Test rules</th><th>Other rules</th><th></th></tr>\n");
+
+ // Separate rule families into language-specific and generic ones.
+ Set<String> languageSpecificRuleFamilies = new TreeSet<>();
+ Set<String> genericRuleFamilies = new TreeSet<>();
+ separateRuleFamilies(docEntries, languageSpecificRuleFamilies, genericRuleFamilies);
+
+ // Create a mapping of rules based on rule type and family.
+ Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping = new HashMap<>();
+ createRuleMapping(docEntries, ruleMapping);
+
+ // Generate the table.
+ for (String ruleFamily : languageSpecificRuleFamilies) {
+ generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily);
+ }
+
+ sb.append("<tr><th> </th></tr>");
+ sb.append("<tr><th colspan=\"5\">Rules that do not apply to a "
+ + "specific programming language</th></tr>");
+ for (String ruleFamily : genericRuleFamilies) {
+ generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily);
+ }
+ sb.append("</table>\n");
+ return ImmutableMap.<String, String>of(DocgenConsts.VAR_HEADER_TABLE, sb.toString(),
+ DocgenConsts.VAR_COMMON_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
+ PredefinedAttributes.COMMON_ATTRIBUTES, DocgenConsts.COMMON_ATTRIBUTES),
+ DocgenConsts.VAR_TEST_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
+ PredefinedAttributes.TEST_ATTRIBUTES, DocgenConsts.TEST_ATTRIBUTES),
+ DocgenConsts.VAR_BINARY_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs(
+ PredefinedAttributes.BINARY_ATTRIBUTES, DocgenConsts.BINARY_ATTRIBUTES),
+ DocgenConsts.VAR_LEFT_PANEL, generateLeftNavigationPanel(docEntries));
+ }
+
+ /**
+ * Create a mapping of rules based on rule type and family.
+ */
+ private void createRuleMapping(Set<RuleDocumentation> docEntries,
+ Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping)
+ throws BuildEncyclopediaDocException {
+ for (RuleDocumentation ruleDoc : docEntries) {
+ RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName());
+ if (ruleClass != null) {
+ String ruleFamily = ruleDoc.getRuleFamily();
+ if (!ruleMapping.containsKey(ruleFamily)) {
+ ruleMapping.put(ruleFamily, LinkedListMultimap.<RuleType, RuleDocumentation>create());
+ }
+ if (ruleClass.isDocumented()) {
+ ruleMapping.get(ruleFamily).put(ruleDoc.getRuleType(), ruleDoc);
+ }
+ } else {
+ throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName());
+ }
+ }
+ }
+
+ /**
+ * Separates all rule families in docEntries into language-specific rules and generic rules.
+ */
+ private void separateRuleFamilies(Set<RuleDocumentation> docEntries,
+ Set<String> languageSpecificRuleFamilies, Set<String> genericRuleFamilies)
+ throws BuildEncyclopediaDocException {
+ for (RuleDocumentation ruleDoc : docEntries) {
+ if (ruleDoc.isLanguageSpecific()) {
+ if (genericRuleFamilies.contains(ruleDoc.getRuleFamily())) {
+ throw ruleDoc.createException("The rule is marked as being language-specific, but other "
+ + "rules of the same family have already been marked as being not.");
+ }
+ languageSpecificRuleFamilies.add(ruleDoc.getRuleFamily());
+ } else {
+ if (languageSpecificRuleFamilies.contains(ruleDoc.getRuleFamily())) {
+ throw ruleDoc.createException("The rule is marked as being generic, but other rules of "
+ + "the same family have already been marked as being language-specific.");
+ }
+ genericRuleFamilies.add(ruleDoc.getRuleFamily());
+ }
+ }
+ }
+
+ private String generateLeftNavigationPanel(Set<RuleDocumentation> docEntries) {
+ // Order the rules alphabetically. At this point they are ordered according to
+ // RuleDocumentation.compareTo() which is not alphabetical.
+ TreeMap<String, String> ruleNames = new TreeMap<>();
+ for (RuleDocumentation ruleDoc : docEntries) {
+ String ruleName = ruleDoc.getRuleName();
+ ruleNames.put(ruleName.toLowerCase(), ruleName);
+ }
+ StringBuilder sb = new StringBuilder();
+ for (String ruleName : ruleNames.values()) {
+ RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleName);
+ Preconditions.checkNotNull(ruleClass);
+ if (ruleClass.isDocumented()) {
+ sb.append(String.format("<a href=\"#%s\">%s</a><br/>\n", ruleName, ruleName));
+ }
+ }
+ return sb.toString();
+ }
+
+ private String generateCommonAttributeDocs(Map<String, RuleDocumentationAttribute> attributes,
+ String attributeGroupName) throws BuildEncyclopediaDocException {
+ RuleDocumentation ruleDoc = new RuleDocumentation(
+ attributeGroupName, "OTHER", null, null, 0, null, ImmutableSet.<String>of(),
+ ruleClassProvider);
+ for (RuleDocumentationAttribute attribute : attributes.values()) {
+ ruleDoc.addAttribute(attribute);
+ }
+ return ruleDoc.generateAttributeDefinitions();
+ }
+
+ private void generateHeaderTableRuleFamily(StringBuilder sb,
+ ListMultimap<RuleType, RuleDocumentation> ruleTypeMap, String ruleFamily) {
+ sb.append("<tr>\n")
+ .append(String.format("<td class=\"lang\">%s</td>\n", ruleFamily));
+ boolean otherRulesSplitted = false;
+ for (RuleType ruleType : DocgenConsts.RuleType.values()) {
+ sb.append("<td>");
+ int i = 0;
+ List<RuleDocumentation> ruleDocList = ruleTypeMap.get(ruleType);
+ for (RuleDocumentation ruleDoc : ruleDocList) {
+ if (i > 0) {
+ if (ruleType.equals(RuleType.OTHER)
+ && ruleDocList.size() >= 4 && i == (ruleDocList.size() + 1) / 2) {
+ // Split 'other rules' into two columns if there are too many of them.
+ sb.append("</td>\n<td>");
+ otherRulesSplitted = true;
+ } else {
+ sb.append("<br/>");
+ }
+ }
+ String ruleName = ruleDoc.getRuleName();
+ String deprecatedString = ruleDoc.hasFlag(DocgenConsts.FLAG_DEPRECATED)
+ ? " class=\"deprecated\"" : "";
+ sb.append(String.format("<a href=\"#%s\"%s>%s</a>", ruleName, deprecatedString, ruleName));
+ i++;
+ }
+ sb.append("</td>\n");
+ }
+ // There should be 6 columns.
+ if (!otherRulesSplitted) {
+ sb.append("<td></td>\n");
+ }
+ sb.append("</tr>\n");
+ }
+
+ private String getRuleDocs(Iterable<RuleDocumentation> docEntries) {
+ StringBuilder sb = new StringBuilder();
+ for (RuleDocumentation doc : docEntries) {
+ sb.append(doc.getHtmlDocumentation());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Goes through all the html files and subdirs under inputPath and collects the rule
+ * and attribute documentations using the ruleDocEntries and attributeDocEntries variable.
+ */
+ public void collectDocs(Set<RuleDocumentation> ruleDocEntries,
+ ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries,
+ File inputPath) throws BuildEncyclopediaDocException, IOException {
+ if (inputPath.isFile()) {
+ if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) {
+ SourceFileReader sfr = new SourceFileReader(
+ ruleClassProvider, inputPath.getAbsolutePath());
+ sfr.readDocsFromComments();
+ ruleDocEntries.addAll(sfr.getRuleDocEntries());
+ if (attributeDocEntries != null) {
+ // Collect all attribute documentations from this file.
+ attributeDocEntries.putAll(sfr.getAttributeDocEntries());
+ }
+ }
+ } else if (inputPath.isDirectory()) {
+ for (File childPath : inputPath.listFiles()) {
+ collectDocs(ruleDocEntries, attributeDocEntries, childPath);
+ }
+ }
+ }
+
+ private File setupDirectories(String outputRootDir) {
+ if (outputRootDir != null) {
+ File outputRootPath = new File(outputRootDir);
+ outputRootPath.mkdirs();
+ return new File(outputRootDir + File.separator + DocgenConsts.BUILD_ENCYCLOPEDIA_NAME);
+ } else {
+ return new File(DocgenConsts.BUILD_ENCYCLOPEDIA_NAME);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/DocCheckerUtils.java b/src/main/java/com/google/devtools/build/docgen/DocCheckerUtils.java
new file mode 100644
index 0000000..3b64362
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/DocCheckerUtils.java
@@ -0,0 +1,95 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.docgen;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A utility class to check the generated documentations.
+ */
+public class DocCheckerUtils {
+
+ // TODO(bazel-team): remove elements from this list and clean up the tested documentations.
+ private static final ImmutableSet<String> UNCHECKED_HTML_TAGS = ImmutableSet.<String>of(
+ "br", "li", "ul", "p");
+
+ private static final Pattern TAG_PATTERN = Pattern.compile(
+ "<([/]?[a-z0-9_]+)"
+ + "([^>]*)"
+ + ">",
+ Pattern.CASE_INSENSITIVE);
+
+ private static final Pattern COMMENT_PATTERN = Pattern.compile(
+ "<!--.*?-->",
+ Pattern.CASE_INSENSITIVE);
+
+ /**
+ * Returns the first unmatched html tag of srcs or null if no such tag exists.
+ * Note that this check is not performed on br, ul, li and p tags. The method also
+ * prints some help in case an unmatched tag is found. The check is performed
+ * inside comments too.
+ */
+ public static String getFirstUnclosedTagAndPrintHelp(String src) {
+ return getFirstUnclosedTag(src, true);
+ }
+
+ static String getFirstUnclosedTag(String src) {
+ return getFirstUnclosedTag(src, false);
+ }
+
+ // TODO(bazel-team): run this on the Skylark docs too.
+ private static String getFirstUnclosedTag(String src, boolean printHelp) {
+ Matcher commentMatcher = COMMENT_PATTERN.matcher(src);
+ src = commentMatcher.replaceAll("");
+ Matcher tagMatcher = TAG_PATTERN.matcher(src);
+ Deque<String> tagStack = new ArrayDeque<>();
+ while (tagMatcher.find()) {
+ String tag = tagMatcher.group(1);
+ String rest = tagMatcher.group(2);
+ String strippedTag = tag.substring(1);
+
+ // Ignoring self closing tags.
+ if (!rest.endsWith("/")
+ // Ignoring unchecked tags.
+ && !UNCHECKED_HTML_TAGS.contains(tag) && !UNCHECKED_HTML_TAGS.contains(strippedTag)) {
+ if (tag.startsWith("/")) {
+ // Closing tag. Removing '/' from the beginning.
+ tag = strippedTag;
+ String lastTag = tagStack.removeLast();
+ if (!lastTag.equals(tag)) {
+ if (printHelp) {
+ System.err.println(
+ "Unclosed tag: " + lastTag + "\n"
+ + "Trying to close with: " + tag + "\n"
+ + "Stack of open tags: " + tagStack + "\n"
+ + "Last 200 characters:\n"
+ + src.substring(Math.max(tagMatcher.start() - 200, 0), tagMatcher.start()));
+ }
+ return lastTag;
+ }
+ } else {
+ // Starting tag.
+ tagStack.addLast(tag);
+ }
+ }
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java b/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java
new file mode 100644
index 0000000..a2e7583
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java
@@ -0,0 +1,169 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * All the constants for the Docgen project.
+ */
+public class DocgenConsts {
+
+ public static final String LS = "\n";
+
+ public static final String HEADER_TEMPLATE = "templates/be-header.html";
+ public static final String FOOTER_TEMPLATE = "templates/be-footer.html";
+ public static final String BODY_TEMPLATE = "templates/be-body.html";
+ public static final String SKYLARK_BODY_TEMPLATE = "templates/skylark-body.html";
+
+ public static final String VAR_LEFT_PANEL = "LEFT_PANEL";
+
+ public static final String VAR_SECTION_BINARY = "SECTION_BINARY";
+ public static final String VAR_SECTION_LIBRARY = "SECTION_LIBRARY";
+ public static final String VAR_SECTION_TEST = "SECTION_TEST";
+ public static final String VAR_SECTION_GENERATE = "SECTION_GENERATE";
+ public static final String VAR_SECTION_OTHER = "SECTION_OTHER";
+
+ public static final String VAR_IMPLICIT_OUTPUTS = "IMPLICIT_OUTPUTS";
+ public static final String VAR_ATTRIBUTE_SIGNATURE = "ATTRIBUTE_SIGNATURE";
+ public static final String VAR_ATTRIBUTE_DEFINITION = "ATTRIBUTE_DEFINITION";
+ public static final String VAR_NAME = "NAME";
+ public static final String VAR_HEADER_TABLE = "HEADER_TABLE";
+ public static final String VAR_COMMON_ATTRIBUTE_DEFINITION = "COMMON_ATTRIBUTE_DEFINITION";
+ public static final String VAR_TEST_ATTRIBUTE_DEFINITION = "TEST_ATTRIBUTE_DEFINITION";
+ public static final String VAR_BINARY_ATTRIBUTE_DEFINITION = "BINARY_ATTRIBUTE_DEFINITION";
+ public static final String VAR_SYNOPSIS = "SYNOPSIS";
+
+ public static final String VAR_SECTION_SKYLARK_BUILTIN = "SECTION_BUILTIN";
+
+ public static final String COMMON_ATTRIBUTES = "common";
+ public static final String TEST_ATTRIBUTES = "test";
+ public static final String BINARY_ATTRIBUTES = "binary";
+
+ /**
+ * Mark the attribute as deprecated in the Build Encyclopedia.
+ */
+ public static final String FLAG_DEPRECATED = "DEPRECATED";
+ public static final String FLAG_GENERIC_RULE = "GENERIC_RULE";
+
+ public static final String HEADER_COMMENT =
+ "<!DOCTYPE html>\n"
+ + "<!--\n"
+ + " This document is synchronized with Blaze releases.\n"
+ + " To edit, submit changes to the Blaze source code.\n"
+ + " Generated by: blaze build java/com/google/devtools/build/docgen:build-encyclopedia.html\n"
+ + "-->\n";
+
+ public static final String BUILD_ENCYCLOPEDIA_NAME = "build-encyclopedia.html";
+
+ public static final FileTypeSet JAVA_SOURCE_FILE_SUFFIX = FileTypeSet.of(FileType.of(".java"));
+
+ public static final String META_KEY_NAME = "NAME";
+ public static final String META_KEY_TYPE = "TYPE";
+ public static final String META_KEY_FAMILY = "FAMILY";
+
+ /**
+ * Types a rule can have (Binary, Library, Test or Other).
+ */
+ public static enum RuleType {
+ BINARY, LIBRARY, TEST, OTHER
+ }
+
+ /**
+ * i.e. <!-- #BLAZE_RULE(NAME = RULE_NAME, TYPE = RULE_TYPE, FAMILY = RULE_FAMILY) -->
+ * i.e. <!-- #BLAZE_RULE(...)[DEPRECATED] -->
+ */
+ public static final Pattern BLAZE_RULE_START = Pattern.compile(
+ "^[\\s]*/\\*[\\s]*\\<!\\-\\-[\\s]*#BLAZE_RULE[\\s]*\\(([\\w\\s=,+/()-]+)\\)"
+ + "(\\[[\\w,]+\\])?[\\s]*\\-\\-\\>");
+ /**
+ * i.e. <!-- #END_BLAZE_RULE -->
+ */
+ public static final Pattern BLAZE_RULE_END = Pattern.compile(
+ "^[\\s]*\\<!\\-\\-[\\s]*#END_BLAZE_RULE[\\s]*\\-\\-\\>[\\s]*\\*/");
+ /**
+ * i.e. <!-- #BLAZE_RULE.EXAMPLE -->
+ */
+ public static final Pattern BLAZE_RULE_EXAMPLE_START = Pattern.compile(
+ "[\\s]*\\<!--[\\s]*#BLAZE_RULE.EXAMPLE[\\s]*--\\>[\\s]*");
+ /**
+ * i.e. <!-- #BLAZE_RULE.END_EXAMPLE -->
+ */
+ public static final Pattern BLAZE_RULE_EXAMPLE_END = Pattern.compile(
+ "[\\s]*\\<!--[\\s]*#BLAZE_RULE.END_EXAMPLE[\\s]*--\\>[\\s]*");
+ /**
+ * i.e. <!-- #BLAZE_RULE(RULE_NAME).VARIABLE_NAME -->
+ */
+ public static final Pattern BLAZE_RULE_VAR_START = Pattern.compile(
+ "^[\\s]*/\\*[\\s]*\\<!\\-\\-[\\s]*#BLAZE_RULE\\(([\\w\\$]+)\\)\\.([\\w]+)[\\s]*\\-\\-\\>");
+ /**
+ * i.e. <!-- #END_BLAZE_RULE.VARIABLE_NAME -->
+ */
+ public static final Pattern BLAZE_RULE_VAR_END = Pattern.compile(
+ "^[\\s]*\\<!\\-\\-[\\s]*#END_BLAZE_RULE\\.([\\w]+)[\\s]*\\-\\-\\>[\\s]*\\*/");
+ /**
+ * i.e. <!-- #BLAZE_RULE(RULE_NAME).ATTRIBUTE(ATTR_NAME) -->
+ * i.e. <!-- #BLAZE_RULE(RULE_NAME).ATTRIBUTE(ATTR_NAME)[DEPRECATED] -->
+ */
+ public static final Pattern BLAZE_RULE_ATTR_START = Pattern.compile(
+ "^[\\s]*/\\*[\\s]*\\<!\\-\\-[\\s]*#BLAZE_RULE\\(([\\w\\$]+)\\)\\."
+ + "ATTRIBUTE\\(([\\w]+)\\)(\\[[\\w,]+\\])?[\\s]*\\-\\-\\>");
+ /**
+ * i.e. <!-- #END_BLAZE_RULE.ATTRIBUTE -->
+ */
+ public static final Pattern BLAZE_RULE_ATTR_END = Pattern.compile(
+ "^[\\s]*\\<!\\-\\-[\\s]*#END_BLAZE_RULE\\.ATTRIBUTE[\\s]*\\-\\-\\>[\\s]*\\*/");
+
+ public static final Pattern BLAZE_RULE_FLAGS = Pattern.compile("^.*\\[(.*)\\].*$");
+
+ public static final Map<String, Integer> ATTRIBUTE_ORDERING = ImmutableMap
+ .<String, Integer>builder()
+ .put("name", -99)
+ .put("deps", -98)
+ .put("src", -97)
+ .put("srcs", -96)
+ .put("data", -95)
+ .put("resource", -94)
+ .put("resources", -93)
+ .put("out", -92)
+ .put("outs", -91)
+ .put("hdrs", -90)
+ .build();
+
+ static String toCommandLineFormat(String cmdDoc) {
+ // Replace html <br> tags with line breaks
+ cmdDoc = cmdDoc.replaceAll("(<br>|<br[\\s]*/>)", "\n") + "\n";
+ // Replace other links <a href=".*">s with human readable links
+ cmdDoc = cmdDoc.replaceAll("\\<a href=\"([^\"]+)\">[^\\<]*\\</a\\>", "$1");
+ // Delete other html tags
+ cmdDoc = cmdDoc.replaceAll("\\<[/]?[^\\>]+\\>", "");
+ // Delete docgen variables
+ cmdDoc = cmdDoc.replaceAll("\\$\\{[\\w_]+\\}", "");
+ // Substitute more than 2 line breaks in a row with 2 line breaks
+ cmdDoc = cmdDoc.replaceAll("[\\n]{2,}", "\n\n");
+ // Ensure that the doc starts and ends with exactly two line breaks
+ cmdDoc = cmdDoc.replaceAll("^[\\n]+", "\n\n");
+ cmdDoc = cmdDoc.replaceAll("[\\n]+$", "\n\n");
+ return cmdDoc;
+ }
+
+ static String removeDuplicatedNewLines(String doc) {
+ return doc.replaceAll("[\\n][\\s]*[\\n]", "\n");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java b/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java
new file mode 100644
index 0000000..c885cc1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java
@@ -0,0 +1,347 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * A class to contain the base definition of common BUILD rule attributes.
+ */
+public class PredefinedAttributes {
+
+ public static final Map<String, RuleDocumentationAttribute> COMMON_ATTRIBUTES = ImmutableMap
+ .<String, RuleDocumentationAttribute>builder()
+ .put("deps", RuleDocumentationAttribute.create("deps", DocgenConsts.COMMON_ATTRIBUTES,
+ "A list of dependencies of this rule.\n"
+ + "<i>(List of <a href=\"build-ref.html#labels\">labels</a>; optional)</i><br/>\n"
+ + "The precise semantics of what it means for this rule to depend on\n"
+ + "another using <code>deps</code> are specific to the kind of this rule,\n"
+ + "and the rule-specific documentation below goes into more detail.\n"
+ + "At a minimum, though, the targets named via <code>deps</code> will\n"
+ + "appear in the <code>*.runfiles</code> area of this rule, if it has\n"
+ + "one.\n"
+ + "<p>Most often, a <code>deps</code> dependency is used to allow one\n"
+ + "module to use symbols defined in another module written in the\n"
+ + "same programming language and separately compiled. Cross-language\n"
+ + "dependencies are also permitted in many cases: for example,\n"
+ + "a <code>java_library</code> rule may depend on C++ code in\n"
+ + "a <code>cc_library</code> rule, by declaring the latter in\n"
+ + "the <code>deps</code> attribute. See the definition\n"
+ + "of <a href=\"build-ref.html#deps\">dependencies</a> for more\n"
+ + "information.</p>\n"
+ + "<p>Almost all rules permit a <code>deps</code> attribute, but where\n"
+ + "this attribute is not allowed, this fact is documented under the\n"
+ + "specific rule.</p>"))
+ .put("data", RuleDocumentationAttribute.create("data", DocgenConsts.COMMON_ATTRIBUTES,
+ "The list of files needed by this rule at runtime.\n"
+ + "<i>(List of <a href=\"build-ref.html#labels\">labels</a>; optional)</i><br/>\n"
+ + "Targets named in the <code>data</code> attribute will appear in\n"
+ + "the <code>*.runfiles</code> area of this rule, if it has one. This\n"
+ + "may include data files needed by a binary or library, or other\n"
+ + "programs needed by it. See the\n"
+ + "<a href=\"build-ref.html#data\">data dependencies</a> section for more\n"
+ + "information about how to depend on and use data files.\n"
+ + "<p>Almost all rules permit a <code>data</code> attribute, but where\n"
+ + "this attribute is not allowed, this fact is documented under the\n"
+ + "specific rule.</p>"))
+ .put("licenses", RuleDocumentationAttribute.create("licenses",
+ DocgenConsts.COMMON_ATTRIBUTES,
+ "<i>(List of strings; optional)</i><br/>\n"
+ + "A list of license-type strings to be used for this particular build rule.\n"
+ + "Overrides the <code>BUILD</code>-file scope defaults defined by the\n"
+ + "<a href=\"#licenses\"><code>licenses()</code></a> directive."))
+ .put("distribs", RuleDocumentationAttribute.create("distribs",
+ DocgenConsts.COMMON_ATTRIBUTES,
+ "<i>(List of strings; optional)</i><br/>\n"
+ + "A list of distribution-method strings to be used for this particular build rule.\n"
+ + "Overrides the <code>BUILD</code>-file scope defaults defined by the\n"
+ + "<a href=\"#distribs\"><code>distribs()</code></a> directive."))
+ .put("deprecation", RuleDocumentationAttribute.create("deprecation",
+ DocgenConsts.COMMON_ATTRIBUTES,
+ "<i>(String; optional)</i><br/>\n"
+ + "An explanatory warning message associated with this rule.\n"
+ + "Typically this is used to notify users that a rule has become obsolete,\n"
+ + "or has become superseded by another rule, is private to a package, or is\n"
+ + "perhaps \"considered harmful\" for some reason. It is a good idea to include\n"
+ + "some reference (like a webpage, a bug number or example migration CLs) so\n"
+ + "that one can easily find out what changes are required to avoid the message.\n"
+ + "If there is a new target that can be used as a drop in replacement, it is a good idea\n"
+ + "to just migrate all users of the old target.\n"
+ + "<p>\n"
+ + "This attribute has no effect on the way things are built, but it\n"
+ + "may affect a build tool's diagnostic output. The build tool issues a\n"
+ + "warning when a rule with a <code>deprecation</code> attribute is\n"
+ + "depended upon by another rule.</p>\n"
+ + "<p>\n"
+ + "Intra-package dependencies are exempt from this warning, so that,\n"
+ + "for example, building the tests of a deprecated rule does not\n"
+ + "encounter a warning.</p>\n"
+ + "<p>\n"
+ + "If a deprecated rule depends on another deprecated rule, no warning\n"
+ + "message is issued.</p>\n"
+ + "<p>\n"
+ + "Once people have stopped using it, the package can be removed or marked as\n"
+ + "<a href=\"#common.obsolete\"><code>obsolete</code></a>.</p>"))
+ .put("obsolete", RuleDocumentationAttribute.create("obsolete",
+ DocgenConsts.COMMON_ATTRIBUTES,
+ "<i>(Boolean; optional; default 0)</i><br/>\n"
+ + "If 1, only obsolete targets can depend on this target. It is an error when\n"
+ + "a non-obsolete target depends on an obsolete target.\n"
+ + "<p>\n"
+ + "As a transition, one can first mark a package as in\n"
+ + "<a href=\"#common.deprecation\"><code>deprecation</code></a>.</p>\n"
+ + "<p>\n"
+ + "This attribute is useful when you want to prevent a target from\n"
+ + "being used but are yet not ready to delete the sources.</p>"))
+ .put("testonly", RuleDocumentationAttribute.create("testonly",
+ DocgenConsts.COMMON_ATTRIBUTES,
+ "<i>(Boolean; optional; default 0 except as noted)</i><br />\n"
+ + "If 1, only testonly targets (such as tests) can depend on this target.\n"
+ + "<p>Equivalently, a rule that is not <code>testonly</code> is not allowed to\n"
+ + "depend on any rule that is <code>testonly</code>.</p>\n"
+ + "<p>Tests (<code>*_test</code> rules)\n"
+ + "and test suites (<a href=\"#test_suite\">test_suite</a> rules)\n"
+ + "are <code>testonly</code> by default.</p>\n"
+ + "<p>By virtue of\n"
+ + "<a href=\"#package.default_testonly\"><code>default_testonly</code></a>,\n"
+ + "targets under <code>javatests</code> are <code>testonly</code> by default.</p>\n"
+ + "<p>This attribute is intended to mean that the target should not be\n"
+ + "contained in binaries that are released to production.</p>\n"
+ + "<p>Because testonly is enforced at build time, not run time, and propagates\n"
+ + "virally through the dependency tree, it should be applied judiciously. For\n"
+ + "example, stubs and fakes that\n"
+ + "are useful for unit tests may also be useful for integration tests\n"
+ + "involving the same binaries that will be released to production, and\n"
+ + "therefore should probably not be marked testonly. Conversely, rules that\n"
+ + "are dangerous to even link in, perhaps because they unconditionally\n"
+ + "override normal behavior, should definitely be marked testonly.</p>"))
+ .put("tags", RuleDocumentationAttribute.create("tags", DocgenConsts.COMMON_ATTRIBUTES,
+ "List of arbitrary text tags. Tags may be any valid string; default is the\n"
+ + "empty list.<br/>\n"
+ + "<i>Tags</i> can be used on any rule; but <i>tags</i> are most useful\n"
+ + "on test and <code>test_suite</code> rules. Tags on non-test rules\n"
+ + "are only useful to humans and/or external programs.\n"
+ + "<i>Tags</i> are generally used to annotate a test's role in your debug\n"
+ + "and release process. Typically, tags are most useful for C++ and\n"
+ + "Python tests, which\n"
+ + "lack any runtime annotation ability. The use of tags and size elements\n"
+ + "gives flexibility in assembling suites of tests based around codebase\n"
+ + "check-in policy.\n"
+ + "<p>\n"
+ + "A few tags have special meaning to the build tool, such as\n"
+ + "indicating that a particular test cannot be run remotely, for\n"
+ + "example. Consult\n"
+ + "the <a href='blaze-user-manual.html#tags_keywords'>Blaze\n"
+ + "documentation</a> for details.\n"
+ + "</p>"))
+ .put("visibility", RuleDocumentationAttribute.create("visibility",
+ DocgenConsts.COMMON_ATTRIBUTES,
+ "<i>(List of <a href=\"build-ref.html#labels\">"
+ + "labels</a>; optional; default private)</i><br/>\n"
+ + "<p>The <code>visibility</code> attribute on a rule controls whether\n"
+ + "the rule can be used by other packages. Rules are always visible to\n"
+ + "other rules declared in the same package.</p>\n"
+ + "<p>There are five forms (and one temporary form) a visibility label can take:\n"
+ + "<ul>\n"
+ + "<li><code>[\"//visibility:public\"]</code>: Anyone can use this rule.</li>\n"
+ + "<li><code>[\"//visibility:private\"]</code>: Only rules in this package\n"
+ + "can use this rule. Rules in <code>javatests/foo/bar</code>\n"
+ + "can always use rules in <code>java/foo/bar</code>.\n"
+ + "</li>\n"
+ + "<li><code>[\"//some/package:__pkg__\", \"//other/package:__pkg__\"]</code>:\n"
+ + "Only rules in <code>some/package</code> and <code>other/package</code>\n"
+ + "(defined in <code>some/package/BUILD</code> and\n"
+ + "<code>other/package/BUILD</code>) have access to this rule. Note that\n"
+ + "sub-packages do not have access to the rule; for example,\n"
+ + "<code>//some/package/foo:bar</code> or\n"
+ + "<code>//other/package/testing:bla</code> wouldn't have access.\n"
+ + "<code>__pkg__</code> is a special target and must be used verbatim.\n"
+ + "It represents all of the rules in the package.\n"
+ + "</li>\n"
+ + "<li><code>[\"//project:__subpackages__\", \"//other:__subpackages__\"]</code>:\n"
+ + "Only rules in packages <code>project</code> or <code>other</code> or\n"
+ + "in one of their sub-packages have access to this rule. For example,\n"
+ + "<code>//project:rule</code>, <code>//project/library:lib</code> or\n"
+ + "<code>//other/testing/internal:munge</code> are allowed to depend on\n"
+ + "this rule (but not <code>//independent:evil</code>)\n"
+ + "</li>\n"
+ + "<li><code>[\"//some/package:my_package_group\"]</code>:\n"
+ + "A <a href=\"#package_group\">package group</a> is\n"
+ + "a named set of package names. Package groups can also grant access rights\n"
+ + "to entire subtrees, e.g.<code>//myproj/...</code>.\n"
+ + "</li>\n"
+ + "<li><code>[\"//visibility:legacy_public\"]</code>: Anyone can use this\n"
+ + "rule (for now). <i>Developer action is needed</i>.\n"
+ + "<p>This value has been used during the transition to the new\n"
+ + "<code>[\"//visibility:private\"]</code> default, on June 6, 2011.\n"
+ + "<i>We will eventually deprecate and then disallow this value.</i>\n"
+ + "</li>\n"
+ + "</ul>\n"
+ + "<p>The visibility specifications of <code>//visibility:public</code>,\n"
+ + "<code>//visibility:private</code> and\n"
+ + "<code>//visibility:legacy_public</code>\n"
+ + "can not be combined with any other visibility specifications.\n"
+ + "A visibility specification may contain a combination of package labels\n"
+ + "(i.e. //foo:__pkg__) and package_groups.</p>\n"
+ + "<p>If a rule does not specify the visibility attribute,\n"
+ + "the <code><a href=\"#package\">default_visibility</a></code>\n"
+ + "attribute of the <code><a href=\"#package\">package</a></code>\n"
+ + "statement in the BUILD file containing the rule is used\n"
+ + "(except <a href=\"#exports_files\">exports_files</a> and\n"
+ + "<a href=\"#cc_public_library\">cc_public_library</a>, which always default to\n"
+ + "public).</p>\n"
+ + "<p>If the default visibility for the package is not specified,\n"
+ + "the rule is private: on June 6, 2011, in order to prevent teams\n"
+ + "from reaching into private code, the default has been changed\n"
+ + "to <code>[\"//visibility:private\"]</code>.</p>\n"
+ + "<p><b>Example</b>:</p>\n"
+ + "<p>\n"
+ + "File <code>//frobber/bin/BUILD</code>:\n"
+ + "</p>\n"
+ + "<pre class=\"code\">\n"
+ + "# This rule is visible to everyone\n"
+ + "py_binary(\n"
+ + " name = \"executable\",\n"
+ + " visibility = [\"//visibility:public\"],\n"
+ + " deps = [\":library\"],\n"
+ + ")\n"
+ + "\n"
+ + "# This rule is visible only to rules declared in the same package\n"
+ + "py_library(\n"
+ + " name = \"library\",\n"
+ + " visibility = [\"//visibility:private\"],\n"
+ + ")\n"
+ + "\n"
+ + "# This rule is visible to rules in package //object and //noun\n"
+ + "py_library(\n"
+ + " name = \"subject\",\n"
+ + " visibility = [\n"
+ + " \"//noun:__pkg__\",\n"
+ + " \"//object:__pkg__\",\n"
+ + " ],\n"
+ + ")\n"
+ + "\n"
+ + "# See package group //frobber:friends (below) for who can access this rule.\n"
+ + "py_library(\n"
+ + " name = \"thingy\",\n"
+ + " visibility = [\"//frobber:friends\"],\n"
+ + ")\n"
+ + "</pre>\n"
+ + "<p>\n"
+ + "File <code>//frobber/BUILD</code>:\n"
+ + "</p>\n"
+ + "<pre class=\"code\">\n"
+ + "# This is the package group declaration to which rule //frobber/bin:thingy refers.\n"
+ + "#\n"
+ + "# Our friends are packages //frobber, //fribber and any subpackage of //fribber.\n"
+ + "package_group(\n"
+ + " name = \"friends\",\n"
+ + " packages = [\n"
+ + " \"//fribber/...\",\n"
+ + " \"//frobber\",\n"
+ + " ],\n"
+ + ")\n"
+ + "</pre>"))
+ .build();
+
+ public static final Map<String, RuleDocumentationAttribute> BINARY_ATTRIBUTES = ImmutableMap.of(
+ "args", RuleDocumentationAttribute.create("args", DocgenConsts.BINARY_ATTRIBUTES,
+ "Add these arguments to the target when executed by\n"
+ + "<code>blaze run</code>.\n"
+ + "<i>(List of strings; optional; subject to\n"
+ + "<a href=\"#make_variables\">\"Make variable\"</a> substitution and\n"
+ + "<a href=\"#sh-tokenization\">Bourne shell tokenization</a>)</i><br/>\n"
+ + "These arguments are passed to the target before the target options\n"
+ + "specified on the <code>blaze run</code> command line.\n"
+ + "<p>Most binary rules permit an <code>args</code> attribute, but where\n"
+ + "this attribute is not allowed, this fact is documented under the\n"
+ + "specific rule.</p>"),
+ "output_licenses", RuleDocumentationAttribute.create("output_licenses",
+ DocgenConsts.BINARY_ATTRIBUTES,
+ "The licenses of the output files that this binary generates.\n"
+ + "<i>(List of strings; optional)</i><br/>\n"
+ + "Describes the licenses of the output of the binary generated by\n"
+ + "the rule. When a binary is referenced in a host attribute (for\n"
+ + "example, the <code>tools</code> attribute of\n"
+ + "a <code>genrule</code>), this license declaration is used rather\n"
+ + "than the union of the licenses of its transitive closure. This\n"
+ + "argument is useful when a binary is used as a tool during the\n"
+ + "build of a rule, and it is not desirable for its license to leak\n"
+ + "into the license of that rule. If this attribute is missing, the\n"
+ + "license computation proceeds as if the host dependency was a\n"
+ + "regular dependency.\n"
+ + "<p>(For more about the distinction between host and target\n"
+ + "configurations,\n"
+ + "see <a href=\"blaze-user-manual.html#configurations\">"
+ + "Build configurations</a> in the Blaze manual.)\n"
+ + "<p><em class=\"harmful\">WARNING: in some cases (specifically, in\n"
+ + "genrules) the build tool cannot guarantee that the binary\n"
+ + "referenced by this attribute is actually used as a tool, and is\n"
+ + "not, for example, copied to the output. In these cases, it is the\n"
+ + "responsibility of the user to make sure that this is\n"
+ + "true.</em></p>"));
+
+ public static final Map<String, RuleDocumentationAttribute> TEST_ATTRIBUTES = ImmutableMap
+ .<String, RuleDocumentationAttribute>builder()
+ .put("args", RuleDocumentationAttribute.create("args", DocgenConsts.TEST_ATTRIBUTES,
+ "Add these arguments to the <code>--test_arg</code>\n"
+ + "when executed by <code>blaze test</code>.\n"
+ + "<i>(List of strings; optional; subject to\n"
+ + "<a href=\"#make_variables\">\"Make variable\"</a> substitution and\n"
+ + "<a href=\"#sh-tokenization\">Bourne shell tokenization</a>)</i><br/>\n"
+ + "These arguments are passed before the <code>--test_arg</code> values\n"
+ + "specified on the <code>blaze test</code> command line."))
+ .put("size", RuleDocumentationAttribute.create("size", DocgenConsts.TEST_ATTRIBUTES,
+ "How \"heavy\" the test is\n"
+ + "<i>(String \"enormous\", \"large\" \"medium\" or \"small\",\n"
+ + "default is \"medium\")</i><br/>\n"
+ + "A classification of the test's \"heaviness\": how much time/resources\n"
+ + "it needs to run."
+ + "Unittests are considered \"small\", integration tests \"medium\", "
+ + "and end-to-end tests \"large\" or \"enormous\". "
+ + "Blaze uses the size only to determine a default timeout."))
+ .put("timeout", RuleDocumentationAttribute.create("timeout", DocgenConsts.TEST_ATTRIBUTES,
+ "How long the test is\n"
+ + "normally expected to run before returning.\n"
+ + "<i>(String \"eternal\", \"long\", \"moderate\", or \"short\"\n"
+ + "with the default derived from a test's size attribute)</i><br/>\n"
+ + "While a test's size attribute controls resource estimation, a test's\n"
+ + "timeout may be set independently. If not explicitly specified, the\n"
+ + "timeout is based on the test's size (with \"small\" ⇒ \"short\",\n"
+ + "\"medium\" ⇒ \"moderate\", etc...). "
+ + "\"short\" means 1 minute, \"moderate\" 5 minutes, and \"long\" 15 minutes."))
+ .put("flaky", RuleDocumentationAttribute.create("flaky", DocgenConsts.TEST_ATTRIBUTES,
+ "Marks test as flaky. <i>(Boolean; optional)</i><br/>\n"
+ + "If set, executes the test up to 3 times before being declared as failed.\n"
+ + "By default this attribute is set to 0 and test is considered to be stable.\n"
+ + "Note, that use of this attribute is generally discouraged - we do prefer\n"
+ + "all tests to be stable."))
+ .put("shard_count", RuleDocumentationAttribute.create("shard_count",
+ DocgenConsts.TEST_ATTRIBUTES,
+ "Specifies the number of parallel shards\n"
+ + "to use to run the test. <i>(Non-negative integer less than or equal to 50;\n"
+ + "optional)</i><br/>\n"
+ + "This value will override any heuristics used to determine the number of\n"
+ + "parallel shards with which to run the test. Note that for some test\n"
+ + "rules, this parameter may be required to enable sharding\n"
+ + "in the first place. Also see --test_sharding_strategy."))
+ .put("local", RuleDocumentationAttribute.create("local", DocgenConsts.TEST_ATTRIBUTES,
+ "Forces the test to be run locally. <i>(Boolean; optional)</i><br/>\n"
+ + "By default this attribute is set to 0 and the default testing strategy is\n"
+ + "used. This is equivalent to providing \"local\" as a tag\n"
+ + "(<code>tags=[\"local\"]</code>)."))
+ .build();
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentation.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentation.java
new file mode 100644
index 0000000..e8835f4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentation.java
@@ -0,0 +1,353 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.docgen.DocgenConsts.RuleType;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RuleClass;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * A class representing the documentation of a rule along with some meta-data. The sole ruleName
+ * field is used as a key for comparison, equals and hashcode.
+ *
+ * <p> The class contains meta information about the rule:
+ * <ul>
+ * <li> Rule type: categorizes the rule based on it's general (language independent) purpose,
+ * see {@link RuleType}.
+ * <li> Rule family: categorizes the rule based on language.
+ * </ul>
+ *
+ * <p> The class also contains physical information about the documentation,
+ * such as declaring file name and the first line of the raw documentation. This can be useful for
+ * proper error signaling during documentation processing.
+ */
+class RuleDocumentation implements Comparable<RuleDocumentation> {
+
+ private final String ruleName;
+ private final RuleType ruleType;
+ private final String ruleFamily;
+ private final String htmlDocumentation;
+ // Store these information for error messages
+ private final int startLineCount;
+ private final String fileName;
+ private final ImmutableSet<String> flags;
+
+ private final Map<String, String> docVariables = new HashMap<>();
+ // Only one attribute per attributeName is allowed
+ private final Set<RuleDocumentationAttribute> attributes = new TreeSet<>();
+ private final ConfiguredRuleClassProvider ruleClassProvider;
+
+ /**
+ * Creates a RuleDocumentation from the rule's name, type, family and raw html documentation
+ * (meaning without expanding the variables in the doc).
+ */
+ RuleDocumentation(String ruleName, String ruleType, String ruleFamily,
+ String htmlDocumentation, int startLineCount, String fileName, ImmutableSet<String> flags,
+ ConfiguredRuleClassProvider ruleClassProvider)
+ throws BuildEncyclopediaDocException {
+ Preconditions.checkNotNull(ruleName);
+ this.ruleName = ruleName;
+ try {
+ this.ruleType = RuleType.valueOf(ruleType);
+ } catch (IllegalArgumentException e) {
+ throw new BuildEncyclopediaDocException(
+ fileName, startLineCount, "Invalid rule type " + ruleType);
+ }
+ this.ruleFamily = ruleFamily;
+ this.htmlDocumentation = htmlDocumentation;
+ this.startLineCount = startLineCount;
+ this.fileName = fileName;
+ this.flags = flags;
+ this.ruleClassProvider = ruleClassProvider;
+ }
+
+ /**
+ * Returns the name of the rule.
+ */
+ String getRuleName() {
+ return ruleName;
+ }
+
+ /**
+ * Returns the type of the rule
+ */
+ RuleType getRuleType() {
+ return ruleType;
+ }
+
+ /**
+ * Returns the family of the rule. The family is usually the corresponding programming language,
+ * except for rules independent of language, such as genrule. E.g. the family of the java_library
+ * rule is 'JAVA', the family of genrule is 'GENERAL'.
+ */
+ String getRuleFamily() {
+ return ruleFamily;
+ }
+
+ /**
+ * Returns the number of first line of the rule documentation in its declaration file.
+ */
+ int getStartLineCount() {
+ return startLineCount;
+ }
+
+ /**
+ * Returns true if this rule documentation has the parameter flag.
+ */
+ boolean hasFlag(String flag) {
+ return flags.contains(flag);
+ }
+
+ /**
+ * Returns true if this rule applies to a specific programming language (e.g. java_library),
+ * returns false if it is a generic action (e.g. genrule, filegroup).
+ *
+ * A rule is considered to be specific to a programming language by default. Generic rules have
+ * to be marked with the flag GENERIC_RULE in their #BLAZE_RULE definition.
+ */
+ boolean isLanguageSpecific() {
+ return !flags.contains(DocgenConsts.FLAG_GENERIC_RULE);
+ }
+
+ /**
+ * Adds a variable name - value pair to the documentation to be substituted.
+ */
+ void addDocVariable(String varName, String value) {
+ docVariables.put(varName, value);
+ }
+
+ /**
+ * Adds a rule documentation attribute to this rule documentation.
+ */
+ void addAttribute(RuleDocumentationAttribute attribute) {
+ attributes.add(attribute);
+ }
+
+ /**
+ * Returns the html documentation in the exact format it should be written into the Build
+ * Encyclopedia (expanding variables).
+ */
+ String getHtmlDocumentation() {
+ String expandedDoc = htmlDocumentation;
+ // Substituting variables
+ for (Entry<String, String> docVariable : docVariables.entrySet()) {
+ expandedDoc = expandedDoc.replace("${" + docVariable.getKey() + "}",
+ expandBuiltInVariables(docVariable.getKey(), docVariable.getValue()));
+ }
+ expandedDoc = expandedDoc.replace("${" + DocgenConsts.VAR_ATTRIBUTE_SIGNATURE + "}",
+ generateAttributeSignatures());
+ expandedDoc = expandedDoc.replace("${" + DocgenConsts.VAR_ATTRIBUTE_DEFINITION + "}",
+ generateAttributeDefinitions(true));
+ return String.format("<h3 id=\"%s\"%s>%s</h3>\n\n%s", ruleName,
+ getDeprecatedString(hasFlag(DocgenConsts.FLAG_DEPRECATED)), ruleName, expandedDoc);
+ }
+
+ /**
+ * Returns the documentation of the rule in a form which is printable on the command line.
+ */
+ String getCommandLineDocumentation() {
+ return "\n" + DocgenConsts.toCommandLineFormat(htmlDocumentation);
+ }
+
+ /**
+ * Returns the html code of the attribute definitions without the header and name
+ * attribute of the rule.
+ */
+ String generateAttributeDefinitions() {
+ return generateAttributeDefinitions(false);
+ }
+
+ private String generateAttributeDefinitions(boolean generateNameAndHeader) {
+ StringBuilder sb = new StringBuilder();
+ if (generateNameAndHeader){
+ String nameExtraHtmlDoc = docVariables.containsKey(DocgenConsts.VAR_NAME)
+ ? docVariables.get(DocgenConsts.VAR_NAME) : "";
+ sb.append(String.format(Joiner.on('\n').join(new String[] {
+ "<h4 id=\"%s_args\">Arguments</h4>",
+ "<ul>",
+ "<li id=\"%s.name\"><code>name</code>: A unique name for this rule.",
+ "<i>(<a href=\"build-ref.html#name\">Name</a>; required)</i>%s</li>\n"}),
+ ruleName, ruleName, nameExtraHtmlDoc));
+ } else {
+ sb.append("<ul>\n");
+ }
+ for (RuleDocumentationAttribute attributeDoc : attributes) {
+ // Only generate attribute documentation here if the rule and the attribute is
+ // either both user defined or built in (of common type).
+ if (isCommonType() == attributeDoc.isCommonType()) {
+ String attrName = attributeDoc.getAttributeName();
+ Attribute attribute = isCommonType() ? null
+ : ruleClassProvider.getRuleClassMap().get(ruleName).getAttributeByName(attrName);
+ sb.append(String.format("<li id=\"%s.%s\"%s><code>%s</code>:\n%s</li>\n",
+ ruleName.toLowerCase(), attrName, getDeprecatedString(
+ attributeDoc.hasFlag(DocgenConsts.FLAG_DEPRECATED)),
+ attrName, attributeDoc.getHtmlDocumentation(attribute)));
+ }
+ }
+ sb.append("</ul>\n");
+ RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleName);
+ if (ruleClass != null && ruleClass.isPublicByDefault()) {
+ sb.append(
+ "The default visibility is public: <code>visibility = [\"//visibility:public\"]</code>.");
+ }
+ return sb.toString();
+ }
+
+ private String getDeprecatedString(boolean deprecated) {
+ return deprecated ? " class=\"deprecated\"" : "";
+ }
+
+ private String generateAttributeSignatures() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(String.format(
+ "<p class=\"rule-signature\">\n%s(<a href=\"#%s.name\">name</a>,\n",
+ ruleName, ruleName));
+ int i = 0;
+ for (RuleDocumentationAttribute attributeDoc : attributes) {
+ String attrName = attributeDoc.getAttributeName();
+ // Generate the link for the attribute documentation
+ sb.append(String.format("<a href=\"#%s.%s\">%s</a>",
+ attributeDoc.getGeneratedInRule(ruleName).toLowerCase(), attrName, attrName));
+ if (i < attributes.size() - 1) {
+ sb.append(",");
+ } else {
+ sb.append(")");
+ }
+ sb.append("\n");
+ i++;
+ }
+ sb.append("</p>\n");
+ return sb.toString();
+ }
+
+ private String expandBuiltInVariables(String key, String value) {
+ // Some built in BLAZE variables need special handling, e.g. adding headers
+ switch (key) {
+ case DocgenConsts.VAR_IMPLICIT_OUTPUTS:
+ return String.format("<h4 id=\"%s_implicit_outputs\">Implicit output targets</h4>\n%s",
+ ruleName.toLowerCase(), value);
+ default:
+ return value;
+ }
+ }
+
+ /**
+ * Returns a set of examples based on markups which can be used as BUILD file
+ * contents for testing.
+ */
+ Set<String> extractExamples() throws BuildEncyclopediaDocException {
+ String[] lines = htmlDocumentation.split(DocgenConsts.LS);
+ Set<String> examples = new HashSet<>();
+ StringBuilder sb = null;
+ boolean inExampleCode = false;
+ int lineCount = 0;
+ for (String line : lines) {
+ if (!inExampleCode) {
+ if (DocgenConsts.BLAZE_RULE_EXAMPLE_START.matcher(line).matches()) {
+ inExampleCode = true;
+ sb = new StringBuilder();
+ } else if (DocgenConsts.BLAZE_RULE_EXAMPLE_END.matcher(line).matches()) {
+ throw new BuildEncyclopediaDocException(fileName, startLineCount + lineCount,
+ "No matching start example tag (#BLAZE_RULE.EXAMPLE) for end example tag.");
+ }
+ } else {
+ if (DocgenConsts.BLAZE_RULE_EXAMPLE_END.matcher(line).matches()) {
+ inExampleCode = false;
+ examples.add(sb.toString());
+ sb = null;
+ } else if (DocgenConsts.BLAZE_RULE_EXAMPLE_START.matcher(line).matches()) {
+ throw new BuildEncyclopediaDocException(fileName, startLineCount + lineCount,
+ "No start example tags (#BLAZE_RULE.EXAMPLE) in a row.");
+ } else {
+ sb.append(line + DocgenConsts.LS);
+ }
+ }
+ lineCount++;
+ }
+ return examples;
+ }
+
+ /**
+ * Return true if the rule doesn't belong to a specific rule family.
+ */
+ private boolean isCommonType() {
+ return ruleFamily == null;
+ }
+
+ /**
+ * Creates a BuildEncyclopediaDocException with the file containing this rule doc and
+ * the number of the first line (where the rule doc is defined). Can be used to create
+ * general BuildEncyclopediaDocExceptions about this rule.
+ */
+ BuildEncyclopediaDocException createException(String msg) {
+ return new BuildEncyclopediaDocException(fileName, startLineCount, msg);
+ }
+
+ @Override
+ public int hashCode() {
+ return ruleName.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RuleDocumentation)) {
+ return false;
+ }
+ return ruleName.equals(((RuleDocumentation) obj).ruleName);
+ }
+
+ private int getTypePriority() {
+ switch (ruleType) {
+ case BINARY:
+ return 1;
+ case LIBRARY:
+ return 2;
+ case TEST:
+ return 3;
+ case OTHER:
+ return 4;
+ }
+ throw new IllegalArgumentException("Illegal value of ruleType: " + ruleType);
+ }
+
+ @Override
+ public int compareTo(RuleDocumentation o) {
+ if (this.getTypePriority() < o.getTypePriority()) {
+ return -1;
+ } else if (this.getTypePriority() > o.getTypePriority()) {
+ return 1;
+ } else {
+ return this.ruleName.compareTo(o.ruleName);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s (TYPE = %s, FAMILY = %s)", ruleName, ruleType, ruleFamily);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java
new file mode 100644
index 0000000..0bdc1f1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java
@@ -0,0 +1,248 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.TriState;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A class storing a rule attribute documentation along with some meta information.
+ * The class provides functionality to compute the ancestry level of this attribute's
+ * generator rule definition class compared to other rule definition classes.
+ *
+ * <p>Warning, two RuleDocumentationAttribute objects are equal based on only the attributeName.
+ */
+class RuleDocumentationAttribute implements Comparable<RuleDocumentationAttribute> {
+
+ private static final Map<Type<?>, String> TYPE_DESC = ImmutableMap.<Type<?>, String>builder()
+ .put(Type.BOOLEAN, "Boolean")
+ .put(Type.INTEGER, "Integer")
+ .put(Type.INTEGER_LIST, "List of Integer")
+ .put(Type.STRING, "String")
+ .put(Type.STRING_LIST, "List of String")
+ .put(Type.TRISTATE, "Integer")
+ .put(Type.LABEL, "<a href=\"build-ref.html#labels\">Label</a>")
+ .put(Type.LABEL_LIST, "List of <a href=\"build-ref.html#labels\">labels</a>")
+ .put(Type.NODEP_LABEL, "<a href=\"build-ref.html#name\">Name</a>")
+ .put(Type.NODEP_LABEL_LIST, "List of <a href=\"build-ref.html#name\">names</a>")
+ .put(Type.OUTPUT, "<a href=\"build-ref.html#filename\">Filename</a>")
+ .put(Type.OUTPUT_LIST, "List of <a href=\"build-ref.html#filename\">filenames</a>")
+ .build();
+
+ private final Class<? extends RuleDefinition> definitionClass;
+ private final String attributeName;
+ private final String htmlDocumentation;
+ private final String commonType;
+ private int startLineCnt;
+ private Set<String> flags;
+
+ /**
+ * Creates common RuleDocumentationAttribute such as deps or data.
+ * These attribute docs have no definitionClass or htmlDocumentation (it's in the BE header).
+ */
+ static RuleDocumentationAttribute create(
+ String attributeName, String commonType, String htmlDocumentation) {
+ RuleDocumentationAttribute docAttribute = new RuleDocumentationAttribute(
+ null, attributeName, htmlDocumentation, 0, ImmutableSet.<String>of(), commonType);
+ return docAttribute;
+ }
+
+ /**
+ * Creates a RuleDocumentationAttribute with all the necessary fields for explicitly
+ * defined rule attributes.
+ */
+ static RuleDocumentationAttribute create(Class<? extends RuleDefinition> definitionClass,
+ String attributeName, String htmlDocumentation, int startLineCnt, Set<String> flags) {
+ return new RuleDocumentationAttribute(definitionClass, attributeName, htmlDocumentation,
+ startLineCnt, flags, null);
+ }
+
+ private RuleDocumentationAttribute(Class<? extends RuleDefinition> definitionClass,
+ String attributeName, String htmlDocumentation, int startLineCnt, Set<String> flags,
+ String commonType) {
+ Preconditions.checkNotNull(attributeName, "AttributeName must not be null.");
+ this.definitionClass = definitionClass;
+ this.attributeName = attributeName;
+ this.htmlDocumentation = htmlDocumentation;
+ this.startLineCnt = startLineCnt;
+ this.flags = flags;
+ this.commonType = commonType;
+ }
+
+ /**
+ * Returns the name of the rule attribute.
+ */
+ String getAttributeName() {
+ return attributeName;
+ }
+
+ /**
+ * Returns the raw html documentation of the rule attribute.
+ */
+ String getHtmlDocumentation(Attribute attribute) {
+ // TODO(bazel-team): this is needed for common type attributes. Fix those and remove this.
+ if (attribute == null) {
+ return htmlDocumentation;
+ }
+ StringBuilder sb = new StringBuilder()
+ .append("<i>(")
+ .append(TYPE_DESC.get(attribute.getType()))
+ .append("; " + (attribute.isMandatory() ? "required" : "optional"))
+ .append(getDefaultValue(attribute))
+ .append(")</i><br/>\n");
+ return htmlDocumentation.replace("${" + DocgenConsts.VAR_SYNOPSIS + "}", sb.toString());
+ }
+
+ private String getDefaultValue(Attribute attribute) {
+ String prefix = "; default is ";
+ Object value = attribute.getDefaultValueForTesting();
+ if (value instanceof Boolean) {
+ return prefix + ((Boolean) value ? "1" : "0");
+ } else if (value instanceof Integer) {
+ return prefix + String.valueOf(value);
+ } else if (value instanceof String && !(((String) value).isEmpty())) {
+ return prefix + "\"" + value + "\"";
+ } else if (value instanceof TriState) {
+ switch((TriState) value) {
+ case AUTO:
+ return prefix + "-1";
+ case NO:
+ return prefix + "0";
+ case YES:
+ return prefix + "1";
+ }
+ } else if (value instanceof Label) {
+ return prefix + "<code>" + value + "</code>";
+ }
+ return "";
+ }
+
+ /**
+ * Returns the number of first line of the attribute documentation in its declaration file.
+ */
+ int getStartLineCnt() {
+ return startLineCnt;
+ }
+
+ /**
+ * Returns true if the attribute doc is of a common attribute type.
+ */
+ boolean isCommonType() {
+ return commonType != null;
+ }
+
+ /**
+ * Returns the common attribute type if this attribute doc is of a common type
+ * otherwise actualRule.
+ */
+ String getGeneratedInRule(String actualRule) {
+ return isCommonType() ? commonType : actualRule;
+ }
+
+ /**
+ * Returns true if this attribute documentation has the parameter flag.
+ */
+ boolean hasFlag(String flag) {
+ return flags.contains(flag);
+ }
+
+ /**
+ * Returns the length of a shortest path from usingClass to the definitionClass of this
+ * RuleDocumentationAttribute in the rule definition ancestry graph. Returns -1
+ * if definitionClass is not the ancestor (transitively) of usingClass.
+ */
+ int getDefinitionClassAncestryLevel(Class<? extends RuleDefinition> usingClass) {
+ if (usingClass.equals(definitionClass)) {
+ return 0;
+ }
+ // Storing nodes (rule class definitions) with the length of the shortest path from usingClass
+ Map<Class<? extends RuleDefinition>, Integer> visited = new HashMap<>();
+ LinkedList<Class<? extends RuleDefinition>> toVisit = new LinkedList<>();
+ visited.put(usingClass, 0);
+ toVisit.add(usingClass);
+ // Searching the shortest path from usingClass to this.definitionClass using BFS
+ do {
+ Class<? extends RuleDefinition> ancestor = toVisit.removeFirst();
+ visitAncestor(ancestor, visited, toVisit);
+ if (ancestor.equals(definitionClass)) {
+ return visited.get(ancestor);
+ }
+ } while (!toVisit.isEmpty());
+ return -1;
+ }
+
+ private void visitAncestor(
+ Class<? extends RuleDefinition> usingClass,
+ Map<Class<? extends RuleDefinition>, Integer> visited,
+ LinkedList<Class<? extends RuleDefinition>> toVisit) {
+ BlazeRule ann = usingClass.getAnnotation(BlazeRule.class);
+ if (ann != null) {
+ for (Class<? extends RuleDefinition> ancestor : ann.ancestors()) {
+ if (!visited.containsKey(ancestor)) {
+ toVisit.addLast(ancestor);
+ visited.put(ancestor, visited.get(usingClass) + 1);
+ }
+ }
+ }
+ }
+
+ private int getAttributeOrderingPriority(RuleDocumentationAttribute attribute) {
+ if (DocgenConsts.ATTRIBUTE_ORDERING.containsKey(attribute.attributeName)) {
+ return DocgenConsts.ATTRIBUTE_ORDERING.get(attribute.attributeName);
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public int compareTo(RuleDocumentationAttribute o) {
+ int thisPriority = getAttributeOrderingPriority(this);
+ int otherPriority = getAttributeOrderingPriority(o);
+ if (thisPriority > otherPriority) {
+ return 1;
+ } else if (thisPriority < otherPriority) {
+ return -1;
+ } else {
+ return this.attributeName.compareTo(o.attributeName);
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RuleDocumentationAttribute)) {
+ return false;
+ }
+ return attributeName.equals(((RuleDocumentationAttribute) obj).attributeName);
+ }
+
+ @Override
+ public int hashCode() {
+ return attributeName.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentationVariable.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationVariable.java
new file mode 100644
index 0000000..21cf9ec
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationVariable.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+/**
+ * Rule documentation variables for modular rule documentation, e.g.
+ * separate section for Implicit outputs.
+ */
+public class RuleDocumentationVariable {
+
+ private String ruleName;
+ private String variableName;
+ private String value;
+ private int startLineCnt;
+
+ public RuleDocumentationVariable(
+ String ruleName, String variableName, String value, int startLineCnt) {
+ this.ruleName = ruleName;
+ this.variableName = variableName;
+ this.value = value;
+ this.startLineCnt = startLineCnt;
+ }
+
+ public String getRuleName() {
+ return ruleName;
+ }
+
+ public String getVariableName() {
+ return variableName;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public int getStartLineCnt() {
+ return startLineCnt;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java
new file mode 100644
index 0000000..c4f8cf3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+
+/**
+ * The main class for the skylark documentation generator.
+ */
+public class SkylarkDocumentationGenerator {
+
+ private static boolean checkArgs(String[] args) {
+ if (args.length < 1) {
+ System.err.println("There has to be one input parameter\n"
+ + " - an output file.");
+ return false;
+ }
+ return true;
+ }
+
+ private static void fail(Throwable e, boolean printStackTrace) {
+ System.err.println("ERROR: " + e.getMessage());
+ if (printStackTrace) {
+ e.printStackTrace();
+ }
+ Runtime.getRuntime().exit(1);
+ }
+
+ public static void main(String[] args) {
+ if (checkArgs(args)) {
+ System.out.println("Generating Skylark documentation...");
+ SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor();
+ try {
+ processor.generateDocumentation(args[0]);
+ } catch (Throwable e) {
+ fail(e, true);
+ }
+ System.out.println("Finished.");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java
new file mode 100644
index 0000000..d2208ea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java
@@ -0,0 +1,437 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkBuiltinMethod;
+import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkJavaMethod;
+import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkModuleDoc;
+import com.google.devtools.build.lib.packages.MethodLibrary;
+import com.google.devtools.build.lib.rules.SkylarkModules;
+import com.google.devtools.build.lib.rules.SkylarkRuleContext;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Environment.NoneType;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+/**
+ * A class to assemble documentation for Skylark.
+ */
+public class SkylarkDocumentationProcessor {
+
+ private static final String TOP_LEVEL_ID = "_top_level";
+
+ private static final boolean USE_TEMPLATE = false;
+
+ @SkylarkModule(name = "Global objects, functions and modules",
+ doc = "Objects, functions and modules registered in the global environment.")
+ private static final class TopLevelModule {}
+
+ static SkylarkModule getTopLevelModule() {
+ return TopLevelModule.class.getAnnotation(SkylarkModule.class);
+ }
+
+ /**
+ * Generates the Skylark documentation to the given output directory.
+ */
+ public void generateDocumentation(String outputPath) throws IOException,
+ BuildEncyclopediaDocException {
+ BufferedWriter bw = null;
+ File skylarkDocPath = new File(outputPath);
+ try {
+ bw = new BufferedWriter(new FileWriter(skylarkDocPath));
+ if (USE_TEMPLATE) {
+ bw.write(SourceFileReader.readTemplateContents(DocgenConsts.SKYLARK_BODY_TEMPLATE,
+ ImmutableMap.<String, String>of(
+ DocgenConsts.VAR_SECTION_SKYLARK_BUILTIN, generateAllBuiltinDoc())));
+ } else {
+ bw.write(generateAllBuiltinDoc());
+ }
+ System.out.println("Skylark documentation generated: " + skylarkDocPath.getAbsolutePath());
+ } finally {
+ if (bw != null) {
+ bw.close();
+ }
+ }
+ }
+
+ @VisibleForTesting
+ Map<String, SkylarkModuleDoc> collectModules() {
+ Map<String, SkylarkModuleDoc> modules = new TreeMap<>();
+ Map<String, SkylarkModuleDoc> builtinModules = collectBuiltinModules();
+ Map<SkylarkModule, Class<?>> builtinJavaObjects = collectBuiltinJavaObjects();
+
+ modules.putAll(builtinModules);
+ SkylarkJavaInterfaceExplorer explorer = new SkylarkJavaInterfaceExplorer();
+ for (SkylarkModuleDoc builtinObject : builtinModules.values()) {
+ // Check the return type for built-in functions, it can be a module previously not added.
+ for (SkylarkBuiltinMethod builtinMethod : builtinObject.getBuiltinMethods().values()) {
+ Class<?> type = builtinMethod.annotation.returnType();
+ if (type.isAnnotationPresent(SkylarkModule.class)) {
+ explorer.collect(type.getAnnotation(SkylarkModule.class), type, modules);
+ }
+ }
+ explorer.collect(builtinObject.getAnnotation(), builtinObject.getClassObject(), modules);
+ }
+ for (Entry<SkylarkModule, Class<?>> builtinModule : builtinJavaObjects.entrySet()) {
+ explorer.collect(builtinModule.getKey(), builtinModule.getValue(), modules);
+ }
+ return modules;
+ }
+
+ private String generateAllBuiltinDoc() {
+ Map<String, SkylarkModuleDoc> modules = collectModules();
+
+ StringBuilder sb = new StringBuilder();
+ // Generate the top level module first in the doc
+ SkylarkModuleDoc topLevelModule = modules.remove(getTopLevelModule().name());
+ generateModuleDoc(topLevelModule, sb);
+ for (SkylarkModuleDoc module : modules.values()) {
+ if (!module.getAnnotation().hidden()) {
+ sb.append("<hr>");
+ generateModuleDoc(module, sb);
+ }
+ }
+ return sb.toString();
+ }
+
+ private void generateModuleDoc(SkylarkModuleDoc module, StringBuilder sb) {
+ SkylarkModule annotation = module.getAnnotation();
+ sb.append(String.format("<h2 id=\"modules.%s\">%s</h2>\n",
+ getModuleId(annotation),
+ annotation.name()))
+ .append(annotation.doc())
+ .append("\n");
+ sb.append("<ul>");
+ // Sort Java and SkylarkBuiltin methods together. The map key is only used for sorting.
+ TreeMap<String, Object> methodMap = new TreeMap<>();
+ for (SkylarkJavaMethod method : module.getJavaMethods()) {
+ methodMap.put(method.name + method.method.getParameterTypes().length, method);
+ }
+ for (SkylarkBuiltinMethod builtin : module.getBuiltinMethods().values()) {
+ methodMap.put(builtin.annotation.name(), builtin);
+ }
+ for (Object object : methodMap.values()) {
+ if (object instanceof SkylarkJavaMethod) {
+ SkylarkJavaMethod method = (SkylarkJavaMethod) object;
+ generateDirectJavaMethodDoc(annotation.name(), method.name, method.method,
+ method.callable, sb);
+ }
+ if (object instanceof SkylarkBuiltinMethod) {
+ generateBuiltinItemDoc(getModuleId(annotation), (SkylarkBuiltinMethod) object, sb);
+ }
+ }
+ sb.append("</ul>");
+ }
+
+ private String getModuleId(SkylarkModule annotation) {
+ if (annotation == getTopLevelModule()) {
+ return TOP_LEVEL_ID;
+ } else {
+ return annotation.name();
+ }
+ }
+
+ private void generateBuiltinItemDoc(
+ String moduleId, SkylarkBuiltinMethod method, StringBuilder sb) {
+ SkylarkBuiltin annotation = method.annotation;
+ if (annotation.hidden()) {
+ return;
+ }
+ sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n",
+ moduleId,
+ annotation.name(),
+ annotation.name()));
+
+ if (com.google.devtools.build.lib.syntax.Function.class.isAssignableFrom(method.fieldClass)) {
+ sb.append(getSignature(moduleId, annotation));
+ } else {
+ if (!annotation.returnType().equals(Object.class)) {
+ sb.append("<code>" + getTypeAnchor(annotation.returnType()) + "</code><br>");
+ }
+ }
+
+ sb.append(annotation.doc() + "\n");
+ printParams(moduleId, annotation, sb);
+ }
+
+ private void printParams(String moduleId, SkylarkBuiltin annotation, StringBuilder sb) {
+ if (annotation.mandatoryParams().length + annotation.optionalParams().length > 0) {
+ sb.append("<h4>Parameters</h4>\n");
+ printParams(moduleId, annotation.name(), annotation.mandatoryParams(), sb);
+ printParams(moduleId, annotation.name(), annotation.optionalParams(), sb);
+ } else {
+ sb.append("<br/>\n");
+ }
+ }
+
+ private void generateDirectJavaMethodDoc(String objectName, String methodName,
+ Method method, SkylarkCallable annotation, StringBuilder sb) {
+ if (annotation.hidden()) {
+ return;
+ }
+
+ sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n%s\n",
+ objectName,
+ methodName,
+ methodName,
+ getSignature(objectName, methodName, method)))
+ .append(annotation.doc())
+ .append(getReturnTypeExtraMessage(annotation))
+ .append("\n");
+ }
+
+ private String getReturnTypeExtraMessage(SkylarkCallable annotation) {
+ if (annotation.allowReturnNones()) {
+ return " May return <code>None</code>.\n";
+ }
+ return "";
+ }
+
+ private String getSignature(String objectName, String methodName, Method method) {
+ String args = method.getAnnotation(SkylarkCallable.class).structField()
+ ? "" : "(" + getParameterString(method) + ")";
+
+ return String.format("<code>%s %s.%s%s</code><br>",
+ getTypeAnchor(method.getReturnType()), objectName, methodName, args);
+ }
+
+ private String getSignature(String objectName, SkylarkBuiltin method) {
+ List<String> argList = new ArrayList<>();
+ for (Param param : method.mandatoryParams()) {
+ argList.add(param.name());
+ }
+ for (Param param : method.optionalParams()) {
+ argList.add(param.name() + "?");
+ }
+ String args = "(" + Joiner.on(", ").join(argList) + ")";
+ if (!objectName.equals(TOP_LEVEL_ID)) {
+ return String.format("<code>%s %s.%s%s</code><br>\n",
+ getTypeAnchor(method.returnType()), objectName, method.name(), args);
+ } else {
+ return String.format("<code>%s %s%s</code><br>\n",
+ getTypeAnchor(method.returnType()), method.name(), args);
+ }
+ }
+
+ private String getTypeAnchor(Class<?> returnType, Class<?> generic1) {
+ return getTypeAnchor(returnType) + " of " + getTypeAnchor(generic1) + "s";
+ }
+
+ private String getTypeAnchor(Class<?> returnType) {
+ if (returnType.equals(String.class)) {
+ return "<a class=\"anchor\" href=\"#modules.string\">string</a>";
+ } else if (Map.class.isAssignableFrom(returnType)) {
+ return "<a class=\"anchor\" href=\"#modules.dict\">dict</a>";
+ } else if (returnType.equals(Void.TYPE) || returnType.equals(NoneType.class)) {
+ return "<a class=\"anchor\" href=\"#modules." + TOP_LEVEL_ID + ".None\">None</a>";
+ } else if (returnType.isAnnotationPresent(SkylarkModule.class)) {
+ // TODO(bazel-team): this can produce dead links for types don't show up in the doc.
+ // The correct fix is to generate those types (e.g. SkylarkFileType) too.
+ String module = returnType.getAnnotation(SkylarkModule.class).name();
+ return "<a class=\"anchor\" href=\"#modules." + module + "\">" + module + "</a>";
+ } else {
+ return EvalUtils.getDataTypeNameFromClass(returnType);
+ }
+ }
+
+ private String getParameterString(Method method) {
+ return Joiner.on(", ").join(Iterables.transform(
+ ImmutableList.copyOf(method.getParameterTypes()), new Function<Class<?>, String>() {
+ @Override
+ public String apply(Class<?> input) {
+ return getTypeAnchor(input);
+ }
+ }));
+ }
+
+ private void printParams(String moduleId, String methodName,
+ Param[] params, StringBuilder sb) {
+ if (params.length > 0) {
+ sb.append("<ul>\n");
+ for (Param param : params) {
+ String paramType = param.type().equals(Object.class) ? ""
+ : (param.generic1().equals(Object.class)
+ ? " (" + getTypeAnchor(param.type()) + ")"
+ : " (" + getTypeAnchor(param.type(), param.generic1()) + ")");
+ sb.append(String.format("\t<li id=\"modules.%s.%s.%s\"><code>%s%s</code>: ",
+ moduleId,
+ methodName,
+ param.name(),
+ param.name(),
+ paramType))
+ .append(param.doc())
+ .append("\n\t</li>\n");
+ }
+ sb.append("</ul>\n");
+ }
+ }
+
+ private Map<String, SkylarkModuleDoc> collectBuiltinModules() {
+ Map<String, SkylarkModuleDoc> modules = new HashMap<>();
+ collectBuiltinDoc(modules, Environment.class.getDeclaredFields());
+ collectBuiltinDoc(modules, MethodLibrary.class.getDeclaredFields());
+ for (Class<?> moduleClass : SkylarkModules.MODULES) {
+ collectBuiltinDoc(modules, moduleClass.getDeclaredFields());
+ }
+ return modules;
+ }
+
+ private Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects() {
+ Map<SkylarkModule, Class<?>> modules = new HashMap<>();
+ collectBuiltinModule(modules, SkylarkRuleContext.class);
+ return modules;
+ }
+
+ /**
+ * Returns the top level modules and functions with their documentation in a command-line
+ * printable format.
+ */
+ public Map<String, String> collectTopLevelModules() {
+ Map<String, String> modules = new TreeMap<>();
+ for (SkylarkModuleDoc doc : collectBuiltinModules().values()) {
+ if (doc.getAnnotation() == getTopLevelModule()) {
+ for (Map.Entry<String, SkylarkBuiltinMethod> entry : doc.getBuiltinMethods().entrySet()) {
+ if (!entry.getValue().annotation.hidden()) {
+ modules.put(entry.getKey(),
+ DocgenConsts.toCommandLineFormat(entry.getValue().annotation.doc()));
+ }
+ }
+ } else {
+ modules.put(doc.getAnnotation().name(),
+ DocgenConsts.toCommandLineFormat(doc.getAnnotation().doc()));
+ }
+ }
+ return modules;
+ }
+
+ /**
+ * Returns the API doc for the specified Skylark object in a command line printable format,
+ * params[0] identifies either a module or a top-level object, the optional params[1] identifies a
+ * method in the module.<br>
+ * Returns null if no Skylark object is found.
+ */
+ public String getCommandLineAPIDoc(String[] params) {
+ Map<String, SkylarkModuleDoc> modules = collectModules();
+ SkylarkModuleDoc toplevelModuleDoc = modules.get(getTopLevelModule().name());
+ if (modules.containsKey(params[0])) {
+ // Top level module
+ SkylarkModuleDoc module = modules.get(params[0]);
+ if (params.length == 1) {
+ String moduleName = module.getAnnotation().name();
+ StringBuilder sb = new StringBuilder();
+ sb.append(moduleName).append("\n\t").append(module.getAnnotation().doc()).append("\n");
+ // Print the signature of all built-in methods
+ for (SkylarkBuiltinMethod method : module.getBuiltinMethods().values()) {
+ printBuiltinFunctionDoc(moduleName, method.annotation, sb);
+ }
+ // Print all Java methods
+ for (SkylarkJavaMethod method : module.getJavaMethods()) {
+ printJavaFunctionDoc(moduleName, method, sb);
+ }
+ return DocgenConsts.toCommandLineFormat(sb.toString());
+ } else {
+ return getFunctionDoc(module.getAnnotation().name(), params[1], module);
+ }
+ } else if (toplevelModuleDoc.getBuiltinMethods().containsKey(params[0])){
+ // Top level object / function
+ return getFunctionDoc(null, params[0], toplevelModuleDoc);
+ }
+ return null;
+ }
+
+ private String getFunctionDoc(String moduleName, String methodName, SkylarkModuleDoc module) {
+ if (module.getBuiltinMethods().containsKey(methodName)) {
+ // Create the doc for the built-in function
+ SkylarkBuiltinMethod method = module.getBuiltinMethods().get(methodName);
+ StringBuilder sb = new StringBuilder();
+ printBuiltinFunctionDoc(moduleName, method.annotation, sb);
+ printParams(moduleName, method.annotation, sb);
+ return DocgenConsts.removeDuplicatedNewLines(DocgenConsts.toCommandLineFormat(sb.toString()));
+ } else {
+ // Search if there are matching Java functions
+ StringBuilder sb = new StringBuilder();
+ boolean foundMatchingMethod = false;
+ for (SkylarkJavaMethod method : module.getJavaMethods()) {
+ if (method.name.equals(methodName)) {
+ printJavaFunctionDoc(moduleName, method, sb);
+ foundMatchingMethod = true;
+ }
+ }
+ if (foundMatchingMethod) {
+ return DocgenConsts.toCommandLineFormat(sb.toString());
+ }
+ }
+ return null;
+ }
+
+ private void printBuiltinFunctionDoc(
+ String moduleName, SkylarkBuiltin annotation, StringBuilder sb) {
+ if (moduleName != null) {
+ sb.append(moduleName).append(".");
+ }
+ sb.append(annotation.name()).append("\n\t").append(annotation.doc()).append("\n");
+ }
+
+ private void printJavaFunctionDoc(String moduleName, SkylarkJavaMethod method, StringBuilder sb) {
+ sb.append(getSignature(moduleName, method.name, method.method))
+ .append("\t").append(method.callable.doc()).append("\n");
+ }
+
+ private void collectBuiltinModule(
+ Map<SkylarkModule, Class<?>> modules, Class<?> moduleClass) {
+ if (moduleClass.isAnnotationPresent(SkylarkModule.class)) {
+ SkylarkModule skylarkModule = moduleClass.getAnnotation(SkylarkModule.class);
+ modules.put(skylarkModule, moduleClass);
+ }
+ }
+
+ private void collectBuiltinDoc(Map<String, SkylarkModuleDoc> modules, Field[] fields) {
+ for (Field field : fields) {
+ if (field.isAnnotationPresent(SkylarkBuiltin.class)) {
+ SkylarkBuiltin skylarkBuiltin = field.getAnnotation(SkylarkBuiltin.class);
+ Class<?> moduleClass = skylarkBuiltin.objectType();
+ SkylarkModule skylarkModule = moduleClass.equals(Object.class)
+ ? getTopLevelModule()
+ : moduleClass.getAnnotation(SkylarkModule.class);
+ if (!modules.containsKey(skylarkModule.name())) {
+ modules.put(skylarkModule.name(), new SkylarkModuleDoc(skylarkModule, moduleClass));
+ }
+ modules.get(skylarkModule.name()).getBuiltinMethods()
+ .put(skylarkBuiltin.name(), new SkylarkBuiltinMethod(skylarkBuiltin, field.getType()));
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/SkylarkJavaInterfaceExplorer.java b/src/main/java/com/google/devtools/build/docgen/SkylarkJavaInterfaceExplorer.java
new file mode 100644
index 0000000..8f58f71
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/SkylarkJavaInterfaceExplorer.java
@@ -0,0 +1,160 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.util.StringUtilities;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * A helper class to collect all the Java objects / methods reachable from Skylark.
+ */
+public class SkylarkJavaInterfaceExplorer {
+ /**
+ * A class representing a Java method callable from Skylark with annotation.
+ */
+ static final class SkylarkJavaMethod {
+ public final String name;
+ public final Method method;
+ public final SkylarkCallable callable;
+
+ private String getName(Method method, SkylarkCallable callable) {
+ return callable.name().isEmpty()
+ ? StringUtilities.toPythonStyleFunctionName(method.getName())
+ : callable.name();
+ }
+
+ SkylarkJavaMethod(Method method, SkylarkCallable callable) {
+ this.name = getName(method, callable);
+ this.method = method;
+ this.callable = callable;
+ }
+ }
+
+ /**
+ * A class representing a Skylark built-in object or method.
+ */
+ static final class SkylarkBuiltinMethod {
+ public final SkylarkBuiltin annotation;
+ public final Class<?> fieldClass;
+
+ public SkylarkBuiltinMethod(SkylarkBuiltin annotation, Class<?> fieldClass) {
+ this.annotation = annotation;
+ this.fieldClass = fieldClass;
+ }
+ }
+
+ /**
+ * A class representing a Skylark built-in object with its {@link SkylarkBuiltin} annotation
+ * and the {@link SkylarkCallable} methods it might have.
+ */
+ static final class SkylarkModuleDoc {
+
+ private final SkylarkModule module;
+ private final Class<?> classObject;
+ private final Map<String, SkylarkBuiltinMethod> builtin;
+ private ArrayList<SkylarkJavaMethod> methods = null;
+
+ SkylarkModuleDoc(SkylarkModule module, Class<?> classObject) {
+ this.module = Preconditions.checkNotNull(module,
+ "Class has to be annotated with SkylarkModule: " + classObject);
+ this.classObject = classObject;
+ this.builtin = new TreeMap<>();
+ }
+
+ SkylarkModule getAnnotation() {
+ return module;
+ }
+
+ Class<?> getClassObject() {
+ return classObject;
+ }
+
+ private boolean javaMethodsNotCollected() {
+ return methods == null;
+ }
+
+ private void setJavaMethods(ArrayList<SkylarkJavaMethod> methods) {
+ this.methods = methods;
+ }
+
+ Map<String, SkylarkBuiltinMethod> getBuiltinMethods() {
+ return builtin;
+ }
+
+ ArrayList<SkylarkJavaMethod> getJavaMethods() {
+ return methods;
+ }
+ }
+
+ /**
+ * Collects and returns all the Java objects reachable in Skylark from (and including)
+ * firstClassObject with the corresponding SkylarkBuiltin annotations.
+ *
+ * <p>Note that the {@link SkylarkBuiltin} annotation for firstClassObject - firstAnnotation -
+ * is also an input parameter, because some top level Skylark built-in objects and methods
+ * are not annotated on the class, but on a field referencing them.
+ */
+ void collect(SkylarkModule firstModule, Class<?> firstClass,
+ Map<String, SkylarkModuleDoc> modules) {
+ Set<Class<?>> processedClasses = new HashSet<>();
+ LinkedList<Class<?>> classesToProcess = new LinkedList<>();
+ Map<Class<?>, SkylarkModule> annotations = new HashMap<>();
+
+ classesToProcess.addLast(firstClass);
+ annotations.put(firstClass, firstModule);
+
+ while (!classesToProcess.isEmpty()) {
+ Class<?> classObject = classesToProcess.removeFirst();
+ SkylarkModule annotation = annotations.get(classObject);
+ processedClasses.add(classObject);
+ if (!modules.containsKey(annotation.name())) {
+ modules.put(annotation.name(), new SkylarkModuleDoc(annotation, classObject));
+ }
+ SkylarkModuleDoc module = modules.get(annotation.name());
+
+ if (module.javaMethodsNotCollected()) {
+ ImmutableMap<Method, SkylarkCallable> methods =
+ FuncallExpression.collectSkylarkMethodsWithAnnotation(classObject);
+ ArrayList<SkylarkJavaMethod> methodList = new ArrayList<>();
+ for (Map.Entry<Method, SkylarkCallable> entry : methods.entrySet()) {
+ methodList.add(new SkylarkJavaMethod(entry.getKey(), entry.getValue()));
+ }
+ module.setJavaMethods(methodList);
+
+ for (Map.Entry<Method, SkylarkCallable> method : methods.entrySet()) {
+ Class<?> returnClass = method.getKey().getReturnType();
+ if (returnClass.isAnnotationPresent(SkylarkModule.class)
+ && !processedClasses.contains(returnClass)) {
+ classesToProcess.addLast(returnClass);
+ annotations.put(returnClass, returnClass.getAnnotation(SkylarkModule.class));
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/SourceFileReader.java b/src/main/java/com/google/devtools/build/docgen/SourceFileReader.java
new file mode 100644
index 0000000..c17b3f6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/SourceFileReader.java
@@ -0,0 +1,322 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+
+/**
+ * A helper class to read and process documentations for rule classes and attributes
+ * from exactly one java source file.
+ */
+public class SourceFileReader {
+
+ private Collection<RuleDocumentation> ruleDocEntries;
+ private ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries;
+ private final ConfiguredRuleClassProvider ruleClassProvider;
+ private final String javaSourceFilePath;
+
+ public SourceFileReader(
+ ConfiguredRuleClassProvider ruleClassProvider, String javaSourceFilePath) {
+ this.ruleClassProvider = ruleClassProvider;
+ this.javaSourceFilePath = javaSourceFilePath;
+ }
+
+ /**
+ * The handler class of the line read from the text file.
+ */
+ public abstract static class ReadAction {
+
+ // Text file line indexing starts from 1
+ private int lineCnt = 1;
+
+ protected abstract void readLineImpl(String line)
+ throws BuildEncyclopediaDocException, IOException;
+
+ protected int getLineCnt() {
+ return lineCnt;
+ }
+
+ public void readLine(String line)
+ throws BuildEncyclopediaDocException, IOException {
+ readLineImpl(line);
+ lineCnt++;
+ }
+ }
+
+ private static final String LS = DocgenConsts.LS;
+
+ /**
+ * Reads the attribute and rule documentation present in the file represented by
+ * SourceFileReader.javaSourceFilePath. The rule doc variables are added to the rule
+ * documentation (which therefore must be defined in the same file). The attribute docs are
+ * stored in a different class member, so they need to be handled outside this method.
+ */
+ public void readDocsFromComments() throws BuildEncyclopediaDocException, IOException {
+ final Map<String, RuleDocumentation> docMap = new HashMap<>();
+ final List<RuleDocumentationVariable> docVariables = new LinkedList<>();
+ final ListMultimap<String, RuleDocumentationAttribute> docAttributes =
+ LinkedListMultimap.create();
+ readTextFile(javaSourceFilePath, new ReadAction() {
+
+ private boolean inBlazeRuleDocs = false;
+ private boolean inBlazeRuleVarDocs = false;
+ private boolean inBlazeAttributeDocs = false;
+ private StringBuilder sb = new StringBuilder();
+ private String ruleName;
+ private String ruleType;
+ private String ruleFamily;
+ private String variableName;
+ private String attributeName;
+ private ImmutableSet<String> flags;
+ private int startLineCnt;
+
+ @Override
+ public void readLineImpl(String line) throws BuildEncyclopediaDocException {
+ // TODO(bazel-team): check if copy paste code can be reduced using inner classes
+ if (inBlazeRuleDocs) {
+ if (DocgenConsts.BLAZE_RULE_END.matcher(line).matches()) {
+ endBlazeRuleDoc(docMap);
+ } else {
+ appendLine(line);
+ }
+ } else if (inBlazeRuleVarDocs) {
+ if (DocgenConsts.BLAZE_RULE_VAR_END.matcher(line).matches()) {
+ endBlazeRuleVarDoc(docVariables);
+ } else {
+ appendLine(line);
+ }
+ } else if (inBlazeAttributeDocs) {
+ if (DocgenConsts.BLAZE_RULE_ATTR_END.matcher(line).matches()) {
+ endBlazeAttributeDoc(docAttributes);
+ } else {
+ appendLine(line);
+ }
+ }
+ Matcher ruleStartMatcher = DocgenConsts.BLAZE_RULE_START.matcher(line);
+ Matcher ruleVarStartMatcher = DocgenConsts.BLAZE_RULE_VAR_START.matcher(line);
+ Matcher ruleAttrStartMatcher = DocgenConsts.BLAZE_RULE_ATTR_START.matcher(line);
+ if (ruleStartMatcher.find()) {
+ startBlazeRuleDoc(line, ruleStartMatcher);
+ } else if (ruleVarStartMatcher.find()) {
+ startBlazeRuleVarDoc(ruleVarStartMatcher);
+ } else if (ruleAttrStartMatcher.find()) {
+ startBlazeAttributeDoc(line, ruleAttrStartMatcher);
+ }
+ }
+
+ private void appendLine(String line) {
+ // Add another line of html code to the building rule documentation
+ // Removing whitespace and java comment asterisk from the beginning of the line
+ sb.append(line.replaceAll("^[\\s]*\\*", "") + LS);
+ }
+
+ private void startBlazeRuleDoc(String line, Matcher matcher)
+ throws BuildEncyclopediaDocException {
+ checkDocValidity();
+ // Start of a new rule
+ String[] metaData = matcher.group(1).split(",");
+
+ ruleName = readMetaData(metaData, DocgenConsts.META_KEY_NAME);
+ ruleType = readMetaData(metaData, DocgenConsts.META_KEY_TYPE);
+ ruleFamily = readMetaData(metaData, DocgenConsts.META_KEY_FAMILY);
+ startLineCnt = getLineCnt();
+ addFlags(line);
+ inBlazeRuleDocs = true;
+ }
+
+ private void endBlazeRuleDoc(final Map<String, RuleDocumentation> documentations)
+ throws BuildEncyclopediaDocException {
+ // End of a rule, create RuleDocumentation object
+ documentations.put(ruleName, new RuleDocumentation(ruleName, ruleType,
+ ruleFamily, sb.toString(), getLineCnt(), javaSourceFilePath, flags,
+ ruleClassProvider));
+ sb = new StringBuilder();
+ inBlazeRuleDocs = false;
+ }
+
+ private void startBlazeRuleVarDoc(Matcher matcher) throws BuildEncyclopediaDocException {
+ checkDocValidity();
+ // Start of a new rule variable
+ ruleName = matcher.group(1).replaceAll("[\\s]", "");
+ variableName = matcher.group(2).replaceAll("[\\s]", "");
+ startLineCnt = getLineCnt();
+ inBlazeRuleVarDocs = true;
+ }
+
+ private void endBlazeRuleVarDoc(final List<RuleDocumentationVariable> docVariables) {
+ // End of a rule, create RuleDocumentationVariable object
+ docVariables.add(
+ new RuleDocumentationVariable(ruleName, variableName, sb.toString(), startLineCnt));
+ sb = new StringBuilder();
+ inBlazeRuleVarDocs = false;
+ }
+
+ private void startBlazeAttributeDoc(String line, Matcher matcher)
+ throws BuildEncyclopediaDocException {
+ checkDocValidity();
+ // Start of a new attribute
+ ruleName = matcher.group(1).replaceAll("[\\s]", "");
+ attributeName = matcher.group(2).replaceAll("[\\s]", "");
+ startLineCnt = getLineCnt();
+ addFlags(line);
+ inBlazeAttributeDocs = true;
+ }
+
+ private void endBlazeAttributeDoc(
+ final ListMultimap<String, RuleDocumentationAttribute> docAttributes) {
+ // End of a attribute, create RuleDocumentationAttribute object
+ docAttributes.put(attributeName, RuleDocumentationAttribute.create(
+ ruleClassProvider.getRuleClassDefinition(ruleName),
+ attributeName, sb.toString(), startLineCnt, flags));
+ sb = new StringBuilder();
+ inBlazeAttributeDocs = false;
+ }
+
+ private void addFlags(String line) {
+ // Add flags if there's any
+ Matcher matcher = DocgenConsts.BLAZE_RULE_FLAGS.matcher(line);
+ if (matcher.find()) {
+ flags = ImmutableSet.<String>copyOf(matcher.group(1).split(","));
+ } else {
+ flags = ImmutableSet.<String>of();
+ }
+ }
+
+ private void checkDocValidity() throws BuildEncyclopediaDocException {
+ if (inBlazeRuleDocs || inBlazeRuleVarDocs || inBlazeAttributeDocs) {
+ throw new BuildEncyclopediaDocException(javaSourceFilePath, getLineCnt(),
+ "Malformed documentation, #BLAZE_RULE started after another #BLAZE_RULE.");
+ }
+ }
+ });
+
+ // Adding rule doc variables to the corresponding rules
+ for (RuleDocumentationVariable docVariable : docVariables) {
+ if (docMap.containsKey(docVariable.getRuleName())) {
+ docMap.get(docVariable.getRuleName()).addDocVariable(
+ docVariable.getVariableName(), docVariable.getValue());
+ } else {
+ throw new BuildEncyclopediaDocException(javaSourceFilePath,
+ docVariable.getStartLineCnt(), String.format(
+ "Malformed rule variable #BLAZE_RULE(%s).%s, "
+ + "rule %s not found in file.", docVariable.getRuleName(),
+ docVariable.getVariableName(), docVariable.getRuleName()));
+ }
+ }
+ ruleDocEntries = docMap.values();
+ attributeDocEntries = docAttributes;
+ }
+
+ public Collection<RuleDocumentation> getRuleDocEntries() {
+ return ruleDocEntries;
+ }
+
+ public ListMultimap<String, RuleDocumentationAttribute> getAttributeDocEntries() {
+ return attributeDocEntries;
+ }
+
+ private String readMetaData(String[] metaData, String metaKey) {
+ for (String metaDataItem : metaData) {
+ String[] metaDataItemParts = metaDataItem.split("=", 2);
+ if (metaDataItemParts.length != 2) {
+ return null;
+ }
+
+ if (metaDataItemParts[0].trim().equals(metaKey)) {
+ return metaDataItemParts[1].trim();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Reads the template file without variable substitution.
+ */
+ public static String readTemplateContents(String templateFilePath)
+ throws BuildEncyclopediaDocException, IOException {
+ return readTemplateContents(templateFilePath, null);
+ }
+
+ /**
+ * Reads the template file and expands the variables. The variables has to have
+ * the following format in the template file: ${VARIABLE_NAME}. In the Map
+ * input parameter the key has to be VARIABLE_NAME. Variables can be null.
+ */
+ public static String readTemplateContents(
+ String templateFilePath, final Map<String, String> variables)
+ throws BuildEncyclopediaDocException, IOException {
+ final StringBuilder sb = new StringBuilder();
+ readTextFile(templateFilePath, new ReadAction() {
+ @Override
+ public void readLineImpl(String line) {
+ sb.append(expandVariables(line, variables) + LS);
+ }
+ });
+ return sb.toString();
+ }
+
+ private static String expandVariables(String line, Map<String, String> variables) {
+ if (variables != null) {
+ for (Entry<String, String> variable : variables.entrySet()) {
+ line = line.replace("${" + variable.getKey() + "}", variable.getValue());
+ }
+ }
+ return line;
+ }
+
+ public static void readTextFile(String filePath, ReadAction action)
+ throws BuildEncyclopediaDocException, IOException {
+ BufferedReader br = null;
+ try {
+ File file = new File(filePath);
+ if (file.exists()) {
+ br = new BufferedReader(new FileReader(file));
+ } else {
+ InputStream is = SourceFileReader.class.getResourceAsStream(filePath);
+ if (is != null) {
+ br = new BufferedReader(new InputStreamReader(is));
+ }
+ }
+ if (br != null) {
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ action.readLine(line);
+ }
+ } else {
+ System.out.println("Couldn't find file or resource: " + filePath);
+ }
+ } finally {
+ if (br != null) {
+ br.close();
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-body.html b/src/main/java/com/google/devtools/build/docgen/templates/be-body.html
new file mode 100644
index 0000000..d1fc7ce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/templates/be-body.html
@@ -0,0 +1,313 @@
+<!-- ============================================
+ binary
+ ============================================
+-->
+<h2 id="binary">*_binary</h2>
+
+<p>A <code>*_binary</code> rule compiles an application. This might be
+ an executable, a <code>.jar</code> file, and/or a collection of scripts.</p>
+
+${SECTION_BINARY}
+
+<!-- ============================================
+ library
+ ============================================
+-->
+<h2 id="library">*_library</h2>
+
+<p>A <code>*_library()</code> rule compiles some sources into a library.
+ In general, a <code><var>language</var>_library</code> rule works like
+ the corresponding <code><var>language</var>_binary</code> rule, but
+ doesn't generate something executable.</p>
+
+${SECTION_LIBRARY}
+
+<!-- ============================================
+ test
+ ============================================
+-->
+<h2 id="test">*_test</h2>
+
+<p>A <code>*_test</code> rule compiles a
+test. See <a href="#common-attributes-tests">Common attributes for
+tests</a> for an explanation of the common attributes.
+
+${SECTION_TEST}
+
+<!-- ============================================
+ generate code and data
+ ============================================
+-->
+<h2>Rules to Generate Code and Data</h2>
+
+${SECTION_GENERATE}
+
+<!-- ============================================
+ variables
+ ============================================
+-->
+<h2 id="make_variables">"Make" Variables</h2>
+
+<p>
+ This section describes how to use or define a special class of string
+ variables that are called the "Make" environment. Bazel defines a set of
+ standard "Make" variables, and you can also define your own.
+</p>
+
+<p>(The reason for the term "Make" is historical: the syntax and semantics of
+ these variables are somewhat similar to those of GNU Make, and in the
+ original implementation, were implemented by GNU Make. The
+ scare-quotes are present because newer build tools support
+ "Make" variables without being implemented using GNU Make; therefore
+ it is important to read the specification below carefully to
+ understand the differences.)
+</p>
+
+<p>To see the list of all common "Make" variables and their values,
+ run <code>bazel info --show_make_env</code>.
+</p>
+
+<p>Build rules can introduce additional rule specific variables. One example is
+ the <a href="#genrule.cmd"><code>cmd</code> attribute of a genrule</a>.
+</p>
+
+<h3 id='make-var-substitution'>"Make" variable substitution</h3>
+
+<p>Variables can be referenced in attributes and other variables using either
+ <code>$(FOO)</code> or <code>varref('FOO')</code>, where <code>FOO</code> is
+ the variable name. In the attribute documentation of rules, it is mentioned
+ when an attribute is subject to "Make" variable substitution. For those
+ attributes this means that any substrings of the form <code>$(X)</code>
+ within those attributes will be interpreted as references to the "Make"
+ variable <var>X</var>, and will be replaced by the appropriate value of that
+ variable for the applicable build configuration. The parens may be omitted
+ for variables whose name is a single character.
+</p>
+<p>
+ It is an error if such attributes contain embedded strings of the
+ form <code>$(X)</code> where <var>X</var> is not the name of a
+ "Make" variable, or unclosed references such as <code>$(</code> not
+ matched by a corresponding <code>)</code>.
+</p>
+<p>
+ Within such attributes, literal dollar signs must be escaped
+ as <code>$$</code> to prevent this expansion.
+</p>
+<p>
+ Those attributes that are subject to this substitution are
+ explicitly indicated as such in their definitions in this document.
+</p>
+
+<h3 id="predefined_variables">Predefined "Make" Variables</h3>
+
+<p>Bazel defines a set of "Make" variables for you.</p>
+
+<p>The build system also provides a consistent PATH for genrules and tests which
+ need to execute shell commands. For genrules, you can indirect your commands
+ using the Make variables below. For basic Unix utilities, prefer relying on
+ the PATH variable to guarantee correct results. For genrules involving
+ compiler and platform invocation, you must use the Make variable syntax.
+ The same basic command set is also available during tests. Simply rely on the
+ PATH.</p>
+
+<p>Bazel uses a tiny Unix distribution to guarantee consistent behavior of
+ build and test steps which execute shell code across all build execution
+ hosting environments but it does not enforce a pure chroot. As such, do
+ <b>not</b> use hard coded paths, such as
+ <code>/usr/bin/foo</code>. Binaries referenced in hardcoded paths are not
+ hermetic and can lead to unexpected and non-reproducible build behavior.</p>
+
+<p><strong>Command Variables for genrules</strong></p>
+
+<p>Note that in general, you should simply refer to many common utilities as
+bare commands that the $PATH variable will correctly resolve to hermetic
+versions for you.</p>
+
+<p><strong>Path Variables</strong></p>
+
+<ul><!-- keep alphabetically sorted -->
+ <li><code>BINDIR</code>: The base of the generated binary tree for the target
+ architecture. (Note that a different tree may be used for
+ programs that run during the build on the host architecture,
+ to support cross-compiling. If you want to run a tool from
+ within a genrule, the recommended way of specifying the path to
+ the tool is to use <code>$(location <i>toolname</i>)</code>,
+ where <i>toolname</i> must be listed in the <code>tools</code>
+ attribute for the genrule.</li>
+ <li><code>GENDIR</code>: The base of the generated code
+ tree for the target architecture.</li>
+ <li><code>JAVABASE</code>:
+ The base directory containing the Java utilities.
+ It will have a "bin" subdirectory.</li>
+</ul>
+
+<p><strong>Architecture Variables</strong></p>
+
+<ul><!-- keep alphabetically sorted -->
+ <li><code>ABI</code>: The C++ ABI version. </li>
+ <li><code>ANDROID_CPU</code>: The Android target architecture's cpu. </li>
+ <li><code>JAVA_CPU</code>: The Java target architecture's cpu. </li>
+ <li> <code>TARGET_CPU</code>: The target architecture's cpu. </li>
+</ul>
+
+<p id="predefined_variables.genrule.cmd">
+ <strong>
+ Other Variables available to <a href="#genrule.cmd">the cmd attribute of a genrule</a>
+ </strong>
+</p>
+<ul><!-- keep alphabetically sorted -->
+ <li><code>OUTS</code>: The <code>outs</code> list. If you have only one output
+ file, you can also use <code>$@</code>.</li>
+ <li><code>SRCS</code>: The <code>srcs</code> list (or more
+ precisely, the pathnames of the files corresponding to
+ labels in the <code>srcs</code> list). If you have only one
+ source file, you can also use <code>$<</code>.</li>
+ <li><code><</code>: <code>srcs</code>, if it is a single file.</li>
+ <li><code>@</code>: <code>outs</code>, if it is a single file.</li>
+ <li><code>@D</code>: The output directory. If there is only
+ one filename in <code>outs</code>, this expands to the
+ directory containing that file. If there are multiple
+ filenames, this variable instead expands to the package's root
+ directory in the <code>genfiles</code> tree, <i>even if all
+ the generated files belong to the same subdirectory</i>!
+ <!-- (as a consequence of the "middleman" implementation) -->
+ If the genrule needs to generate temporary intermediate files
+ (perhaps as a result of using some other tool like a compiler)
+ then it should attempt to write the temporary files to
+ <code>@D</code> (although <code>/tmp</code> will also be
+ writable), and to remove any such generated temporary files.
+ Especially, avoid writing to directories containing inputs -
+ they may be on read-only filesystems. </li>
+</ul>
+
+</ul>
+
+<h3 id="define_your_own_make_vars">Defining Your Own Variables</h3>
+
+<p>
+You may want to use Python-style variable assignments rather than "Make"
+variables, because they work in more use cases and are less surprising. "Make"
+variables will work in the <a href="#genrule.cmd">cmd</a> attribute of genrules
+and in the key of the <a href="#cc_library.abi_deps">abi_deps</a> attribute of
+a limited number of rules, but only in very few other places.
+
+</p>
+<p>To define your "Make" own variables, first call <a
+ href="#vardef">vardef()</a> to define your variable, then call <a
+ href="#varref">varref(name)</a> to retrieve it. varref can be embedded as part
+ of a larger string. Custom "Make" variables differ from ordinary "Python"
+ variables in the BUILD language in two important ways:
+</p>
+<ul>
+ <li>Only string values are supported,</li>
+ <li>The "Make" environment is parameterized over the build
+ platform, so that variables can be conditionally defined based on
+ the target architecture, ABI or compiler, and</li>
+ <li>The values of custom "Make" variables are <i>not available</i> during
+ BUILD-file evaluation. To work around this, you must call <a
+ href="#varref">varref()</a> to retrieve the value of a variable (unlike
+ predefined values, which can be retrieved using <code>$(FOO)</code>.
+ varref defers evaluation until after BUILD file evaluation.</li>
+</ul>
+
+<h4 id="vardef">vardef()</h4>
+
+<p><code>vardef(name, value, platform)</code></p>
+
+ <p>
+ Define a variable for use within this <code>BUILD</code> file only.
+ This variable can then be used by <a href="#varref">varref()</a>.
+ The value of the variable can be overridden on the command line by using the
+ <code class='flag'><a href='bazel-user-manual.html#flag--define'>--define</a></code>
+ flag.
+ </p>
+
+ <p id="vardef_args"><strong>Arguments</strong></p>
+<ul>
+ <li><code>name</code>: The name of the variable.
+ <i>(String; required)</i><br/>
+ Convention is to use names consisting of ALL CAPS. This name must
+ be a unique identifier in this package.
+ </li>
+ <li><code>value</code>: The value to assign to this variable.
+ <i>(String; required)</i><br/>
+ The value may make use of variables you know are defined in the "Make"
+ environment.
+ </li>
+ <li><code>platform</code>: Conditionally define this variable for a given
+ platform.
+ <i>(String; optional)</i><br/>
+
+ <code>vardef</code> binds the <code>name</code> to <code>value</code> if we're
+ compiling for <code>platform</code>.
+ </li>
+</ul>
+
+<p id="vardef_notes"><strong>Notes</strong></p>
+<p>
+ Because references to "Make" variables are expanded <i>after</i>
+ BUILD file evaluation, the relative order of <code>vardef</code>
+ statements and rule declarations is unimportant; it is order of
+ <code>vardef</code> statements relative to each other, and hence the
+ state of the "Make" environment at the end of evaluation that
+ matters.
+</p>
+<p>
+ If there are multiple matching <code>vardef</code> definitions for
+ the same variable, the definition that wins is
+ the <strong>last</strong> matching definition
+ <strong>that specifies a platform</strong>, unless there are no matching
+ definitions that specify a platform, in which case the definition
+ that wins is the <strong>last</strong> definition <strong>without
+ a platform</strong>.
+</p>
+
+<!-- =================================================================
+ varref()
+ =================================================================
+ -->
+
+<h4 id="varref">varref</h4>
+
+<p><code>varref(name)</code></p>
+
+<p>
+<code>varref("FOO")</code> is equivalent of writing "$(FOO)". It is used to
+dereference variables defined with <a href="#vardef"><code>vardef</code></a>
+as well as <a href="#predefined_variables">predefined variables</a>.
+</p>
+
+<p>
+ In rule attributes that are subject to "Make" variable
+ substitution, the string produced by <code>varref(<i>name</i>)</code>
+ will expand to the value of variable <i>name</i>.
+</p>
+
+<p><code>varref(name)</code> may not be used in rule attributes that are
+not subject to "Make" variable substitution.</p>
+
+<p id="varref_args"><strong>Arguments</strong></p>
+<ul>
+ <li><code>name</code>: The name of the variable to dereference.
+ </li>
+</ul>
+
+<p id="varref_notes"><strong>Notes</strong></p>
+<ul>
+ <li><code>varref</code> can access either local or global variables.
+ It prefers the local variable, if both a local and a global exist with
+ the same name.
+ </li>
+</ul>
+
+<p id="varref_examples"><strong>Examples</strong></p>
+<p>See <a href="#vardef_examples">vardef()</a> examples.</p>
+
+
+<!-- ============================================
+ other
+ ============================================
+-->
+<h2 id="misc">Other Stuff</h2>
+
+${SECTION_OTHER}
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html b/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html
new file mode 100644
index 0000000..fb4cc67
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html
@@ -0,0 +1,407 @@
+<!-- =================================================================
+ package()
+ =================================================================
+-->
+
+<h3 id="package">package</h3>
+
+<p>This function declares metadata that applies to every subsequent rule in the
+package.</p>
+
+<p>The <a href="build-ref.html#package">package</a>
+ function is used at most once within a build package (BUILD file).
+ It is recommended that the package function is called at the top of the
+ file, before any rule.</p>
+
+<h4 id="package_args">Arguments</h4>
+
+<ul>
+
+ <li id="package.default_visibility"><code>default_visibility</code>:
+ The default visibility of the rules in this package.
+ <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/>
+
+ <p>Every rule in this package has the visibility specified in this
+ attribute, unless otherwise specified in the <code>visibility</code>
+ attribute of the rule. For detailed information about the syntax of this
+ attribute, see the documentation of the <a href="#common.visibility">visibility</a>
+ attribute.
+ </li>
+
+ <li id="package.default_obsolete"><code>default_obsolete</code>:
+ The default value of <a href="#common.obsolete"><code>obsolete</code></a> property
+ for all rules in this package. <i>(Boolean; optional; default is 0)</i>
+ </li>
+
+ <li id="package.default_deprecation"><code>default_deprecation</code>:
+ Sets the default <a href="#common.deprecation"><code>deprecation</code></a> message
+ for all rules in this package. <i>(String; optional)</i>
+ </li>
+
+ <li id="package.default_testonly"><code>default_testonly</code>:
+ Sets the default <a href="#common.testonly"><code>testonly</code></a> property
+ for all rules in this package. <i>(Boolean; optional; default is 0 except as noted)</i>
+
+ <p>In packages under <code>javatests</code> the default value is 1.</p>
+ </li>
+
+</ul>
+
+<h4 id="package_example">Examples</h4>
+
+The declaration below declares that the rules in this package are
+visible only to members of package
+group <code>//foo:target</code>. Individual visibility declarations
+on a rule, if present, override this specification.
+
+<pre class="code">
+package(default_visibility = ["//foo:target"])
+</pre>
+
+<!-- =================================================================
+ package_group()
+ =================================================================
+-->
+
+<h3 id="package_group">package_group</h3>
+
+<p><code>package_group(name, packages, includes)</code></p>
+
+<p>This function defines a set of build packages.
+
+Package groups are used for visibility control. You can grant access to a rule
+to one or more package groups, every rule, or only to rules declared
+in the same package. For more detailed description of the visibility system, see
+the <a href="#common.visibility">visibility</a> attribute.
+
+<h4 id="package_group_args">Arguments</h4>
+
+<ul>
+ <li id="package_group.name"><code>name</code>:
+ A unique name for this rule.
+ <i>(<a href="build-ref.html#name">Name</a>; required)</i>
+ </li>
+
+ <li id="package_group.packages"><code>packages</code>:
+ A complete enumeration of packages in this group.
+ <i>(List of <a href="build-ref.html#s4">Package</a>; optional)</i><br/>
+
+ <p>Packages should be referred to using their full names,
+ starting with a double slash. For
+ example, <code>//foo/bar/main</code> is a valid element
+ of this list.</p>
+
+ <p>You can also specify wildcards: the specification
+ <code>//foo/...</code> specifies every package under
+ <code>//foo</code>, including <code>//foo</code> itself.
+
+ <p>If this attribute is missing, the package group itself will contain
+ no packages (but it can still include other package groups).</p>
+ </li>
+
+ <li id="package_group.includes"><code>includes</code>:
+ Other package groups that are included in this one.
+ <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/>
+
+ <p>The labels in this attribute must refer to other package
+ groups. Packages in referenced package groups are taken to be part
+ of this package group. This is transitive, that is, if package
+ group <code>a</code> contains package group <code>b</code>,
+ and <code>b</code> contains package group <code>c</code>, every
+ package in <code>c</code> will also be a member of <code>a</code>.</p>
+ </li>
+</ul>
+
+
+<h4 id="package_group_example">Examples</h4>
+
+The following <code>package_group</code> declaration specifies a
+package group called "tropical" that contains tropical fruits.
+
+<pre class="code">
+package_group(
+ name = "tropical",
+ packages = [
+ "//fruits/mango",
+ "//fruits/orange",
+ "//fruits/papaya/...",
+ ],
+)
+</pre>
+
+The following declarations specify the package groups of a fictional
+application:
+
+<pre class="code">
+package_group(
+ name = "fooapp",
+ includes = [
+ ":controller",
+ ":model",
+ ":view",
+ ],
+)
+
+package_group(
+ name = "model",
+ packages = ["//fooapp/database"],
+)
+
+package_group(
+ name = "view",
+ packages = [
+ "//fooapp/swingui",
+ "//fooapp/webui",
+ ],
+)
+
+package_group(
+ name = "controller",
+ packages = ["//fooapp/algorithm"],
+)
+</pre>
+
+
+<!-- =================================================================
+ DESCRIPTION
+ =================================================================
+ -->
+
+<h3 id="description">Description</h3>
+
+<p><code># Description: <var>...</var></code></p>
+
+ <p>
+ Each BUILD file should contain a <code>Description</code> comment.
+ </p>
+
+ <p>
+ Description comments may contain references to other
+ documentation. A string that starts with "http" will become a
+ link. HTML markup is
+ allowed in description comments, but please keep the BUILD files readable.
+ We encourage you to list the URLs of relevant design docs and howtos
+ in these description comments.
+ </p>
+
+<h3 id="distribs">distribs</h3>
+
+<p><code>distribs(distrib_methods)</code></p>
+
+<p><code>distribs()</code> specifies the default distribution method (or
+ methods) of the build rules in a <code>BUILD</code> file. The <code>distribs()</code>
+ directive
+ should appear close to the beginning of the <code>BUILD</code> file,
+ before any build rules, as it sets the <code>BUILD</code>-file scope
+ default for build rule distribution methods.
+</p>
+
+<h4 id="distribs_args">Arguments</h4>
+
+<p>The argument, <code id="distribs.distrib_methods">distrib_methods</code>,
+ is a list of distribution-method strings.
+</p>
+
+<!-- =================================================================
+ exports_files([label, ...])
+ =================================================================
+ -->
+
+<h3 id="exports_files">exports_files</h3>
+
+<p><code>exports_files([<i>label</i>, ...], visibility, licenses)</code></p>
+
+<p>
+ <code>exports_files()</code> specifies a list of files belonging to
+ this package that are exported to other packages but not otherwise
+ mentioned in the BUILD file.
+</p>
+
+<p>
+ The BUILD file for a package may only refer to files belonging to another
+ package if they are mentioned somewhere in the other packages's BUILD file,
+ whether as an input to a rule or an explicit or implicit output from a rule.
+ The remaining files are not associated with a specific rule but are just "data",
+ and for these, <code>exports_files</code> ensures that they may be referenced by
+ other packages. (One kind of data for which this is particularly useful are
+ shell and Perl scripts.)
+</p>
+
+<p>
+ Note: A BUILD file only consisting of <code>exports_files()</code> statements
+ is needless though, as there are no BUILD rules that could own any files.
+ The files listed can already be accessed through the containing package and
+ exported from there if needed.
+</p>
+
+<h4 id="exports_files_args">Arguments</h4>
+
+<p>
+ The argument is a list of names of files within the current package. A
+ visibility declaration can also be specified; in this case, the files will be
+ visible to the targets specified. If no visibility is specified, the files
+ will be visible to every package, even if a package default visibility was
+ specified in the <code><a href="#package">package</a></code> function. The
+ <a href="#common.licenses">licenses</a> can also be specified.
+</p>
+
+<!-- =================================================================
+ glob()
+ =================================================================
+ -->
+
+<h3 id="glob">glob</h3>
+
+<p><code>glob(include, exclude=[], exclude_directories=1)</code>
+</p>
+
+<p>
+Glob is a helper function that can be used anywhere a list of filenames
+is expected. It takes one or two lists of filename patterns containing
+the <code>*</code> wildcard: as per the Unix shell, this wildcard
+matches any string excluding the directory separator <code>/</code>.
+In addition filename patterns can contain the recursive <code>**</code>
+wildcard. This wildcard will match zero or more complete
+path segments separated by the directory separator <code>/</code>.
+This wildcard can only be used as a complete path segment. For example,
+<code>"x/**/*.java"</code> is legal, but <code>"test**/testdata.xml"</code>
+and <code>"**.java"</code> are both illegal. No other wildcards are supported.
+</p>
+<p>
+Glob returns a list of every file in the current build package that:
+</p>
+<ul>
+ <li style="margin-bottom: 0">
+ Matches at least one pattern in <code>include</code>. </li>
+ <li>
+ Does not match any of the patterns in <code>exclude</code> (default []).</li>
+</ul>
+<p>
+If the <code>exclude_directories</code> argument is enabled (1), files of
+type directory will be omitted from the results (default 1).
+</p>
+<p>
+There are several important limitations and caveats:
+</p>
+
+<ol>
+ <li>
+ Globs only match files in your source tree, never
+ generated files. If you are building a target that requires both
+ source and generated files, create an explicit list of generated
+ files, and use <code>+</code> to add it to the result of the
+ <code>glob()</code> call.
+ </li>
+
+ <li>
+ Globs may match files in subdirectories. And subdirectory names
+ may be wildcarded. However...
+ </li>
+
+ <li>
+ Labels are not allowed to cross the package boundary and glob does
+ not match files in subpackages.
+
+ For example, the glob expression <code>**/*.cc</code> in package
+ <code>x</code> does not include <code>x/y/z.cc</code> if
+ <code>x/y</code> exists as a package (either as
+ <code>x/y/BUILD</code>, or somewhere else on the package-path). This
+ means that the result of the glob expression actually depends on the
+ existence of BUILD files — that is, the same glob expression would
+ include <code>x/y/z.cc</code> if there was no package called
+ <code>x/y</code>.
+ </li>
+
+ <li>
+ The restriction above applies to all glob expressions,
+ no matter which wildcards they use.
+ </li>
+</ol>
+
+<p>
+In general, you should <b>try to provide an appropriate extension (e.g. *.html)
+instead of using a bare '*'</b> for a glob pattern. The more explicit name
+is both self documenting and ensures that you don't accidentally match backup
+files, or emacs/vi/... auto-save files.
+</p>
+
+<h4 id="glob_example">Glob Examples</h4>
+
+<p>Include all txt files in directory testdata except experimental.txt.
+Note that files in subdirectories of testdata will not be included. If
+you want those files to be included, use a recursive glob (**).</p>
+<pre class="code">
+java_test(
+ name = "myprog",
+ srcs = ["myprog.java"],
+ data = glob(
+ ["testdata/*.txt"],
+ exclude = ["testdata/experimental.txt"],
+ ),
+)
+</pre>
+
+<h4 id="recursive_glob_example">Recursive Glob Examples</h4>
+
+<p>Create a library built from all java files in this directory and all
+subdirectories except those whose path includes a directory named testing.
+Subdirectories containing a BUILD file are ignored.
+<b>This should be a very common pattern.</b>
+</p>
+<pre class="code">
+java_library(
+ name = "mylib",
+ srcs = glob(
+ ["**/*.java"],
+ exclude = ["**/testing/**"],
+ ),
+)
+</pre>
+
+<p>Make the test depend on all txt files in the testdata directory,
+ its subdirectories</p>
+<pre class="code">
+java_test(
+ name = "mytest",
+ srcs = ["mytest.java"],
+ data = glob(["testdata/**/*.txt"]),
+)
+</pre>
+
+<!-- =================================================================
+ licenses()
+ =================================================================
+-->
+
+<h3 id="licenses">licenses</h3>
+
+<p><code>licenses(license_types)</code></p>
+
+<p><code>licenses()</code> specifies the default license type (or types)
+ of the build rules in a <code>BUILD</code> file. The <code>licenses()</code>
+ directive should appear close to the
+ beginning of the <code>BUILD</code> file, before any build rules, as it
+ sets the <code>BUILD</code>-file scope default for build rule license
+ types.</p>
+
+<h4 id="licenses_args">Arguments</h4>
+
+<p>The argument, <code id="licenses.licence_types">license_types</code>,
+ is a list of license-type strings.
+</p>
+
+<!-- =================================================================
+ include()
+ =================================================================
+-->
+
+<h3 id="include">include</h3>
+
+<p><code>include(name)</code></p>
+
+<p><code>include()</code> incorporates build
+ language definitions from an external file into the evaluation of the
+ current <code>BUILD</code> file.</p>
+
+</body>
+</html>
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-header.html b/src/main/java/com/google/devtools/build/docgen/templates/be-header.html
new file mode 100644
index 0000000..cdebf9b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/docgen/templates/be-header.html
@@ -0,0 +1,612 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Bazel BUILD Encyclopedia of Functions</title>
+
+ <style type="text/css" id="internalStyle">
+ body {
+ background-color: #ffffff;
+ color: black;
+ margin-right: 10%;
+ margin-left: 10%;
+ }
+
+ h1, h2, h3, h4, h5, h6 {
+ color: #dd7755;
+ font-family: sans-serif;
+ }
+ @media print {
+ /* Darker version for printing */
+ h1, h2, h3, h4, h5, h6 {
+ color: #008000;
+ font-family: helvetica, sans-serif;
+ }
+ }
+
+ h1 {
+ text-align: center;
+ }
+ h2 {
+ margin-left: -0.5in;
+ }
+ h3 {
+ margin-left: -0.25in;
+ }
+ h4 {
+ margin-left: -0.125in;
+ }
+ hr {
+ margin-left: -1in;
+ }
+ address {
+ text-align: right;
+ }
+
+ /* A compact unordered list */
+ ul.tight > li {
+ margin-bottom: 0;
+ }
+
+ /* Use the <code> tag for bits of code and <var> for variable and object names. */
+ code,pre,samp,var {
+ color: #006000;
+ }
+ /* Use the <file> tag for file and directory paths and names. */
+ file {
+ color: #905050;
+ font-family: monospace;
+ }
+ /* Use the <kbd> tag for stuff the user should type. */
+ kbd {
+ color: #600000;
+ }
+ div.note p {
+ float: right;
+ width: 3in;
+ margin-right: 0%;
+ padding: 1px;
+ border: 2px solid #60a060;
+ background-color: #fffff0;
+ }
+
+ table.grid {
+ background-color: #ffffee;
+ border: 1px solid black;
+ border-collapse: collapse;
+ margin-left: 2mm;
+ margin-right: 2mm;
+ }
+
+ table.grid th,
+ table.grid td {
+ border: 1px solid black;
+ padding: 0 2mm 0 2mm;
+ }
+
+ /* Use pre.code for code listings.
+ Use pre.interaction for "Here's what you see when you run a.out.".
+ (Within pre.interaction, use <kbd> things the user types)
+ */
+ pre.code {
+ background-color: #FFFFEE;
+ border: 1px solid black;
+ color: #004000;
+ font-size: 10pt;
+ margin-left: 2mm;
+ margin-right: 2mm;
+ padding: 2mm;
+ -moz-border-radius: 12px 0px 0px 0px;
+ }
+
+ pre.interaction {
+ background-color: #EEFFEE;
+ color: #004000;
+ padding: 2mm;
+ }
+
+ pre.interaction kbd {
+ font-weight: bold;
+ color: #000000;
+ }
+
+ /* legacy style */
+ pre.interaction b.astyped {
+ color: #000000;
+ }
+
+ h1 { margin-bottom: 5px; }
+ .toc { margin: 0px; }
+ ul li { margin-bottom: 1em; }
+ ul.toc li { margin-bottom: 0px; }
+ em.harmful { color: red; }
+
+ .deprecated { text-decoration: line-through; }
+ .discouraged { text-decoration: line-through; }
+
+ #rules { width: 980px; border-collapse: collapse; }
+ #rules td { border-top: 1px solid gray; padding: 4px; vertical-align: top; }
+ #rules th { text-align: left; padding: 4px; }
+
+ table.layout { width: 980px; }
+ table.layout td { vertical-align: top; }
+
+ #maintainer { text-align: right; }
+
+ dt {
+ font-weight: bold;
+ margin-top: 0.5em;
+ margin-bottom: 0.5em;
+ }
+ dd dt {
+ font-weight: normal;
+ text-decoration: underline;
+ color: gray;
+ }
+ </style>
+
+ <style type="text/css">
+ .rule-signature {
+ color: #006000;
+ font-family: monospace;
+ }
+ </style>
+</head>
+
+<body>
+
+<h1>Bazel BUILD Encyclopedia of Functions</h1>
+
+<h2>Contents</h2>
+
+ <h3>Concepts and terminology</h3>
+ <table class="layout"><tr><td>
+ <ul class="toc">
+ <li><a href="#common-definitions">Common definitions</a>:
+ <ul>
+ <li><a href="#sh-tokenization">Bourne shell tokenization</a></li>
+ <li><a href="#label-expansion">Label expansion</a></li>
+ <li><a href="#common-attributes">Common attributes</a></li>
+ <li><a href="#common-attributes-tests">Common attributes for tests</a></li>
+ <li><a href="#common-attributes-binaries">Common attributes for binaries</a></li>
+ <li><a href="#implicit-outputs">Implicit output targets</a></li>
+ </ul>
+ </li>
+ </ul>
+ </td><td>
+ <ul class="toc">
+ <li><a href="#make_variables">"Make" variables</a>
+ <ul class="toc">
+ <li><a href="#make-var-substitution">"Make" variable substitution</a></li>
+ <li><a href="#predefined_variables">Predefined variables</a></li>
+ <li><a href="#define_your_own_make_vars">Defining your own variables</a>
+ <ul>
+ <li><a href="#vardef">vardef</a></li>
+ <li><a href="#varref">varref</a></li>
+ </ul>
+ </li>
+ </ul>
+ <li><a href="#predefined-python-variables">Predefined Python Variables</a></li>
+ </ul>
+ </td><td>
+ <ul class="toc">
+ <li><a href="#include">include</a></li>
+ <li><a href="#package">package</a></li>
+ <li><a href="#package_group">package_group</a></li>
+ <li><a href="#description">Description</a></li>
+ <li><a href="#distribs">distribs</a></li>
+ <li><a href="#licenses">licenses</a></li>
+ <li><a href="#exports_files">exports_files</a></li>
+ <li><a href="#glob">glob</a></li>
+ </ul>
+ </td></tr></table>
+
+${HEADER_TABLE}
+
+<h2 id="common-definitions">Common definitions</h2>
+
+<p>This section defines various terms and concepts that are common to
+many functions or build rules below.
+</p>
+
+<!-- we haven't defined 'rules' or 'attributes' yet. -->
+
+<h3 id='sh-tokenization'>Bourne shell tokenization</h3>
+<p>
+ Certain string attributes of some rules are split into multiple
+ words according to the tokenization rules of the Bourne shell:
+ unquoted spaces delimit separate words, and single- and
+ double-quotes characters and backslashes are used to prevent
+ tokenization.
+</p>
+<p>
+ Those attributes that are subject to this tokenization are
+ explicitly indicated as such in their definitions in this document.
+</p>
+<p>
+ Attributes subject to "Make" variable expansion and Bourne shell
+ tokenization are typically used for passing arbitrary options to
+ compilers and other tools, such as the <code>copts</code> attribute
+ of <code>cc_library</code>, or the <code>javacopts</code> attribute of
+ <code>java_library</code>. Together these substitutions allow a
+ single string variable to expand into a configuration-specific list
+ of option words.
+</p>
+
+<h3 id='label-expansion'>Label expansion</h3>
+<p>
+ Some string attributes of a very few rules are subject to label
+ expansion: if those strings contain a valid build label as a
+ substring, such as <code>//mypkg:target</code>, and that label is a
+ declared prerequisite of the current rule, it is expanded into the
+ pathname of the file represented by the target <code>//mypkg:target</code>.
+</p>
+<p>
+ Example attributes include the <code>cmd</code> attribute of
+ genrule, and the <code>linkopts</code> attribute
+ of <code>cc_library</code>. The details may vary significantly in
+ each case, over such issues as: whether relative labels are
+ expanded; how labels that expand to multiple files are
+ treated, etc. Consult the rule attribute documentation for
+ specifics.
+</p>
+
+<h3 id="common-attributes">Attributes common to all build rules</h3>
+
+<p>This section describes attributes that are common to all build rules.<br/>
+Please note that it is an error to list the same label twice in a list of
+labels attribute.
+</p>
+
+<ul>
+<li id="common.deps"><code>deps</code>:
+A list of dependencies of this rule.
+<i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/>
+The precise semantics of what it means for this rule to depend on
+another using <code>deps</code> are specific to the kind of this rule,
+and the rule-specific documentation below goes into more detail.
+At a minimum, though, the targets named via <code>deps</code> will
+appear in the <code>*.runfiles</code> area of this rule, if it has
+one.
+<p>Most often, a <code>deps</code> dependency is used to allow one
+module to use symbols defined in another module written in the
+same programming language and separately compiled. Cross-language
+dependencies are also permitted in many cases: for example,
+a <code>java_library</code> rule may depend on C++ code in
+a <code>cc_library</code> rule, by declaring the latter in
+the <code>deps</code> attribute. See the definition
+of <a href="build-ref.html#deps">dependencies</a> for more
+information.</p>
+<p>Almost all rules permit a <code>deps</code> attribute, but where
+this attribute is not allowed, this fact is documented under the
+specific rule.</p></li>
+<li id="common.data"><code>data</code>:
+The list of files needed by this rule at runtime.
+<i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/>
+Targets named in the <code>data</code> attribute will appear in
+the <code>*.runfiles</code> area of this rule, if it has one. This
+may include data files needed by a binary or library, or other
+programs needed by it. See the
+<a href="build-ref.html#data">data dependencies</a> section for more
+information about how to depend on and use data files.
+<p>Almost all rules permit a <code>data</code> attribute, but where
+this attribute is not allowed, this fact is documented under the
+specific rule.</p></li>
+<li id="common.deprecation"><code>deprecation</code>:
+<i>(String; optional)</i><br/>
+An explanatory warning message associated with this rule.
+Typically this is used to notify users that a rule has become obsolete,
+or has become superseded by another rule, is private to a package, or is
+perhaps "considered harmful" for some reason. It is a good idea to include
+some reference (like a webpage, a bug number or example migration CLs) so
+that one can easily find out what changes are required to avoid the message.
+If there is a new target that can be used as a drop in replacement, it is a good idea
+to just migrate all users of the old target.
+<p>
+This attribute has no effect on the way things are built, but it
+may affect a build tool's diagnostic output. The build tool issues a
+warning when a rule with a <code>deprecation</code> attribute is
+depended upon by another rule.</p>
+<p>
+Intra-package dependencies are exempt from this warning, so that,
+for example, building the tests of a deprecated rule does not
+encounter a warning.</p>
+<p>
+If a deprecated rule depends on another deprecated rule, no warning
+message is issued.</p>
+<p>
+Once people have stopped using it, the package can be removed or marked as
+<a href="#common.obsolete"><code>obsolete</code></a>.</p></li>
+<li id="common.distribs"><code>distribs</code>:
+<i>(List of strings; optional)</i><br/>
+A list of distribution-method strings to be used for this particular build rule.
+Overrides the <code>BUILD</code>-file scope defaults defined by the
+<a href="#distribs"><code>distribs()</code></a> directive.</li>
+<li id="common.licenses"><code>licenses</code>:
+<i>(List of strings; optional)</i><br/>
+A list of license-type strings to be used for this particular build rule.
+Overrides the <code>BUILD</code>-file scope defaults defined by the
+<a href="#licenses"><code>licenses()</code></a> directive.</li>
+<li id="common.obsolete"><code>obsolete</code>:
+<i>(Boolean; optional; default 0)</i><br/>
+If 1, only obsolete targets can depend on this target. It is an error when
+a non-obsolete target depends on an obsolete target.
+<p>
+As a transition, one can first mark a package as in
+<a href="#common.deprecation"><code>deprecation</code></a>.</p>
+<p>
+This attribute is useful when you want to prevent a target from
+being used but are yet not ready to delete the sources.</p></li>
+<li id="common.tags"><code>tags</code>:
+List of arbitrary text tags. Tags may be any valid string; default is the
+empty list.<br/>
+<i>Tags</i> can be used on any rule; but <i>tags</i> are most useful
+on test and <code>test_suite</code> rules. Tags on non-test rules
+are only useful to humans and/or external programs.
+<i>Tags</i> are generally used to annotate a test's role in your debug
+and release process. The use of tags and size elements
+gives flexibility in assembling suites of tests based around codebase
+check-in policy.
+<p>
+A few tags have special meaning to the build tool; consult
+the <a href='bazel-user-manual.html#tags_keywords'>Bazel
+documentation</a> for details.
+</p></li>
+<li id="common.testonly"><code>testonly</code>:
+<i>(Boolean; optional; default 0 except as noted)</i><br />
+If 1, only testonly targets (such as tests) can depend on this target.
+<p>Equivalently, a rule that is not <code>testonly</code> is not allowed to
+depend on any rule that is <code>testonly</code>.</p>
+<p>Tests (<code>*_test</code> rules)
+and test suites (<a href="#test_suite">test_suite</a> rules)
+are <code>testonly</code> by default.</p>
+<p>By virtue of
+<a href="#package.default_testonly"><code>default_testonly</code></a>,
+targets under <code>javatests</code> are <code>testonly</code> by default.</p>
+<p>This attribute is intended to mean that the target should not be
+contained in binaries that are released to production.</p>
+<p>Because testonly is enforced at build time, not run time, and propagates
+virally through the dependency tree, it should be applied judiciously. For
+example, stubs and fakes that
+are useful for unit tests may also be useful for integration tests
+involving the same binaries that will be released to production, and
+therefore should probably not be marked testonly. Conversely, rules that
+are dangerous to even link in, perhaps because they unconditionally
+override normal behavior, should definitely be marked testonly.</p></li>
+<li id="common.visibility"><code>visibility</code>:
+<i>(List of <a href="build-ref.html#labels">labels</a>; optional; default private)</i><br/>
+<p>The <code>visibility</code> attribute on a rule controls whether
+the rule can be used by other packages. Rules are always visible to
+other rules declared in the same package.</p>
+<p>There are five forms (and one temporary form) a visibility label can take:
+<ul>
+<li><code>['//visibility:public']</code>: Anyone can use this rule.</li>
+<li><code>['//visibility:private']</code>: Only rules in this package
+can use this rule. Rules in <code>javatests/foo/bar</code>
+can always use rules in <code>java/foo/bar</code>.
+</li>
+<li><code>['//some/package:__pkg__', '//other/package:__pkg__']</code>:
+Only rules in <code>some/package</code> and <code>other/package</code>
+(defined in <code>some/package/BUILD</code> and
+<code>other/package/BUILD</code>) have access to this rule. Note that
+sub-packages do not have access to the rule; for example,
+<code>//some/package/foo:bar</code> or
+<code>//other/package/testing:bla</code> wouldn't have access.
+<code>__pkg__</code> is a special target and must be used verbatim.
+It represents all of the rules in the package.
+</li>
+<li><code>['//project:__subpackages__', '//other:__subpackages__']</code>:
+Only rules in packages <code>project</code> or <code>other</code> or
+in one of their sub-packages have access to this rule. For example,
+<code>//project:rule</code>, <code>//project/library:lib</code> or
+<code>//other/testing/internal:munge</code> are allowed to depend on
+this rule (but not <code>//independent:evil</code>)
+</li>
+<li><code>['//some/package:my_package_group']</code>:
+A <a href="#package_group">package group</a> is
+a named set of package names. Package groups can also grant access rights
+to entire subtrees, e.g.<code>//myproj/...</code>.
+</li>
+</ul>
+<p>The visibility specifications of <code>//visibility:public</code>,
+<code>//visibility:private</code> and
+<code>//visibility:legacy_public</code>
+can not be combined with any other visibility specifications.
+A visibility specification may contain a combination of package labels
+(i.e. //foo:__pkg__) and package_groups.</p>
+<p>If a rule does not specify the visibility attribute,
+the <code><a href="#package">default_visibility</a></code>
+attribute of the <code><a href="#package">package</a></code>
+statement in the BUILD file containing the rule is used
+(except <a href="#exports_files">exports_files</a> and
+<a href="#cc_public_library">cc_public_library</a>, which always default to
+public).</p>
+<p><b>Example</b>:</p>
+<p>
+File <code>//frobber/bin/BUILD</code>:
+</p>
+<pre class="code">
+# This rule is visible to everyone
+java_binary(
+ name = "executable",
+ visibility = ["//visibility:public"],
+ deps = [":library"],
+)
+
+# This rule is visible only to rules declared in the same package
+java_library(
+ name = "library",
+ visibility = ["//visibility:private"],
+)
+
+# This rule is visible to rules in package //object and //noun
+java_library(
+ name = "subject",
+ visibility = [
+ "//noun:__pkg__",
+ "//object:__pkg__",
+ ],
+)
+
+# See package group //frobber:friends (below) for who can access this rule.
+java_library(
+ name = "thingy",
+ visibility = ["//frobber:friends"],
+)
+</pre>
+<p>
+File <code>//frobber/BUILD</code>:
+</p>
+<pre class="code">
+# This is the package group declaration to which rule //frobber/bin:thingy refers.
+#
+# Our friends are packages //frobber, //fribber and any subpackage of //fribber.
+package_group(
+ name = "friends",
+ packages = [
+ "//fribber/...",
+ "//frobber",
+ ],
+)
+</pre></li>
+</ul>
+
+
+<h3 id="common-attributes-tests">Attributes common to all test rules (*_test)</h3>
+
+<p>This section describes attributes that are common to all test rules.</p>
+
+<ul>
+<li id="test.args"><code>args</code>:
+Add these arguments to the <code>--test_arg</code>
+when executed by <code>bazel test</code>.
+<i>(List of strings; optional; subject to
+<a href="#make_variables">"Make variable"</a> substitution and
+<a href="#sh-tokenization">Bourne shell tokenization</a>)</i><br/>
+These arguments are passed before the <code>--test_arg</code> values
+specified on the <code>bazel test</code> command line.</li>
+<li id="test.flaky"><code>flaky</code>:
+Marks test as flaky. <i>(Boolean; optional)</i><br/>
+If set, executes the test up to 3 times before being declared as failed.
+By default this attribute is set to 0 and test is considered to be stable.
+Note, that use of this attribute is generally discouraged - we do prefer
+all tests to be stable.</li>
+<li id="test.local"><code>local</code>:
+Forces the test to be run locally. <i>(Boolean; optional)</i><br/>
+By default this attribute is set to 0 and the default testing strategy is
+used. This is equivalent to providing 'local' as a tag
+(<code>tags=["local"]</code>).</li>
+<li id="test.shard_count"><code>shard_count</code>:
+Specifies the number of parallel shards
+to use to run the test. <i>(Non-negative integer less than or equal to 50;
+optional)</i><br/>
+This value will override any heuristics used to determine the number of
+parallel shards with which to run the test.</li>
+<li id="test.size"><code>size</code>:
+How "heavy" the test is
+<i>(String "enormous", "large" "medium" or "small",
+default is "medium")</i><br/>
+A classification of the test's "heaviness": how much time/resources
+it needs to run. This is useful when deciding which tests to run.
+Before checking in a change, you might run the small tests.
+Before a big release, you might run the large tests.
+</li>
+<li id="test.timeout"><code>timeout</code>:
+How long the test is
+normally expected to run before returning.
+<i>(String "eternal", "long", "moderate", or "short"
+with the default derived from a test's size attribute)</i><br/>
+While a test's size attribute controls resource estimation, a test's
+timeout may be set independently. If not explicitly specified, the
+timeout is based on the test's size (with "small" ⇒ "short",
+"medium" ⇒ "moderate", etc...). While size and runtime are generally
+heavily correlated, they are not strictly causal, hence the ability to set
+them independently.</li>
+</ul>
+
+
+<h3 id="common-attributes-binaries">Attributes common to all binary rules (*_binary)</h3>
+
+<p>This section describes attributes that are common to all binary rules.</p>
+
+<ul>
+<li id="binary.args"><code>args</code>:
+Add these arguments to the target when executed by
+<code>bazel run</code>.
+<i>(List of strings; optional; subject to
+<a href="#make_variables">"Make variable"</a> substitution and
+<a href="#sh-tokenization">Bourne shell tokenization</a>)</i><br/>
+These arguments are passed to the target before the target options
+specified on the <code>bazel run</code> command line.
+<p>Most binary rules permit an <code>args</code> attribute, but where
+this attribute is not allowed, this fact is documented under the
+specific rule.</p></li>
+<li id="binary.output_licenses"><code>output_licenses</code>:
+The licenses of the output files that this binary generates.
+<i>(List of strings; optional)</i><br/>
+Describes the licenses of the output of the binary generated by
+the rule. When a binary is referenced in a host attribute (for
+example, the <code>tools</code> attribute of
+a <code>genrule</code>), this license declaration is used rather
+than the union of the licenses of its transitive closure. This
+argument is useful when a binary is used as a tool during the
+build of a rule, and it is not desirable for its license to leak
+into the license of that rule. If this attribute is missing, the
+license computation proceeds as if the host dependency was a
+regular dependency.
+<p><em class="harmful">WARNING: in some cases (specifically, in
+genrules) the build tool cannot guarantee that the binary
+referenced by this attribute is actually used as a tool, and is
+not, for example, copied to the output. In these cases, it is the
+responsibility of the user to make sure that this is
+true.</em></p></li>
+</ul>
+
+
+<h3 id="implicit-outputs">Implicit output targets</h3>
+
+<p>When you define a build rule in a BUILD file, you are explicitly
+ declaring a new, named rule target in a package. Many build rule
+ functions also <i>implicitly</i> entail one or more output file
+ targets, whose contents and meaning are rule-specific.
+
+ For example, when you explicitly declare a
+ <code>java_binary(name='foo', ...)</code> rule, you are also
+ <i>implicitly</i> declaring an output file
+ target <code>foo_deploy.jar</code> as a member of the same package.
+ (This particular target is a self-contained Java archive suitable
+ for deployment.)
+</p>
+
+<p>
+ Implicit output targets are first-class members of the build
+ target graph. Just like other targets, they are built on demand,
+ either when specified in the top-level built command, or when they
+ are necessary prerequisites for other build targets. They can be
+ referenced as dependencies in BUILD files, and can be observed in
+ the output of analysis tools such as <code>bazel query</code>.
+</p>
+
+<p>
+ For each kind of build rule, the rule's documentation contains a
+ special section detailing the names and contents of any implicit
+ outputs entailed by a declaration of that kind of rule.
+</p>
+
+<p>
+ Please note an important but somewhat subtle distinction between the
+ two namespaces used by the build system. Build
+ <a href="build-ref.html#labels">labels</a> identify <em>targets</em>,
+ which may be rules or files, and file targets may be divided into
+ either source (or input) file targets and derived (or output) file
+ targets. These are the things you can mention in BUILD files,
+ build from the command-line, or examine using <code>bazel query</code>;
+ this is the <em>target namespace</em>. Each file target corresponds
+ to one actual file on disk (the "file system namespace"); each rule
+ target may correspond to zero, one or more actual files on disk.
+ There may be files on disk that have no corresponding target; for
+ example, <code>.o</code> object files produced during C++ compilation
+ cannot be referenced from within BUILD files or from the command line.
+ In this way, the build tool may hide certain implementation details of
+ how it does its job. This is explained more fully in
+ the <a href="build-ref.html">BUILD Concept Reference</a>.
+</p>
diff --git a/src/main/java/com/google/devtools/build/lib/Constants.java b/src/main/java/com/google/devtools/build/lib/Constants.java
new file mode 100644
index 0000000..052b090
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/Constants.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Various constants required by Bazel.
+ *
+ * <p>The extra {@code .toString()} calls are there so that javac doesn't inline these constants
+ * so that we can replace this class file in the .jar after Bazel was built.
+ */
+public class Constants {
+ private Constants() {
+ }
+
+ public static final String PRODUCT_NAME = "bazel".toString();
+ public static final ImmutableList<String> DEFAULT_PACKAGE_PATH = ImmutableList.of("%workspace%");
+ public static final String MAIN_RULE_CLASS_PROVIDER =
+ "com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider".toString();
+ public static final ImmutableList<String> IGNORED_TEST_WARNING_PREFIXES = ImmutableList.of();
+ public static final String RUNFILES_PREFIX = "".toString();
+
+ public static final ImmutableList<String> WATCHFS_BLACKLIST = ImmutableList.of();
+
+ public static final String PRELUDE_FILE_DEPOT_RELATIVE_PATH = "tools/build_rules/prelude_bazel";
+}
diff --git a/src/main/java/com/google/devtools/build/lib/UnixJniLoader.java b/src/main/java/com/google/devtools/build/lib/UnixJniLoader.java
new file mode 100644
index 0000000..ca3ca3b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/UnixJniLoader.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib;
+
+import java.io.File;
+
+/**
+ * A class to load JNI dependencies for Bazel.
+ */
+public class UnixJniLoader {
+ public static void loadJni() {
+ try {
+ System.loadLibrary("unix");
+ } catch (UnsatisfiedLinkError ex) {
+ // We are probably in tests, let's try to find the library relative to where we are.
+ File cwd = new File(System.getProperty("user.dir"));
+ String libunix = "src" + File.separator + "main" + File.separator + "native" + File.separator
+ + System.mapLibraryName("unix");
+ File toTest = new File(cwd, libunix);
+ if (toTest.exists()) {
+ System.load(toTest.toString());
+ } else {
+ throw ex;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java b/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java
new file mode 100644
index 0000000..87105d5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java
@@ -0,0 +1,420 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * Abstract implementation of Action which implements basic functionality: the
+ * inputs, outputs, and toString method. Both input and output sets are
+ * immutable.
+ */
+@Immutable @ThreadSafe
+public abstract class AbstractAction implements Action {
+
+ /**
+ * An arbitrary default resource set. Currently 250MB of memory, 50% CPU and 0% of total I/O.
+ */
+ public static final ResourceSet DEFAULT_RESOURCE_SET = new ResourceSet(250, 0.5, 0);
+
+ // owner/inputs/outputs attributes below should never be directly accessed even
+ // within AbstractAction itself. The appropriate getter methods should be used
+ // instead. This has to be done due to the fact that the getter methods can be
+ // overridden in subclasses.
+ private final ActionOwner owner;
+ // The variable inputs is non-final only so that actions that discover their inputs can modify it.
+ private Iterable<Artifact> inputs;
+ private final ImmutableSet<Artifact> outputs;
+
+ private int cachedInputCount = -1;
+ private String cachedKey;
+
+ /**
+ * Construct an abstract action with the specified inputs and outputs;
+ */
+ protected AbstractAction(ActionOwner owner,
+ Iterable<Artifact> inputs,
+ Iterable<Artifact> outputs) {
+ Preconditions.checkNotNull(owner);
+ // TODO(bazel-team): Use RuleContext.actionOwner here instead
+ this.owner = new ActionOwnerDescription(owner);
+ this.inputs = CollectionUtils.makeImmutable(inputs);
+ this.outputs = ImmutableSet.copyOf(outputs);
+ Preconditions.checkArgument(!this.outputs.isEmpty(), owner);
+ }
+
+ @Override
+ public final ActionOwner getOwner() {
+ return owner;
+ }
+
+ @Override
+ public boolean inputsKnown() {
+ return true;
+ }
+
+ @Override
+ public boolean discoversInputs() {
+ return false;
+ }
+
+ @Override
+ public void discoverInputs(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ throw new IllegalStateException("discoverInputs cannot be called for " + this.prettyPrint()
+ + " since it does not discover inputs");
+ }
+
+ @Override
+ public void updateInputsFromCache(
+ ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths) {
+ throw new IllegalStateException(
+ "Method must be overridden for actions that may have unknown inputs.");
+ }
+
+ /**
+ * Should only be overridden by actions that need to optionally insert inputs. Actions that
+ * discover their inputs should use {@link #setInputs} to set the new iterable of inputs when they
+ * know it.
+ */
+ @Override
+ public Iterable<Artifact> getInputs() {
+ return inputs;
+ }
+
+ /**
+ * Set the inputs of the action. May only be used by an action that {@link #discoversInputs()}.
+ * The iterable passed in is automatically made immutable.
+ */
+ public void setInputs(Iterable<Artifact> inputs) {
+ Preconditions.checkState(discoversInputs());
+ this.inputs = CollectionUtils.makeImmutable(inputs);
+ cachedInputCount = -1;
+ }
+
+ /*
+ * Get count of inputs.
+ *
+ * <p>Computes the count on first invocation, returns cached value for further invocations.
+ */
+ @Override
+ @ThreadSafe
+ public synchronized int getInputCount() {
+ if (cachedInputCount == -1) {
+ cachedInputCount = Iterables.size(getInputs());
+ }
+ return cachedInputCount;
+ }
+
+ @Override
+ public ImmutableSet<Artifact> getOutputs() {
+ return outputs;
+ }
+
+ @Override
+ public Artifact getPrimaryInput() {
+ // The default behavior is to return the first input artifact.
+ // Call through the method, not the field, because it may be overridden.
+ return Iterables.getFirst(getInputs(), null);
+ }
+
+ @Override
+ public Artifact getPrimaryOutput() {
+ // Default behavior is to return the first output artifact.
+ // Use the method rather than field in case of overriding in subclasses.
+ return Iterables.getFirst(getOutputs(), null);
+ }
+
+ @Override
+ public Iterable<Artifact> getMandatoryInputs() {
+ return getInputs();
+ }
+
+ @Override
+ public String toString() {
+ return prettyPrint() + " (" + getMnemonic() + "[" + ImmutableList.copyOf(getInputs())
+ + (inputsKnown() ? " -> " : ", unknown inputs -> ")
+ + getOutputs() + "]" + ")";
+ }
+
+ @Override
+ public abstract String getMnemonic();
+ protected abstract String computeKey();
+
+ @Override
+ public synchronized final String getKey() {
+ if (cachedKey == null) {
+ cachedKey = computeKey();
+ }
+ return cachedKey;
+ }
+
+ @Override
+ public String describeKey() {
+ return null;
+ }
+
+ @Override
+ public boolean executeUnconditionally() {
+ return false;
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return false;
+ }
+
+ @Override
+ public boolean showsOutputUnconditionally() {
+ return false;
+ }
+
+ @Override
+ public final String getProgressMessage() {
+ String message = getRawProgressMessage();
+ if (message == null) {
+ return null;
+ }
+ String additionalInfo = getOwner().getAdditionalProgressInfo();
+ return additionalInfo == null ? message : message + " [" + additionalInfo + "]";
+ }
+
+ /**
+ * Returns a progress message string that is specific for this action. This is
+ * then annotated with additional information, currently the string '[for host]'
+ * for actions in the host configurations.
+ *
+ * <p>A return value of null indicates no message should be reported.
+ */
+ protected String getRawProgressMessage() {
+ // A cheesy default implementation. Subclasses are invited to do something
+ // more meaningful.
+ return defaultProgressMessage();
+ }
+
+ private String defaultProgressMessage() {
+ return getMnemonic() + " " + getPrimaryOutput().prettyPrint();
+ }
+
+ @Override
+ public String prettyPrint() {
+ return "action '" + describe() + "'";
+ }
+
+ /**
+ * Deletes all of the action's output files, if they exist. If any of the
+ * Artifacts refers to a directory recursively removes the contents of the
+ * directory.
+ *
+ * @param execRoot the exec root in which this action is executed
+ */
+ protected void deleteOutputs(Path execRoot) throws IOException {
+ for (Artifact output : getOutputs()) {
+ deleteOutput(output);
+ }
+ }
+
+ /**
+ * Helper method to remove an Artifact. If the Artifact refers to a directory
+ * recursively removes the contents of the directory.
+ */
+ protected void deleteOutput(Artifact output) throws IOException {
+ Path path = output.getPath();
+ try {
+ // Optimize for the common case: output artifacts are files.
+ path.delete();
+ } catch (IOException e) {
+ // Only try to recursively delete a directory if the output root is known. This is just a
+ // sanity check so that we do not start deleting random files on disk.
+ // TODO(bazel-team): Strengthen this test by making sure that the output is part of the
+ // output tree.
+ if (path.isDirectory(Symlinks.NOFOLLOW) && output.getRoot() != null) {
+ FileSystemUtils.deleteTree(path);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * If the action might read directories as inputs in a way that is unsound wrt dependency
+ * checking, this method must be called.
+ */
+ protected void checkInputsForDirectories(EventHandler eventHandler,
+ MetadataHandler metadataHandler) {
+ // Report "directory dependency checking" warning only for non-generated directories (generated
+ // ones will be reported earlier).
+ for (Artifact input : getMandatoryInputs()) {
+ // Assume that if the file did not exist, we would not have gotten here.
+ if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) {
+ eventHandler.handle(Event.warn(getOwner().getLocation(), "input '"
+ + input.prettyPrint() + "' to " + getOwner().getLabel()
+ + " is a directory; dependency checking of directories is unsound"));
+ }
+ }
+ }
+
+ @Override
+ public MiddlemanType getActionType() {
+ return MiddlemanType.NORMAL;
+ }
+
+ /**
+ * If the action might create directories as outputs this method must be called.
+ */
+ protected void checkOutputsForDirectories(EventHandler eventHandler) {
+ for (Artifact output : getOutputs()) {
+ Path path = output.getPath();
+ String ownerString = Label.print(getOwner().getLabel());
+ if (path.isDirectory()) {
+ eventHandler.handle(new Event(EventKind.WARNING, getOwner().getLocation(),
+ "output '" + output.prettyPrint() + "' of " + ownerString
+ + " is a directory; dependency checking of directories is unsound",
+ ownerString));
+ }
+ }
+ }
+
+ @Override
+ public void prepare(Path execRoot) throws IOException {
+ deleteOutputs(execRoot);
+ }
+
+ @Override
+ public String describe() {
+ String progressMessage = getProgressMessage();
+ return progressMessage != null ? progressMessage : defaultProgressMessage();
+ }
+
+ @Override
+ public abstract ResourceSet estimateResourceConsumption(Executor executor);
+
+ @Override
+ public boolean shouldReportPathPrefixConflict(Action action) {
+ return this != action;
+ }
+
+ @Override
+ public ExtraActionInfo.Builder getExtraActionInfo() {
+ return ExtraActionInfo.newBuilder()
+ .setOwner(getOwner().getLabel().toString())
+ .setId(getKey())
+ .setMnemonic(getMnemonic());
+ }
+
+ /**
+ * Returns input files that need to be present to allow extra_action rules to shadow this action
+ * correctly when run remotely. This is at least the normal inputs of the action, but may include
+ * other files as well. For example C(++) compilation may perform include file header scanning.
+ * This needs to be mirrored by the extra_action rule. Called by
+ * {@link com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time.
+ *
+ * <p>As this method is called from the ExtraAction, make sure it is ok to call
+ * this method from a different thread than the one this action is executed on.
+ *
+ * @param actionExecutionContext Services in the scope of the action, like the Out/Err streams.
+ * @throws ActionExecutionException only when code called from this method
+ * throws that exception.
+ * @throws InterruptedException if interrupted
+ */
+ public Iterable<Artifact> getInputFilesForExtraAction(
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ return getInputs();
+ }
+
+ /**
+ * A copying implementation of {@link ActionOwner}.
+ *
+ * <p>ConfiguredTargets implement ActionOwner themselves, but we do not want actions
+ * to keep direct references to configured targets just for a label and a few strings.
+ */
+ @Immutable
+ private static class ActionOwnerDescription implements ActionOwner {
+
+ private final Location location;
+ private final Label label;
+ private final String configurationName;
+ private final String configurationMnemonic;
+ private final String configurationKey;
+ private final String targetKind;
+ private final String additionalProgressInfo;
+
+ private ActionOwnerDescription(ActionOwner originalOwner) {
+ this.location = originalOwner.getLocation();
+ this.label = originalOwner.getLabel();
+ this.configurationName = originalOwner.getConfigurationName();
+ this.configurationMnemonic = originalOwner.getConfigurationMnemonic();
+ this.configurationKey = originalOwner.getConfigurationShortCacheKey();
+ this.targetKind = originalOwner.getTargetKind();
+ this.additionalProgressInfo = originalOwner.getAdditionalProgressInfo();
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public String getConfigurationName() {
+ return configurationName;
+ }
+
+ @Override
+ public String getConfigurationMnemonic() {
+ return configurationMnemonic;
+ }
+
+ @Override
+ public String getConfigurationShortCacheKey() {
+ return configurationKey;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return targetKind;
+ }
+
+ @Override
+ public String getAdditionalProgressInfo() {
+ return additionalProgressInfo;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java
new file mode 100644
index 0000000..0272160
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * An action owner base class that provides default implementations for some of
+ * the {@link ActionOwner} methods.
+ */
+public abstract class AbstractActionOwner implements ActionOwner {
+
+ @Override
+ public String getAdditionalProgressInfo() {
+ return null;
+ }
+
+ @Override
+ public Location getLocation() {
+ return null;
+ }
+
+ @Override
+ public Label getLabel() {
+ return null;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return "empty target kind";
+ }
+
+ @Override
+ public String getConfigurationName() {
+ return "empty configuration";
+ }
+
+ /**
+ * An action owner for special cases. Usage is strongly discouraged.
+ */
+ public static final ActionOwner SYSTEM_ACTION_OWNER = new AbstractActionOwner() {
+ @Override
+ public final String getConfigurationName() {
+ return "system";
+ }
+
+ @Override
+ public String getConfigurationMnemonic() {
+ return "system";
+ }
+
+ @Override
+ public final String getConfigurationShortCacheKey() {
+ return "system";
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Action.java b/src/main/java/com/google/devtools/build/lib/actions/Action.java
new file mode 100644
index 0000000..4970203
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/Action.java
@@ -0,0 +1,188 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadCompatible;
+import com.google.devtools.build.lib.profiler.Describable;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+
+/**
+ * An Action represents a function from Artifacts to Artifacts executed as an
+ * atomic build step. Examples include compilation of a single C++ source
+ * file, or linking a single library.
+ */
+public interface Action extends ActionMetadata, Describable {
+
+ /**
+ * Prepares for executing this action; called by the Builder prior to
+ * executing the Action itself. This method should prepare the file system, so
+ * that the execution of the Action can write the output files. At a minimum
+ * any pre-existing and write protected output files should be removed or the
+ * permissions should be changed, so that they can be safely overwritten by
+ * the action.
+ *
+ * @throws IOException if there is an error deleting the outputs.
+ */
+ void prepare(Path execRoot) throws IOException;
+
+ /**
+ * Executes this action; called by the Builder when all of this Action's
+ * inputs have been successfully created. (Behaviour is undefined if the
+ * prerequisites are not up to date.) This method <i>actually does the work
+ * of the Action, unconditionally</i>; in other words, it is invoked by the
+ * Builder only when dependency analysis has deemed it necessary.</p>
+ *
+ * <p>The framework guarantees that the output directory for each file in
+ * <code>getOutputs()</code> has already been created, and will check to
+ * ensure that each of those files is indeed created.</p>
+ *
+ * <p>Implementations of this method should try to honour the {@link
+ * java.lang.Thread#interrupted} contract: if an interrupt is delivered to
+ * the thread in which execution occurs, the action should detect this on a
+ * best-effort basis and terminate as quickly as possible by throwing an
+ * ActionExecutionException.
+ *
+ * <p>Action execution must be ThreadCompatible in order to be safely used
+ * with a concurrent Builder implementation such as ParallelBuilder.
+ *
+ * @param actionExecutionContext Services in the scope of the action, like the output and error
+ * streams to use for messages arising during action execution.
+ * @throws ActionExecutionException if execution fails for any reason.
+ * @throws InterruptedException
+ */
+ @ConditionallyThreadCompatible
+ void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException;
+
+ /**
+ * Returns true iff action must be executed regardless of its current state.
+ * Default implementation can be overridden by some actions that might be
+ * executed unconditionally under certain circumstances - e.g., if caching of
+ * test results is not requested, this method could be used to force test
+ * execution even if all dependencies are up-to-date.
+ *
+ * <p>Note, it is <b>very</b> important not to abuse this method, since it
+ * completely overrides dependency checking. Any use of this method must
+ * be carefully reviewed and proved to be necessary.
+ *
+ * <p>Note that the definition of {@link #isVolatile} depends on the
+ * definition of this method, so be sure to consider both methods together
+ * when making changes.
+ */
+ boolean executeUnconditionally();
+
+ /**
+ * Returns true if it's ever possible that {@link #executeUnconditionally}
+ * could evaluate to true during the lifetime of this instance, false
+ * otherwise.
+ */
+ boolean isVolatile();
+
+ /**
+ * Method used to find inputs before execution for an action that
+ * {@link ActionMetadata#discoversInputs}.
+ */
+ public void discoverInputs(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException;
+
+ /**
+ * Method used to update action inputs based on the information contained in
+ * the action cache. It will be called iff inputsKnown() is false for the
+ * given action instance and there is a related cache entry in the action
+ * cache.
+ *
+ * Method must be redefined for any action that may return
+ * inputsKnown() == false. It also expects that implementation will ensure
+ * that inputsKnown() returns true after call to this method.
+ *
+ * @param artifactResolver the artifact factory that can be used to manufacture artifacts
+ * @param inputPaths List of relative (to the execution root) input paths
+ */
+ public void updateInputsFromCache(
+ ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths);
+
+ /**
+ * Return a best-guess estimate of the operation's resource consumption on the
+ * local host itself for use in scheduling.
+ *
+ * @param executor the application-specific value passed to the
+ * executor parameter of the top-level call to
+ * Builder.buildArtifacts().
+ */
+ @Nullable ResourceSet estimateResourceConsumption(Executor executor);
+
+ /**
+ * @return true iff path prefix conflict (conflict where two actions generate
+ * two output artifacts with one of the artifact's path being the
+ * prefix for another) between this action and another action should
+ * be reported.
+ */
+ boolean shouldReportPathPrefixConflict(Action action);
+
+ /**
+ * Returns true if the output should bypass output filtering. This is used for test actions.
+ */
+ boolean showsOutputUnconditionally();
+
+ /**
+ * Called by {@link com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time to
+ * extract information from this action into a protocol buffer to be used by extra_action rules.
+ *
+ * <p>As this method is called from the ExtraAction, make sure it is ok to call this method from
+ * a different thread than the one this action is executed on.
+ */
+ ExtraActionInfo.Builder getExtraActionInfo();
+
+ /**
+ * Returns the action type. Must not be {@code null}.
+ */
+ MiddlemanType getActionType();
+
+ /**
+ * The action type.
+ */
+ public enum MiddlemanType {
+
+ /** A normal action. */
+ NORMAL,
+
+ /** A normal middleman, which just encapsulates a list of artifacts. */
+ AGGREGATING_MIDDLEMAN,
+
+ /**
+ * A middleman that enforces action ordering, is not validated by the dependency checker, but
+ * allows errors to be propagated.
+ */
+ ERROR_PROPAGATING_MIDDLEMAN,
+
+ /**
+ * A runfiles middleman, which is validated by the dependency checker, but is not expanded
+ * in blaze. Instead, the runfiles manifest is sent to remote execution client, which
+ * performs the expansion.
+ */
+ RUNFILES_MIDDLEMAN;
+
+ public boolean isMiddleman() {
+ return this != NORMAL;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
new file mode 100644
index 0000000..2525c8d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
@@ -0,0 +1,341 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.cache.Digest;
+import com.google.devtools.build.lib.actions.cache.Metadata;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Checks whether an {@link Action} needs to be executed, or whether it has not changed since it was
+ * last stored in the action cache. Must be informed of the new Action data after execution as well.
+ *
+ * <p>The fingerprint, input files names, and metadata (either mtimes or MD5sums) of each action are
+ * cached in the action cache to avoid unnecessary rebuilds. Middleman artifacts are handled
+ * specially, avoiding the need to create actual files corresponding to the middleman artifacts.
+ * Instead of that, results of MiddlemanAction dependency checks are cached internally and then
+ * reused whenever an input middleman artifact is encountered.
+ *
+ * <p>While instances of this class hold references to action and metadata cache instances, they are
+ * otherwise lightweight, and should be constructed anew and discarded for each build request.
+ */
+public class ActionCacheChecker {
+ private final ActionCache actionCache;
+ private final Predicate<? super Action> executionFilter;
+ private final ArtifactResolver artifactResolver;
+ // True iff --verbose_explanations flag is set.
+ private final boolean verboseExplanations;
+
+ public ActionCacheChecker(ActionCache actionCache, ArtifactResolver artifactResolver,
+ Predicate<? super Action> executionFilter, boolean verboseExplanations) {
+ this.actionCache = actionCache;
+ this.executionFilter = executionFilter;
+ this.artifactResolver = artifactResolver;
+ this.verboseExplanations = verboseExplanations;
+ }
+
+ public boolean isActionExecutionProhibited(Action action) {
+ return !executionFilter.apply(action);
+ }
+
+ /**
+ * Checks whether one of existing output paths is already used as a key.
+ * If yes, returns it - otherwise uses first output file as a key
+ */
+ private ActionCache.Entry getCacheEntry(Action action) {
+ for (Artifact output : action.getOutputs()) {
+ ActionCache.Entry entry = actionCache.get(output.getExecPathString());
+ if (entry != null) {
+ return entry;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Validate metadata state for action input or output artifacts.
+ *
+ * @param entry cached action information.
+ * @param action action to be validated.
+ * @param metadataHandler provider of metadata for the artifacts this action interacts with.
+ * @param checkOutput true to validate output artifacts, Otherwise, just
+ * validate inputs.
+ *
+ * @return true if at least one artifact has changed, false - otherwise.
+ */
+ private boolean validateArtifacts(ActionCache.Entry entry, Action action,
+ MetadataHandler metadataHandler, boolean checkOutput) {
+ Iterable<Artifact> artifacts = checkOutput
+ ? Iterables.concat(action.getOutputs(), action.getInputs())
+ : action.getInputs();
+ Map<String, Metadata> mdMap = new HashMap<>();
+ for (Artifact artifact : artifacts) {
+ mdMap.put(artifact.getExecPathString(), metadataHandler.getMetadataMaybe(artifact));
+ }
+ return !Digest.fromMetadata(mdMap).equals(entry.getFileDigest());
+ }
+
+ private void reportCommand(EventHandler handler, Action action) {
+ if (handler != null) {
+ if (verboseExplanations) {
+ String keyDescription = action.describeKey();
+ reportRebuild(handler, action,
+ keyDescription == null ? "action command has changed" :
+ "action command has changed.\nNew action: " + keyDescription);
+ } else {
+ reportRebuild(handler, action,
+ "action command has changed (try --verbose_explanations for more info)");
+ }
+ }
+ }
+
+ protected boolean unconditionalExecution(Action action) {
+ return !isActionExecutionProhibited(action) && action.executeUnconditionally();
+ }
+
+ /**
+ * Checks whether {@code action} needs to be executed and returns a non-null Token if so.
+ *
+ * <p>The method checks if any of the action's inputs or outputs have changed. Returns a non-null
+ * {@link Token} if the action needs to be executed, and null otherwise.
+ *
+ * <p>If this method returns non-null, indicating that the action will be executed, the
+ * metadataHandler's {@link MetadataHandler#discardMetadata} method must be called, so that it
+ * does not serve stale metadata for the action's outputs after the action is executed.
+ */
+ // Note: the handler should only be used for DEPCHECKER events; there's no
+ // guarantee it will be available for other events.
+ public Token getTokenIfNeedToExecute(Action action, EventHandler handler,
+ MetadataHandler metadataHandler) {
+ // TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that
+ // produce only symlinks we should not check whether inputs are valid at all - all that matters
+ // that inputs and outputs are still exist (and new inputs have not appeared). All other checks
+ // are unnecessary. In other words, the only metadata we should check for them is file existence
+ // itself.
+
+ MiddlemanType middlemanType = action.getActionType();
+ if (middlemanType.isMiddleman()) {
+ // Some types of middlemen are not checked because they should not
+ // propagate invalidation of their inputs.
+ if (middlemanType != MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN) {
+ checkMiddlemanAction(action, handler, metadataHandler);
+ }
+ return null;
+ }
+ ActionCache.Entry entry = null; // Populated lazily.
+
+ // Update action inputs from cache, if necessary.
+ boolean inputsKnown = action.inputsKnown();
+ if (!inputsKnown) {
+ Preconditions.checkState(action.discoversInputs());
+ entry = getCacheEntry(action);
+ updateActionInputs(action, entry);
+ }
+ if (mustExecute(action, entry, handler, metadataHandler)) {
+ return new Token(getKeyString(action));
+ }
+ return null;
+ }
+
+ protected boolean mustExecute(Action action, @Nullable ActionCache.Entry entry,
+ EventHandler handler, MetadataHandler metadataHandler) {
+ // Unconditional execution can be applied only for actions that are allowed to be executed.
+ if (unconditionalExecution(action)) {
+ Preconditions.checkState(action.isVolatile());
+ reportUnconditionalExecution(handler, action);
+ return true; // must execute - unconditional execution is requested.
+ }
+
+ if (entry == null) {
+ entry = getCacheEntry(action);
+ }
+ if (entry == null) {
+ reportNewAction(handler, action);
+ return true; // must execute -- no cache entry (e.g. first build)
+ }
+
+ if (entry.isCorrupted()) {
+ reportCorruptedCacheEntry(handler, action);
+ return true; // cache entry is corrupted - must execute
+ } else if (validateArtifacts(entry, action, metadataHandler, true)) {
+ reportChanged(handler, action);
+ return true; // files have changed
+ } else if (!entry.getActionKey().equals(action.getKey())){
+ reportCommand(handler, action);
+ return true; // must execute -- action key is different
+ }
+
+ entry.getFileDigest();
+ return false; // cache hit
+ }
+
+ public void afterExecution(Action action, Token token, MetadataHandler metadataHandler)
+ throws IOException {
+ Preconditions.checkArgument(token != null);
+ String key = token.cacheKey;
+ ActionCache.Entry entry = actionCache.createEntry(action.getKey());
+ for (Artifact output : action.getOutputs()) {
+ // Remove old records from the cache if they used different key.
+ String execPath = output.getExecPathString();
+ if (!key.equals(execPath)) {
+ actionCache.remove(key);
+ }
+ // Output files *must* exist and be accessible after successful action execution.
+ Metadata metadata = metadataHandler.getMetadata(output);
+ Preconditions.checkState(metadata != null);
+ entry.addFile(output.getExecPath(), metadata);
+ }
+ for (Artifact input : action.getInputs()) {
+ entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input));
+ }
+ entry.getFileDigest();
+ actionCache.put(key, entry);
+ }
+
+ protected void updateActionInputs(Action action, ActionCache.Entry entry) {
+ if (entry == null || entry.isCorrupted()) {
+ return;
+ }
+
+ List<PathFragment> outputs = new ArrayList<>();
+ for (Artifact output : action.getOutputs()) {
+ outputs.add(output.getExecPath());
+ }
+ List<PathFragment> inputs = new ArrayList<>();
+ for (String path : entry.getPaths()) {
+ PathFragment execPath = new PathFragment(path);
+ // Code assumes that action has only 1-2 outputs and ArrayList.contains() will be
+ // most efficient.
+ if (!outputs.contains(execPath)) {
+ inputs.add(execPath);
+ }
+ }
+ action.updateInputsFromCache(artifactResolver, inputs);
+ }
+
+ /**
+ * Special handling for the MiddlemanAction. Since MiddlemanAction output
+ * artifacts are purely fictional and used only to stay within dependency
+ * graph model limitations (action has to depend on artifacts, not on other
+ * actions), we do not need to validate metadata for the outputs - only for
+ * inputs. We also do not need to validate MiddlemanAction key, since action
+ * cache entry key already incorporates that information for the middlemen
+ * and we will experience a cache miss when it is different. Whenever it
+ * encounters middleman artifacts as input artifacts for other actions, it
+ * consults with the aggregated middleman digest computed here.
+ */
+ protected void checkMiddlemanAction(Action action, EventHandler handler,
+ MetadataHandler metadataHandler) {
+ Artifact middleman = action.getPrimaryOutput();
+ String cacheKey = middleman.getExecPathString();
+ ActionCache.Entry entry = actionCache.get(cacheKey);
+ boolean changed = false;
+ if (entry != null) {
+ if (entry.isCorrupted()) {
+ reportCorruptedCacheEntry(handler, action);
+ changed = true;
+ } else if (validateArtifacts(entry, action, metadataHandler, false)) {
+ reportChanged(handler, action);
+ changed = true;
+ }
+ } else {
+ reportChangedDeps(handler, action);
+ changed = true;
+ }
+ if (changed) {
+ // Compute the aggregated middleman digest.
+ // Since we never validate action key for middlemen, we should not store
+ // it in the cache entry and just use empty string instead.
+ entry = actionCache.createEntry("");
+ for (Artifact input : action.getInputs()) {
+ entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input));
+ }
+ }
+
+ metadataHandler.setDigestForVirtualArtifact(middleman, entry.getFileDigest());
+ if (changed) {
+ actionCache.put(cacheKey, entry);
+ }
+ }
+
+ /**
+ * Returns an action key. It is always set to the first output exec path string.
+ */
+ private static String getKeyString(Action action) {
+ Preconditions.checkState(!action.getOutputs().isEmpty());
+ return action.getOutputs().iterator().next().getExecPathString();
+ }
+
+
+ /**
+ * In most cases, this method should not be called directly - reportXXX() methods
+ * should be used instead. This is done to avoid cost associated with building
+ * the message.
+ */
+ private static void reportRebuild(@Nullable EventHandler handler, Action action, String message) {
+ // For MiddlemanAction, do not report rebuild.
+ if (handler != null && !action.getActionType().isMiddleman()) {
+ handler.handle(new Event(
+ EventKind.DEPCHECKER, null, "Executing " + action.prettyPrint() + ": " + message + "."));
+ }
+ }
+
+ // Called by IncrementalDependencyChecker.
+ protected static void reportUnconditionalExecution(
+ @Nullable EventHandler handler, Action action) {
+ reportRebuild(handler, action, "unconditional execution is requested");
+ }
+
+ private static void reportChanged(@Nullable EventHandler handler, Action action) {
+ reportRebuild(handler, action, "One of the files has changed");
+ }
+
+ private static void reportChangedDeps(@Nullable EventHandler handler, Action action) {
+ reportRebuild(handler, action, "the set of files on which this action depends has changed");
+ }
+
+ private static void reportNewAction(@Nullable EventHandler handler, Action action) {
+ reportRebuild(handler, action, "no entry in the cache (action is new)");
+ }
+
+ private static void reportCorruptedCacheEntry(@Nullable EventHandler handler, Action action) {
+ reportRebuild(handler, action, "cache entry is corrupted");
+ }
+
+ /** Wrapper for all context needed by the ActionCacheChecker to handle a single action. */
+ public static final class Token {
+ private final String cacheKey;
+
+ private Token(String cacheKey) {
+ this.cacheKey = Preconditions.checkNotNull(cacheKey);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java
new file mode 100644
index 0000000..a2cb577
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An event that is fired after an action completes (either successfully or not).
+ */
+public final class ActionCompletionEvent {
+
+ private final ActionMetadata actionMetadata;
+
+ public ActionCompletionEvent(ActionMetadata actionMetadata) {
+ this.actionMetadata = actionMetadata;
+ }
+
+ /**
+ * Returns the action metadata.
+ */
+ public ActionMetadata getActionMetadata() {
+ return actionMetadata;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java
new file mode 100644
index 0000000..4c0eaa2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+import java.util.Map;
+
+/**
+ * An object describing that actions require a particular implementation of an
+ * {@link ActionContext}.
+ *
+ * <p>This is expected to be implemented by modules that also implement actions which need these
+ * contexts. Other modules will provide implementations for various action contexts by implementing
+ * {@link ActionContextProvider}.
+ *
+ * <p>Example: a module requires {@code SpawnActionContext} to do its job, and it creates
+ * actions with the mnemonic <code>C++</code>. Then the {@link #getSpawnActionContexts} method of
+ * this module would return a map with the key <code>"C++"</code> in it.
+ *
+ * <p>The module can either decide for itself which implementation is needed and make the value
+ * associated with this key a constant or defer that decision to the user, for example, by
+ * providing a command line option and setting the value in the map based on that.
+ *
+ * <p>Other modules are free to provide different implementations of {@code SpawnActionContext}.
+ * This can be used, for example, to implement sandboxed or distributed execution of
+ * {@code SpawnAction}s in different ways, while giving the user control over how exactly they
+ * are executed.
+ */
+public interface ActionContextConsumer {
+ /**
+ * Returns a map from spawn action mnemonics created by this module to the name of the
+ * implementation of {@code SpawnActionContext} that the module wants to use for executing
+ * it.
+ *
+ * <p>If a spawn action is executed whose mnemonic maps to the empty string or is not
+ * present in the map at all, the choice of the implementation is left to Blaze.
+ */
+ public Map<String, String> getSpawnActionContexts();
+
+ /**
+ * Returns a map from action context class to the implementation required by the module.
+ *
+ * <p>If the implementation name is the empty string, the choice is left to Blaze.
+ */
+ public Map<Class<? extends ActionContext>, String> getActionContexts();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java
new file mode 100644
index 0000000..fcb2e3d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation for action contexts. Actions contexts should also implement {@link ActionContext}.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ActionContextMarker {
+ String name();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java
new file mode 100644
index 0000000..6e52a4a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+/**
+ * An object that provides execution strategies to {@link BlazeExecutor}.
+ *
+ * <p>For more information, see {@link ActionContextConsumer}.
+ */
+public interface ActionContextProvider {
+ /**
+ * Returns the execution strategies that are provided by this object.
+ *
+ * <p>These may or may not actually end up in the executor depending on the command line options
+ * and other factors influencing how the executor is set up.
+ */
+ Iterable<ActionContext> getActionContexts();
+
+ /**
+ * Called when the executor is constructed. The parameter contains all the contexts that were
+ * selected for this execution phase.
+ */
+ void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException;
+
+ /**
+ * Called when the execution phase is started.
+ */
+ void executionPhaseStarting(
+ ActionInputFileCache actionInputFileCache,
+ ActionGraph actionGraph,
+ Iterable<Artifact> topLevelArtifacts)
+ throws ExecutorInitException, InterruptedException;
+
+ /**
+ * Called when the execution phase is finished.
+ */
+ void executionPhaseEnding();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java
new file mode 100644
index 0000000..00ef9b4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * This event is fired during the build, when an action is executed. It contains information about
+ * the action: the Action itself, and the output file names its stdout and stderr are recorded in.
+ */
+public class ActionExecutedEvent {
+ private final Action action;
+ private final ActionExecutionException exception;
+ private final String stdout;
+ private final String stderr;
+
+ public ActionExecutedEvent(Action action,
+ ActionExecutionException exception, String stdout, String stderr) {
+ this.action = action;
+ this.exception = exception;
+ this.stdout = stdout;
+ this.stderr = stderr;
+ }
+
+ public Action getAction() {
+ return action;
+ }
+
+ // null if action succeeded
+ public ActionExecutionException getException() {
+ return exception;
+ }
+
+ public String getStdout() {
+ return stdout;
+ }
+
+ public String getStderr() {
+ return stderr;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java
new file mode 100644
index 0000000..d6d08fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+
+/**
+ * A class that groups services in the scope of the action. Like the FileOutErr object.
+ */
+public class ActionExecutionContext {
+
+ private final Executor executor;
+ private final ActionInputFileCache actionInputFileCache;
+ private final MetadataHandler metadataHandler;
+ private final FileOutErr fileOutErr;
+ private final MiddlemanExpander middlemanExpander;
+
+ public ActionExecutionContext(Executor executor, ActionInputFileCache actionInputFileCache,
+ MetadataHandler metadataHandler, FileOutErr fileOutErr, MiddlemanExpander middlemanExpander) {
+ this.actionInputFileCache = actionInputFileCache;
+ this.metadataHandler = metadataHandler;
+ this.fileOutErr = fileOutErr;
+ this.executor = executor;
+ this.middlemanExpander = middlemanExpander;
+ }
+
+ public ActionInputFileCache getActionInputFileCache() {
+ return actionInputFileCache;
+ }
+
+ public MetadataHandler getMetadataHandler() {
+ return metadataHandler;
+ }
+
+ public Executor getExecutor() {
+ return executor;
+ }
+
+ public MiddlemanExpander getMiddlemanExpander() {
+ return middlemanExpander;
+ }
+
+ /**
+ * Provide that {@code FileOutErr} that the action should use for redirecting the output and error
+ * stream.
+ */
+ public FileOutErr getFileOutErr() {
+ return fileOutErr;
+ }
+
+ /**
+ * Allows us to create a new context that overrides the FileOutErr with another one. This is
+ * useful for muting the output for example.
+ */
+ public ActionExecutionContext withFileOutErr(FileOutErr fileOutErr) {
+ return new ActionExecutionContext(executor, actionInputFileCache, metadataHandler, fileOutErr,
+ middlemanExpander);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java
new file mode 100644
index 0000000..0d0d908
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * This exception gets thrown if {@link Action#execute(ActionExecutionContext)} is unsuccessful.
+ * Typically these are re-raised ExecException throwables.
+ */
+@ThreadSafe
+public class ActionExecutionException extends Exception {
+
+ private final Action action;
+ private final NestedSet<Label> rootCauses;
+ private final boolean catastrophe;
+
+ public ActionExecutionException(Throwable cause, Action action, boolean catastrophe) {
+ super(cause.getMessage(), cause);
+ this.action = action;
+ this.rootCauses = rootCausesFromAction(action);
+ this.catastrophe = catastrophe;
+ }
+
+ public ActionExecutionException(String message,
+ Throwable cause, Action action, boolean catastrophe) {
+ super(message + ": " + cause.getMessage(), cause);
+ this.action = action;
+ this.rootCauses = rootCausesFromAction(action);
+ this.catastrophe = catastrophe;
+ }
+
+ public ActionExecutionException(String message, Action action, boolean catastrophe) {
+ super(message);
+ this.action = action;
+ this.rootCauses = rootCausesFromAction(action);
+ this.catastrophe = catastrophe;
+ }
+
+ public ActionExecutionException(String message, Action action,
+ NestedSet<Label> rootCauses, boolean catastrophe) {
+ super(message);
+ this.action = action;
+ this.rootCauses = rootCauses;
+ this.catastrophe = catastrophe;
+ }
+
+ public ActionExecutionException(String message, Throwable cause, Action action,
+ NestedSet<Label> rootCauses, boolean catastrophe) {
+ super(message, cause);
+ this.action = action;
+ this.rootCauses = rootCauses;
+ this.catastrophe = catastrophe;
+ }
+
+ static NestedSet<Label> rootCausesFromAction(Action action) {
+ return action == null || action.getOwner() == null || action.getOwner().getLabel() == null
+ ? NestedSetBuilder.<Label>emptySet(Order.STABLE_ORDER)
+ : NestedSetBuilder.create(Order.STABLE_ORDER, action.getOwner().getLabel());
+ }
+
+ /**
+ * Returns the action that failed.
+ */
+ public Action getAction() {
+ return action;
+ }
+
+ /**
+ * Return the root causes that should be reported. Usually the owner of the action, but it can
+ * be the label of a missing artifact.
+ */
+ public NestedSet<Label> getRootCauses() {
+ return rootCauses;
+ }
+
+ /**
+ * Returns the location of the owner of this action. May be null.
+ */
+ public Location getLocation() {
+ return action.getOwner().getLocation();
+ }
+
+ /**
+ * Catastrophic exceptions should stop builds, even if --keep_going.
+ */
+ public boolean isCatastrophe() {
+ return catastrophe;
+ }
+
+ /**
+ * Returns true if the error should be shown.
+ */
+ public boolean showError() {
+ return getMessage() != null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java
new file mode 100644
index 0000000..34aadc4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java
@@ -0,0 +1,262 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * Implements "Still waiting..." message functionality, displaying current status for "in-flight"
+ * actions. Used by the ParallelBuilder.
+ *
+ * TODO(bazel-team): (2010) It would be nice if "duplicated" actions (e.g. test shards and multiple
+ * test runs) were merged into the single line.
+ */
+@ThreadSafe
+public final class ActionExecutionStatusReporter {
+ // Maximum number of lines to output per each status category before truncation.
+ private static final int MAX_LINES = 10;
+
+ private final EventHandler eventHandler;
+ private final Executor executor;
+ private final EventBus eventBus;
+ private final Clock clock;
+
+ /**
+ * The status of each action "in flight", i.e. whose ExecuteBuildAction.call() method is active.
+ * Used for implementing the "still waiting" message.
+ */
+ private final Map<ActionMetadata, Pair<String, Long>> actionStatus =
+ new ConcurrentHashMap<>(100);
+
+ public static ActionExecutionStatusReporter create(EventHandler eventHandler) {
+ return create(eventHandler, null, null);
+ }
+
+ @VisibleForTesting
+ static ActionExecutionStatusReporter create(EventHandler eventHandler, Clock clock) {
+ return create(eventHandler, null, null, clock);
+ }
+
+ public static ActionExecutionStatusReporter create(EventHandler eventHandler,
+ @Nullable Executor executor, @Nullable EventBus eventBus) {
+ return create(eventHandler, executor, eventBus, null);
+ }
+
+ private static ActionExecutionStatusReporter create(EventHandler eventHandler,
+ @Nullable Executor executor, @Nullable EventBus eventBus, @Nullable Clock clock) {
+ ActionExecutionStatusReporter result = new ActionExecutionStatusReporter(eventHandler, executor,
+ eventBus, clock == null ? BlazeClock.instance() : clock);
+ if (eventBus != null) {
+ eventBus.register(result);
+ }
+ return result;
+ }
+
+ private ActionExecutionStatusReporter(EventHandler eventHandler, @Nullable Executor executor,
+ @Nullable EventBus eventBus, Clock clock) {
+ this.eventHandler = Preconditions.checkNotNull(eventHandler);
+ this.executor = executor;
+ this.eventBus = eventBus;
+ this.clock = Preconditions.checkNotNull(clock);
+ }
+
+ public void unregisterFromEventBus() {
+ if (eventBus != null) {
+ eventBus.unregister(this);
+ }
+ }
+
+ private void setStatus(ActionMetadata action, String message) {
+ actionStatus.put(action, Pair.of(message, clock.nanoTime()));
+ }
+
+ /**
+ * Remove action from the list of active actions.
+ */
+ public void remove(Action action) {
+ Preconditions.checkNotNull(actionStatus.remove(action), action);
+ }
+
+ /**
+ * Set "Preparing" status.
+ */
+ public void setPreparing(Action action) {
+ updateStatus(ActionStatusMessage.preparingStrategy(action));
+ }
+
+ public void setRunningFromBuildData(ActionMetadata action) {
+ updateStatus(ActionStatusMessage.runningStrategy(action));
+ }
+
+ @Subscribe
+ public void updateStatus(ActionStatusMessage statusMsg) {
+ String message = statusMsg.getMessage();
+ ActionMetadata action = statusMsg.getActionMetadata();
+ if (statusMsg.needsStrategy()) {
+ String strategy = action.describeStrategy(executor);
+ if (strategy == null) {
+ return;
+ }
+ message = String.format(message, strategy);
+ }
+ setStatus(action, message);
+ }
+
+ public int getCount() {
+ return actionStatus.size();
+ }
+
+ private static void appendGroupStatus(StringBuilder buffer,
+ Map<ActionMetadata, Pair<String, Long>> statusMap, String status, long currentTime) {
+ List<Pair<Long, ActionMetadata>> actions = new ArrayList<>();
+ for (Map.Entry<ActionMetadata, Pair<String, Long>> entry : statusMap.entrySet()) {
+ if (entry.getValue().first.equals(status)) {
+ actions.add(Pair.of(entry.getValue().second, entry.getKey()));
+ }
+ }
+ if (actions.size() == 0) {
+ return;
+ }
+ Collections.sort(actions, Pair.<Long, ActionMetadata>compareByFirst());
+
+ buffer.append("\n " + status + ":");
+
+ boolean truncateList = actions.size() > MAX_LINES;
+ for (Pair<Long, ActionMetadata> entry : actions.subList(0,
+ truncateList ? MAX_LINES - 1 : actions.size())) {
+ String message = entry.second.getProgressMessage();
+ if (message == null) {
+ // Actions will a null progress message should run so
+ // fast we never see them here. In any case...
+ message = entry.second.prettyPrint();
+ }
+ buffer.append("\n ").append(message);
+ long runTime = (currentTime - entry.first) / 1000000000L; // Convert to seconds.
+ buffer.append(", ").append(runTime).append(" s");
+ }
+ if (truncateList) {
+ buffer.append("\n ... ").append(actions.size() - MAX_LINES + 1).append(" more jobs");
+ }
+ }
+
+ /**
+ * Get message showing currently executing actions.
+ */
+ private String getExecutionStatusMessage(Map<ActionMetadata, Pair<String, Long>> statusMap) {
+ int count = statusMap.size();
+ StringBuilder s = count != 1
+ ? new StringBuilder("Still waiting for ").append(count).append(" jobs to complete:")
+ : new StringBuilder("Still waiting for 1 job to complete:");
+
+ long currentTime = clock.nanoTime();
+
+ // A tree is just as fast as HashSet for small data sets.
+ Set<String> statuses = new TreeSet<String>();
+ for (Map.Entry<ActionMetadata, Pair<String, Long>> entry : statusMap.entrySet()) {
+ statuses.add(entry.getValue().first);
+ }
+
+ for (String status : statuses) {
+ appendGroupStatus(s, statusMap, status, currentTime);
+ }
+ return s.toString();
+ }
+
+ /**
+ * Show currently executing actions.
+ */
+ public void showCurrentlyExecutingActions(String progressPercentageMessage) {
+ // Defensive copy to ensure thread safety.
+ Map<ActionMetadata, Pair<String, Long>> statusMap = new HashMap<>(actionStatus);
+ if (statusMap.size() > 0) {
+ eventHandler.handle(
+ Event.progress(progressPercentageMessage + getExecutionStatusMessage(statusMap)));
+ }
+ }
+
+ /**
+ * Warn about actions that are still being executed.
+ * Method is used to produce informative message when build is interrupted.
+ */
+ void warnAboutCurrentlyExecutingActions() {
+ // Defensive copy to ensure thread safety.
+ Map<ActionMetadata, Pair<String, Long>> statusMap = new HashMap<>(actionStatus);
+ if (statusMap.size() == 0) {
+ // There are no tasks in the queue so there is nothing to report.
+ eventHandler.handle(Event.warn("There are no active jobs - stopping the build"));
+ return;
+ }
+ Iterator<ActionMetadata> iterator = statusMap.keySet().iterator();
+ while (iterator.hasNext()) {
+ // Filter out actions that are not executed yet.
+ if (statusMap.get(iterator.next()).first.equals(ActionStatusMessage.PREPARING)) {
+ iterator.remove();
+ }
+ }
+ if (statusMap.size() > 0) {
+ eventHandler.handle(Event.warn(getExecutionStatusMessage(statusMap)
+ + "\nBuild will be stopped after these tasks terminate"));
+ } else {
+ // It is possible that one or more tasks in "Preparing" state just started being executed.
+ // So warn user just in case.
+ eventHandler.handle(Event.warn("Still waiting for unfinished jobs"));
+ }
+ }
+
+ /**
+ * Returns the number of seconds to wait before reporting slow progress again.
+ *
+ * @param userSpecifiedProgressInterval value of the --progress_report_interval flag; 0 means
+ * use default 10, then 30, then 60 seconds wait times
+ * @param previousWaitTime previous value returned by this method
+ */
+ public static int getWaitTime(int userSpecifiedProgressInterval, int previousWaitTime) {
+ if (userSpecifiedProgressInterval > 0) {
+ return userSpecifiedProgressInterval;
+ }
+
+ // Increase waitTime to 10, then to 30 and then to 60 seconds to reduce
+ // spamming during long wait periods. If the user specified a
+ // waitTime directly through progressReportInterval, then use
+ // that value.
+ if (previousWaitTime == 0) {
+ return 10;
+ } else if (previousWaitTime == 10) {
+ return 30;
+ } else {
+ return 60;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java
new file mode 100644
index 0000000..bb2b707
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import javax.annotation.Nullable;
+
+/**
+ * An action graph.
+ *
+ * <p>Provides lookups of generating actions for artifacts.
+ */
+public interface ActionGraph {
+
+ /**
+ * Returns the Action that, when executed, gives rise to this file.
+ *
+ * <p>If this Artifact is a source file, null is returned. (We don't try to return a "no-op
+ * action" because that would require creating a new no-op Action for every source file, since
+ * each Action knows its outputs, so sharing all the no-ops is not an option.)
+ *
+ * <p>It's also possible for derived Artifacts to have null generating Actions when these actions
+ * are unknown.
+ */
+ @Nullable
+ Action getGeneratingAction(Artifact artifact);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java b/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java
new file mode 100644
index 0000000..4ddda4c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An abstract visitor for the action graph. Specializes {@link BipartiteVisitor} for artifacts and
+ * actions, and takes care of visiting the complete transitive closure.
+ */
+public abstract class ActionGraphVisitor extends BipartiteVisitor<Action, Artifact> {
+
+ private final ActionGraph actionGraph;
+
+ public ActionGraphVisitor(ActionGraph actionGraph) {
+ this.actionGraph = actionGraph;
+ }
+
+ /**
+ * Called for all artifacts in the visitation. Hook for subclasses.
+ *
+ * @param artifact
+ */
+ protected void visitArtifact(Artifact artifact) {}
+
+ /**
+ * Called for all actions in the visitation. Hook for subclasses.
+ *
+ * @param action
+ */
+ protected void visitAction(Action action) {}
+
+ /**
+ * Whether the given action should be visited. If this returns false, the visitation stops here,
+ * so the dependencies of this action are also not visited.
+ *
+ * @param action
+ */
+ protected boolean shouldVisit(Action action) {
+ return true;
+ }
+
+ /**
+ * Whether the given artifact should be visited. If this returns false, the visitation stops here,
+ * so dependencies of this artifact (if it is a generated one) are also not visited.
+ *
+ * @param artifact
+ */
+ protected boolean shouldVisit(Artifact artifact) {
+ return true;
+ }
+
+ @SuppressWarnings("unused")
+ protected final void visitArtifacts(Iterable<Artifact> artifacts) {
+ for (Artifact artifact : artifacts) {
+ visitArtifact(artifact);
+ }
+ }
+
+ @Override protected void white(Artifact artifact) {
+ Action action = actionGraph.getGeneratingAction(artifact);
+ visitArtifact(artifact);
+ if (action != null && shouldVisit(action)) {
+ visitBlackNode(action);
+ }
+ }
+
+ @Override protected void black(Action action) {
+ visitAction(action);
+ for (Artifact input : action.getInputs()) {
+ if (shouldVisit(input)) {
+ visitWhiteNode(input);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java
new file mode 100644
index 0000000..c370591
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * Represents an input file to a build action, with an appropriate relative path and digest
+ * value.
+ *
+ * <p>Artifact is the only notable implementer of the interface, but the interface remains
+ * because 1) some Google specific rules ship files that could be Artifacts to remote execution
+ * by instantiating ad-hoc derived classes of ActionInput. 2) historically, Google C++ rules
+ * allow underspecified C++ builds. For that case, we have extra logic to guess the undeclared
+ * header inclusions (eg. computed inclusions). The extra logic lives in a file that is not
+ * needed for remote execution, but is a dependency, and it is inserted as a non-Artifact
+ * ActionInput.
+ *
+ * <p>ActionInput is used as a cache "key" for ActionInputFileCache: for Artifacts, the
+ * digest/size is already stored in Artifact, but for non-artifacts, we use getExecPathString
+ * to find this data in a filesystem related cache.
+ */
+public interface ActionInput {
+
+ /**
+ * @return the relative path to the input file.
+ */
+ public String getExecPathString();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java
new file mode 100644
index 0000000..b45e9cd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * The interface for Action inputs metadata (Digest and size).
+ *
+ * NOTE: Implementations must be thread safe.
+ */
+@ThreadSafe
+public interface ActionInputFileCache {
+ /**
+ * Returns digest for the given artifact. This digest is current as of some time t >= the start of
+ * the present build. If the artifact is an output of an action that already executed at time p,
+ * then t >= p. Aside from these properties, t can be any value and may vary arbitrarily across
+ * calls.
+ *
+ * @param input the input to retrieve the digest for
+ * @return the artifact's digest or null if digest cannot be obtained (due to artifact
+ * non-existence, lookup errors, or any other reason)
+ *
+ * @throws DigestOfDirectoryException in case {@code input} is a directory.
+ * @throws IOException If the file cannot be digested.
+ *
+ */
+ @Nullable
+ ByteString getDigest(ActionInput input) throws IOException;
+
+ /**
+ * Retrieve the size of the file at the given path. Will usually return 0 on failure instead of
+ * throwing an IOException. Returns 0 for files inaccessible to user, but available to the
+ * execution environment.
+ *
+ * @param input the input.
+ * @return the file size in bytes.
+ * @throws IOException on failure.
+ */
+ long getSizeInBytes(ActionInput input) throws IOException;
+
+ /**
+ * Checks if the file is available locally, based on the assumption that previous operations on
+ * the ActionInputFileCache would have created a cache entry for it.
+ *
+ * @param digest the digest to lookup.
+ * @return true if the specified digest is backed by a locally-readable file, false otherwise
+ */
+ boolean contentsAvailableLocally(ByteString digest);
+
+ /**
+ * Concrete subclasses must implement this to provide a mapping from digest to file path,
+ * based on files previously seen as inputs.
+ *
+ * @param digest the digest.
+ * @return a File path.
+ */
+ @Nullable
+ File getFileFromDigest(ByteString digest) throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java
new file mode 100644
index 0000000..0fed928
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java
@@ -0,0 +1,160 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Helper utility to create ActionInput instances.
+ */
+public final class ActionInputHelper {
+ private ActionInputHelper() {
+ }
+
+ @VisibleForTesting
+ public static Artifact.MiddlemanExpander actionGraphMiddlemanExpander(
+ final ActionGraph actionGraph) {
+ return new Artifact.MiddlemanExpander() {
+ @Override
+ public void expand(Artifact mm, Collection<? super Artifact> output) {
+ // Skyframe is stricter in that it checks that "mm" is a input of the action, because
+ // it cannot expand arbitrary middlemen without access to a global action graph.
+ // We could check this constraint here too, but it seems unnecessary. This code is
+ // going away anyway.
+ Preconditions.checkArgument(mm.isMiddlemanArtifact(),
+ "%s is not a middleman artifact", mm);
+ Action middlemanAction = actionGraph.getGeneratingAction(mm);
+ Preconditions.checkState(middlemanAction != null, mm);
+ // TODO(bazel-team): Consider expanding recursively or throwing an exception here.
+ // Most likely, this code will cause silent errors if we ever have a middleman that
+ // contains a middleman.
+ if (middlemanAction.getActionType() == Action.MiddlemanType.AGGREGATING_MIDDLEMAN) {
+ Artifact.addNonMiddlemanArtifacts(middlemanAction.getInputs(), output,
+ Functions.<Artifact>identity());
+ }
+
+ }
+ };
+ }
+
+ /**
+ * Most ActionInputs are created and never used again. On the off chance that one is, however, we
+ * implement equality via path comparison. Since file caches are keyed by ActionInput, equality
+ * checking does come up.
+ */
+ private static class BasicActionInput implements ActionInput {
+ private final String path;
+ public BasicActionInput(String path) {
+ this.path = Preconditions.checkNotNull(path);
+ }
+
+ @Override
+ public String getExecPathString() {
+ return path;
+ }
+
+ @Override
+ public int hashCode() {
+ return path.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null) {
+ return false;
+ }
+ if (!this.getClass().equals(other.getClass())) {
+ return false;
+ }
+ return this.path.equals(((BasicActionInput) other).path);
+ }
+
+ @Override
+ public String toString() {
+ return "BasicActionInput: " + path;
+ }
+ }
+
+ /**
+ * Creates an ActionInput with just the given relative path and no digest.
+ *
+ * @param path the relative path of the input.
+ * @return a ActionInput.
+ */
+ public static ActionInput fromPath(String path) {
+ return new BasicActionInput(path);
+ }
+
+ private static final Function<String, ActionInput> FROM_PATH =
+ new Function<String, ActionInput>() {
+ @Override
+ public ActionInput apply(String path) {
+ return fromPath(path);
+ }
+ };
+
+ /**
+ * Creates a sequence of {@link ActionInput}s from a sequence of string paths.
+ */
+ public static Collection<ActionInput> fromPaths(Collection<String> paths) {
+ return Collections2.transform(paths, FROM_PATH);
+ }
+
+ /**
+ * Expands middleman artifacts in a sequence of {@link ActionInput}s.
+ *
+ * <p>Non-middleman artifacts are returned untouched.
+ */
+ public static List<ActionInput> expandMiddlemen(Iterable<? extends ActionInput> inputs,
+ Artifact.MiddlemanExpander middlemanExpander) {
+
+ List<ActionInput> result = new ArrayList<>();
+ List<Artifact> containedArtifacts = new ArrayList<>();
+ for (ActionInput input : inputs) {
+ if (!(input instanceof Artifact)) {
+ result.add(input);
+ continue;
+ }
+ containedArtifacts.add((Artifact) input);
+ }
+ Artifact.addExpandedArtifacts(containedArtifacts, result, middlemanExpander);
+ return result;
+ }
+
+ /** Formatter for execPath String output. Public because Artifact uses it directly. */
+ public static final Function<ActionInput, String> EXEC_PATH_STRING_FORMATTER =
+ new Function<ActionInput, String>() {
+ @Override
+ public String apply(ActionInput input) {
+ return input.getExecPathString();
+ }
+ };
+
+ public static Iterable<String> toExecPaths(Iterable<? extends ActionInput> artifacts) {
+ return Iterables.transform(artifacts, EXEC_PATH_STRING_FORMATTER);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java b/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java
new file mode 100644
index 0000000..2c80e8a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A source for generating unique action log paths.
+ */
+public final class ActionLogBufferPathGenerator {
+
+ private final AtomicInteger actionCounter = new AtomicInteger();
+
+ private final Path actionOutputRoot;
+
+ public ActionLogBufferPathGenerator(Path actionOutputRoot) {
+ this.actionOutputRoot = actionOutputRoot;
+ }
+
+ /**
+ * Generates a unique filename for an action to store its output.
+ */
+ public FileOutErr generate() {
+ int actionId = actionCounter.incrementAndGet();
+ return new FileOutErr(actionOutputRoot.getRelative("stdout-" + actionId),
+ actionOutputRoot.getRelative("stderr-" + actionId));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java b/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java
new file mode 100644
index 0000000..3569f07
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java
@@ -0,0 +1,217 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import javax.annotation.Nullable;
+
+/**
+ * Side-effect free query methods for information about an {@link Action}.
+ *
+ * <p>This method is intended for use in situations when the intention is to pass around information
+ * about an action without allowing actual execution of the action.
+ *
+ * <p>The split between {@link Action} and {@link ActionMetadata} is somewhat arbitrary, other than
+ * that all methods with side effects must belong to the former.
+ */
+public interface ActionMetadata {
+ /**
+ * If this executable can supply verbose information, returns a string that can be used as a
+ * progress message while this executable is running. A return value of {@code null} indicates no
+ * message should be reported.
+ */
+ @Nullable
+ public String getProgressMessage();
+
+ /**
+ * Returns the owner of this executable if this executable can supply verbose information. This is
+ * typically the rule that constructed it; see ActionOwner class comment for details. Returns
+ * {@code null} if no owner can be determined.
+ *
+ * <p>If this executable does not supply verbose information, this function may throw an
+ * IllegalStateException.
+ */
+ public ActionOwner getOwner();
+
+ /**
+ * Returns a mnemonic (string constant) for this kind of action; written into
+ * the master log so that the appropriate parser can be invoked for the output
+ * of the action. Effectively a public method as the value is used by the
+ * extra_action feature to match actions.
+ */
+ String getMnemonic();
+
+ /**
+ * Returns a pretty string representation of this action, suitable for use in
+ * progress messages or error messages.
+ */
+ String prettyPrint();
+
+ /**
+ * Returns a string that can be used to describe the execution strategy.
+ * For example, "local".
+ *
+ * May return null if the action chooses to update its strategy
+ * locality "manually", via ActionLocalityMessage.
+ *
+ * @param executor the application-specific value passed to the
+ * executor parameter of the top-level call to
+ * Builder.buildArtifacts().
+ */
+ public String describeStrategy(Executor executor);
+
+ /**
+ * Returns true iff the getInputs set is known to be complete.
+ *
+ * <p>For most Actions, this always returns true, but in some cases (e.g. C++ compilation), inputs
+ * are dynamically discovered from the previous execution of the Action, and so before the initial
+ * execution, this method will return false in those cases.
+ *
+ * <p>Any builder <em>must</em> unconditionally execute an Action for which inputsKnown() returns
+ * false, regardless of all other inferences made by its dependency analysis. In addition, all
+ * prerequisites mentioned in the (possibly incomplete) value returned by getInputs must also be
+ * built first, as usual.
+ */
+ @ThreadSafe
+ boolean inputsKnown();
+
+ /**
+ * Returns true iff inputsKnown() may ever return false.
+ */
+ @ThreadSafe
+ boolean discoversInputs();
+
+ /**
+ * Returns the input Artifacts that this Action depends upon. May be empty.
+ *
+ * <p>For subclasses overriding getInputs(), if getInputs() could return different values in the
+ * lifetime of an object, {@link #getInputCount()} must also be overridden.
+ *
+ * <p>During execution, the {@link Iterable} returned by {@code getInputs} <em>must not</em> be
+ * concurrently modified before the value is fully read in {@code JavaDistributorDriver#exec} (via
+ * the {@code Iterable<ActionInput>} argument there). Violating this would require somewhat
+ * pathological behavior by the {@link Action}, since it would have to modify its inputs, as a
+ * list, say, without reassigning them. This should never happen with any Action subclassing
+ * AbstractAction, since AbstractAction's implementation of getInputs() returns an immutable
+ * iterable.
+ */
+ Iterable<Artifact> getInputs();
+
+ /**
+ * Returns the number of input Artifacts that this Action depends upon.
+ *
+ * <p>Must be consistent with {@link #getInputs()}.
+ */
+ int getInputCount();
+
+ /**
+ * Returns the (unordered, immutable) set of output Artifacts that
+ * this action generates. (It would not make sense for this to be empty.)
+ */
+ ImmutableSet<Artifact> getOutputs();
+
+ /**
+ * Returns the "primary" input of this action, if applicable.
+ *
+ * <p>For example, a C++ compile action would return the .cc file which is being compiled,
+ * irrespective of the other inputs.
+ *
+ * <p>May return null.
+ */
+ Artifact getPrimaryInput();
+
+ /**
+ * Returns the "primary" output of this action.
+ *
+ * <p>For example, the linked library would be the primary output of a LinkAction.
+ *
+ * <p>Never returns null.
+ */
+ Artifact getPrimaryOutput();
+
+ /**
+ * Returns an iterable of input Artifacts that MUST exist prior to executing an action. In other
+ * words, in case when action is scheduled for execution, builder will ensure that all artifacts
+ * returned by this method are present in the filesystem (artifact.getPath().exists() is true) or
+ * action execution will be aborted with an error that input file does not exist. While in
+ * majority of cases this method will return all action inputs, for some actions (e.g.
+ * CppCompileAction) it can return a subset of inputs because that not all action inputs might be
+ * mandatory for action execution to succeed (e.g. header files retrieved from *.d file from the
+ * previous build).
+ */
+ Iterable<Artifact> getMandatoryInputs();
+
+ /**
+ * <p>Returns a string encoding all of the significant behaviour of this
+ * Action that might affect the output. The general contract of
+ * <code>getKey</code> is this: if the work to be performed by the
+ * execution of this action changes, the key must change. </p>
+ *
+ * <p>As a corollary, the build system is free to omit the execution of an
+ * Action <code>a1</code> if (a) at some time in the past, it has already
+ * executed an Action <code>a0</code> with the same key as
+ * <code>a1</code>, and (b) the names and contents of the input files listed
+ * by <code>a1.getInputs()</code> are identical to the names and contents of
+ * the files listed by <code>a0.getInputs()</code>. </p>
+ *
+ * <p>Examples of changes that should affect the key are:
+ * <ul>
+ * <li>Changes to the BUILD file that materially affect the rule which gave
+ * rise to this Action.</li>
+ *
+ * <li>Changes to the command-line options, environment, or other global
+ * configuration resources which affect the behaviour of this kind of Action
+ * (other than changes to the names of the input/output files, which are
+ * handled externally).</li>
+ *
+ * <li>An upgrade to the build tools which changes the program logic of this
+ * kind of Action (typically this is achieved by incorporating a UUID into
+ * the key, which is changed each time the program logic of this action
+ * changes).</li>
+ *
+ * </ul></p>
+ */
+ String getKey();
+
+ /**
+ * Returns a human-readable description of the inputs to {@link #getKey()}.
+ * Used in the output from '--explain', and in error messages for
+ * '--check_up_to_date' and '--check_tests_up_to_date'.
+ * May return null, meaning no extra information is available.
+ *
+ * <p>If the return value is non-null, for consistency it should be a multiline message of the
+ * form:
+ * <pre>
+ * <var>Summary</var>
+ * <var>Fieldname</var>: <var>value</var>
+ * <var>Fieldname</var>: <var>value</var>
+ * ...
+ * </pre>
+ * where each line after the first one is intended two spaces, and where any fields that might
+ * contain newlines or other funny characters are escaped using {@link
+ * com.google.devtools.build.lib.shell.ShellUtils#shellEscape}.
+ * For example:
+ * <pre>
+ * Compiling foo.cc
+ * Command: /usr/bin/gcc
+ * Argument: '-c'
+ * Argument: foo.cc
+ * Argument: '-o'
+ * Argument: foo.o
+ * </pre>
+ */
+ @Nullable String describeKey();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java
new file mode 100644
index 0000000..855d97a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java
@@ -0,0 +1,54 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * This event is fired during the build, when a middleman action is executed. Middleman actions
+ * don't usually do any computation but we need them in the critical path because they depend on
+ * other actions.
+ */
+public class ActionMiddlemanEvent {
+
+ private final Action action;
+ private final long nanoTimeStart;
+
+ /**
+ * Create an event for action that has been started.
+ *
+ * @param action the middleman action.
+ * @param nanoTimeStart the time when the action was started. This allow us to record more
+ * accurately the time spent by the middleman action, since even for middleman actions we execute
+ * some.
+ */
+ public ActionMiddlemanEvent(Action action, long nanoTimeStart) {
+ Preconditions.checkArgument(action.getActionType().isMiddleman(),
+ "Only middleman actions should be passed: %s", action);
+ this.action = action;
+ this.nanoTimeStart = nanoTimeStart;
+ }
+
+ /**
+ * Returns the associated action.
+ */
+ public Action getAction() {
+ return action;
+ }
+
+ public long getNanoTimeStart() {
+ return nanoTimeStart;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java
new file mode 100644
index 0000000..ab59f63
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * The owner of an action is responsible for reporting conflicts in the action
+ * graph (two actions attempting to generate the same artifact).
+ *
+ * Typically an action's owner is the RuleConfiguredTarget instance responsible
+ * for creating it, but to avoid coupling between the view and actions
+ * packages, the RuleConfiguredTarget is hidden behind this interface, which
+ * exposes only the error reporting functionality.
+ */
+public interface ActionOwner {
+
+ /**
+ * Returns the location of this ActionOwner, if any; null otherwise.
+ */
+ Location getLocation();
+
+ /**
+ * Returns the label for this ActionOwner, if any; null otherwise.
+ */
+ Label getLabel();
+
+ /**
+ * Returns the name of the configuration of the action owner.
+ */
+ String getConfigurationName();
+
+ /**
+ * Returns the configuration's mnemonic.
+ */
+ String getConfigurationMnemonic();
+
+ /**
+ * Returns the short cache key for the configuration of the action owner.
+ *
+ * <p>Special action owners that are not targets can return any string here as long as it is
+ * constant. If the configuration is null, this should return "null".
+ *
+ * <p>These requirements exist so that {@link ActionOwner} instances are consistent with
+ * {@code BuildView.ActionOwnerIdentity(ConfiguredTargetValue)}.
+ */
+ String getConfigurationShortCacheKey();
+
+ /**
+ * Returns the target kind (rule class name) for this ActionOwner, if any; null otherwise.
+ */
+ String getTargetKind();
+
+ /**
+ * Returns additional information that should be displayed in progress messages, or {@code null}
+ * if nothing should be added.
+ */
+ String getAdditionalProgressInfo();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java b/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java
new file mode 100644
index 0000000..db7e350
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * An interface for registering actions.
+ */
+public interface ActionRegistry {
+ /**
+ * This method notifies the registry new actions.
+ */
+ void registerAction(Action... actions);
+
+ /**
+ * Get the (Label and BuildConfiguration) of the ConfiguredTarget ultimately responsible for all
+ * these actions.
+ */
+ ArtifactOwner getOwner();
+
+ /**
+ * An action registry that does exactly nothing.
+ */
+ @VisibleForTesting
+ public static final ActionRegistry NOP = new ActionRegistry() {
+ @Override
+ public void registerAction(Action... actions) {}
+
+ @Override
+ public ArtifactOwner getOwner() {
+ return ArtifactOwner.NULL_OWNER;
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java
new file mode 100644
index 0000000..a2a978f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * This event is fired during the build, when an action is started.
+ */
+public class ActionStartedEvent {
+ private final Action action;
+ private final long nanoTimeStart;
+
+ /**
+ * Create an event for action that has been started.
+ *
+ * @param action the started action.
+ * @param nanoTimeStart the time when the action was started. This allow us to
+ * record more accurately the time spend by the action, since we execute some code before
+ * deciding if we execute the action or not.
+ */
+ public ActionStartedEvent(Action action, long nanoTimeStart) {
+ this.action = action;
+ this.nanoTimeStart = nanoTimeStart;
+ }
+
+ /**
+ * Returns the associated action.
+ */
+ public Action getAction() {
+ return action;
+ }
+
+ public long getNanoTimeStart() {
+ return nanoTimeStart;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java
new file mode 100644
index 0000000..c932a9e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * A message used to update in-flight action status. An action's status may change low down in the
+ * execution stack (for instance, from running remotely to running locally), so this message can be
+ * used to notify any interested parties.
+ */
+public class ActionStatusMessage {
+ private final ActionMetadata action;
+ private final String message;
+ public static final String PREPARING = "Preparing";
+
+ public ActionStatusMessage(ActionMetadata action, String message) {
+ this.action = action;
+ this.message = message;
+ }
+
+ public ActionMetadata getActionMetadata() {
+ return action;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ /** Returns whether the message needs further interpolation of a 'strategy' when printed. */
+ public boolean needsStrategy() {
+ return false;
+ }
+
+ /** Creates "Analyzing" status message. */
+ public static ActionStatusMessage analysisStrategy(ActionMetadata action) {
+ return new ActionStatusMessage(action, "Analyzing");
+ }
+
+ /** Creates "Preparing" status message. */
+ public static ActionStatusMessage preparingStrategy(ActionMetadata action) {
+ return new ActionStatusMessage(action, PREPARING);
+ }
+
+ /** Creates "Scheduling" status message. */
+ public static ActionStatusMessage schedulingStrategy(ActionMetadata action) {
+ return new ActionStatusMessage(action, "Scheduling");
+ }
+
+ /** Creates "Running (%s)" status message (needs strategy interpolated). */
+ public static ActionStatusMessage runningStrategy(ActionMetadata action) {
+ return new ActionStatusMessage(action, "Running (%s)") {
+ @Override
+ public boolean needsStrategy() {
+ return true;
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Actions.java b/src/main/java/com/google/devtools/build/lib/actions/Actions.java
new file mode 100644
index 0000000..fb2b834
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/Actions.java
@@ -0,0 +1,79 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.Iterables;
+import com.google.common.escape.Escaper;
+import com.google.common.escape.Escapers;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Helper class for actions.
+ */
+@ThreadSafe
+public final class Actions {
+ private static final Escaper PATH_ESCAPER = Escapers.builder()
+ .addEscape('_', "_U")
+ .addEscape('/', "_S")
+ .addEscape('\\', "_B")
+ .addEscape(':', "_C")
+ .build();
+
+ /**
+ * Checks if the two actions are equivalent. This method exists to support sharing actions between
+ * configured targets for cases where there is no canonical target that could own the action. In
+ * the action graph construction this case shows up as two actions generating the same output
+ * file.
+ *
+ * <p>This method implements an equivalence relationship across actions, based on the action
+ * class, the key, and the list of inputs and outputs.
+ */
+ public static boolean canBeShared(Action a, Action b) {
+ if (!a.getMnemonic().equals(b.getMnemonic())) {
+ return false;
+ }
+ if (!a.getKey().equals(b.getKey())) {
+ return false;
+ }
+ // Don't bother to check input and output counts first; the expected result for these tests is
+ // to always be true (i.e., that this method returns true).
+ if (!Iterables.elementsEqual(a.getMandatoryInputs(), b.getMandatoryInputs())) {
+ return false;
+ }
+ if (!Iterables.elementsEqual(a.getOutputs(), b.getOutputs())) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns the escaped name for a given relative path as a string. This takes
+ * a short relative path and turns it into a string suitable for use as a
+ * filename. Invalid filename characters are escaped with an '_' + a single
+ * character token.
+ */
+ public static String escapedPath(String path) {
+ return PATH_ESCAPER.escape(path);
+ }
+
+ /**
+ * Returns a string that is usable as a unique path component for a label. It is guaranteed
+ * that no other label maps to this string.
+ */
+ public static String escapeLabel(Label label) {
+ return PATH_ESCAPER.escape(label.getPackageName() + ":" + label.getName());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java b/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java
new file mode 100644
index 0000000..4ce258b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * This wrapper exception is used as a marker class to already reported errors. Errors are reported
+ * at {@code AbstractBuilder.executeActionTask()} method in case of the builder not aborting in case
+ * of exceptions (For example keepgoing).
+ *
+ * <p>Then In upper levels we wrap catch the exception and throw a BuildFailedException
+ * unconditionally, that is caught and shown as error in AbstractBuildCommand (because the message
+ * of the exception is !=null).
+ *
+ * With this exception we detect that the error was already shown and we wrap it in a
+ * BuildFailedException without message.
+ */
+public class AlreadyReportedActionExecutionException extends ActionExecutionException {
+
+ public AlreadyReportedActionExecutionException(ActionExecutionException cause) {
+ super(cause.getMessage(), cause.getCause(), cause.getAction(), cause.getRootCauses(),
+ cause.isCatastrophe());
+ }
+
+ @Override
+ public boolean showError() {
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
new file mode 100644
index 0000000..2f2272b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
@@ -0,0 +1,654 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * An Artifact represents a file used by the build system, whether it's a source
+ * file or a derived (output) file. Not all Artifacts have a corresponding
+ * FileTarget object in the <code>build.packages</code> API: for example,
+ * low-level intermediaries internal to a given rule, such as a Java class files
+ * or C++ object files. However all FileTargets have a corresponding Artifact.
+ *
+ * <p>In any given call to Builder#buildArtifacts(), no two Artifacts in the
+ * action graph may refer to the same path.
+ *
+ * <p>Artifacts generally fall into two classifications, source and derived, but
+ * there exist a few other cases that are fuzzy and difficult to classify. The
+ * following cases exist:
+ * <ul>
+ * <li>Well-formed source Artifacts will have null generating Actions and a root
+ * that is orthogonal to execRoot. (With the root coming from the package path.)
+ * <li>Well-formed derived Artifacts will have non-null generating Actions, and
+ * a root that is below execRoot.
+ * <li>Symlinked include source Artifacts under the output/include tree will
+ * appear to be derived artifacts with null generating Actions.
+ * <li>Some derived Artifacts, mostly in the genfiles tree and mostly discovered
+ * during include validation, will also have null generating Actions.
+ * </ul>
+ *
+ * <p>This class is "theoretically" final; it should not be subclassed except by
+ * {@link SpecialArtifact}.
+ */
+@Immutable
+@SkylarkModule(name = "File",
+ doc = "This type represents a file used by the build system. It can be "
+ + "either a source file or a derived file produced by a rule.")
+public class Artifact implements FileType.HasFilename, Comparable<Artifact>, ActionInput {
+
+ /** An object that can expand middleman artifacts. */
+ public interface MiddlemanExpander {
+
+ /**
+ * Expands the middleman artifact "mm", and populates "output" with the result.
+ *
+ * <p>{@code mm.isMiddlemanArtifact()} must be true. Only aggregating middlemen are expanded.
+ */
+ void expand(Artifact mm, Collection<? super Artifact> output);
+ }
+
+ public static final ImmutableList<Artifact> NO_ARTIFACTS = ImmutableList.of();
+
+ /**
+ * A Predicate that evaluates to true if the Artifact is not a middleman artifact.
+ */
+ public static final Predicate<Artifact> MIDDLEMAN_FILTER = new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact input) {
+ return !input.isMiddlemanArtifact();
+ }
+ };
+
+ private final Path path;
+ private final Root root;
+ private final PathFragment execPath;
+ private final PathFragment rootRelativePath;
+ // Non-final only for use when dealing with deserialized artifacts.
+ private ArtifactOwner owner;
+
+ /**
+ * Constructs an artifact for the specified path, root and execPath. The root must be an ancestor
+ * of path, and execPath must be a non-absolute tail of path. Outside of testing, this method
+ * should only be called by ArtifactFactory. The ArtifactOwner may be null.
+ *
+ * <p>In a source Artifact, the path tail after the root will be identical to the execPath, but
+ * the root will be orthogonal to execRoot.
+ * <pre>
+ * [path] == [/root/][execPath]
+ * </pre>
+ *
+ * <p>In a derived Artifact, the execPath will overlap with part of the root, which in turn will
+ * be below of the execRoot.
+ * <pre>
+ * [path] == [/root][pathTail] == [/execRoot][execPath] == [/execRoot][rootPrefix][pathTail]
+ * <pre>
+ */
+ @VisibleForTesting
+ public Artifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner) {
+ if (root == null || !path.startsWith(root.getPath())) {
+ throw new IllegalArgumentException(root + ": illegal root for " + path);
+ }
+ if (execPath == null || execPath.isAbsolute() || !path.asFragment().endsWith(execPath)) {
+ throw new IllegalArgumentException(execPath + ": illegal execPath for " + path);
+ }
+ this.path = path;
+ this.root = root;
+ this.execPath = execPath;
+ // These two lines establish the invariant that
+ // execPath == rootRelativePath <=> execPath.equals(rootRelativePath)
+ // This is important for isSourceArtifact.
+ PathFragment rootRel = path.relativeTo(root.getPath());
+ if (!execPath.endsWith(rootRel)) {
+ throw new IllegalArgumentException(execPath + ": illegal execPath doesn't end with "
+ + rootRel + " at " + path + " with root " + root);
+ }
+ this.rootRelativePath = rootRel.equals(execPath) ? execPath : rootRel;
+ this.owner = Preconditions.checkNotNull(owner, path);
+ }
+
+ /**
+ * Constructs an artifact for the specified path, root and execPath. The root must be an ancestor
+ * of path, and execPath must be a non-absolute tail of path. Should only be called for testing.
+ *
+ * <p>In a source Artifact, the path tail after the root will be identical to the execPath, but
+ * the root will be orthogonal to execRoot.
+ * <pre>
+ * [path] == [/root/][execPath]
+ * </pre>
+ *
+ * <p>In a derived Artifact, the execPath will overlap with part of the root, which in turn will
+ * be below of the execRoot.
+ * <pre>
+ * [path] == [/root][pathTail] == [/execRoot][execPath] == [/execRoot][rootPrefix][pathTail]
+ * <pre>
+ */
+ @VisibleForTesting
+ public Artifact(Path path, Root root, PathFragment execPath) {
+ this(path, root, execPath, ArtifactOwner.NULL_OWNER);
+ }
+
+ /**
+ * Constructs a source or derived Artifact for the specified path and specified root. The root
+ * must be an ancestor of the path.
+ */
+ @VisibleForTesting // Only exists for testing.
+ public Artifact(Path path, Root root) {
+ this(path, root, root.getExecPath().getRelative(path.relativeTo(root.getPath())),
+ ArtifactOwner.NULL_OWNER);
+ }
+
+ /**
+ * Constructs a source or derived Artifact for the specified root-relative path and root.
+ */
+ @VisibleForTesting // Only exists for testing.
+ public Artifact(PathFragment rootRelativePath, Root root) {
+ this(root.getPath().getRelative(rootRelativePath), root,
+ root.getExecPath().getRelative(rootRelativePath), ArtifactOwner.NULL_OWNER);
+ }
+
+ /**
+ * Returns the location of this Artifact on the filesystem.
+ */
+ public final Path getPath() {
+ return path;
+ }
+
+ /**
+ * Returns the base file name of this artifact.
+ */
+ @Override
+ public final String getFilename() {
+ return getExecPath().getBaseName();
+ }
+
+ /**
+ * Returns the artifact owner. May be null.
+ */
+ @Nullable public final Label getOwner() {
+ return owner.getLabel();
+ }
+
+ /**
+ * Get the {@code LabelAndConfiguration} of the {@code ConfiguredTarget} that owns this artifact,
+ * if it was set. Otherwise, this should be a dummy value -- either {@link
+ * ArtifactOwner#NULL_OWNER} or a dummy owner set in tests. Such a dummy value should only occur
+ * for source artifacts if created without specifying the owner, or for special derived artifacts,
+ * such as target completion middleman artifacts, build info artifacts, and the like.
+ *
+ * When deserializing artifacts we end up with a dummy owner. In that case, it must be set using
+ * {@link #setArtifactOwner} before this method is called.
+ */
+ public final ArtifactOwner getArtifactOwner() {
+ Preconditions.checkState(owner != DESERIALIZED_MARKER_OWNER, this);
+ return owner;
+ }
+
+ /**
+ * Sets the artifact owner of this artifact. Should only be called for artifacts that were created
+ * through deserialization, and so their owner was unknown at the time of creation.
+ */
+ public final void setArtifactOwner(ArtifactOwner owner) {
+ if (this.owner == DESERIALIZED_MARKER_OWNER) {
+ // We tolerate multiple calls of this method to accommodate shared actions.
+ this.owner = Preconditions.checkNotNull(owner, this);
+ }
+ }
+
+ /**
+ * Returns the root beneath which this Artifact resides, if any. This may be one of the
+ * package-path entries (for source Artifacts), or one of the bin, genfiles or includes dirs
+ * (for derived Artifacts). It will always be an ancestor of getPath().
+ */
+ public final Root getRoot() {
+ return root;
+ }
+
+ /**
+ * Returns the exec path of this Artifact. The exec path is a relative path
+ * that is suitable for accessing this artifact relative to the execution
+ * directory for this build.
+ */
+ public final PathFragment getExecPath() {
+ return execPath;
+ }
+
+ /**
+ * Returns true iff this is a source Artifact as determined by its path and
+ * root relationships. Note that this will report all Artifacts in the output
+ * tree, including in the include symlink tree, as non-source.
+ */
+ public final boolean isSourceArtifact() {
+ return execPath == rootRelativePath;
+ }
+
+ /**
+ * Returns true iff this is a middleman Artifact as determined by its root.
+ */
+ public final boolean isMiddlemanArtifact() {
+ return getRoot().isMiddlemanRoot();
+ }
+
+ /**
+ * Returns whether the artifact represents a Fileset.
+ */
+ public boolean isFileset() {
+ return false;
+ }
+
+ /**
+ * Returns true iff metadata cache must return constant metadata for the
+ * given artifact.
+ */
+ public boolean isConstantMetadata() {
+ return false;
+ }
+
+ /**
+ * Special artifact types.
+ *
+ * @see SpecialArtifact
+ */
+ static enum SpecialArtifactType {
+ FILESET,
+ CONSTANT_METADATA,
+ }
+
+ /**
+ * A special kind of artifact that either is a fileset or needs special metadata caching behavior.
+ *
+ * <p>We subclass {@link Artifact} instead of storing the special attributes inside in order
+ * to save memory. The proportion of artifacts that are special is very small, and by not having
+ * to keep around the attribute for the rest we save some memory.
+ */
+ @Immutable
+ @VisibleForTesting
+ public static final class SpecialArtifact extends Artifact {
+ private final SpecialArtifactType type;
+
+ SpecialArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner,
+ SpecialArtifactType type) {
+ super(path, root, execPath, owner);
+ this.type = type;
+ }
+
+ @Override
+ public final boolean isFileset() {
+ return type == SpecialArtifactType.FILESET;
+ }
+
+ @Override
+ public boolean isConstantMetadata() {
+ return type == SpecialArtifactType.CONSTANT_METADATA;
+ }
+ }
+
+ /**
+ * Returns the relative path to this artifact relative to its root. (Useful
+ * when deriving output filenames from input files, etc.)
+ */
+ public final PathFragment getRootRelativePath() {
+ return rootRelativePath;
+ }
+
+ /**
+ * Returns this.getExecPath().getPathString().
+ */
+ @Override
+ @SkylarkCallable(name = "path", structField = true,
+ doc = "The execution path of this file, relative to the execution directory.")
+ public final String getExecPathString() {
+ return getExecPath().getPathString();
+ }
+
+ @SkylarkCallable(name = "short_path", structField = true,
+ doc = "The path of this file relative to its root.")
+ public final String getRootRelativePathString() {
+ return getRootRelativePath().getPathString();
+ }
+
+ /**
+ * Returns a pretty string representation of the path denoted by this artifact, suitable for use
+ * in user error messages. Artifacts beneath a root will be printed relative to that root; other
+ * artifacts will be printed as an absolute path.
+ *
+ * <p>(The toString method is intended for developer messages since its more informative.)
+ */
+ public final String prettyPrint() {
+ // toDetailString would probably be more useful to users, but lots of tests rely on the
+ // current values.
+ return rootRelativePath.toString();
+ }
+
+ @Override
+ public final boolean equals(Object other) {
+ if (!(other instanceof Artifact)) {
+ return false;
+ }
+ // We don't bother to check root in the equivalence relation, because we
+ // assume that 'root' is an ancestor of 'path', and that all possible roots
+ // are disjoint, so unless things are really screwed up, it's ok.
+ Artifact that = (Artifact) other;
+ return this.path.equals(that.path);
+ }
+
+ @Override
+ public final int compareTo(Artifact o) {
+ // The artifact factory ensures that there is a unique artifact for a given path.
+ return this.path.compareTo(o.path);
+ }
+
+ @Override
+ public final int hashCode() {
+ return path.hashCode();
+ }
+
+ @Override
+ public final String toString() {
+ return "Artifact:" + toDetailString();
+ }
+
+ /**
+ * Returns the root-part of a given path by trimming off the end specified by
+ * a given tail. Assumes that the tail is known to match, and simply relies on
+ * the segment lengths.
+ */
+ private static PathFragment trimTail(PathFragment path, PathFragment tail) {
+ return path.subFragment(0, path.segmentCount() - tail.segmentCount());
+ }
+
+ /**
+ * Returns a string representing the complete artifact path information.
+ */
+ public final String toDetailString() {
+ if (isSourceArtifact()) {
+ // Source Artifact: relPath == execPath, & real path is not under execRoot
+ return "[" + root + "]" + rootRelativePath;
+ } else {
+ // Derived Artifact: path and root are under execRoot
+ PathFragment execRoot = trimTail(path.asFragment(), execPath);
+ return "[[" + execRoot + "]" + root.getPath().asFragment().relativeTo(execRoot) + "]"
+ + rootRelativePath;
+ }
+ }
+
+ /**
+ * Serializes this artifact to a string that has enough data to reconstruct the artifact.
+ */
+ public final String serializeToString() {
+ // In theory, it should be enough to serialize execPath and rootRelativePath (which is a suffix
+ // of execPath). However, in practice there is code around that uses other attributes which
+ // needs cleaning up.
+ String result = execPath + " /" + rootRelativePath.toString().length();
+ if (getOwner() != null) {
+ result += " " + getOwner();
+ }
+ return result;
+ }
+
+ //---------------------------------------------------------------------------
+ // Static methods to assist in working with Artifacts
+
+ /**
+ * Formatter for execPath PathFragment output.
+ */
+ private static final Function<Artifact, PathFragment> EXEC_PATH_FORMATTER =
+ new Function<Artifact, PathFragment>() {
+ @Override
+ public PathFragment apply(Artifact input) {
+ return input.getExecPath();
+ }
+ };
+
+ private static final Function<Artifact, String> ROOT_RELATIVE_PATH_STRING =
+ new Function<Artifact, String>() {
+ @Override
+ public String apply(Artifact artifact) {
+ return artifact.getRootRelativePath().getPathString();
+ }
+ };
+
+ /**
+ * Converts a collection of artifacts into execution-time path strings, and
+ * adds those to a given collection. Middleman artifacts are ignored by this
+ * method.
+ */
+ public static void addExecPaths(Iterable<Artifact> artifacts, Collection<String> output) {
+ addNonMiddlemanArtifacts(artifacts, output, ActionInputHelper.EXEC_PATH_STRING_FORMATTER);
+ }
+
+ /**
+ * Converts a collection of artifacts into the outputs computed by
+ * outputFormatter and adds them to a given collection. Middleman artifacts
+ * are ignored.
+ */
+ static <E> void addNonMiddlemanArtifacts(Iterable<Artifact> artifacts,
+ Collection<? super E> output, Function<? super Artifact, E> outputFormatter) {
+ for (Artifact artifact : artifacts) {
+ if (MIDDLEMAN_FILTER.apply(artifact)) {
+ output.add(outputFormatter.apply(artifact));
+ }
+ }
+ }
+
+ /**
+ * Lazily converts artifacts into root-relative path strings. Middleman artifacts are ignored by
+ * this method.
+ */
+ public static Iterable<String> toRootRelativePaths(Iterable<Artifact> artifacts) {
+ return Iterables.transform(
+ Iterables.filter(artifacts, MIDDLEMAN_FILTER),
+ ROOT_RELATIVE_PATH_STRING);
+ }
+
+ /**
+ * Lazily converts artifacts into execution-time path strings. Middleman artifacts are ignored by
+ * this method.
+ */
+ public static Iterable<String> toExecPaths(Iterable<Artifact> artifacts) {
+ return ActionInputHelper.toExecPaths(Iterables.filter(artifacts, MIDDLEMAN_FILTER));
+ }
+
+ /**
+ * Converts a collection of artifacts into execution-time path strings, and
+ * returns those as an immutable list. Middleman artifacts are ignored by this method.
+ */
+ public static List<String> asExecPaths(Iterable<Artifact> artifacts) {
+ return ImmutableList.copyOf(toExecPaths(artifacts));
+ }
+
+ /**
+ * Renders a collection of artifacts as execution-time paths and joins
+ * them into a single string. Middleman artifacts are ignored by this method.
+ */
+ public static String joinExecPaths(String delimiter, Iterable<Artifact> artifacts) {
+ return Joiner.on(delimiter).join(toExecPaths(artifacts));
+ }
+
+ /**
+ * Renders a collection of artifacts as root-relative paths and joins
+ * them into a single string. Middleman artifacts are ignored by this method.
+ */
+ public static String joinRootRelativePaths(String delimiter, Iterable<Artifact> artifacts) {
+ return Joiner.on(delimiter).join(toRootRelativePaths(artifacts));
+ }
+
+ /**
+ * Adds a collection of artifacts to a given collection, with
+ * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions expanded once.
+ */
+ public static void addExpandedArtifacts(Iterable<Artifact> artifacts,
+ Collection<? super Artifact> output, MiddlemanExpander middlemanExpander) {
+ addExpandedArtifacts(artifacts, output, Functions.<Artifact>identity(), middlemanExpander);
+ }
+
+ /**
+ * Converts a collection of artifacts into execution-time path strings, and
+ * adds those to a given collection. Middleman artifacts for
+ * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions are expanded
+ * once.
+ */
+ @VisibleForTesting
+ public static void addExpandedExecPathStrings(Iterable<Artifact> artifacts,
+ Collection<String> output,
+ MiddlemanExpander middlemanExpander) {
+ addExpandedArtifacts(artifacts, output, ActionInputHelper.EXEC_PATH_STRING_FORMATTER,
+ middlemanExpander);
+ }
+
+ /**
+ * Converts a collection of artifacts into execution-time path fragments, and
+ * adds those to a given collection. Middleman artifacts for
+ * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions are expanded
+ * once.
+ */
+ public static void addExpandedExecPaths(Iterable<Artifact> artifacts,
+ Collection<PathFragment> output, MiddlemanExpander middlemanExpander) {
+ addExpandedArtifacts(artifacts, output, EXEC_PATH_FORMATTER, middlemanExpander);
+ }
+
+ /**
+ * Converts a collection of artifacts into the outputs computed by
+ * outputFormatter and adds them to a given collection. Middleman artifacts
+ * are expanded once.
+ */
+ private static <E> void addExpandedArtifacts(Iterable<Artifact> artifacts,
+ Collection<? super E> output,
+ Function<? super Artifact, E> outputFormatter,
+ MiddlemanExpander middlemanExpander) {
+ for (Artifact artifact : artifacts) {
+ if (artifact.isMiddlemanArtifact()) {
+ expandMiddlemanArtifact(artifact, output, outputFormatter, middlemanExpander);
+ } else {
+ output.add(outputFormatter.apply(artifact));
+ }
+ }
+ }
+
+ private static <E> void expandMiddlemanArtifact(Artifact middleman,
+ Collection<? super E> output,
+ Function<? super Artifact, E> outputFormatter,
+ MiddlemanExpander middlemanExpander) {
+ Preconditions.checkArgument(middleman.isMiddlemanArtifact());
+ List<Artifact> artifacts = new ArrayList<>();
+ middlemanExpander.expand(middleman, artifacts);
+ for (Artifact artifact : artifacts) {
+ output.add(outputFormatter.apply(artifact));
+ }
+ }
+
+ /**
+ * Converts a collection of artifacts into execution-time path strings, and
+ * returns those as a list. Middleman artifacts are expanded once. The
+ * returned list is mutable.
+ */
+ public static List<String> asExpandedExecPathStrings(Iterable<Artifact> artifacts,
+ MiddlemanExpander middlemanExpander) {
+ List<String> result = new ArrayList<>();
+ addExpandedExecPathStrings(artifacts, result, middlemanExpander);
+ return result;
+ }
+
+ /**
+ * Converts a collection of artifacts into execution-time path fragments, and
+ * returns those as a list. Middleman artifacts are expanded once. The
+ * returned list is mutable.
+ */
+ public static List<PathFragment> asExpandedExecPaths(Iterable<Artifact> artifacts,
+ MiddlemanExpander middlemanExpander) {
+ List<PathFragment> result = new ArrayList<>();
+ addExpandedExecPaths(artifacts, result, middlemanExpander);
+ return result;
+ }
+
+ /**
+ * Converts a collection of artifacts into execution-time path strings with
+ * the root-break delimited with a colon ':', and adds those to a given list.
+ * <pre>
+ * Source: sourceRoot/rootRelative => :rootRelative
+ * Derived: execRoot/rootPrefix/rootRelative => rootPrefix:rootRelative
+ * </pre>
+ */
+ public static void addRootPrefixedExecPaths(Iterable<Artifact> artifacts,
+ List<String> output) {
+ for (Artifact artifact : artifacts) {
+ output.add(asRootPrefixedExecPath(artifact));
+ }
+ }
+
+ /**
+ * Convenience method to filter the files to build for a certain filetype.
+ *
+ * @param artifacts the files to filter
+ * @param allowedType the allowed filetype
+ * @return all members of filesToBuild that are of one of the
+ * allowed filetypes
+ */
+ public static List<Artifact> filterFiles(Iterable<Artifact> artifacts, FileType allowedType) {
+ List<Artifact> filesToBuild = new ArrayList<>();
+ for (Artifact artifact : artifacts) {
+ if (allowedType.matches(artifact.getFilename())) {
+ filesToBuild.add(artifact);
+ }
+ }
+ return filesToBuild;
+ }
+
+ @VisibleForTesting
+ static String asRootPrefixedExecPath(Artifact artifact) {
+ PathFragment execPath = artifact.getExecPath();
+ PathFragment rootRel = artifact.getRootRelativePath();
+ if (execPath.equals(rootRel)) {
+ return ":" + rootRel.getPathString();
+ } else { //if (execPath.endsWith(rootRel)) {
+ PathFragment rootPrefix = trimTail(execPath, rootRel);
+ return rootPrefix.getPathString() + ":" + rootRel.getPathString();
+ }
+ }
+
+ /**
+ * Converts artifacts into their exec paths. Returns an immutable list.
+ */
+ public static List<PathFragment> asPathFragments(Iterable<Artifact> artifacts) {
+ return ImmutableList.copyOf(Iterables.transform(artifacts, EXEC_PATH_FORMATTER));
+ }
+
+ static final ArtifactOwner DESERIALIZED_MARKER_OWNER = new ArtifactOwner() {
+ @Override
+ public Label getLabel() {
+ return null;
+ }};
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java
new file mode 100644
index 0000000..40996ac
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * An interface for creating artifacts from their serialized representations.
+ *
+ * @see ArtifactSerializer
+ */
+public interface ArtifactDeserializer {
+
+ /**
+ * Looks up an artifact by an integer id.
+ *
+ * <p>This is a dual of {@link ArtifactSerializer#getArtifactId}.
+ */
+ Artifact lookupArtifactById(int artifactId);
+
+ /**
+ * Maps a list of artifact ids to a list of artifacts.
+ *
+ * <p>This is a batch version of {@link #lookupArtifactById}, provided for efficiency. It takes
+ * an iterable of boxed integers because that's what the proto wrapper provides.
+ */
+ ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java
new file mode 100644
index 0000000..99a53cb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java
@@ -0,0 +1,339 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * A cache of Artifacts, keyed by Path.
+ */
+@ThreadSafe
+public class ArtifactFactory implements ArtifactResolver, ArtifactSerializer, ArtifactDeserializer {
+
+ private final Path execRoot;
+
+ /**
+ * The main Path to source artifact cache. There will always be exactly one canonical
+ * artifact for a given source path.
+ */
+ private final Map<PathFragment, Artifact> pathToSourceArtifact = new HashMap<>();
+
+ /**
+ * Map of package names to source root paths so that we can create source
+ * artifact paths given execPaths in the symlink forest.
+ */
+ private ImmutableMap<PackageIdentifier, Root> packageRoots;
+
+ /**
+ * Reverse-ordered list of derived roots for use in looking up or (in rare cases) creating
+ * derived artifacts from execPaths. The reverse order is only significant for overlapping roots
+ * so that the longest is found first.
+ */
+ private ImmutableCollection<Root> derivedRoots = ImmutableList.of();
+
+ private ArtifactIdRegistry artifactIdRegistry = new ArtifactIdRegistry();
+
+ /**
+ * Constructs a new artifact factory that will use a given execution root when
+ * creating artifacts.
+ *
+ * @param execRoot the execution root Path to use
+ */
+ public ArtifactFactory(Path execRoot) {
+ this.execRoot = execRoot;
+ }
+
+ /**
+ * Clear the cache.
+ */
+ public synchronized void clear() {
+ pathToSourceArtifact.clear();
+ packageRoots = null;
+ derivedRoots = ImmutableList.of();
+ artifactIdRegistry = new ArtifactIdRegistry();
+ clearDeserializedArtifacts();
+ }
+
+ /**
+ * Set the set of known packages and their corresponding source artifact
+ * roots. Must be called exactly once after construction or clear().
+ *
+ * @param packageRoots the map of package names to source artifact roots to
+ * use.
+ */
+ public synchronized void setPackageRoots(Map<PackageIdentifier, Root> packageRoots) {
+ this.packageRoots = ImmutableMap.copyOf(packageRoots);
+ }
+
+ /**
+ * Set the set of known derived artifact roots. Must be called exactly once
+ * after construction or clear().
+ *
+ * @param roots the set of derived artifact roots to use
+ */
+ public synchronized void setDerivedArtifactRoots(Collection<Root> roots) {
+ derivedRoots = ImmutableSortedSet.<Root>reverseOrder().addAll(roots).build();
+ }
+
+ @Override
+ public Artifact getSourceArtifact(PathFragment execPath, Root root, ArtifactOwner owner) {
+ Preconditions.checkArgument(!execPath.isAbsolute());
+ Preconditions.checkNotNull(owner, execPath);
+ execPath = execPath.normalize();
+ return getArtifact(root.getPath().getRelative(execPath), root, execPath, owner, null);
+ }
+
+ @Override
+ public Artifact getSourceArtifact(PathFragment execPath, Root root) {
+ return getSourceArtifact(execPath, root, ArtifactOwner.NULL_OWNER);
+ }
+
+ /**
+ * Only for use by BinTools! Returns an artifact for a tool at the given path
+ * fragment, relative to the exec root, creating it if not found. This method
+ * only works for normalized, relative paths.
+ */
+ public Artifact getDerivedArtifact(PathFragment execPath) {
+ Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
+ Preconditions.checkArgument(execPath.isNormalized(), execPath);
+ // TODO(bazel-team): Check that either BinTools do not change over the life of the Blaze server,
+ // or require that a legitimate ArtifactOwner be passed in here to allow for ownership.
+ return getArtifact(execRoot.getRelative(execPath), Root.execRootAsDerivedRoot(execRoot),
+ execPath, ArtifactOwner.NULL_OWNER, null);
+ }
+
+ private void validatePath(PathFragment rootRelativePath, Root root) {
+ Preconditions.checkArgument(!rootRelativePath.isAbsolute(), rootRelativePath);
+ Preconditions.checkArgument(rootRelativePath.isNormalized(), rootRelativePath);
+ Preconditions.checkArgument(root.getPath().startsWith(execRoot), "%s %s", root, execRoot);
+ Preconditions.checkArgument(!root.getPath().equals(execRoot), "%s %s", root, execRoot);
+ // TODO(bazel-team): this should only accept roots from derivedRoots.
+ //Preconditions.checkArgument(derivedRoots.contains(root), "%s not in %s", root, derivedRoots);
+ }
+
+ /**
+ * Returns an artifact for a tool at the given root-relative path under the given root, creating
+ * it if not found. This method only works for normalized, relative paths.
+ *
+ * <p>The root must be below the execRoot, and the execPath of the resulting Artifact is computed
+ * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}.
+ */
+ // TODO(bazel-team): Don't allow root == execRoot.
+ public Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root,
+ ArtifactOwner owner) {
+ validatePath(rootRelativePath, root);
+ Path path = root.getPath().getRelative(rootRelativePath);
+ return getArtifact(path, root, path.relativeTo(execRoot), owner, null);
+ }
+
+ /**
+ * Returns an artifact that represents the output directory of a Fileset at the given
+ * root-relative path under the given root, creating it if not found. This method only works for
+ * normalized, relative paths.
+ *
+ * <p>The root must be below the execRoot, and the execPath of the resulting Artifact is computed
+ * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}.
+ */
+ public Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root,
+ ArtifactOwner owner) {
+ validatePath(rootRelativePath, root);
+ Path path = root.getPath().getRelative(rootRelativePath);
+ return getArtifact(path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.FILESET);
+ }
+
+ public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root,
+ ArtifactOwner owner) {
+ validatePath(rootRelativePath, root);
+ Path path = root.getPath().getRelative(rootRelativePath);
+ return getArtifact(
+ path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.CONSTANT_METADATA);
+ }
+
+ /**
+ * Returns the Artifact for the specified path, creating one if not found and
+ * setting the <code>root</code> and <code>execPath</code> to the
+ * specified values.
+ */
+ private synchronized Artifact getArtifact(Path path, Root root, PathFragment execPath,
+ ArtifactOwner owner, @Nullable SpecialArtifactType type) {
+ Preconditions.checkNotNull(root);
+ Preconditions.checkNotNull(execPath);
+
+ if (!root.isSourceRoot()) {
+ return createArtifact(path, root, execPath, owner, type);
+ }
+
+ Artifact artifact = pathToSourceArtifact.get(execPath);
+
+ if (artifact == null || !Objects.equals(artifact.getArtifactOwner(), owner)) {
+ // There really should be a safety net that makes it impossible to create two Artifacts
+ // with the same exec path but a different Owner, but we also need to reuse Artifacts from
+ // previous builds.
+ artifact = createArtifact(path, root, execPath, owner, type);
+ pathToSourceArtifact.put(execPath, artifact);
+ } else {
+ // TODO(bazel-team): Maybe we should check for equality of the fileset bit. However, that
+ // would require us to differentiate between artifact-creating and artifact-getting calls to
+ // getDerivedArtifact().
+ Preconditions.checkState(root.equals(artifact.getRoot()),
+ "root for path %s changed from %s to %s", path, artifact.getRoot(), root);
+ Preconditions.checkState(execPath.equals(artifact.getExecPath()),
+ "execPath for path %s changed from %s to %s", path, artifact.getExecPath(), execPath);
+ }
+ return artifact;
+ }
+
+ private Artifact createArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner,
+ @Nullable SpecialArtifactType type) {
+ Preconditions.checkNotNull(owner, path);
+ if (type == null) {
+ return new Artifact(path, root, execPath, owner);
+ } else {
+ return new Artifact.SpecialArtifact(path, root, execPath, owner, type);
+ }
+ }
+
+ @Override
+ public synchronized Artifact resolveSourceArtifact(PathFragment execPath) {
+ execPath = execPath.normalize();
+ // First try a quick map lookup to see if the artifact already exists.
+ Artifact a = pathToSourceArtifact.get(execPath);
+ if (a != null) {
+ return a;
+ }
+ // Don't create an artifact if it's derived.
+ if (findDerivedRoot(execRoot.getRelative(execPath)) != null) {
+ return null;
+ }
+ // Must be a new source artifact, so probe the known packages to find the longest package
+ // prefix, and then use the corresponding source root to create a new artifact.
+ for (PathFragment dir = execPath.getParentDirectory(); dir != null;
+ dir = dir.getParentDirectory()) {
+ Root sourceRoot = packageRoots.get(PackageIdentifier.createInDefaultRepo(dir));
+ if (sourceRoot != null) {
+ return getSourceArtifact(execPath, sourceRoot, ArtifactOwner.NULL_OWNER);
+ }
+ }
+ return null; // not a path that we can find...
+ }
+
+ /**
+ * Finds the derived root for a full path by comparing against the known
+ * derived artifact roots.
+ *
+ * @param path a Path to resolve the root for
+ * @return the root for the path or null if no root can be determined
+ */
+ @VisibleForTesting // for our own unit tests only.
+ synchronized Root findDerivedRoot(Path path) {
+ for (Root prefix : derivedRoots) {
+ if (path.startsWith(prefix.getPath())) {
+ return prefix;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns all source artifacts created by the artifact factory.
+ */
+ public synchronized Iterable<Artifact> getSourceArtifacts() {
+ return ImmutableList.copyOf(pathToSourceArtifact.values());
+ }
+
+ // Non-final only because clear()ing a map does not actually free the memory it took up, so we
+ // assign it to a new map in lieu of clearing.
+ private ConcurrentMap<PathFragment, Artifact> deserializedArtifacts =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Returns the map of all artifacts that were deserialized this build. The caller should process
+ * them and then call {@link #clearDeserializedArtifacts}.
+ */
+ public Map<PathFragment, Artifact> getDeserializedArtifacts() {
+ return deserializedArtifacts;
+ }
+
+ /** Clears the map of deserialized artifacts. */
+ public void clearDeserializedArtifacts() {
+ deserializedArtifacts = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Resolves an artifact based on its deserialized representation. The artifact can be either a
+ * source or a derived one.
+ *
+ * <p>Note: this method represents a hole in the usual contract that artifacts with a random path
+ * cannot be created. Unfortunately, we currently need this in some cases.
+ *
+ * @param execPath the exec path of the artifact
+ */
+ public Artifact deserializeArtifact(PathFragment execPath, PackageRootResolver resolver) {
+ Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
+ Path path = execRoot.getRelative(execPath);
+ Root root = findDerivedRoot(path);
+
+ Artifact result;
+ if (root != null) {
+ result = getDerivedArtifact(path.relativeTo(root.getPath()), root,
+ Artifact.DESERIALIZED_MARKER_OWNER);
+ Artifact oldResult = deserializedArtifacts.putIfAbsent(execPath, result);
+ if (oldResult != null) {
+ result = oldResult;
+ }
+ return result;
+ } else {
+ Map<PathFragment, Root> sourceRoots = resolver.findPackageRoots(Lists.newArrayList(execPath));
+ if (sourceRoots == null || sourceRoots.get(execPath) == null) {
+ return null;
+ }
+ return getSourceArtifact(execPath, sourceRoots.get(execPath), ArtifactOwner.NULL_OWNER);
+ }
+ }
+
+ @Override
+ public Artifact lookupArtifactById(int artifactId) {
+ return artifactIdRegistry.lookupArtifactById(artifactId);
+ }
+
+ @Override
+ public ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds) {
+ return artifactIdRegistry.lookupArtifactsByIds(artifactIds);
+ }
+
+ @Override
+ public int getArtifactId(Artifact artifact) {
+ return artifactIdRegistry.getArtifactId(artifact);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java
new file mode 100644
index 0000000..9edf684
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.MapMaker;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * A registry that keeps a map of artifacts to unique integer ids.
+ */
+class ArtifactIdRegistry implements ArtifactSerializer, ArtifactDeserializer {
+
+ /**
+ * A sequence of registered artifacts. The position in the list is the artifact's id.
+ *
+ * <p>Synchronized using {@link #artifactIdsLock}.
+ */
+ private final List<Artifact> serializedArtifactList = new ArrayList<>();
+
+ /**
+ * A map of artifacts to unique integer ids.
+ *
+ * <p>Writes to this map must be synchronized using {@link #artifactIdsLock}, in order to
+ * maintain consistency with {@link #serializedArtifactList}.
+ */
+ private final ConcurrentMap<Artifact, Integer> serializedArtifactIds =
+ new MapMaker().concurrencyLevel(1).makeMap();
+
+ /**
+ * A lock for keeping {@code serializedArtifactList} and {@code serializedArtifactIds} in sync.
+ */
+ private ReadWriteLock artifactIdsLock = new ReentrantReadWriteLock();
+
+ ArtifactIdRegistry() {
+ }
+
+ @Override
+ public int getArtifactId(Artifact artifact) {
+ Integer artifactId = serializedArtifactIds.get(artifact);
+ if (artifactId == null) {
+ artifactId = assignArtifactId(artifact);
+ }
+ return artifactId;
+ }
+
+ private Integer assignArtifactId(Artifact artifact) {
+ artifactIdsLock.writeLock().lock();
+ try {
+ Integer artifactId = serializedArtifactIds.get(artifact);
+ if (artifactId == null) {
+ artifactId = serializedArtifactList.size();
+ serializedArtifactList.add(artifact);
+ serializedArtifactIds.put(artifact, artifactId);
+ }
+ return artifactId;
+ } finally {
+ artifactIdsLock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public Artifact lookupArtifactById(int artifactId) {
+ artifactIdsLock.readLock().lock();
+ try {
+ return serializedArtifactList.get(artifactId);
+ } finally {
+ artifactIdsLock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds) {
+ int size = Iterables.size(artifactIds);
+ Artifact[] result = new Artifact[size];
+
+ int i = 0;
+
+ artifactIdsLock.readLock().lock();
+ try {
+ for (int artifactId : artifactIds) {
+ result[i] = serializedArtifactList.get(artifactId);
+ i++;
+ }
+ } finally {
+ artifactIdsLock.readLock().unlock();
+ }
+
+ return ImmutableList.copyOf(result);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java
new file mode 100644
index 0000000..458fb42
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * An interface for {@code LabelAndConfiguration}, or at least for a {@link Label}. Only tests and
+ * internal {@link Artifact}-generators should implement this interface -- otherwise,
+ * {@code LabelAndConfiguration} should be the only implementation.
+ */
+public interface ArtifactOwner {
+ Label getLabel();
+
+ @VisibleForTesting
+ public static final ArtifactOwner NULL_OWNER = new ArtifactOwner() {
+ @Override
+ public Label getLabel() {
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "NULL_OWNER";
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java
new file mode 100644
index 0000000..42ad285
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Exception to indicate that one {@link Action} has an output artifact whose path is a prefix of an
+ * output of another action. Since the first path cannot be both a directory and a file, this would
+ * lead to an error if both actions were executed in the same build.
+ */
+public class ArtifactPrefixConflictException extends Exception {
+ public ArtifactPrefixConflictException(PathFragment firstPath, PathFragment secondPath,
+ Label firstOwner, Label secondOwner) {
+ super(String.format(
+ "output path '%s' (belonging to %s) is a prefix of output path '%s' (belonging to %s). "
+ + "These actions cannot be simultaneously present; please rename one of the output files "
+ + "or, as a last resort, run 'blaze clean' and then build just one of them",
+ firstPath, firstOwner, secondPath, secondOwner));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java
new file mode 100644
index 0000000..b74c17b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * An interface for resolving artifact names to {@link Artifact} objects. Should only be used
+ * in the internal machinery of Blaze: rule implementations are not allowed to do this.
+ */
+public interface ArtifactResolver {
+ /**
+ * Returns the source Artifact for the specified path, creating it if not found and setting its
+ * root and execPath.
+ *
+ * @param execPath the path of the source artifact relative to the source root
+ * @param root the source root prefix of the path
+ * @param owner the artifact owner.
+ * @return the canonical source artifact for the given path
+ */
+ Artifact getSourceArtifact(PathFragment execPath, Root root, ArtifactOwner owner);
+
+ /**
+ * Returns the source Artifact for the specified path, creating it if not found and setting its
+ * root and execPath.
+ *
+ * @see #getSourceArtifact(PathFragment, Root, ArtifactOwner)
+ */
+ Artifact getSourceArtifact(PathFragment execPath, Root root);
+
+ /**
+ * Resolves a source Artifact given an execRoot-relative path.
+ *
+ * <p>Never creates or returns derived artifacts, only source artifacts.
+ *
+ * <p>Note: this method should only be used when the roots are unknowable, such as from the
+ * post-compile .d or manifest scanning methods.
+ *
+ * @param execPath the exec path of the artifact to resolve
+ * @return an existing or new source Artifact for the given execPath. Returns null if
+ * the root can not be determined and the artifact did not exist before.
+ */
+ Artifact resolveSourceArtifact(PathFragment execPath);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java
new file mode 100644
index 0000000..703eeb7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An interface for creating artifacts from their serialized representations.
+ *
+ * @see ArtifactDeserializer
+ */
+public interface ArtifactSerializer {
+
+ /**
+ * Returns a number that uniquely identifies an artifact.
+ *
+ * <p>The artifact can be retrieved again later by calling
+ * {@link ArtifactDeserializer#lookupArtifactById}.
+ */
+ int getArtifactId(Artifact artifact);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java b/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java
new file mode 100644
index 0000000..dd879c2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java
@@ -0,0 +1,214 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.extra.EnvironmentVariable;
+import com.google.devtools.build.lib.actions.extra.SpawnInfo;
+import com.google.devtools.build.lib.util.CommandDescriptionForm;
+import com.google.devtools.build.lib.util.CommandFailureUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Base implementation of a Spawn.
+ */
+@Immutable
+public class BaseSpawn implements Spawn {
+ private final ImmutableList<String> arguments;
+ private final ImmutableMap<String, String> environment;
+ private final ImmutableMap<String, String> executionInfo;
+ private final ImmutableMap<PathFragment, Artifact> runfilesManifests;
+ private final ActionMetadata action;
+ private final ResourceSet localResources;
+
+ /**
+ * Returns a new Spawn. The caller must not modify the parameters after the call; neither will
+ * this method.
+ */
+ public BaseSpawn(List<String> arguments,
+ Map<String, String> environment,
+ Map<String, String> executionInfo,
+ Map<PathFragment, Artifact> runfilesManifests,
+ ActionMetadata action,
+ ResourceSet localResources) {
+ this.arguments = ImmutableList.copyOf(arguments);
+ this.environment = ImmutableMap.copyOf(environment);
+ this.executionInfo = ImmutableMap.copyOf(executionInfo);
+ this.runfilesManifests = ImmutableMap.copyOf(runfilesManifests);
+ this.action = action;
+ this.localResources = localResources;
+ }
+
+ /**
+ * Returns a new Spawn.
+ */
+ public BaseSpawn(List<String> arguments,
+ Map<String, String> environment,
+ Map<String, String> executionInfo,
+ // TODO(bazel-team): have this always be non-null.
+ @Nullable Artifact runfilesManifest,
+ ActionMetadata action,
+ ResourceSet localResources) {
+ this(arguments, environment, executionInfo,
+ ((runfilesManifest != null)
+ ? ImmutableMap.of(runfilesForFragment(new PathFragment(arguments.get(0))),
+ runfilesManifest)
+ : ImmutableMap.<PathFragment, Artifact>of()),
+ action, localResources);
+ }
+
+ public static PathFragment runfilesForFragment(PathFragment pathFragment) {
+ return pathFragment.getParentDirectory().getChild(pathFragment.getBaseName() + ".runfiles");
+ }
+
+ /**
+ * Returns a new Spawn.
+ */
+ public BaseSpawn(List<String> arguments,
+ Map<String, String> environment,
+ Map<String, String> executionInfo,
+ ActionMetadata action,
+ ResourceSet localResources) {
+ this(arguments, environment, executionInfo,
+ ImmutableMap.<PathFragment, Artifact>of(), action, localResources);
+ }
+
+ @Override
+ public boolean isRemotable() {
+ return !executionInfo.containsKey("local");
+ }
+
+ @Override
+ public final ImmutableMap<String, String> getExecutionInfo() {
+ return executionInfo;
+ }
+
+ @Override
+ public String asShellCommand(Path workingDir) {
+ return asShellCommand(getArguments(), workingDir, getEnvironment());
+ }
+
+ @Override
+ public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() {
+ return runfilesManifests;
+ }
+
+ @Override
+ public ImmutableList<Artifact> getFilesetManifests() {
+ return ImmutableList.<Artifact>of();
+ }
+
+ @Override
+ public SpawnInfo getExtraActionInfo() {
+ SpawnInfo.Builder info = SpawnInfo.newBuilder();
+
+ info.addAllArgument(getArguments());
+ for (Map.Entry<String, String> variable : getEnvironment().entrySet()) {
+ info.addVariable(EnvironmentVariable.newBuilder()
+ .setName(variable.getKey())
+ .setValue(variable.getValue()).build());
+ }
+ for (ActionInput input : getInputFiles()) {
+ // Explicitly ignore middleman artifacts here.
+ if (!(input instanceof Artifact) || !((Artifact) input).isMiddlemanArtifact()) {
+ info.addInputFile(input.getExecPathString());
+ }
+ }
+ info.addAllOutputFile(ActionInputHelper.toExecPaths(getOutputFiles()));
+ return info.build();
+ }
+
+ @Override
+ public ImmutableList<String> getArguments() {
+ // TODO(bazel-team): this method should be final, as the correct value of the args can be
+ // injected in the ctor.
+ return arguments;
+ }
+
+ @Override
+ public ImmutableMap<String, String> getEnvironment() {
+ if (getRunfilesManifests().size() != 1) {
+ return environment;
+ }
+
+ ImmutableMap.Builder<String, String> env = ImmutableMap.builder();
+ env.putAll(environment);
+ for (Map.Entry<PathFragment, Artifact> e : getRunfilesManifests().entrySet()) {
+ // TODO(bazel-team): Unify these into a single env variable.
+ env.put("JAVA_RUNFILES", e.getKey().getPathString() + "/");
+ env.put("PYTHON_RUNFILES", e.getKey().getPathString() + "/");
+ }
+ return env.build();
+ }
+
+ @Override
+ public Iterable<? extends ActionInput> getInputFiles() {
+ return action.getInputs();
+ }
+
+ @Override
+ public Collection<? extends ActionInput> getOutputFiles() {
+ return action.getOutputs();
+ }
+
+ @Override
+ public ActionMetadata getResourceOwner() {
+ return action;
+ }
+
+ @Override
+ public ResourceSet getLocalResources() {
+ return localResources;
+ }
+
+ @Override
+ public ActionOwner getOwner() { return action.getOwner(); }
+
+ @Override
+ public String getMnemonic() { return action.getMnemonic(); }
+
+ /**
+ * Convert a working dir + environment map + arg list into a Bourne shell
+ * command.
+ */
+ public static String asShellCommand(Collection<String> arguments,
+ Path workingDirectory,
+ Map<String, String> environment) {
+ // We print this command out in such a way that it can safely be
+ // copied+pasted as a Bourne shell command. This is extremely valuable for
+ // debugging.
+ return CommandFailureUtils.describeCommand(CommandDescriptionForm.COMPLETE,
+ arguments, environment, workingDirectory.getPathString());
+ }
+
+ /**
+ * A local spawn requiring zero resources.
+ */
+ public static class Local extends BaseSpawn {
+ public Local(List<String> arguments, Map<String, String> environment, ActionMetadata action) {
+ super(arguments, environment, ImmutableMap.<String, String>of("local", ""),
+ action, ResourceSet.ZERO);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java b/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java
new file mode 100644
index 0000000..61803c8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java
@@ -0,0 +1,98 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A visitor helper class for bipartite graphs. The alternate kinds of nodes
+ * are arbitrarily designated "black" or "white".
+ *
+ * <p> Subclasses implement the black() and white() hook functions which are
+ * called as nodes are visited. The class holds a mapping from each node to a
+ * small integer; this is available to subclasses if they wish.
+ */
+public abstract class BipartiteVisitor<BLACK, WHITE> {
+
+ protected BipartiteVisitor() {}
+
+ private int nextNodeId = 0;
+
+ // Maps each visited black node to a small integer.
+ protected final Map<BLACK, Integer> visitedBlackNodes = new HashMap<>();
+
+ // Maps each visited white node to a small integer.
+ protected final Map<WHITE, Integer> visitedWhiteNodes = new HashMap<>();
+
+ /**
+ * Visit the specified black node. If this node has not already been
+ * visited, the black() hook is called and true is returned; otherwise,
+ * false is returned.
+ */
+ public final boolean visitBlackNode(BLACK blackNode) {
+ if (blackNode == null) { throw new NullPointerException(); }
+ if (!visitedBlackNodes.containsKey(blackNode)) {
+ visitedBlackNodes.put(blackNode, nextNodeId++);
+ black(blackNode);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Visit all specified black nodes.
+ */
+ public final void visitBlackNodes(Iterable<BLACK> blackNodes) {
+ for (BLACK blackNode : blackNodes) {
+ visitBlackNode(blackNode);
+ }
+ }
+
+ /**
+ * Visit the specified white node. If this node has not already been
+ * visited, the white() hook is called and true is returned; otherwise,
+ * false is returned.
+ */
+ public final boolean visitWhiteNode(WHITE whiteNode) {
+ if (whiteNode == null) {
+ throw new NullPointerException();
+ }
+ if (!visitedWhiteNodes.containsKey(whiteNode)) {
+ visitedWhiteNodes.put(whiteNode, nextNodeId++);
+ white(whiteNode);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Visit all specified white nodes.
+ */
+ public final void visitWhiteNodes(Iterable<WHITE> whiteNodes) {
+ for (WHITE whiteNode : whiteNodes) {
+ visitWhiteNode(whiteNode);
+ }
+ }
+
+ /**
+ * Called whenever a white node is visited. Hook for subclasses.
+ */
+ protected abstract void white(WHITE whiteNode);
+
+ /**
+ * Called whenever a black node is visited. Hook for subclasses.
+ */
+ protected abstract void black(BLACK blackNode);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java b/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java
new file mode 100644
index 0000000..fd3c3d9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java
@@ -0,0 +1,233 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.OptionsClassProvider;
+
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * The Executor class provides a dynamic abstraction of the various actual primitive system
+ * operations that might be performed during a build step.
+ *
+ * <p>Constructions of this class might perform distributed execution, "virtual" execution for
+ * testing purposes, or just print out the sequence of commands that would be executed, like Make's
+ * "-n" option.
+ */
+@ThreadSafe
+public final class BlazeExecutor implements Executor {
+
+ private final Path outputPath;
+ private final boolean verboseFailures;
+ private final boolean showSubcommands;
+ private final Path execRoot;
+ private final Reporter reporter;
+ private final EventBus eventBus;
+ private final Clock clock;
+ private final OptionsClassProvider options;
+ private AtomicBoolean inExecutionPhase;
+
+ private final Map<String, SpawnActionContext> spawnActionContextMap;
+ private final Map<Class<? extends ActionContext>, ActionContext> contextMap =
+ new HashMap<>();
+
+ /**
+ * Constructs an Executor, bound to a specified output base path, and which
+ * will use the specified reporter to announce SUBCOMMAND events,
+ * the given event bus to delegate events and the given output streams
+ * for streaming output. The list of
+ * strategy implementation classes is used to construct instances of the
+ * strategies mapped by their declared abstract type. This list is uniquified
+ * before using. Each strategy instance is created with a reference to this
+ * Executor as well as the given options object.
+ * <p>
+ * Don't forget to call startBuildRequest() and stopBuildRequest() for each
+ * request, and shutdown() when you're done with this executor.
+ */
+ public BlazeExecutor(Path execRoot,
+ Path outputPath,
+ Reporter reporter,
+ EventBus eventBus,
+ Clock clock,
+ OptionsClassProvider options,
+ boolean verboseFailures,
+ boolean showSubcommands,
+ List<ActionContext> contextImplementations,
+ Map<String, ActionContext> spawnContextMap,
+ Iterable<ActionContextProvider> contextProviders)
+ throws ExecutorInitException {
+ this.outputPath = outputPath;
+ this.verboseFailures = verboseFailures;
+ this.showSubcommands = showSubcommands;
+ this.execRoot = execRoot;
+ this.reporter = reporter;
+ this.eventBus = eventBus;
+ this.clock = clock;
+ this.options = options;
+ this.inExecutionPhase = new AtomicBoolean(false);
+
+ // We need to keep only the last occurrences of the entries in contextImplementations
+ // (so we respect insertion order but also instantiate them only once).
+ LinkedHashSet<ActionContext> allContexts = new LinkedHashSet<>();
+ allContexts.addAll(contextImplementations);
+
+ ImmutableMap.Builder<String, SpawnActionContext> spawnMapBuilder = ImmutableMap.builder();
+ for (Map.Entry<String, ActionContext> entry: spawnContextMap.entrySet()) {
+ spawnMapBuilder.put(entry.getKey(), (SpawnActionContext) entry.getValue());
+ allContexts.add(entry.getValue());
+ }
+
+ for (ActionContext context : contextImplementations) {
+ ExecutionStrategy annotation = context.getClass().getAnnotation(ExecutionStrategy.class);
+ if (annotation != null) {
+ contextMap.put(annotation.contextType(), context);
+ }
+ }
+ this.spawnActionContextMap = spawnMapBuilder.build();
+
+ for (ActionContextProvider factory : contextProviders) {
+ factory.executorCreated(allContexts);
+ }
+ }
+
+ @Override
+ public Path getExecRoot() {
+ return execRoot;
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ return reporter;
+ }
+
+ @Override
+ public EventBus getEventBus() {
+ return eventBus;
+ }
+
+ @Override
+ public Clock getClock() {
+ return clock;
+ }
+
+ @Override
+ public boolean reportsSubcommands() {
+ return showSubcommands;
+ }
+
+ /**
+ * Report a subcommand event to this Executor's Reporter and, if action
+ * logging is enabled, post it on its EventBus.
+ */
+ @Override
+ public void reportSubcommand(String reason, String message) {
+ reporter.handle(new Event(EventKind.SUBCOMMAND, null, "# " + reason + "\n" + message));
+ }
+
+ /**
+ * This method is called before the start of the execution phase of each
+ * build request.
+ */
+ public void executionPhaseStarting() {
+ Preconditions.checkState(!inExecutionPhase.getAndSet(true));
+ Profiler.instance().startTask(ProfilerTask.INFO, "Initializing executors");
+ Profiler.instance().completeTask(ProfilerTask.INFO);
+ }
+
+ /**
+ * This method is called after the end of the execution phase of each build
+ * request (even if there was an interrupt).
+ */
+ public void executionPhaseEnding() {
+ if (!inExecutionPhase.get()) {
+ return;
+ }
+
+ Profiler.instance().startTask(ProfilerTask.INFO, "Shutting down executors");
+ Profiler.instance().completeTask(ProfilerTask.INFO);
+ inExecutionPhase.set(false);
+ }
+
+ public static void shutdownHelperPool(EventHandler reporter, ExecutorService pool,
+ String name) {
+ pool.shutdownNow();
+
+ boolean interrupted = false;
+ while (true) {
+ try {
+ if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
+ reporter.handle(Event.warn(name + " threadpool shutdown took greater than ten seconds"));
+ }
+ break;
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public <T extends ActionContext> T getContext(Class<? extends T> type) {
+ Preconditions.checkArgument(type != SpawnActionContext.class,
+ "should use getSpawnActionContext instead");
+ return type.cast(contextMap.get(type));
+ }
+
+ /**
+ * Returns the {@link SpawnActionContext} to use for the given mnemonic. If no execution mode is
+ * set, then it returns the default strategy for spawn actions.
+ */
+ @Override
+ public SpawnActionContext getSpawnActionContext(String mnemonic) {
+ SpawnActionContext context = spawnActionContextMap.get(mnemonic);
+ return context == null ? spawnActionContextMap.get("") : context;
+ }
+
+ /** Returns true iff the --verbose_failures option was enabled. */
+ @Override
+ public boolean getVerboseFailures() {
+ return verboseFailures;
+ }
+
+ /** Returns the options associated with the execution. */
+ @Override
+ public OptionsClassProvider getOptions() {
+ return options;
+ }
+
+ public Path getOutputPath() {
+ return outputPath;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java b/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java
new file mode 100644
index 0000000..088ba60
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * This exception gets thrown if there were errors during the execution phase of
+ * the build.
+ *
+ * <p>The argument to the constructor may be null if the thrower has already
+ * printed an error message; in this case, no error message should be printed by
+ * the catcher. (Typically, this happens when the builder is unsuccessful and
+ * {@code --keep_going} was specified. This error corresponds to one or more
+ * actions failing, but since those actions' failures will be reported
+ * separately, the exception carries no message and is just used for control
+ * flow.)
+ */
+@ThreadSafe
+public class BuildFailedException extends Exception {
+ private final boolean catastrophic;
+ private final Action action;
+ private final Iterable<Label> rootCauses;
+ private final boolean errorAlreadyShown;
+
+ public BuildFailedException() {
+ this(null);
+ }
+
+ public BuildFailedException(String message) {
+ this(message, false, null, ImmutableList.<Label>of());
+ }
+
+ public BuildFailedException(String message, boolean catastrophic) {
+ this(message, catastrophic, null, ImmutableList.<Label>of());
+ }
+
+ public BuildFailedException(String message, boolean catastrophic,
+ Action action, Iterable<Label> rootCauses) {
+ this(message, catastrophic, action, rootCauses, false);
+ }
+
+ public BuildFailedException(String message, boolean catastrophic,
+ Action action, Iterable<Label> rootCauses, boolean errorAlreadyShown) {
+ super(message);
+ this.catastrophic = catastrophic;
+ this.rootCauses = ImmutableList.copyOf(rootCauses);
+ this.action = action;
+ this.errorAlreadyShown = errorAlreadyShown;
+ }
+
+ public boolean isCatastrophic() {
+ return catastrophic;
+ }
+
+ public Action getAction() {
+ return action;
+ }
+
+ public Iterable<Label> getRootCauses() {
+ return rootCauses;
+ }
+
+ public boolean isErrorAlreadyShown() {
+ return errorAlreadyShown || getMessage() == null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java b/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java
new file mode 100644
index 0000000..72ce13335
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * Methods needed by {@code SkyframeBuilder}.
+ */
+public final class BuilderUtils {
+
+ private BuilderUtils() {}
+
+ /**
+ * Figure out why an action's execution failed and rethrow the right kind of exception.
+ */
+ public static void rethrowCause(Exception e) throws BuildFailedException, TestExecException {
+ Throwable cause = e.getCause();
+ Throwable innerCause = cause.getCause();
+ if (innerCause instanceof TestExecException) {
+ throw (TestExecException) innerCause;
+ }
+ if (cause instanceof ActionExecutionException) {
+ ActionExecutionException actionExecutionCause = (ActionExecutionException) cause;
+ // Sometimes ActionExecutionExceptions are caused by Actions with no owner.
+ String message =
+ (actionExecutionCause.getLocation() != null) ?
+ (actionExecutionCause.getLocation().print() + " " + cause.getMessage()) :
+ e.getMessage();
+ throw new BuildFailedException(message, actionExecutionCause.isCatastrophe(),
+ actionExecutionCause.getAction(), actionExecutionCause.getRootCauses(),
+ /*errorAlreadyShown=*/ !actionExecutionCause.showError());
+ } else if (cause instanceof MissingInputFileException) {
+ throw new BuildFailedException(cause.getMessage());
+ } else if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ } else if (cause instanceof Error) {
+ throw (Error) cause;
+ } else {
+ /*
+ * This should never happen - we should only get exceptions listed in the exception
+ * specification for ExecuteBuildAction.call().
+ */
+ throw new IllegalArgumentException("action terminated with "
+ + "unexpected exception: " + cause.getMessage(), cause);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java b/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java
new file mode 100644
index 0000000..35db7d6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * This event is fired during the build if an action was in the action cache.
+ */
+public class CachedActionEvent {
+
+ private final Action action;
+ private final long nanoTimeStart;
+
+ /**
+ * Create an event for an action that was cached.
+ *
+ * @param action the cached action
+ * @param nanoTimeStart the time when the action was started. This allow us to
+ * record more accurately the time spend by the action, since we execute some code before
+ * deciding if we execute the action or not.
+ */
+ public CachedActionEvent(Action action, long nanoTimeStart) {
+ this.action = action;
+ this.nanoTimeStart = nanoTimeStart;
+ }
+
+ public Action getAction() {
+ return action;
+ }
+
+ public long getNanoTimeStart() {
+ return nanoTimeStart;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java
new file mode 100644
index 0000000..9177cba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Set;
+
+/**
+ * Used to signal when the incremental builder has found the set of changed artifacts.
+ */
+public class ChangedArtifactsMessage {
+
+ private final Set<Artifact> artifacts;
+
+ public ChangedArtifactsMessage(Set<Artifact> changedArtifacts) {
+ this.artifacts = ImmutableSet.copyOf(changedArtifacts);
+ }
+
+ public Set<Artifact> getChangedArtifacts() {
+ return artifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java
new file mode 100644
index 0000000..56f6882
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Set;
+
+/**
+ * A message sent conveying a set of changed files.
+ */
+public class ChangedFilesMessage {
+
+ private final Set<PathFragment> changedFiles;
+
+ public ChangedFilesMessage(Set<PathFragment> changedFiles) {
+ this.changedFiles = ImmutableSet.copyOf(changedFiles);
+ }
+
+ public Set<PathFragment> getChangedFiles() {
+ return changedFiles;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java b/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java
new file mode 100644
index 0000000..d9d875d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java
@@ -0,0 +1,220 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * A Multimap-like object that is actually a {@link ConcurrentMap} of {@code SmallSet}s to avoid
+ * the memory penalties of a {@code Multimap} while preserving concurrency guarantees, and
+ * retrieving a consistent "head" element. Operations are guaranteed to reflect a consistent view of
+ * a {@code SetMultimap}, although most methods are not implemented.
+ */
+final class ConcurrentMultimapWithHeadElement<K, V> {
+ private final ConcurrentMap<K, SmallSet<V>> map = Maps.newConcurrentMap();
+
+ /**
+ * Remove (key, val) pair from the multimap. If this removes the current 'head' element
+ * for a key, then another randomly chosen element becomes the current head.
+ *
+ * <p>Until the next (possibly concurrent) {@link #putAndGet}(key, val) call, {@link #get}(key)
+ * will never return val.
+ */
+ void remove(K key, V val) {
+ SmallSet<V> entry = getEntry(key);
+ if (entry != null) {
+ entry.remove(val);
+ if (entry.get() == null) {
+ // Remove entry completely from map if dead.
+ map.remove(key, entry);
+ }
+ }
+ }
+
+ /**
+ * Return some value val such that (key, val) is in the multimap. If there is always at least one
+ * entry for key in the multimap during the lifetime of this method call, it will not return null.
+ */
+ @Nullable V get(K key) {
+ SmallSet<V> entry = getEntry(key);
+ return (entry != null) ? entry.get() : null;
+ }
+
+ /**
+ * Adds (key, val) to the multimap. Returns the head element for key, either val or another
+ * already-stored value.
+ */
+ V putAndGet(K key, V val) {
+ V result = null;
+ while (result == null) {
+ // If another thread concurrently removes the only remaining value from the entry, this
+ // putAndGet will return null, since the entry is about to be removed from the map. In that
+ // case, we obtain a fresh entry from the map and do the put on it.
+ result = getOrCreateEntry(key).putAndGet(val);
+ }
+ return result;
+ }
+
+ /**
+ * Obtain the entry for key, adding it to the underlying map if no entry was previously present.
+ */
+ private SmallSet<V> getOrCreateEntry(K key) {
+ SmallSet<V> entry = new SmallSet<V>();
+ SmallSet<V> oldEntry = map.putIfAbsent(key, entry);
+ if (oldEntry != null) {
+ return oldEntry;
+ }
+ return entry;
+ }
+
+ /**
+ * Obtain the entry for key, returning null if no entry was present in the underlying map.
+ */
+ private SmallSet<V> getEntry(K key) {
+ return map.get(key);
+ }
+
+ /**
+ * Clears the multimap. May not be called concurrently with any other methods.
+ */
+ @ThreadHostile
+ void clear() {
+ map.clear();
+ }
+
+ /**
+ * Wrapper for a {@code #Set} that will probably have at most one element. Keeps the first element
+ * in a separate variable for fast reading/writing and to save space if more than one element is
+ * never written to this set. We always have the invariant that {@link #first} is null only if
+ * {@link #rest} is null.
+ */
+ private static class SmallSet<T> {
+ /*
+ * What is this 'volatile' on first and where's the lock on the read path?
+ *
+ * Volatile is an alternative to locking that works only in very limited situations, such as
+ * simple field reads and writes. Writes from one thread to 'first' happen before reads from
+ * other threads. When used correctly, it can have the same correctness properties as a
+ * 'synchronized' but is much faster on most hardware.
+ *
+ * Here, volatile is used to eliminate locks on the read path. Since get() is merely fetching
+ * the contents of 'first', it meets the criteria for a safe volatile read. In the mutator
+ * methods, care is taken to write only correct values to 'first'; intermediate and incomplete
+ * values do not get written to the field. This means that whenever 'first' is replaced, it is
+ * immediately replaced with the next correct value. Therefore, it is a safe volatile write.
+ *
+ * Other more complex relationships that need to be maintained during the mutate are maintained
+ * with the Object monitor. Since they do not impact the read path (only 'first' matters), the
+ * lock is sufficient for writes and unnecessary for 'first' reads.
+ *
+ * Documentation on volatile:
+ * http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/package-summary.html#MemoryVisibility
+ * (java.util.concurrent package docs)
+ */
+
+ private volatile T first = null;
+ private Set<T> rest = null;
+
+ /*
+ * We may have a race where one thread tries to remove a small set from the map while another
+ * thread tries to add to it. If the second thread loses the race, it will add to a set that is
+ * no longer in the map. To prevent that, once a small set is ever empty, we mark it "dead" by
+ * setting {@code rest} to a {@code TOMBSTONE} value, and (and subsequently remove it from the
+ * map). No modifications to a set can happen after the {@code TOMBSTONE} value is set. Thus,
+ * the thread trying to add a new value to a set will fail, and knows to retrieve the entry anew
+ * from the map and try again.
+ */
+ private static final Set<Object> TOMBSTONE = ImmutableSet.of();
+
+ /**
+ * Return some value in the SmallSet.
+ *
+ * <p>If there is always at least one value in the SmallSet during the lifetime of this call,
+ * it will not return null, since by the invariant, {@link #first} must be non-null.
+ */
+ private T get() {
+ return first;
+ }
+
+ /**
+ * Adds val to the SmallSet. Returns some element of the SmallSet.
+ */
+ private synchronized T putAndGet(T elt) {
+ Preconditions.checkNotNull(elt);
+ if (isDead()) {
+ return null;
+ }
+ if (elt.equals(first)) {
+ return first;
+ }
+ if (first == null) {
+ Preconditions.checkState(rest == null, elt);
+ first = elt;
+ return first;
+ }
+ if (rest == null) {
+ rest = Sets.newHashSet();
+ }
+ rest.add(elt);
+ return first;
+ }
+
+ /**
+ * Remove val from the SmallSet, if it is present.
+ */
+ private synchronized void remove(T elt) {
+ Preconditions.checkNotNull(elt);
+ if (isDead()) {
+ return;
+ }
+ if (elt.equals(first)) {
+ // Normalize to enforce invariant "first is null only if rest is empty."
+ if (rest != null) {
+ Iterator<T> it = rest.iterator();
+ first = it.next();
+ it.remove();
+ if (!it.hasNext()) {
+ rest = null;
+ }
+ } else {
+ first = null;
+ markDead();
+ }
+ } else if ((rest != null) && rest.remove(elt) && rest.isEmpty()) { // side-effect: remove
+ rest = null;
+ }
+ }
+
+ private boolean isDead() {
+ Preconditions.checkState(rest != TOMBSTONE || first == null,
+ "%s present in tombstoned SmallSet, but tombstoned SmallSets should be empty", first);
+ return rest == TOMBSTONE;
+ }
+
+ @SuppressWarnings("unchecked") // Cast of TOMBSTONE. Ok since TOMBSTONE is empty immutable set.
+ private void markDead() {
+ rest = (Set<T>) TOMBSTONE;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java b/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java
new file mode 100644
index 0000000..4b4048b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java
@@ -0,0 +1,106 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.extra.SpawnInfo;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+
+/**
+ * A delegating spawn that allow us to overwrite certain methods while maintaining the original
+ * behavior for non-overwritten methods.
+ */
+public class DelegateSpawn implements Spawn {
+
+ private final Spawn spawn;
+
+ public DelegateSpawn(Spawn spawn){
+ this.spawn = spawn;
+ }
+
+ @Override
+ public final ImmutableMap<String, String> getExecutionInfo() {
+ return spawn.getExecutionInfo();
+ }
+
+ @Override
+ public boolean isRemotable() {
+ return spawn.isRemotable();
+ }
+
+ @Override
+ public ImmutableList<Artifact> getFilesetManifests() {
+ return spawn.getFilesetManifests();
+ }
+
+ @Override
+ public String asShellCommand(Path workingDir) {
+ return spawn.asShellCommand(workingDir);
+ }
+
+ @Override
+ public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() {
+ return spawn.getRunfilesManifests();
+ }
+
+ @Override
+ public SpawnInfo getExtraActionInfo() {
+ return spawn.getExtraActionInfo();
+ }
+
+ @Override
+ public ImmutableList<String> getArguments() {
+ return spawn.getArguments();
+ }
+
+ @Override
+ public ImmutableMap<String, String> getEnvironment() {
+ return spawn.getEnvironment();
+ }
+
+ @Override
+ public Iterable<? extends ActionInput> getInputFiles() {
+ return spawn.getInputFiles();
+ }
+
+ @Override
+ public Collection<? extends ActionInput> getOutputFiles() {
+ return spawn.getOutputFiles();
+ }
+
+ @Override
+ public ActionMetadata getResourceOwner() {
+ return spawn.getResourceOwner();
+ }
+
+ @Override
+ public ResourceSet getLocalResources() {
+ return spawn.getLocalResources();
+ }
+
+ @Override
+ public ActionOwner getOwner() {
+ return spawn.getOwner();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return spawn.getMnemonic();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java b/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java
new file mode 100644
index 0000000..92cc12b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import java.io.IOException;
+
+/**
+ * Exception thrown when we try to digest a directory in {@code ActionInputFileCache}.
+ *
+ */
+public class DigestOfDirectoryException extends IOException {
+
+ public DigestOfDirectoryException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java b/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java
new file mode 100644
index 0000000..e2c493c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An ExecException which is results from an external problem on the user's
+ * local system.
+ *
+ * <p>Note that this is fundamentally different exception then the higher level
+ * LocalEnvironmentException, which is thrown from the BuildTool. That exception
+ * is thrown when the higher levels of Blaze decide to exit.
+ *
+ * <p>This exception is thrown when a low level error is encountered in the
+ * strategy or client protocol layers. This does not necessarily mean we will
+ * exit; we may just retry the action.
+ */
+public class EnvironmentalExecException extends ExecException {
+
+ public EnvironmentalExecException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public EnvironmentalExecException(String message) {
+ super(message);
+ }
+
+ public EnvironmentalExecException(String message, Throwable cause, boolean catastrophe) {
+ super(message, cause, catastrophe);
+ }
+
+ public EnvironmentalExecException(String message, boolean catastrophe) {
+ super(message, catastrophe);
+ }
+
+ @Override
+ public ActionExecutionException toActionExecutionException(String messagePrefix,
+ boolean verboseFailures, Action action) {
+ if (verboseFailures) {
+ return new ActionExecutionException(messagePrefix + " failed" + getMessage(), this, action,
+ isCatastrophic());
+ } else {
+ return new ActionExecutionException(messagePrefix + " failed" + getMessage(), action,
+ isCatastrophic());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecException.java b/src/main/java/com/google/devtools/build/lib/actions/ExecException.java
new file mode 100644
index 0000000..a2edc84
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ExecException.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An exception indication that the execution of an action has failed OR could
+ * not be attempted OR could not be finished OR had something else wrong.
+ *
+ * <p>The four main kinds of failure are broadly defined as follows:
+ *
+ * <p>USER_INPUT which means it had something to do with what the user told us
+ * to do. This failure should satisfy the invariant that it would happen
+ * identically again if all other things are equal.
+ *
+ * <p>ENVIRONMENT which is loosely defined as anything which is generally out of
+ * scope for a blaze evaluation. As a rule of thumb, these are any errors would
+ * not necessarily happen again given constant input.
+ *
+ * <p>INTERRUPTION conditions arise from being unable to complete an evaluation
+ * for whatever reason.
+ *
+ * <p>INTERNAL_ERROR would happen because of anything which arises from within
+ * blaze itself but is generally unexpected to ever occur for any user input.
+ *
+ * <p>The class is a catch-all for both failures of actions and failures to
+ * evaluate actions properly.
+ *
+ * <p>Invariably, all low level ExecExceptions are caught by various specific
+ * ConfigurationAction classes and re-raised as ActionExecutionExceptions.
+ */
+public abstract class ExecException extends Exception {
+
+ private final boolean catastrophe;
+
+ public ExecException(String message, boolean catastrophe) {
+ super(message);
+ this.catastrophe = catastrophe;
+ }
+
+ public ExecException(String message) {
+ this(message, false);
+ }
+
+ public ExecException(String message, Throwable cause, boolean catastrophe) {
+ super(message + ": " + cause.getMessage(), cause);
+ this.catastrophe = catastrophe;
+ }
+
+ public ExecException(String message, Throwable cause) {
+ this(message, cause, false);
+ }
+
+ /**
+ * Catastrophic exceptions should stop the build, even if --keep_going.
+ */
+ public boolean isCatastrophic() {
+ return catastrophe;
+ }
+
+ /**
+ * Returns a new ActionExecutionException without a message prefix.
+ * @param action failed action
+ * @return ActionExecutionException object describing the action failure
+ */
+ public ActionExecutionException toActionExecutionException(Action action) {
+ // In all ExecException implementations verboseFailures argument used only to determine should
+ // we pass ExecException as cause of ActionExecutionException. So use this method only
+ // if you need this information inside of ActionExecutionexception.
+ return toActionExecutionException("", true, action);
+ }
+
+ /**
+ * Returns a new ActionExecutionException given a message prefix describing the action type as a
+ * noun. When appropriate (we use some heuristics to decide), produces an abbreviated message
+ * incorporating just the termination status if available.
+ *
+ * @param messagePrefix describes the action type as noun
+ * @param verboseFailures true if user requested verbose output with flag --verbose_failures
+ * @param action failed action
+ * @return ActionExecutionException object describing the action failure
+ */
+ public abstract ActionExecutionException toActionExecutionException(String messagePrefix,
+ boolean verboseFailures, Action action);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java
new file mode 100644
index 0000000..d86ea6c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation that marks strategies that extend the execution phase behavior of Blaze.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ExecutionStrategy {
+ /**
+ * The names this strategy is available under on the command line.
+ */
+ String[] name() default {};
+
+ /**
+ * Returns the action context this strategy implements.
+ */
+ Class<? extends ActionContext> contextType();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Executor.java b/src/main/java/com/google/devtools/build/lib/actions/Executor.java
new file mode 100644
index 0000000..f3904bb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/Executor.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.OptionsClassProvider;
+
+/**
+ * The Executor provides the context for the execution of actions. It is only valid during the
+ * execution phase, and references should not be cached.
+ *
+ * <p>This class provides the actual logic to execute actions. The platonic ideal of this system
+ * is that {@link Action}s are immutable objects that tell Blaze <b>what</b> to do and
+ * <link>ActionContext</link>s tell Blaze <b>how</b> to do it (however, we do have an "execute"
+ * method on actions now).
+ *
+ * <p>In theory, most of the methods below would not exist and they would be methods on action
+ * contexts, but in practice, that would require some refactoring work so we are stuck with these
+ * for the time being.
+ *
+ * <p>In theory, we could also merge {@link Executor} with {@link ActionExecutionContext}, since
+ * they both provide services to actions being executed and are passed to almost the same places.
+ */
+public interface Executor {
+ /**
+ * A marker interface for classes that provide services for actions during execution.
+ *
+ * <p>Interfaces extending this one should also be annotated with {@link ActionContextMarker}.
+ */
+ public interface ActionContext {
+ }
+
+ /**
+ * Returns the execution root. This is the directory underneath which Blaze builds its entire
+ * output working tree, including the source symlink forest. All build actions are executed
+ * relative to this directory.
+ */
+ Path getExecRoot();
+
+ /**
+ * Returns a clock. This is not hermetic, and should only be used for build info actions or
+ * performance measurements / reporting.
+ */
+ Clock getClock();
+
+ /**
+ * The EventBus for the current build.
+ */
+ EventBus getEventBus();
+
+ /**
+ * Returns whether failures should have verbose error messages.
+ */
+ boolean getVerboseFailures();
+
+ /**
+ * Returns the command line options of the Blaze command being executed.
+ */
+ OptionsClassProvider getOptions();
+
+ /**
+ * Whether this Executor reports subcommands. If not, reportSubcommand has no effect.
+ * This is provided so the caller of reportSubcommand can avoid wastefully constructing the
+ * subcommand string.
+ */
+ boolean reportsSubcommands();
+
+ /**
+ * Report a subcommand event to this Executor's Reporter and, if action
+ * logging is enabled, post it on its EventBus.
+ */
+ void reportSubcommand(String reason, String message);
+
+ /**
+ * An event listener to report messages to. Errors that signal a action failure should
+ * use ActionExecutionException.
+ */
+ EventHandler getEventHandler();
+
+ /**
+ * Looks up and returns an action context implementation of the given interface type.
+ */
+ <T extends ActionContext> T getContext(Class<? extends T> type);
+
+ /**
+ * Returns the action context implementation for spawn actions with a given mnemonic.
+ */
+ SpawnActionContext getSpawnActionContext(String mnemonic);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java
new file mode 100644
index 0000000..21369fb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+
+/**
+ * An exception that is thrown when an executor can't be initialized.
+ */
+public class ExecutorInitException extends AbruptExitException {
+
+ public ExecutorInitException(String message) {
+ this(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
+ }
+
+ public ExecutorInitException(String message, ExitCode exitCode) {
+ super(message, exitCode);
+ }
+
+ public ExecutorInitException(String message, Throwable cause) {
+ super(message + ": " + cause.getMessage(),
+ ExitCode.LOCAL_ENVIRONMENTAL_ERROR,
+ cause);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FailAction.java b/src/main/java/com/google/devtools/build/lib/actions/FailAction.java
new file mode 100644
index 0000000..6c15b31
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/FailAction.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+/**
+ * FailAction is an Action that always fails to execute. (Used as scaffolding
+ * for rules we haven't yet implemented. Also useful for testing.)
+ */
+@ThreadSafe
+public final class FailAction extends AbstractAction {
+
+ private static final String GUID = "626cb78a-810f-4af3-979c-ee194955f04c";
+
+ private final String errorMessage;
+
+ public FailAction(ActionOwner owner, Iterable<Artifact> outputs, String errorMessage) {
+ super(owner, ImmutableList.<Artifact>of(), outputs);
+ this.errorMessage = errorMessage;
+ }
+
+ @Override
+ public Artifact getPrimaryInput() {
+ return null;
+ }
+
+ @Override
+ public void execute(
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ throw new ActionExecutionException(errorMessage, this, false);
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ protected String computeKey() {
+ return GUID;
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Building unsupported rule " + getOwner().getLabel()
+ + " located at " + getOwner().getLocation();
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "Fail";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java
new file mode 100644
index 0000000..1ae15ba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/** Definition of a symlink in the output tree of a Fileset rule. */
+public final class FilesetOutputSymlink {
+ private static final String STRIPPED_METADATA = "<stripped-for-testing>";
+
+ /** Final name of the symlink relative to the Fileset's output directory. */
+ public final PathFragment name;
+
+ /** Target of the symlink. Depending on FilesetEntry.symlinks it may be relative or absolute. */
+ public final PathFragment target;
+
+ /** Opaque metadata about the link and its target; should change if either of them changes. */
+ public final String metadata;
+
+ @VisibleForTesting
+ public FilesetOutputSymlink(PathFragment name, PathFragment target) {
+ this.name = name;
+ this.target = target;
+ this.metadata = STRIPPED_METADATA;
+ }
+
+ /**
+ * @param name relative path under the Fileset's output directory, including FilesetEntry.destdir
+ * with and FilesetEntry.strip_prefix applied (if applicable)
+ * @param target relative or absolute value of the link
+ * @param metadata opaque metadata about the link and its target; should change if either the link
+ * or its target changes
+ */
+ public FilesetOutputSymlink(PathFragment name, PathFragment target, String metadata) {
+ this.name = Preconditions.checkNotNull(name);
+ this.target = Preconditions.checkNotNull(target);
+ this.metadata = Preconditions.checkNotNull(metadata);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || !obj.getClass().equals(getClass())) {
+ return false;
+ }
+ FilesetOutputSymlink o = (FilesetOutputSymlink) obj;
+ return name.equals(o.name) && target.equals(o.target) && metadata.equals(o.metadata);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(name, target, metadata);
+ }
+
+ @Override
+ public String toString() {
+ if (metadata.equals(STRIPPED_METADATA)) {
+ return String.format("FilesetOutputSymlink(%s -> %s)",
+ name.getPathString(), target.getPathString());
+ } else {
+ return String.format("FilesetOutputSymlink(%s -> %s | metadata=%s)",
+ name.getPathString(), target.getPathString(), metadata);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java
new file mode 100644
index 0000000..1464f73
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java
@@ -0,0 +1,165 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Optional;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+
+import java.util.Set;
+
+/**
+ * Parameters of a filesystem traversal requested by a Fileset rule.
+ *
+ * <p>This object stores the details of the traversal request, e.g. whether it's a direct or nested
+ * traversal (see {@link #getDirectTraversal()} and {@link #getNestedTraversal()}) or who the owner
+ * of the traversal is.
+ */
+public interface FilesetTraversalParams {
+
+ /**
+ * Abstraction of the root directory of a {@link DirectTraversal}.
+ *
+ * <ul>
+ * <li>The root of package traversals is the package directory, i.e. the parent of the BUILD file.
+ * <li>The root of "recursive" directory traversals is the directory's path.
+ * <li>The root of "file" traversals is the path of the file (or directory, or symlink) itself.
+ * </ul>
+ *
+ * <p>For the meaning of "recursive" and "file" traversals see {@link DirectTraversal}.
+ */
+ interface DirectTraversalRoot {
+
+ /**
+ * Returns the root part of the full path.
+ *
+ * <p>This is typically the workspace root or some output tree's root (e.g. genfiles, binfiles).
+ */
+ Path getRootPart();
+
+ /**
+ * Returns the {@link #getRootPart() root}-relative part of the path.
+ *
+ * <p>This is typically the source directory under the workspace or the output file under an
+ * output directory.
+ */
+ PathFragment getRelativePart();
+
+ /** Returns a {@link RootedPath} composed of the root and relative parts. */
+ RootedPath asRootedPath();
+ }
+
+ /**
+ * Describes a request for a direct filesystem traversal.
+ *
+ * <p>"Direct" means this corresponds to an actual filesystem traversal as opposed to traversing
+ * another Fileset rule, which is called a "nested" traversal.
+ *
+ * <p>Direct traversals can further be divided into two categories, "file" traversals and
+ * "recursive" traversals.
+ *
+ * <p>File traversal requests are created when the FilesetEntry.files attribute is defined; one
+ * file traversal request is created for each entry.
+ *
+ * <p>Recursive traversal requests are created when the FilesetEntry.files attribute is
+ * unspecified; one recursive traversal request is created for the FilesetEntry.srcdir.
+ *
+ * <p>See {@link DirectTraversal#getRoot()} for more details.
+ */
+ interface DirectTraversal {
+
+ /** Returns the root of the traversal; see {@link DirectTraversalRoot}. */
+ DirectTraversalRoot getRoot();
+
+ /**
+ * Returns true if this traversal refers to a whole package.
+ *
+ * <p>In that case the root (see {@link #getRoot()}) refers to the path of the package.
+ *
+ * <p>Package traversals are always recursive (see {@link #isRecursive()}) and are never
+ * generated (see {@link #isGenerated()}).
+ */
+ boolean isPackage();
+
+ /**
+ * Returns true if this is a "recursive traversal", i.e. created from FilesetEntry.srcdir.
+ *
+ * <p>This type of traversal is created when the FilesetEntry doesn't define a "files" list.
+ * When it does, the traversal is referred to as a "file traversal". When it doesn't, but the
+ * srcdir points to another Fileset, it is called a "nested" traversal.
+ *
+ * <p>Recursive traversals got their name from recursively traversing a directory structure.
+ * These are usually whole-package traversals, i.e. when FilesetEntry.srcdir refers to a BUILD
+ * file (see {@link #isPackage()}), but sometimes the srcdir references a input or output
+ * directory (the latter being generated by a local genrule) or a symlink (which must point to a
+ * directory; enforced during action execution).
+ *
+ * <p>The files in the results of a recursive traversal are all under the {@link #getRoot()
+ * root}. The root's path is stripped from the results.
+ *
+ * <p>N.B.: "file traversals" can also be recursive if the entry in FilesetEntry.files, for
+ * which the traversal parameters were created, turned out to be a directory. The difference
+ * lies in how the output paths are computed (with recursive traversals, the directory's name
+ * is stripped; with file traversals it is not, modulo usage of strip_prefix and the excludes
+ * attributes), and how directory symlinks are handled (in "recursive traversals" they are
+ * expanded just like normal directories, subsequent directory symlinks under them are *not*
+ * expanded though; they are not expanded at all in "file traversals").
+ */
+ boolean isRecursive();
+
+ /** Returns true if the root points to a generated file, symlink or directory. */
+ boolean isGenerated();
+
+ /** Returns true if input symlinks should be dereferenced; false if copied. */
+ boolean isFollowingSymlinks();
+
+ /** Returns the desired behavior when the traversal hits a subpackage. */
+ boolean getCrossPackageBoundary();
+ }
+
+ /** Label of the Fileset rule that owns this traversal. */
+ Label getOwnerLabel();
+
+ /** Returns the directory under the output path where the files will be mapped. May be empty. */
+ PathFragment getDestPath();
+
+ /** Returns a list of file basenames to be excluded from the output. May be empty. */
+ Set<String> getExcludedFiles();
+
+ /**
+ * Returns the parameters of the direct traversal request, if any.
+ *
+ * <p>A direct traversal is anything that's not a nested traversal, e.g. traversal of a package or
+ * directory (when FilesetEntry.srcdir is specified) or traversal of a single file (when
+ * FilesetEntry.files is specified). See {@link DirectTraversal} for more detail.
+ *
+ * <p>The value is present if and only if {@link #getNestedTraversal} is absent.
+ */
+ Optional<DirectTraversal> getDirectTraversal();
+
+ /**
+ * Returns the parameters of the nested traversal request, if any.
+ *
+ * <p>A nested traversal is the traversal of another Fileset referenced by FilesetEntry.srcdir.
+ *
+ * <p>The value is present if and only if {@link #getDirectTraversal} is absent.
+ */
+ Optional<FilesetTraversalParams> getNestedTraversal();
+
+ /** Adds the fingerprint of this traversal object. */
+ void fingerprint(Fingerprint fp);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java
new file mode 100644
index 0000000..acc0e86
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java
@@ -0,0 +1,314 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Ordering;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversal;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversalRoot;
+import com.google.devtools.build.lib.syntax.FilesetEntry.SymlinkBehavior;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/** Factory of {@link FilesetTraversalParams}. */
+public final class FilesetTraversalParamsFactory {
+
+ /**
+ * Creates parameters for a recursive traversal request in a package.
+ *
+ * <p>"Recursive" means that a directory is traversed along with all of its subdirectories. Such
+ * a traversal is created when FilesetEntry.files is unspecified.
+ *
+ * @param ownerLabel the rule that created this object
+ * @param buildFile path of the BUILD file of the package to traverse
+ * @param destPath path in the Fileset's output directory that will be the root of files found
+ * in this directory
+ * @param excludes optional; set of files directly under this package's directory to exclude;
+ * files in subdirectories cannot be excluded
+ * @param symlinkBehaviorMode what to do with symlinks
+ * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains
+ * a BUILD file)
+ */
+ public static FilesetTraversalParams recursiveTraversalOfPackage(Label ownerLabel,
+ Artifact buildFile, PathFragment destPath, @Nullable Set<String> excludes,
+ SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) {
+ Preconditions.checkState(buildFile.isSourceArtifact(), "%s", buildFile);
+ return new DirectoryTraversalParams(ownerLabel, DirectTraversalRootImpl.forPackage(buildFile),
+ true, destPath, excludes, symlinkBehaviorMode, crossPkgBoundary, true, false);
+ }
+
+ /**
+ * Creates parameters for a recursive traversal request in a directory.
+ *
+ * <p>"Recursive" means that a directory is traversed along with all of its subdirectories. Such
+ * a traversal is created when FilesetEntry.files is unspecified.
+ *
+ * @param ownerLabel the rule that created this object
+ * @param directoryToTraverse path of the directory to traverse
+ * @param destPath path in the Fileset's output directory that will be the root of files found
+ * in this directory
+ * @param excludes optional; set of files directly below this directory to exclude; files in
+ * subdirectories cannot be excluded
+ * @param symlinkBehaviorMode what to do with symlinks
+ * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains
+ * a BUILD file)
+ */
+ public static FilesetTraversalParams recursiveTraversalOfDirectory(Label ownerLabel,
+ Artifact directoryToTraverse, PathFragment destPath, @Nullable Set<String> excludes,
+ SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) {
+ return new DirectoryTraversalParams(ownerLabel,
+ DirectTraversalRootImpl.forFileOrDirectory(directoryToTraverse), false, destPath,
+ excludes, symlinkBehaviorMode, crossPkgBoundary, true,
+ !directoryToTraverse.isSourceArtifact());
+ }
+
+ /**
+ * Creates parameters for a file traversal request.
+ *
+ * <p>Such a traversal is created for every entry in FilesetEntry.files, when it is specified.
+ *
+ * @param ownerLabel the rule that created this object
+ * @param fileToTraverse the file to traverse; "traversal" means that if this file is actually a
+ * directory or a symlink to one then it'll be traversed as one
+ * @param destPath path in the Fileset's output directory that will be the name of this file's
+ * respective symlink there, or the root of files found (in case this is a directory)
+ * @param symlinkBehaviorMode what to do with symlinks
+ * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains
+ * a BUILD file)
+ */
+ public static FilesetTraversalParams fileTraversal(Label ownerLabel, Artifact fileToTraverse,
+ PathFragment destPath, SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) {
+ return new DirectoryTraversalParams(ownerLabel,
+ DirectTraversalRootImpl.forFileOrDirectory(fileToTraverse), false, destPath, null,
+ symlinkBehaviorMode, crossPkgBoundary, false, !fileToTraverse.isSourceArtifact());
+ }
+
+ /**
+ * Creates traversal request parameters for a FilesetEntry wrapping another Fileset.
+ *
+ * @param ownerLabel the rule that created this object
+ * @param nested the traversal params that were used for the nested (inner) Fileset
+ * @param destDir path in the Fileset's output directory that will be the root of files coming
+ * from the nested Fileset
+ * @param excludes optional; set of files directly below (not in a subdirectory of) the nested
+ * Fileset that should be excluded from the outer Fileset
+ */
+ public static FilesetTraversalParams nestedTraversal(Label ownerLabel,
+ FilesetTraversalParams nested, PathFragment destDir, @Nullable Set<String> excludes) {
+ // When srcdir is another Fileset, then files must be null so strip_prefix must also be null.
+ return new NestedTraversalParams(ownerLabel, nested, destDir, excludes);
+ }
+
+ private abstract static class ParamsCommon implements FilesetTraversalParams {
+ private final Label ownerLabel;
+ private final PathFragment destDir;
+ private final ImmutableSet<String> excludes;
+
+ ParamsCommon(Label ownerLabel, PathFragment destDir, @Nullable Set<String> excludes) {
+ this.ownerLabel = ownerLabel;
+ this.destDir = destDir;
+ if (excludes == null) {
+ this.excludes = ImmutableSet.<String>of();
+ } else {
+ // Order the set for the sake of deterministic fingerprinting.
+ this.excludes = ImmutableSet.copyOf(Ordering.natural().immutableSortedCopy(excludes));
+ }
+ }
+
+ @Override
+ public Label getOwnerLabel() {
+ return ownerLabel;
+ }
+
+ @Override
+ public Set<String> getExcludedFiles() {
+ return excludes;
+ }
+
+ @Override
+ public PathFragment getDestPath() {
+ return destDir;
+ }
+
+ protected final void commonFingerprint(Fingerprint fp) {
+ fp.addPath(destDir);
+ if (!excludes.isEmpty()) {
+ fp.addStrings(excludes);
+ }
+ }
+ }
+
+ private static final class DirectTraversalImpl implements DirectTraversal {
+ private final DirectTraversalRoot root;
+ private final boolean isPackage;
+ private final boolean followSymlinks;
+ private final boolean crossPkgBoundary;
+ private final boolean isRecursive;
+ private final boolean isGenerated;
+
+ DirectTraversalImpl(DirectTraversalRoot root, boolean isPackage, boolean followSymlinks,
+ boolean crossPkgBoundary, boolean isRecursive, boolean isGenerated) {
+ this.root = root;
+ this.isPackage = isPackage;
+ this.followSymlinks = followSymlinks;
+ this.crossPkgBoundary = crossPkgBoundary;
+ this.isRecursive = isRecursive;
+ this.isGenerated = isGenerated;
+ }
+
+ @Override
+ public DirectTraversalRoot getRoot() {
+ return root;
+ }
+
+ @Override
+ public boolean isPackage() {
+ return isPackage;
+ }
+
+ @Override
+ public boolean isRecursive() {
+ return isRecursive;
+ }
+
+ @Override
+ public boolean isGenerated() {
+ return isGenerated;
+ }
+
+ @Override
+ public boolean isFollowingSymlinks() {
+ return followSymlinks;
+ }
+
+ @Override
+ public boolean getCrossPackageBoundary() {
+ return crossPkgBoundary;
+ }
+
+ void fingerprint(Fingerprint fp) {
+ fp.addPath(root.asRootedPath().asPath());
+ fp.addBoolean(isPackage);
+ fp.addBoolean(followSymlinks);
+ fp.addBoolean(isRecursive);
+ fp.addBoolean(isGenerated);
+ fp.addBoolean(crossPkgBoundary);
+ }
+ }
+
+ private static final class DirectoryTraversalParams extends ParamsCommon {
+ private final DirectTraversalImpl traversal;
+
+ DirectoryTraversalParams(Label ownerLabel,
+ DirectTraversalRoot root,
+ boolean isPackage,
+ PathFragment destPath,
+ @Nullable Set<String> excludes,
+ SymlinkBehavior symlinkBehaviorMode,
+ boolean crossPkgBoundary,
+ boolean isRecursive,
+ boolean isGenerated) {
+ super(ownerLabel, destPath, excludes);
+ traversal = new DirectTraversalImpl(root, isPackage,
+ symlinkBehaviorMode == SymlinkBehavior.DEREFERENCE, crossPkgBoundary, isRecursive,
+ isGenerated);
+ }
+
+ @Override
+ public Optional<DirectTraversal> getDirectTraversal() {
+ return Optional.<DirectTraversal>of(traversal);
+ }
+
+ @Override
+ public Optional<FilesetTraversalParams> getNestedTraversal() {
+ return Optional.absent();
+ }
+
+ @Override
+ public void fingerprint(Fingerprint fp) {
+ commonFingerprint(fp);
+ traversal.fingerprint(fp);
+ }
+ }
+
+ private static final class NestedTraversalParams extends ParamsCommon {
+ private final FilesetTraversalParams nested;
+
+ public NestedTraversalParams(Label ownerLabel, FilesetTraversalParams nested,
+ PathFragment destDir, @Nullable Set<String> excludes) {
+ super(ownerLabel, destDir, excludes);
+ this.nested = nested;
+ }
+
+ @Override
+ public Optional<DirectTraversal> getDirectTraversal() {
+ return Optional.absent();
+ }
+
+ @Override
+ public Optional<FilesetTraversalParams> getNestedTraversal() {
+ return Optional.of(nested);
+ }
+
+ @Override
+ public void fingerprint(Fingerprint fp) {
+ commonFingerprint(fp);
+ nested.fingerprint(fp);
+ }
+ }
+
+ private static final class DirectTraversalRootImpl implements DirectTraversalRoot {
+ private final Path rootDir;
+ private final PathFragment relativeDir;
+
+ static DirectTraversalRoot forPackage(Artifact buildFile) {
+ return new DirectTraversalRootImpl(buildFile.getRoot().getPath(),
+ buildFile.getRootRelativePath().getParentDirectory());
+ }
+
+ static DirectTraversalRoot forFileOrDirectory(Artifact fileOrDirectory) {
+ return new DirectTraversalRootImpl(fileOrDirectory.getRoot().getPath(),
+ fileOrDirectory.getRootRelativePath());
+ }
+
+ private DirectTraversalRootImpl(Path rootDir, PathFragment relativeDir) {
+ this.rootDir = rootDir;
+ this.relativeDir = relativeDir;
+ }
+
+ @Override
+ public Path getRootPart() {
+ return rootDir;
+ }
+
+ @Override
+ public PathFragment getRelativePart() {
+ return relativeDir;
+ }
+
+ @Override
+ public RootedPath asRootedPath() {
+ return RootedPath.toRootedPath(rootDir, relativeDir);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
new file mode 100644
index 0000000..5fc6013
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
@@ -0,0 +1,302 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.ProcMeminfoParser;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This class estimates the local host's resource capacity.
+ */
+@ThreadCompatible
+public final class LocalHostCapacity {
+
+ private static final Logger LOG = Logger.getLogger(LocalHostCapacity.class.getName());
+
+ /**
+ * Stores parsed /proc/stat CPU time counters.
+ * See {@link LocalHostCapacity#getCpuTimes(String)} for details.
+ */
+ @Immutable
+ private final static class CpuTimes {
+ private final long idleJiffies;
+ private final long totalJiffies;
+
+ CpuTimes(long idleJiffies, long totalJiffies) {
+ this.idleJiffies = idleJiffies;
+ this.totalJiffies = totalJiffies;
+ }
+
+ /**
+ * Return idle CPU ratio using current and previous CPU readings or 0 if
+ * ratio is undefined.
+ */
+ double getIdleRatio(CpuTimes prevTimes) {
+ if (prevTimes.totalJiffies == 0 || totalJiffies == prevTimes.totalJiffies) {
+ return 0;
+ }
+ return ((double)(idleJiffies - prevTimes.idleJiffies) /
+ (double)(totalJiffies - prevTimes.totalJiffies));
+ }
+ }
+
+ /**
+ * Used to store available local CPU and RAM resources information.
+ * See {@link LocalHostCapacity#getFreeResources(FreeResources)} for details.
+ */
+ public static final class FreeResources {
+
+ private final Clock clock;
+ private final CpuTimes cpuTimes;
+ private final long lastTimestamp;
+ private final double freeCpu;
+ private final double freeMb;
+ private final long interval;
+
+ private FreeResources(Clock localClock, ProcMeminfoParser memInfo, String statContent,
+ FreeResources prevStats) {
+ clock = localClock;
+ lastTimestamp = localClock.nanoTime();
+ freeMb = ProcMeminfoParser.kbToMb(memInfo.getFreeRamKb());
+ cpuTimes = getCpuTimes(statContent);
+ if (prevStats == null) {
+ interval = 0;
+ freeCpu = 0.0;
+ } else {
+ interval = lastTimestamp - prevStats.lastTimestamp;
+ freeCpu = getLocalHostCapacity().getCpuUsage() * cpuTimes.getIdleRatio(prevStats.cpuTimes);
+ }
+ }
+
+ /**
+ * Returns amount of available RAM in MB.
+ */
+ public double getFreeMb() { return freeMb; }
+
+ /**
+ * Returns average available CPU resources (as a fraction of the CPU core,
+ * so one fully CPU-bound thread should consume exactly 1.0 CPU resource).
+ */
+ public double getAvgFreeCpu() { return freeCpu; }
+
+ /**
+ * Returns interval in ms between CPU load measurements used to calculate
+ * average available CPU resources.
+ */
+ public long getInterval() { return interval / 1000000; }
+
+ /**
+ * Returns age of available resource data in ms.
+ */
+ public long getReadingAge() {
+ return (clock.nanoTime() - lastTimestamp) / 1000000;
+ }
+ }
+
+ // Disables getFreeResources() if error occured during reading or parsing
+ // /proc/* information.
+ @VisibleForTesting
+ static boolean isDisabled;
+
+ // If /proc/* information is not available, assume 3000 MB and 2 CPUs.
+ private static ResourceSet DEFAULT_RESOURCES = new ResourceSet(3000.0, 2.0, 1.0);
+
+ private LocalHostCapacity() {}
+
+ /**
+ * Estimates of the local host's resource capacity,
+ * obtained by reading /proc/cpuinfo and /proc/meminfo.
+ */
+ private static ResourceSet localHostCapacity;
+
+ /**
+ * Estimates of the local host's resource capacity,
+ * obtained by reading /proc/cpuinfo and /proc/meminfo.
+ */
+ public static ResourceSet getLocalHostCapacity() {
+ if (localHostCapacity == null) {
+ localHostCapacity = getLocalHostCapacity("/proc/cpuinfo", "/proc/meminfo");
+ }
+ return localHostCapacity;
+ }
+
+ /**
+ * Returns new FreeResources object populated with free RAM information from
+ * /proc/meminfo and CPU load information from the /proc/stat. First call
+ * should be made with null parameter to instantiate new FreeResources object.
+ * Subsequent calls will use information inside it to calculate average CPU
+ * load over the time between calls and to calculate amount of free CPU
+ * resources and generate new FreeResources() instance.
+ *
+ * If information is not available due to error, functionality will be disabled
+ * and method will always return null.
+ */
+ public static FreeResources getFreeResources(FreeResources stats) {
+ return getFreeResources(BlazeClock.instance(), "/proc/meminfo", "/proc/stat", stats);
+ }
+
+ private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
+
+ @VisibleForTesting
+ static int getLogicalCpuCount(String cpuinfoContent) {
+ Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent);
+ int count = 0;
+ for (String line : lines) {
+ if(line.startsWith("processor")) {
+ count++;
+ }
+ }
+ if (count == 0) {
+ throw new IllegalArgumentException("Can't locate processor in the /proc/cpuinfo");
+ }
+ return count;
+ }
+
+ @VisibleForTesting
+ static int getPhysicalCpuCount(String cpuinfoContent, int logicalCpuCount) {
+ Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent);
+ Set<String> uniq = new HashSet<>();
+ for (String line : lines) {
+ if(line.startsWith("physical id")) {
+ uniq.add(line);
+ }
+ }
+ int physicalCpuCount = uniq.size();
+ if (physicalCpuCount == 0) {
+ physicalCpuCount = logicalCpuCount;
+ }
+ return physicalCpuCount;
+ }
+
+ @VisibleForTesting
+ static int getCoresPerCpu(String cpuinfoFileContent) {
+ Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoFileContent);
+ Set<String> uniq = new HashSet<>();
+ for (String line : lines) {
+ if(line.startsWith("core id")) {
+ uniq.add(line);
+ }
+ }
+ int coresPerCpu = uniq.size();
+ if (coresPerCpu == 0) {
+ coresPerCpu = 1;
+ }
+ return coresPerCpu;
+ }
+
+ /**
+ * Parses cpu line of the /proc/stats, calculates number of idle and total
+ * CPU jiffies and returns CpuTimes instance with that information.
+ *
+ * Total CPU time includes <b>all</b> time reported to be spent by the CPUs,
+ * including so-called "stolen" time - time spent by other VMs on the same
+ * workstation.
+ */
+ private static CpuTimes getCpuTimes(String statContent) {
+ String[] cpuStats = statContent.substring(0, statContent.indexOf('\n')).trim().split(" +");
+ // Supported versions of /proc/stat (Linux kernel 2.6.x) must contain either
+ // 9 or 10 fields:
+ // "cpu" utime ultime stime idle iowait irq softirq steal(since 2.6.11) 0
+ // We are interested in total time (sum of all columns) and idle time.
+ if (cpuStats.length < 9 | cpuStats.length > 10) {
+ throw new IllegalArgumentException("Unrecognized /proc/stat format");
+ }
+ if (!cpuStats[0].equals("cpu")) {
+ throw new IllegalArgumentException("/proc/stat does not start with cpu keyword");
+ }
+ long idleCpuJiffies = Long.parseLong(cpuStats[4]); // "idle" column.
+ long totalJiffies = 0;
+ for (int i = 1; i < cpuStats.length; i++) {
+ totalJiffies += Long.parseLong(cpuStats[i]);
+ }
+ long totalCpuJiffies = totalJiffies;
+ return new CpuTimes(idleCpuJiffies, totalCpuJiffies);
+ }
+
+ @VisibleForTesting
+ static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) {
+ try {
+ String cpuinfoContent = readContent(cpuinfoFile);
+ ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile);
+ int logicalCpuCount = getLogicalCpuCount(cpuinfoContent);
+ int physicalCpuCount = getPhysicalCpuCount(cpuinfoContent, logicalCpuCount);
+ int coresPerCpu = getCoresPerCpu(cpuinfoContent);
+ int totalCores = coresPerCpu * physicalCpuCount;
+ boolean hyperthreading = (logicalCpuCount != totalCores);
+ double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
+ final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
+ return new ResourceSet(
+ ramMb,
+ logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU
+ : 1.0),
+ 1.0);
+ } catch (IOException | IllegalArgumentException e) {
+ disableProcFsUse(e);
+ return DEFAULT_RESOURCES;
+ }
+ }
+
+ @VisibleForTesting
+ static FreeResources getFreeResources(Clock localClock, String meminfoFile, String statFile,
+ FreeResources prevStats) {
+ if (isDisabled) { return null; }
+ try {
+ String statContent = readContent(statFile);
+ return new FreeResources(localClock, new ProcMeminfoParser(meminfoFile),
+ statContent, prevStats);
+ } catch (IOException | IllegalArgumentException e) {
+ disableProcFsUse(e);
+ return null;
+ }
+ }
+
+ /**
+ * For testing purposes only. Do not use it.
+ */
+ @VisibleForTesting
+ static void setLocalHostCapacity(ResourceSet resources) {
+ localHostCapacity = resources;
+ isDisabled = false;
+ }
+
+ private static String readContent(String filename) throws IOException {
+ return Files.toString(new File(filename), Charset.defaultCharset());
+ }
+
+ /**
+ * Disables use of /proc filesystem. Called internally when unexpected
+ * exception is caught.
+ */
+ private static void disableProcFsUse(Throwable cause) {
+ LoggingUtil.logToRemote(Level.WARNING, "Unable to read system load or capacity", cause);
+ LOG.log(Level.WARNING, "Unable to read system load or capacity", cause);
+ isDisabled = true;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java
new file mode 100644
index 0000000..2788f2f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * An action graph that resolves generating actions by looking them up in a map.
+ */
+@ThreadSafe
+public final class MapBasedActionGraph implements MutableActionGraph {
+
+ private final ConcurrentMultimapWithHeadElement<Artifact, Action> generatingActionMap =
+ new ConcurrentMultimapWithHeadElement<Artifact, Action>();
+
+ @Override
+ @Nullable
+ public Action getGeneratingAction(Artifact artifact) {
+ return generatingActionMap.get(artifact);
+ }
+
+ @Override
+ public void registerAction(Action action) throws ActionConflictException {
+ for (Artifact artifact : action.getOutputs()) {
+ Action previousAction = generatingActionMap.putAndGet(artifact, action);
+ if (previousAction != null && previousAction != action
+ && !Actions.canBeShared(action, previousAction)) {
+ generatingActionMap.remove(artifact, action);
+ throw new ActionConflictException(artifact, previousAction, action);
+ }
+ }
+ }
+
+ @Override
+ public void unregisterAction(Action action) {
+ for (Artifact artifact : action.getOutputs()) {
+ generatingActionMap.remove(artifact, action);
+ Action otherAction = generatingActionMap.get(artifact);
+ Preconditions.checkState(otherAction == null
+ || (otherAction != action && Actions.canBeShared(action, otherAction)),
+ "%s %s", action, otherAction);
+ }
+ }
+
+ @Override
+ public void clear() {
+ generatingActionMap.clear();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java
new file mode 100644
index 0000000..9d3a2b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java
@@ -0,0 +1,107 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * An action that depends on a set of inputs and creates a single output file whenever it
+ * runs. This is useful for bundling up a bunch of dependencies that are shared
+ * between individual targets in the action graph; for example generated header files.
+ */
+public class MiddlemanAction extends AbstractAction {
+
+ public static final String MIDDLEMAN_MNEMONIC = "Middleman";
+ private final String description;
+ private final MiddlemanType middlemanType;
+
+ /**
+ * Constructs a new {@link MiddlemanAction}.
+ *
+ * @param owner the owner of the action, usually a {@code ConfiguredTarget}
+ * @param inputs inputs of the middleman, i.e. the files it acts as a placeholder for
+ * @param stampFile the output of the middleman expansion; must be a middleman artifact (see
+ * {@link Artifact#isMiddlemanArtifact()})
+ * @param description a short description for the action, for progress messages
+ * @param middlemanType the type of the middleman
+ * @throws IllegalArgumentException if {@code stampFile} is not a middleman artifact
+ */
+ public MiddlemanAction(ActionOwner owner, Iterable<Artifact> inputs, Artifact stampFile,
+ String description, MiddlemanType middlemanType) {
+ super(owner, inputs, ImmutableList.of(stampFile));
+ Preconditions.checkNotNull(middlemanType);
+ Preconditions.checkArgument(stampFile.isMiddlemanArtifact(), stampFile);
+ this.description = description;
+ this.middlemanType = middlemanType;
+ }
+
+ @Override
+ public final void execute(
+ ActionExecutionContext actionExecutionContext) {
+ throw new IllegalStateException("MiddlemanAction should never be executed");
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ protected String computeKey() {
+ // TODO(bazel-team): Need to take middlemanType into account here.
+ // Only the set of inputs matters, and the dependency checker is
+ // responsible for considering those.
+ return "";
+ }
+
+ /**
+ * Returns the type of the middleman.
+ */
+ @Override
+ public MiddlemanType getActionType() {
+ return middlemanType;
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return null; // users don't really want to know about Middlemen.
+ }
+
+ @Override
+ public String prettyPrint() {
+ return description + " for " + Label.print(getOwner().getLabel());
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public String getMnemonic() {
+ return MIDDLEMAN_MNEMONIC;
+ }
+
+ /**
+ * Creates a new middleman action.
+ */
+ public static Action create(ActionRegistry env, ActionOwner owner,
+ Iterable<Artifact> inputs, Artifact stampFile, String purpose, MiddlemanType middlemanType) {
+ MiddlemanAction action = new MiddlemanAction(owner, inputs, stampFile, purpose, middlemanType);
+ env.registerAction(action);
+ return action;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java
new file mode 100644
index 0000000..85a1450
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java
@@ -0,0 +1,188 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Iterator;
+
+/**
+ * A factory to create middleman objects.
+ */
+@ThreadSafe
+public final class MiddlemanFactory {
+
+ private final ArtifactFactory artifactFactory;
+ private final ActionRegistry actionRegistry;
+
+ public MiddlemanFactory(
+ ArtifactFactory artifactFactory, ActionRegistry actionRegistry) {
+ this.artifactFactory = Preconditions.checkNotNull(artifactFactory);
+ this.actionRegistry = Preconditions.checkNotNull(actionRegistry);
+ }
+
+ /**
+ * Creates a {@link MiddlemanType#AGGREGATING_MIDDLEMAN aggregating} middleman.
+ *
+ * @param owner the owner of the action that will be created; must not be null
+ * @param purpose the purpose for which this middleman is created. This should be a string which
+ * is suitable for use as a filename. A single rule may have many middlemen with distinct
+ * purposes.
+ * @param inputs the set of artifacts for which the created artifact is to be the middleman.
+ * @param middlemanDir the directory in which to place the middleman.
+ * @return null iff {@code inputs} is empty; the single element of {@code inputs} if there's only
+ * one; a new aggregating middleman for the {@code inputs} otherwise
+ */
+ public Artifact createAggregatingMiddleman(
+ ActionOwner owner, String purpose, Iterable<Artifact> inputs, Root middlemanDir) {
+ if (hasExactlyOneInput(inputs)) { // Optimization: No middleman for just one input.
+ return Iterables.getOnlyElement(inputs);
+ }
+ Pair<Artifact, Action> result = createMiddleman(
+ owner, Label.print(owner.getLabel()), purpose, inputs, middlemanDir,
+ MiddlemanType.AGGREGATING_MIDDLEMAN);
+ return result == null ? null : result.getFirst();
+ }
+
+ /**
+ * Returns <code>null</code> iff inputs is empty. Returns the sole element
+ * of inputs iff <code>inputs.size()==1</code>. Otherwise, returns a
+ * middleman artifact and creates a middleman action that generates that
+ * artifact.
+ *
+ * @param owner the owner of the action that will be created.
+ * @param owningArtifact the artifact of the file for which the runfiles
+ * should be created. There may be at most one set of runfiles for
+ * an owning artifact, unless the owning artifact is null. There
+ * may be at most one set of runfiles per owner with a null
+ * owning artifact.
+ * Further, if the owning Artifact is non-null, the owning Artifacts'
+ * root-relative path must be unique and the artifact must be part
+ * of the runfiles tree for which this middleman is created. Usually
+ * this artifact will be an executable program.
+ * @param inputs the set of artifacts for which the created artifact is to be
+ * the middleman.
+ * @param middlemanDir the directory in which to place the middleman.
+ */
+ public Artifact createRunfilesMiddleman(
+ ActionOwner owner, Artifact owningArtifact, Iterable<Artifact> inputs, Root middlemanDir) {
+ if (hasExactlyOneInput(inputs)) { // Optimization: No middleman for just one input.
+ return Iterables.getOnlyElement(inputs);
+ }
+ String middlemanPath = owningArtifact == null
+ ? Label.print(owner.getLabel())
+ : owningArtifact.getRootRelativePath().getPathString();
+ return createMiddleman(owner, middlemanPath, "runfiles", inputs, middlemanDir,
+ MiddlemanType.RUNFILES_MIDDLEMAN).getFirst();
+ }
+
+ private <T> boolean hasExactlyOneInput(Iterable<T> iterable) {
+ Iterator<T> it = iterable.iterator();
+ if (!it.hasNext()) {
+ return false;
+ }
+ it.next();
+ return !it.hasNext();
+ }
+
+ /**
+ * Creates a {@link MiddlemanType#ERROR_PROPAGATING_MIDDLEMAN error-propagating} middleman.
+ *
+ * @param owner the owner of the action that will be created. May not be null.
+ * @param middlemanName a unique file name for the middleman artifact in the {@code middlemanDir};
+ * in practice this is usually the owning rule's label (so it gets escaped as such)
+ * @param purpose the purpose for which this middleman is created. This should be a string which
+ * is suitable for use as a filename. A single rule may have many middlemen with distinct
+ * purposes.
+ * @param inputs the set of artifacts for which the created artifact is to be the middleman; must
+ * not be null or empty
+ * @param middlemanDir the directory in which to place the middleman.
+ * @return a middleman that enforces scheduling order (just like a scheduling middleman) and
+ * propagates errors, but is ignored by the dependency checker
+ * @throws IllegalArgumentException if {@code inputs} is null or empty
+ */
+ public Artifact createErrorPropagatingMiddleman(ActionOwner owner, String middlemanName,
+ String purpose, Iterable<Artifact> inputs, Root middlemanDir) {
+ Preconditions.checkArgument(inputs != null);
+ Preconditions.checkArgument(!Iterables.isEmpty(inputs));
+ // We must always create this middleman even if there is only one input.
+ return createMiddleman(owner, middlemanName, purpose, inputs, middlemanDir,
+ MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN).getFirst();
+ }
+
+ /**
+ * Returns the same artifact as {@code createErrorPropagatingMiddleman} would return,
+ * but doesn't create any action.
+ */
+ public Artifact getErrorPropagatingMiddlemanArtifact(String middlemanName, String purpose,
+ Root middlemanDir) {
+ return getStampFileArtifact(middlemanName, purpose, middlemanDir);
+ }
+
+ /**
+ * Creates both normal and scheduling middlemen.
+ *
+ * <p>Note: there's no need to synchronize this method; the only use of a field is via a call to
+ * another synchronized method (getArtifact()).
+ *
+ * @return null iff {@code inputs} is null or empty; the middleman file and the middleman action
+ * otherwise
+ */
+ private Pair<Artifact, Action> createMiddleman(
+ ActionOwner owner, String middlemanName, String purpose, Iterable<Artifact> inputs,
+ Root middlemanDir, MiddlemanType middlemanType) {
+ if (inputs == null || Iterables.isEmpty(inputs)) {
+ return null;
+ }
+
+ Artifact stampFile = getStampFileArtifact(middlemanName, purpose, middlemanDir);
+ Action action = new MiddlemanAction(owner, inputs, stampFile, purpose, middlemanType);
+ actionRegistry.registerAction(action);
+ return Pair.of(stampFile, action);
+ }
+
+ /**
+ * Creates a normal middleman.
+ *
+ * <p>If called multiple times, it always returns the same object depending on the {@code
+ * purpose}. It does not check that the list of inputs is identical. In contrast to other
+ * middleman methods, this one also returns an object if the list of inputs is empty.
+ *
+ * <p>Note: there's no need to synchronize this method; the only use of a field is via a call to
+ * another synchronized method (getArtifact()).
+ */
+ public Artifact createMiddlemanAllowMultiple(ActionRegistry registry,
+ ActionOwner owner, String purpose, Iterable<Artifact> inputs, Root middlemanDir) {
+ PathFragment stampName = new PathFragment("_middlemen/" + purpose);
+ Artifact stampFile = artifactFactory.getDerivedArtifact(stampName, middlemanDir,
+ actionRegistry.getOwner());
+ MiddlemanAction.create(
+ registry, owner, inputs, stampFile, purpose, MiddlemanType.AGGREGATING_MIDDLEMAN);
+ return stampFile;
+ }
+
+ private Artifact getStampFileArtifact(String middlemanName, String purpose, Root middlemanDir) {
+ String escapedFilename = Actions.escapedPath(middlemanName);
+ PathFragment stampName = new PathFragment("_middlemen/" + escapedFilename + "-" + purpose);
+ Artifact stampFile = artifactFactory.getDerivedArtifact(stampName, middlemanDir,
+ actionRegistry.getOwner());
+ return stampFile;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java b/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java
new file mode 100644
index 0000000..52f9f27
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.events.Location;
+
+/**
+ * This exception is thrown during a build when an input file is missing, but the file
+ * is not the input to any action being executed.
+ *
+ * If a missing input file is an input
+ * to an action, an {@link ActionExecutionException} is thrown instead.
+ */
+public class MissingInputFileException extends BuildFailedException {
+ private final Location location;
+
+ public MissingInputFileException(String message, Location location) {
+ super(message);
+ this.location = location;
+ }
+
+ /**
+ * Return a location where this input file is referenced. If there
+ * are multiple such locations, one is chosen arbitrarily. If there
+ * are none, return null.
+ */
+ public Location getLocation() {
+ return location;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java
new file mode 100644
index 0000000..8b84c31
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java
@@ -0,0 +1,151 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Sets.SetView;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.StringUtil;
+
+import java.util.Set;
+
+/**
+ * A mutable action graph. Implementations of this interface must be thread-safe.
+ */
+public interface MutableActionGraph extends ActionGraph {
+
+ /**
+ * Attempts to register the action. If any of the action's outputs already has a generating
+ * action, and the two actions are not compatible, then an {@link ActionConflictException} is
+ * thrown. The internal data structure may be partially modified when that happens; it is not
+ * guaranteed that all potential conflicts are detected, but at least one of them is.
+ *
+ * <p>For example, take three actions A, B, and C, where A creates outputs a and b, B creates just
+ * b, and C creates c and b. There are two potential conflicts in this case, between A and B, and
+ * between B and C. Depending on the ordering of calls to this method and the ordering of outputs
+ * in the action output lists, either one or two conflicts are detected: if B is registered first,
+ * then both conflicts are detected; if either A or C is registered first, then only one conflict
+ * is detected.
+ */
+ void registerAction(Action action) throws ActionConflictException;
+
+ /**
+ * Removes an action from this action graph if it is present.
+ *
+ * <p>Throws {@link IllegalStateException} if one of the outputs of the action is in fact
+ * generated by a different {@link Action} instance (even if they are sharable).
+ */
+ void unregisterAction(Action action);
+
+ /**
+ * Clear the action graph.
+ */
+ void clear();
+
+ /**
+ * This exception is thrown when a conflict between actions is detected. It contains information
+ * about the artifact for which the conflict is found, and data about the two conflicting actions
+ * and their owners.
+ */
+ public static final class ActionConflictException extends Exception {
+
+ private final Artifact artifact;
+ private final Action previousAction;
+ private final Action attemptedAction;
+
+ public ActionConflictException(Artifact artifact, Action previousAction,
+ Action attemptedAction) {
+ super("for " + artifact);
+ this.artifact = artifact;
+ this.previousAction = previousAction;
+ this.attemptedAction = attemptedAction;
+ }
+
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void reportTo(EventHandler eventListener) {
+ String msg = "file '" + artifact.prettyPrint()
+ + "' is generated by these conflicting actions:\n" +
+ suffix(attemptedAction, previousAction);
+ eventListener.handle(Event.error(msg));
+ }
+
+ private void addStringDetail(StringBuilder sb, String key, String valueA, String valueB) {
+ valueA = valueA != null ? valueA : "(null)";
+ valueB = valueB != null ? valueB : "(null)";
+
+ sb.append(key).append(": ").append(valueA);
+ if (!valueA.equals(valueB)) {
+ sb.append(", ").append(valueB);
+ }
+ sb.append("\n");
+ }
+
+ private void addListDetail(StringBuilder sb, String key,
+ Iterable<Artifact> valueA, Iterable<Artifact> valueB) {
+ Set<Artifact> setA = ImmutableSet.copyOf(valueA);
+ Set<Artifact> setB = ImmutableSet.copyOf(valueB);
+ SetView<Artifact> diffA = Sets.difference(setA, setB);
+ SetView<Artifact> diffB = Sets.difference(setB, setA);
+
+ sb.append(key).append(": ");
+ if (diffA.isEmpty() && diffB.isEmpty()) {
+ sb.append("are equal");
+ } else {
+ if (!diffA.isEmpty() && !diffB.isEmpty()) {
+ sb.append("attempted action contains artifacts not in previous action and "
+ + "previous action contains artifacts not in attempted action.");
+ } else if (!diffA.isEmpty()) {
+ sb.append("attempted action contains artifacts not in previous action: ");
+ sb.append(StringUtil.joinEnglishList(diffA, "and"));
+ } else if (!diffB.isEmpty()) {
+ sb.append("previous action contains artifacts not in attempted action: ");
+ sb.append(StringUtil.joinEnglishList(diffB, "and"));
+ }
+ }
+ sb.append("\n");
+ }
+
+ // See also Actions.canBeShared()
+ private String suffix(Action a, Action b) {
+ // Note: the error message reveals to users the names of intermediate files that are not
+ // documented in the BUILD language. This error-reporting logic is rather elaborate but it
+ // does help to diagnose some tricky situations.
+ StringBuilder sb = new StringBuilder();
+ ActionOwner aOwner = a.getOwner();
+ ActionOwner bOwner = b.getOwner();
+ boolean aNull = aOwner == null;
+ boolean bNull = bOwner == null;
+
+ addStringDetail(sb, "Label", aNull ? null : Label.print(aOwner.getLabel()),
+ bNull ? null : Label.print(bOwner.getLabel()));
+ addStringDetail(sb, "RuleClass", aNull ? null : aOwner.getTargetKind(),
+ bNull ? null : bOwner.getTargetKind());
+ addStringDetail(sb, "Configuration", aNull ? null : aOwner.getConfigurationName(),
+ bNull ? null : bOwner.getConfigurationName());
+ addStringDetail(sb, "Mnemonic", a.getMnemonic(), b.getMnemonic());
+
+ addListDetail(sb, "MandatoryInputs", a.getMandatoryInputs(), b.getMandatoryInputs());
+ addListDetail(sb, "Outputs", a.getOutputs(), b.getOutputs());
+
+ return sb.toString();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java b/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java
new file mode 100644
index 0000000..fa9b54e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An action which must know when it is skipped due to an action cache hit.
+ *
+ * Use should be rare, as the action graph is a functional model.
+ */
+public interface NotifyOnActionCacheHit extends Action {
+
+ /**
+ * Called when action has "cache hit", and therefore need not be executed.
+ *
+ * @param executor the executor
+ */
+ void actionCacheHit(Executor executor);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java b/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java
new file mode 100644
index 0000000..90af136
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Represents logic that evaluates the root of the package containing path.
+ */
+public interface PackageRootResolver {
+
+ /**
+ * Returns mapping from execPath to Root. Some roots can equal null if the corresponding
+ * package can't be found. Returns null if for some reason we can't evaluate it.
+ */
+ @Nullable
+ Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java b/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java
new file mode 100644
index 0000000..80df9e2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java
@@ -0,0 +1,114 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+
+/**
+ * Support for parameter file generation (as used by gcc and other tools, e.g.
+ * {@code gcc @param_file}. Note that the parameter file needs to be explicitly
+ * deleted after use. Different tools require different parameter file formats,
+ * which can be selected via the {@link ParameterFileType} enum.
+ *
+ * <p>The default charset is ISO-8859-1 (latin1). This also has to match the
+ * expectation of the tool.
+ *
+ * <p>Don't use this class for new code. Use the ParameterFileWriteAction
+ * instead!
+ */
+public class ParameterFile {
+
+ /**
+ * Different styles of parameter files.
+ */
+ public static enum ParameterFileType {
+ /**
+ * A parameter file with every parameter on a separate line. This format
+ * cannot handle newlines in parameters. It is currently used for most
+ * tools, but may not be interpreted correctly if parameters contain
+ * white space or other special characters. It should be avoided for new
+ * development.
+ */
+ UNQUOTED,
+
+ /**
+ * A parameter file where each parameter is correctly quoted for shell
+ * use, and separated by white space (space, tab, newline). This format is
+ * safe for all characters, but must be specially supported by the tool. In
+ * particular, it must not be used with gcc and related tools, which do not
+ * support this format as it is.
+ */
+ SHELL_QUOTED;
+ }
+
+ // Parameter file location.
+ private final Path execRoot;
+ private final PathFragment execPath;
+ private final Charset charset;
+ private final ParameterFileType type;
+
+ @VisibleForTesting
+ public static final FileType PARAMETER_FILE = FileType.of(".params");
+
+ /**
+ * Creates a parameter file with the given parameters.
+ */
+ public ParameterFile(Path execRoot, PathFragment execPath, Charset charset,
+ ParameterFileType type) {
+ Preconditions.checkNotNull(type);
+ this.execRoot = execRoot;
+ this.execPath = execPath;
+ this.charset = Preconditions.checkNotNull(charset);
+ this.type = Preconditions.checkNotNull(type);
+ }
+
+ /**
+ * Derives an exec path from a given exec path by appending <code>".params"</code>.
+ */
+ public static PathFragment derivePath(PathFragment original) {
+ return original.replaceName(original.getBaseName() + "-2.params");
+ }
+
+ /**
+ * Returns the path for the parameter file.
+ */
+ public Path getPath() {
+ return execRoot.getRelative(execPath);
+ }
+
+ /**
+ * Writes the arguments from the list into the parameter file according to
+ * the style selected in the constructor.
+ */
+ public void writeContent(List<String> arguments) throws ExecException {
+ Iterable<String> actualArgs = (type == ParameterFileType.SHELL_QUOTED) ?
+ ShellEscaper.escapeAll(arguments) : arguments;
+ Path file = getPath();
+ try {
+ FileSystemUtils.writeLinesAs(file, charset, actualArgs);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("could not write param file '" + file + "'", e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
new file mode 100644
index 0000000..929a106
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
@@ -0,0 +1,472 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.CountDownLatch;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * CPU/RAM resource manager. Used to keep track of resources consumed by the Blaze action execution
+ * threads and throttle them when necessary.
+ *
+ * <p>Threads which are known to consume a significant amount of the local CPU or RAM resources
+ * should call {@link #acquireResources} method. This method will check whether requested resources
+ * are available and will either mark them as used and allow thread to proceed or will block the
+ * thread until requested resources will become available. When thread completes it task, it must
+ * release allocated resources by calling {@link #releaseResources} method.
+ *
+ * <p>Available resources can be calculated using one of three ways:
+ * <ol>
+ * <li>They can be preset using {@link #setAvailableResources(ResourceSet)} method. This is used
+ * mainly by the unit tests (however it is possible to provide a future option that would
+ * artificially limit amount of CPU/RAM consumed by the Blaze).
+ * <li>They can be preset based on the /proc/cpuinfo and /proc/meminfo information. Blaze will
+ * calculate amount of available CPU cores (adjusting for hyperthreading logical cores) and
+ * amount of the total available memory and will limit itself to the number of effective cores
+ * and 2/3 of the available memory. For details, please look at the {@link
+ * LocalHostCapacity#getLocalHostCapacity} method.
+ * <li>Blaze will periodically (every 3 seconds) poll {@code /proc/meminfo} and {@code /proc/stat}
+ * information to obtain how much RAM and CPU resources are currently idle at that moment. For
+ * calculation details, please look at the {@link LocalHostCapacity#getFreeResources}
+ * implementation.
+ * </ol>
+ *
+ * <p>The resource manager also allows a slight overallocation of the resources to account for the
+ * fact that requested resources are usually estimated using a pessimistic approximation. It also
+ * guarantees that at least one thread will always be able to acquire any amount of requested
+ * resources (even if it is greater than amount of available resources). Therefore, assuming that
+ * threads correctly release acquired resources, Blaze will never be fully blocked.
+ */
+@ThreadSafe
+public class ResourceManager {
+
+ private static final Logger LOG = Logger.getLogger(ResourceManager.class.getName());
+ private final boolean FINE;
+
+ private EventBus eventBus;
+
+ private final ThreadLocal<Boolean> threadLocked = new ThreadLocal<Boolean>() {
+ @Override
+ protected Boolean initialValue() {
+ return false;
+ }
+ };
+
+ /**
+ * Singleton reference defined in a separate class to ensure thread-safe lazy
+ * initialization.
+ */
+ private static class Singleton {
+ static ResourceManager instance = new ResourceManager();
+ }
+
+ /**
+ * Returns singleton instance of the resource manager.
+ */
+ public static ResourceManager instance() {
+ return Singleton.instance;
+ }
+
+ // Allocated resources are allowed to go "negative", but at least
+ // MIN_AVAILABLE_CPU_RATIO portion of CPU and MIN_AVAILABLE_RAM_RATIO portion
+ // of RAM should be available.
+ // Please note that this value is purely empirical - we assume that generally
+ // requested resources are somewhat pessimistic and thread would end up
+ // using less than requested amount.
+ private final static double MIN_NECESSARY_CPU_RATIO = 0.6;
+ private final static double MIN_NECESSARY_RAM_RATIO = 1.0;
+ private final static double MIN_NECESSARY_IO_RATIO = 1.0;
+
+ // List of blocked threads. Associated CountDownLatch object will always
+ // be initialized to 1 during creation in the acquire() method.
+ private final List<Pair<ResourceSet, CountDownLatch>> requestList;
+
+ // The total amount of resources on the local host. Must be set by
+ // an explicit call to setAvailableResources(), often using
+ // LocalHostCapacity.getLocalHostCapacity() as an argument.
+ private ResourceSet staticResources = null;
+
+ private ResourceSet availableResources = null;
+ private LocalHostCapacity.FreeResources freeReading = null;
+
+ // Used amount of CPU capacity (where 1.0 corresponds to the one fully
+ // occupied CPU core. Corresponds to the CPU resource definition in the
+ // ResourceSet class.
+ private double usedCpu;
+
+ // Used amount of RAM capacity in MB. Corresponds to the RAM resource
+ // definition in the ResourceSet class.
+ private double usedRam;
+
+ // Used amount of I/O resources. Corresponds to the I/O resource
+ // definition in the ResourceSet class.
+ private double usedIo;
+
+ // Specifies how much of the RAM in staticResources we should allow to be used.
+ public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67;
+ private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE;
+
+ // Timer responsible for the periodic polling of the current system load.
+ private Timer timer = null;
+
+ private ResourceManager() {
+ FINE = LOG.isLoggable(Level.FINE);
+ requestList = new LinkedList<Pair<ResourceSet, CountDownLatch>>();
+ }
+
+ @VisibleForTesting public static ResourceManager instanceForTestingOnly() {
+ return new ResourceManager();
+ }
+
+ /**
+ * Resets resource manager state and releases all thread locks.
+ * Note - it does not reset auto-sensing or available resources. Use
+ * separate call to setAvailableResoures() or to setAutoSensing().
+ */
+ public synchronized void resetResourceUsage() {
+ usedCpu = 0;
+ usedRam = 0;
+ usedIo = 0;
+ for (Pair<ResourceSet, CountDownLatch> request : requestList) {
+ // CountDownLatch can be set only to 0 or 1.
+ request.second.countDown();
+ }
+ requestList.clear();
+ }
+
+ /**
+ * Sets available resources using given resource set. Must be called
+ * at least once before using resource manager.
+ * <p>
+ * Method will also disable auto-sensing if it was enabled.
+ */
+ public synchronized void setAvailableResources(ResourceSet resources) {
+ Preconditions.checkNotNull(resources);
+ staticResources = resources;
+ setAutoSensing(false);
+ }
+
+ public synchronized boolean isAutoSensingEnabled() {
+ return timer != null;
+ }
+
+ /**
+ * Specify how much of the available RAM we should allow to be used.
+ * This has no effect if autosensing is enabled.
+ */
+ public synchronized void setRamUtilizationPercentage(int percentage) {
+ ramUtilizationPercentage = percentage;
+ }
+
+ /**
+ * Enables or disables secondary resource allocation algorithm that will
+ * periodically (when needed but at most once per 3 seconds) checks real
+ * amount of available memory (based on /proc/meminfo) and current CPU load
+ * (based on 1 second difference of /proc/stat) and allows additional resource
+ * acquisition if previous requests were overly pessimistic.
+ */
+ public synchronized void setAutoSensing(boolean enable) {
+ // Create new Timer instance only if it does not exist already.
+ if (enable && !isAutoSensingEnabled()) {
+ Profiler.instance().logEvent(ProfilerTask.INFO, "Enable auto sensing");
+ if(refreshFreeResources()) {
+ timer = new Timer("AutoSenseTimer", true);
+ timer.schedule(new TimerTask() {
+ @Override public void run() { refreshFreeResources(); }
+ }, 3000, 3000);
+ }
+ } else if (!enable) {
+ if (isAutoSensingEnabled()) {
+ Profiler.instance().logEvent(ProfilerTask.INFO, "Disable auto sensing");
+ timer.cancel();
+ timer = null;
+ }
+ if (staticResources != null) {
+ updateAvailableResources(false);
+ }
+ }
+ }
+
+ /**
+ * Acquires requested resource set. Will block if resource is not available.
+ * NB! This method must be thread-safe!
+ */
+ public void acquireResources(ActionMetadata owner, ResourceSet resources)
+ throws InterruptedException {
+ Preconditions.checkArgument(resources != null);
+ long startTime = Profiler.nanoTimeMaybe();
+ CountDownLatch latch = null;
+ try {
+ waiting(owner);
+ latch = acquire(resources);
+ if (latch != null) {
+ latch.await();
+ }
+ } finally {
+ threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0
+ || resources.getIoUsage() != 0);
+ acquired(owner);
+
+ // Profile acquisition only if it waited for resource to become available.
+ if (latch != null) {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.ACTION_LOCK, owner);
+ }
+ }
+ }
+
+ /**
+ * Acquires the given resources if available immediately. Does not block.
+ * @return true iff the given resources were locked (all or nothing).
+ */
+ public boolean tryAcquire(ActionMetadata owner, ResourceSet resources) {
+ boolean acquired = false;
+ synchronized (this) {
+ if (areResourcesAvailable(resources)) {
+ incrementResources(resources);
+ acquired = true;
+ }
+ }
+
+ if (acquired) {
+ threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0);
+ acquired(owner);
+ }
+
+ return acquired;
+ }
+
+ private void incrementResources(ResourceSet resources) {
+ usedCpu += resources.getCpuUsage();
+ usedRam += resources.getMemoryMb();
+ usedIo += resources.getIoUsage();
+ }
+
+ /**
+ * Return true if any resources have been claimed through this manager.
+ */
+ public synchronized boolean inUse() {
+ return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || requestList.size() > 0;
+ }
+
+
+ /**
+ * Return true iff this thread has a lock on non-zero resources.
+ */
+ public boolean threadHasResources() {
+ return threadLocked.get();
+ }
+
+ public void setEventBus(EventBus eventBus) {
+ Preconditions.checkState(this.eventBus == null);
+ this.eventBus = Preconditions.checkNotNull(eventBus);
+ }
+
+ public void unsetEventBus() {
+ Preconditions.checkState(this.eventBus != null);
+ this.eventBus = null;
+ }
+
+ private void waiting(ActionMetadata owner) {
+ if (eventBus != null) {
+ // Null only in tests.
+ eventBus.post(ActionStatusMessage.schedulingStrategy(owner));
+ }
+ }
+
+ private void acquired(ActionMetadata owner) {
+ if (eventBus != null) {
+ // Null only in tests.
+ eventBus.post(ActionStatusMessage.runningStrategy(owner));
+ }
+ }
+
+ /**
+ * Releases previously requested resource set.
+ *
+ * <p>NB! This method must be thread-safe!
+ */
+ public void releaseResources(ActionMetadata owner, ResourceSet resources) {
+ boolean isConflict = false;
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ isConflict = release(resources);
+ } finally {
+ threadLocked.set(false);
+
+ // Profile resource release only if it resolved at least one allocation request.
+ if (isConflict) {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.ACTION_RELEASE, owner);
+ }
+ }
+ }
+
+ private synchronized CountDownLatch acquire(ResourceSet resources) {
+ if (areResourcesAvailable(resources)) {
+ incrementResources(resources);
+ return null;
+ }
+ Pair<ResourceSet, CountDownLatch> request =
+ new Pair<>(resources, new CountDownLatch(1));
+ requestList.add(request);
+
+ // If we use auto sensing and there has not been an update within last
+ // 30 seconds, something has gone really wrong - disable it.
+ if (isAutoSensingEnabled() && freeReading.getReadingAge() > 30000) {
+ LoggingUtil.logToRemote(Level.WARNING, "Free resource readings were " +
+ "not updated for 30 seconds - auto-sensing is disabled",
+ new IllegalStateException());
+ LOG.warning("Free resource readings were not updated for 30 seconds - "
+ + "auto-sensing is disabled");
+ setAutoSensing(false);
+ }
+ return request.second;
+ }
+
+ private synchronized boolean release(ResourceSet resources) {
+ usedCpu -= resources.getCpuUsage();
+ usedRam -= resources.getMemoryMb();
+ usedIo -= resources.getIoUsage();
+
+ // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being
+ // e.g. 1E-15. So if it is small enough, we set it to 0. But maybe there is a better solution.
+ if (usedCpu < 0.0001) {
+ usedCpu = 0;
+ }
+ if (usedRam < 0.0001) {
+ usedRam = 0;
+ }
+ if (usedIo < 0.0001) {
+ usedIo = 0;
+ }
+ if (requestList.size() > 0) {
+ processWaitingThreads();
+ return true;
+ }
+ return false;
+ }
+
+
+ /**
+ * Tries to unblock one or more waiting threads if there are sufficient resources available.
+ */
+ private synchronized void processWaitingThreads() {
+ Iterator<Pair<ResourceSet, CountDownLatch>> iterator = requestList.iterator();
+ while (iterator.hasNext()) {
+ Pair<ResourceSet, CountDownLatch> request = iterator.next();
+ if (areResourcesAvailable(request.first)) {
+ incrementResources(request.first);
+ request.second.countDown();
+ iterator.remove();
+ }
+ }
+ }
+
+ // Method will return true if all requested resources are considered to be available.
+ private boolean areResourcesAvailable(ResourceSet resources) {
+ Preconditions.checkNotNull(availableResources);
+ // Comparison below is robust, since any calculation errors will be fixed
+ // by the release() method.
+ if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0) {
+ return true;
+ }
+ // Use only MIN_NECESSARY_???_RATIO of the resource value to check for
+ // allocation. This is necessary to account for the fact that most of the
+ // requested resource sets use pessimistic estimations. Note that this
+ // ratio is used only during comparison - for tracking we will actually
+ // mark whole requested amount as used.
+ double cpu = resources.getCpuUsage() * MIN_NECESSARY_CPU_RATIO;
+ double ram = resources.getMemoryMb() * MIN_NECESSARY_RAM_RATIO;
+ double io = resources.getIoUsage() * MIN_NECESSARY_IO_RATIO;
+
+ double availableCpu = availableResources.getCpuUsage();
+ double availableRam = availableResources.getMemoryMb();
+ double availableIo = availableResources.getIoUsage();
+
+ // Resources are considered available if any one of the conditions below is true:
+ // 1) If resource is not requested at all, it is available.
+ // 2) If resource is not used at the moment, it is considered to be
+ // available regardless of how much is requested. This is necessary to
+ // ensure that at any given time, at least one thread is able to acquire
+ // resources even if it requests more than available.
+ // 3) If used resource amount is less than total available resource amount.
+ return (cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu) &&
+ (ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam) &&
+ (io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo);
+ }
+
+ private synchronized void updateAvailableResources(boolean useFreeReading) {
+ Preconditions.checkNotNull(staticResources);
+ if (useFreeReading && isAutoSensingEnabled()) {
+ availableResources = new ResourceSet(
+ usedRam + freeReading.getFreeMb(),
+ usedCpu + freeReading.getAvgFreeCpu(),
+ staticResources.getIoUsage());
+ if(FINE) {
+ LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB,"
+ + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU");
+ }
+ processWaitingThreads();
+ } else {
+ availableResources = new ResourceSet(
+ staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0,
+ staticResources.getCpuUsage(),
+ staticResources.getIoUsage());
+ processWaitingThreads();
+ }
+ }
+
+ /**
+ * Called by the timer thread to update system load information.
+ *
+ * @return true if update was successful and false if error was detected and
+ * autosensing was disabled.
+ */
+ private boolean refreshFreeResources() {
+ freeReading = LocalHostCapacity.getFreeResources(freeReading);
+ if (freeReading == null) { // Unable to read or parse /proc/* information.
+ LOG.warning("Unable to obtain system load - autosensing is disabled");
+ setAutoSensing(false);
+ return false;
+ }
+ updateAvailableResources(
+ freeReading.getInterval() >= 1000 && freeReading.getInterval() <= 10000);
+ return true;
+ }
+
+ @VisibleForTesting
+ synchronized int getWaitCount() {
+ return requestList.size();
+ }
+
+ @VisibleForTesting
+ synchronized boolean isAvailable(double ram, double cpu, double io) {
+ return areResourcesAvailable(new ResourceSet(ram, cpu, io));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java
new file mode 100644
index 0000000..e7ab98f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Splitter;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Instances of this class represent an estimate of the resource consumption
+ * for a particular Action, or the total available resources. We plan to
+ * use this to do smarter scheduling of actions, for example making sure
+ * that we don't schedule jobs concurrently if they would use so much
+ * memory as to cause the machine to thrash.
+ */
+@Immutable
+public class ResourceSet {
+
+ /** For actions that consume negligible resources. */
+ public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0);
+
+ /** The amount of real memory (resident set size). */
+ private final double memoryMb;
+
+ /** The number of CPUs, or fractions thereof. */
+ private final double cpuUsage;
+
+ /**
+ * Relative amount of used I/O resources (with 1.0 being total available amount on an "average"
+ * workstation.
+ */
+ private final double ioUsage;
+
+ public ResourceSet(double memoryMb, double cpuUsage, double ioUsage) {
+ this.memoryMb = memoryMb;
+ this.cpuUsage = cpuUsage;
+ this.ioUsage = ioUsage;
+ }
+
+ /** Returns the amount of real memory (resident set size) used in MB. */
+ public double getMemoryMb() {
+ return memoryMb;
+ }
+
+ /**
+ * Returns the number of CPUs (or fractions thereof) used.
+ * For a CPU-bound single-threaded process, this will be 1.0.
+ * For a single-threaded process which spends part of its
+ * time waiting for I/O, this will be somewhere between 0.0 and 1.0.
+ * For a multi-threaded or multi-process application,
+ * this may be more than 1.0.
+ */
+ public double getCpuUsage() {
+ return cpuUsage;
+ }
+
+ /**
+ * Returns the amount of I/O used.
+ * Full amount of available I/O resources on the "average" workstation is
+ * considered to be 1.0.
+ */
+ public double getIoUsage() {
+ return ioUsage;
+ }
+
+ public static class ResourceSetConverter implements Converter<ResourceSet> {
+ private static final Splitter SPLITTER = Splitter.on(',');
+
+ @Override
+ public ResourceSet convert(String input) throws OptionsParsingException {
+ Iterator<String> values = SPLITTER.split(input).iterator();
+ try {
+ double memoryMb = Double.parseDouble(values.next());
+ double cpuUsage = Double.parseDouble(values.next());
+ double ioUsage = Double.parseDouble(values.next());
+ if (values.hasNext()) {
+ throw new OptionsParsingException("Expected exactly 3 comma-separated float values");
+ }
+ if (memoryMb <= 0.0 || cpuUsage <= 0.0 || ioUsage <= 0.0) {
+ throw new OptionsParsingException("All resource values must be positive");
+ }
+ return new ResourceSet(memoryMb, cpuUsage, ioUsage);
+ } catch (NumberFormatException nfe) {
+ throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nfe);
+ } catch (NoSuchElementException nsee) {
+ throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nsee);
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "comma-separated available amount of RAM (in MB), CPU (in cores) and "
+ + "available I/O (1.0 being average workstation)";
+ }
+
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Root.java b/src/main/java/com/google/devtools/build/lib/actions/Root.java
new file mode 100644
index 0000000..284b85f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/Root.java
@@ -0,0 +1,163 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A root for an artifact. The roots are the directories containing artifacts, and they are mapped
+ * together into a single directory tree to form the execution environment. There are two kinds of
+ * roots, source roots and derived roots. Source roots correspond to entries of the package path,
+ * and they can be anywhere on disk. Derived roots correspond to output directories; there are
+ * generally different output directories for different configurations, and different types of
+ * output (bin, genfiles, includes, etc.).
+ *
+ * <p>When mapping the roots into a single directory tree, the source roots are merged, such that
+ * each package is accessed in its entirety from a single source root. The package cache is
+ * responsible for determining that mapping. The derived roots, on the other hand, have to be
+ * distinct. (It is currently allowed to have a derived root that is the prefix of another one.)
+ *
+ * <p>The derived roots must have paths that point inside the exec root, i.e. below the directory
+ * that is the root of the merged directory tree.
+ */
+@SkylarkModule(name = "root",
+ doc = "A root for files. The roots are the directories containing files, and they are mapped "
+ + "together into a single directory tree to form the execution environment.")
+public final class Root implements Comparable<Root>, Serializable {
+
+ /**
+ * Returns the given path as a source root. The path may not be {@code null}.
+ */
+ public static Root asSourceRoot(Path path) {
+ return new Root(null, path);
+ }
+
+ /**
+ * DO NOT USE IN PRODUCTION CODE!
+ *
+ * <p>Returns the given path as a derived root. This method only exists as a convenience for
+ * tests, which don't need a proper Root object.
+ */
+ @VisibleForTesting
+ public static Root asDerivedRoot(Path path) {
+ return new Root(path, path);
+ }
+
+ /**
+ * Returns the given path as a derived root, relative to the given exec root. The root must be a
+ * proper sub-directory of the exec root (i.e. not equal). Neither may be {@code null}.
+ *
+ * <p>Be careful with this method - all derived roots must be registered with the artifact factory
+ * before the analysis phase.
+ */
+ public static Root asDerivedRoot(Path execRoot, Path root) {
+ Preconditions.checkArgument(root.startsWith(execRoot));
+ Preconditions.checkArgument(!root.equals(execRoot));
+ return new Root(execRoot, root);
+ }
+
+ public static Root middlemanRoot(Path execRoot, Path outputDir) {
+ Path root = outputDir.getRelative("internal");
+ Preconditions.checkArgument(root.startsWith(execRoot));
+ Preconditions.checkArgument(!root.equals(execRoot));
+ return new Root(execRoot, root, true);
+ }
+
+ /**
+ * Returns the exec root as a derived root. The exec root should never be treated as a derived
+ * root, but this is currently allowed. Do not add any further uses besides the ones that already
+ * exist!
+ */
+ static Root execRootAsDerivedRoot(Path execRoot) {
+ return new Root(execRoot, execRoot);
+ }
+
+ @Nullable private final Path execRoot;
+ private final Path path;
+ private final boolean isMiddlemanRoot;
+
+ private Root(@Nullable Path execRoot, Path path, boolean isMiddlemanRoot) {
+ this.execRoot = execRoot;
+ this.path = Preconditions.checkNotNull(path);
+ this.isMiddlemanRoot = isMiddlemanRoot;
+ }
+
+ private Root(@Nullable Path execRoot, Path path) {
+ this(execRoot, path, false);
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ /**
+ * Returns the path fragment from the exec root to the actual root. For source roots, this returns
+ * the empty fragment.
+ */
+ public PathFragment getExecPath() {
+ return isSourceRoot() ? PathFragment.EMPTY_FRAGMENT : path.relativeTo(execRoot);
+ }
+
+ @SkylarkCallable(name = "path", structField = true,
+ doc = "Returns the relative path from the exec root to the actual root.")
+ public String getExecPathString() {
+ return getExecPath().getPathString();
+ }
+
+ public boolean isSourceRoot() {
+ return execRoot == null;
+ }
+
+ public boolean isMiddlemanRoot() {
+ return isMiddlemanRoot;
+ }
+
+ @Override
+ public int compareTo(Root o) {
+ return path.compareTo(o.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(execRoot, path.hashCode());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof Root)) {
+ return false;
+ }
+ Root r = (Root) o;
+ return path.equals(r.path) && Objects.equals(execRoot, r.execRoot);
+ }
+
+ @Override
+ public String toString() {
+ return path.toString() + (isSourceRoot() ? "[source]" : "[derived]");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Spawn.java b/src/main/java/com/google/devtools/build/lib/actions/Spawn.java
new file mode 100644
index 0000000..2f905d0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/Spawn.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.extra.SpawnInfo;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+
+/**
+ * An object representing a subprocess to be invoked, including its command and
+ * arguments, its working directory, its environment, a boolean indicating
+ * whether remote execution is appropriate for this command, and if so, the set
+ * of files it is expected to read and write.
+ */
+public interface Spawn {
+
+ /**
+ * Returns true iff this command may be executed remotely.
+ */
+ boolean isRemotable();
+
+ /**
+ * Out-of-band data for this spawn. This can be used to signal hints (hardware requirements,
+ * local vs. remote) to the execution subsystem.
+ *
+ * <p>String tags from {@link
+ * com.google.devtools.build.lib.rules.test.TestTargetProperties#getExecutionInfo()} can be added
+ * as keys with arbitrary values to this map too.
+ */
+ ImmutableMap<String, String> getExecutionInfo();
+
+ /**
+ * Returns this Spawn as a Bourne shell command.
+ *
+ * @param workingDir the initial working directory of the command
+ */
+ String asShellCommand(Path workingDir);
+
+ /**
+ * Returns the runfiles data for remote execution. Format is (directory, manifest file).
+ */
+ ImmutableMap<PathFragment, Artifact> getRunfilesManifests();
+
+ /**
+ * Returns artifacts for filesets, so they can be scheduled on remote execution.
+ */
+ ImmutableList<Artifact> getFilesetManifests();
+
+ /**
+ * Returns a protocol buffer describing this spawn for use by the extra_action functionality.
+ */
+ SpawnInfo getExtraActionInfo();
+
+ /**
+ * Returns the command (the first element) and its arguments.
+ */
+ ImmutableList<String> getArguments();
+
+ /**
+ * Returns the initial environment of the process.
+ * If null, the environment is inherited from the parent process.
+ */
+ ImmutableMap<String, String> getEnvironment();
+
+ /**
+ * Returns the list of files that this command may read.
+ *
+ * <p>This method explicitly does not expand middleman artifacts. Pass the result
+ * to an appropriate utility method on {@link com.google.devtools.build.lib.actions.Artifact} to
+ * expand the middlemen.
+ *
+ * <p>This is for use with remote execution, so we can ship inputs before starting the
+ * command. Order stability across multiple calls should be upheld for performance reasons.
+ */
+ Iterable<? extends ActionInput> getInputFiles();
+
+ /**
+ * Returns the collection of files that this command must write. Callers should not mutate
+ * the result.
+ *
+ * <p>This is for use with remote execution, so remote execution does not have to guess what
+ * outputs the process writes. While the order does not affect the semantics, it should be
+ * stable so it can be cached.
+ */
+ Collection<? extends ActionInput> getOutputFiles();
+
+ /**
+ * Returns the resource owner for local fallback.
+ */
+ ActionMetadata getResourceOwner();
+
+ /**
+ * Returns the amount of resources needed for local fallback.
+ */
+ ResourceSet getLocalResources();
+
+ /**
+ * Returns the owner for this action. Production code should supply a non-null owner.
+ */
+ ActionOwner getOwner();
+
+ /**
+ * Returns a mnemonic (string constant) for this kind of spawn.
+ */
+ String getMnemonic();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java
new file mode 100644
index 0000000..c2ea1b0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+/**
+ * A context that allows execution of {@link Spawn} instances.
+ */
+@ActionContextMarker(name = "spawn")
+public interface SpawnActionContext extends Executor.ActionContext {
+
+ /**
+ * Executes the given spawn.
+ */
+ void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException;
+
+ /** Returns the locality of running the spawn, i.e., "local". */
+ String strategyLocality(String mnemonic, boolean remotable);
+
+ /**
+ * This implements a tri-state mode. There are three possible cases: (1) implementations of this
+ * class can unconditionally execute spawns locally, (2) they can follow whatever is set for the
+ * corresponding spawn (see {@link Spawn#isRemotable}), or (3) they can unconditionally execute
+ * spawns remotely, i.e., force remote execution.
+ *
+ * <p>Passing the spawns remotable flag to this method returns whether the spawn will actually be
+ * executed remotely.
+ */
+ boolean isRemotable(String mnemonic, boolean remotable);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java b/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java
new file mode 100644
index 0000000..9092ab2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An exception indicating that a target is out of date.
+ */
+public class TargetOutOfDateException extends ActionExecutionException {
+
+ public TargetOutOfDateException(Action action) {
+ super (action.prettyPrint() + " is not up-to-date", action, false);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java b/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java
new file mode 100644
index 0000000..c861338
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An TestExecException that is related to the failure of a TestAction.
+ */
+public final class TestExecException extends ExecException {
+
+ public TestExecException(String message) {
+ super(message);
+ }
+
+ @Override
+ public ActionExecutionException toActionExecutionException(String messagePrefix,
+ boolean verboseFailures, Action action) {
+ String message = messagePrefix + " failed" + getMessage();
+ if (verboseFailures) {
+ return new ActionExecutionException(message, this, action, isCatastrophic());
+ } else {
+ return new ActionExecutionException(message, action, isCatastrophic());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java b/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java
new file mode 100644
index 0000000..128ac20
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+/**
+ * Used as a notification mechanism for the scheduling middleman mutations
+ * related to scheduling exclusive tests.
+ */
+public interface TestMiddlemanObserver {
+
+ /**
+ * Called when the test removes the stale middleman.
+ *
+ * @param action the test action.
+ * @param middleman the scheduling middleman.
+ * @param middlemanAction the action generating the scheduling middleman
+ */
+ void remove(Action action, Artifact middleman, Action middlemanAction);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java b/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java
new file mode 100644
index 0000000..86a6eb0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * An ExecException that is related to the failure of an Action and therefore
+ * very likely the user's fault.
+ */
+public class UserExecException extends ExecException {
+
+ public UserExecException(String message) {
+ super(message);
+ }
+
+ public UserExecException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ @Override
+ public ActionExecutionException toActionExecutionException(String messagePrefix,
+ boolean verboseFailures, Action action) {
+ String message = messagePrefix + " failed: " + getMessage();
+ if (verboseFailures) {
+ return new ActionExecutionException(message, this, action, isCatastrophic());
+ } else {
+ return new ActionExecutionException(message, action, isCatastrophic());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
new file mode 100644
index 0000000..2ac0ab8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
@@ -0,0 +1,173 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An interface defining a cache of already-executed Actions.
+ *
+ * <p>This class' naming is misleading; it doesn't cache the actual actions, but it stores a
+ * fingerprint of the action state (ie. a hash of the input and output files on disk), so
+ * we can tell if we need to rerun an action given the state of the file system.
+ *
+ * <p>Each action entry uses one of its output paths as a key (after conversion
+ * to the string).
+ */
+@ThreadCompatible
+public interface ActionCache {
+
+ /**
+ * Updates the cache entry for the specified key.
+ */
+ void put(String key, ActionCache.Entry entry);
+
+ /**
+ * Returns the corresponding cache entry for the specified key, if any, or
+ * null if not found.
+ */
+ ActionCache.Entry get(String key);
+
+ /**
+ * Removes entry from cache
+ */
+ void remove(String key);
+
+ /**
+ * Returns a new Entry instance. This method allows ActionCache subclasses to
+ * define their own Entry implementation.
+ */
+ ActionCache.Entry createEntry(String key);
+
+ /**
+ * An entry in the ActionCache that contains all action input and output
+ * artifact paths and their metadata plus action key itself.
+ *
+ * Cache entry operates under assumption that once it is fully initialized
+ * and getFileDigest() method is called, it becomes logically immutable (all methods
+ * will continue to return same result regardless of internal data transformations).
+ */
+ public final class Entry {
+ private final String actionKey;
+ private final List<String> files;
+ // If null, digest is non-null and the entry is immutable.
+ private Map<String, Metadata> mdMap;
+ private Digest digest;
+
+ public Entry(String key) {
+ actionKey = key;
+ files = new ArrayList<>();
+ mdMap = new HashMap<>();
+ }
+
+ public Entry(String key, List<String> files, Digest digest) {
+ actionKey = key;
+ this.files = files;
+ this.digest = digest;
+ mdMap = null;
+ }
+
+ /**
+ * Adds the artifact, specified by the executable relative path and its
+ * metadata into the cache entry.
+ */
+ public void addFile(PathFragment relativePath, Metadata md) {
+ Preconditions.checkState(mdMap != null);
+ Preconditions.checkState(!isCorrupted());
+ Preconditions.checkState(digest == null);
+
+ String execPath = relativePath.getPathString();
+ files.add(execPath);
+ mdMap.put(execPath, md);
+ }
+
+ /**
+ * @return action key string.
+ */
+ public String getActionKey() {
+ return actionKey;
+ }
+
+ /**
+ * Returns the combined digest of the action's inputs and outputs.
+ *
+ * This may compresses the data into a more compact representation, and
+ * makes the object immutable.
+ */
+ public Digest getFileDigest() {
+ if (digest == null) {
+ digest = Digest.fromMetadata(mdMap);
+ mdMap = null;
+ }
+ return digest;
+ }
+
+ /**
+ * Returns true if this cache entry is corrupted and should be ignored.
+ */
+ public boolean isCorrupted() {
+ return actionKey == null;
+ }
+
+ /**
+ * @return stored path strings.
+ */
+ public Collection<String> getPaths() {
+ return files;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(" actionKey = ").append(actionKey).append("\n");
+ builder.append(" digestKey = ");
+ if (digest == null) {
+ builder.append(Digest.fromMetadata(mdMap)).append(" (from mdMap)\n");
+ } else {
+ builder.append(digest).append("\n");
+ }
+ List<String> fileInfo = Lists.newArrayListWithCapacity(files.size());
+ fileInfo.addAll(files);
+ Collections.sort(fileInfo);
+ for (String info : fileInfo) {
+ builder.append(" ").append(info).append("\n");
+ }
+ return builder.toString();
+ }
+ }
+
+ /**
+ * Give persistent cache implementations a notification to write to disk.
+ * @return size in bytes of the serialized cache.
+ */
+ long save() throws IOException;
+
+ /**
+ * Dumps action cache content into the given PrintStream.
+ */
+ void dump(PrintStream out);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
new file mode 100644
index 0000000..24eb42e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
@@ -0,0 +1,389 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.CompactStringIndexer;
+import com.google.devtools.build.lib.util.PersistentMap;
+import com.google.devtools.build.lib.util.StringIndexer;
+import com.google.devtools.build.lib.util.VarInt;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An implementation of the ActionCache interface that uses
+ * {@link CompactStringIndexer} to reduce memory footprint and saves
+ * cached actions using the {@link PersistentMap}.
+ *
+ * <p>This cache is not fully correct: as hashes are xor'd together, a permutation of input
+ * file contents will erroneously be considered up to date.
+ */
+@ConditionallyThreadSafe // condition: each instance must instantiated with
+ // different cache root
+public class CompactPersistentActionCache implements ActionCache {
+ private static final int SAVE_INTERVAL_SECONDS = 3;
+ private static final long NANOS_PER_SECOND = 1000 * 1000 * 1000;
+
+ // Key of the action cache record that holds information used to verify referential integrity
+ // between action cache and string indexer. Must be < 0 to avoid conflict with real action
+ // cache records.
+ private static final int VALIDATION_KEY = -10;
+
+ private static final int VERSION = 10;
+
+ private final class ActionMap extends PersistentMap<Integer, byte[]> {
+ private final Clock clock;
+ private long nextUpdate;
+
+ public ActionMap(Map<Integer, byte[]> map, Clock clock, Path mapFile, Path journalFile)
+ throws IOException {
+ super(VERSION, map, mapFile, journalFile);
+ this.clock = clock;
+ // Using nanoTime. currentTimeMillis may not provide enough granularity.
+ nextUpdate = clock.nanoTime() / NANOS_PER_SECOND + SAVE_INTERVAL_SECONDS;
+ load();
+ }
+
+ @Override
+ protected boolean updateJournal() {
+ // Using nanoTime. currentTimeMillis may not provide enough granularity.
+ long time = clock.nanoTime() / NANOS_PER_SECOND;
+ if (SAVE_INTERVAL_SECONDS == 0 || time > nextUpdate) {
+ nextUpdate = time + SAVE_INTERVAL_SECONDS;
+ // Force flushing of the PersistentStringIndexer instance. This is needed to ensure
+ // that filename index data on disk is always up-to-date when we save action cache
+ // data.
+ indexer.flush();
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ protected boolean keepJournal() {
+ // We must first flush the journal to get an accurate measure of its size.
+ forceFlush();
+ try {
+ return journalSize() * 100 < cacheSize();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected Integer readKey(DataInputStream in) throws IOException {
+ return in.readInt();
+ }
+
+ @Override
+ protected byte[] readValue(DataInputStream in)
+ throws IOException {
+ int size = in.readInt();
+ if (size < 0) {
+ throw new IOException("found negative array size: " + size);
+ }
+ byte[] data = new byte[size];
+ in.readFully(data);
+ return data;
+ }
+
+ @Override
+ protected void writeKey(Integer key, DataOutputStream out)
+ throws IOException {
+ out.writeInt(key);
+ }
+
+ @Override
+ // TODO(bazel-team): (2010) This method, writeKey() and related Metadata methods
+ // should really use protocol messages. Doing so would allow easy inspection
+ // of the action cache content and, more importantly, would cut down on the
+ // need to change VERSION to different number every time we touch those
+ // methods. Especially when we'll start to add stuff like statistics for
+ // each action.
+ protected void writeValue(byte[] value, DataOutputStream out)
+ throws IOException {
+ out.writeInt(value.length);
+ out.write(value);
+ }
+ }
+
+ private final PersistentMap<Integer, byte[]> map;
+ private final PersistentStringIndexer indexer;
+ static final ActionCache.Entry CORRUPTED = new ActionCache.Entry(null);
+
+ public CompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException {
+ Path cacheFile = cacheFile(cacheRoot);
+ Path journalFile = journalFile(cacheRoot);
+ Path indexFile = cacheRoot.getChild("filename_index_v" + VERSION + ".blaze");
+ // we can now use normal hash map as backing map, since dependency checker
+ // will manually purge records from the action cache.
+ Map<Integer, byte[]> backingMap = new HashMap<>();
+
+ try {
+ indexer = PersistentStringIndexer.newPersistentStringIndexer(indexFile, clock);
+ } catch (IOException e) {
+ renameCorruptedFiles(cacheRoot);
+ throw new IOException("Failed to load filename index data", e);
+ }
+
+ try {
+ map = new ActionMap(backingMap, clock, cacheFile, journalFile);
+ } catch (IOException e) {
+ renameCorruptedFiles(cacheRoot);
+ throw new IOException("Failed to load action cache data", e);
+ }
+
+ // Validate referential integrity between two collections.
+ if (!map.isEmpty()) {
+ String integrityError = validateIntegrity(indexer.size(), map.get(VALIDATION_KEY));
+ if (integrityError != null) {
+ renameCorruptedFiles(cacheRoot);
+ throw new IOException("Failed action cache referential integrity check: " + integrityError);
+ }
+ }
+ }
+
+ /**
+ * Rename corrupted files so they could be analyzed later. This would also ensure
+ * that next initialization attempt will create empty cache.
+ */
+ private static void renameCorruptedFiles(Path cacheRoot) {
+ try {
+ for (Path path : UnixGlob.forPath(cacheRoot).addPattern("action_*_v" + VERSION + ".*")
+ .glob()) {
+ path.renameTo(path.getParentDirectory().getChild(path.getBaseName() + ".bad"));
+ }
+ for (Path path : UnixGlob.forPath(cacheRoot).addPattern("filename_*_v" + VERSION + ".*")
+ .glob()) {
+ path.renameTo(path.getParentDirectory().getChild(path.getBaseName() + ".bad"));
+ }
+ } catch (IOException e) {
+ // do nothing
+ }
+ }
+
+ /**
+ * @return false iff indexer contains no data or integrity check has failed.
+ */
+ private static String validateIntegrity(int indexerSize, byte[] validationRecord) {
+ if (indexerSize == 0) {
+ return "empty index";
+ }
+ if (validationRecord == null) {
+ return "no validation record";
+ }
+ try {
+ int validationSize = ByteBuffer.wrap(validationRecord).asIntBuffer().get();
+ if (validationSize <= indexerSize) {
+ return null;
+ } else {
+ return String.format("Validation mismatch: validation entry %d is too large " +
+ "compared to index size %d", validationSize, indexerSize);
+ }
+ } catch (BufferUnderflowException e) {
+ return e.getMessage();
+ }
+
+ }
+
+ public static Path cacheFile(Path cacheRoot) {
+ return cacheRoot.getChild("action_cache_v" + VERSION + ".blaze");
+ }
+
+ public static Path journalFile(Path cacheRoot) {
+ return cacheRoot.getChild("action_journal_v" + VERSION + ".blaze");
+ }
+
+ @Override
+ public ActionCache.Entry createEntry(String key) {
+ return new ActionCache.Entry(key);
+ }
+
+ @Override
+ public ActionCache.Entry get(String key) {
+ int index = indexer.getIndex(key);
+ if (index < 0) {
+ return null;
+ }
+ byte[] data;
+ synchronized (this) {
+ data = map.get(index);
+ }
+ try {
+ return data != null ? CompactPersistentActionCache.decode(indexer, data) : null;
+ } catch (IOException e) {
+ // return entry marked as corrupted.
+ return CORRUPTED;
+ }
+ }
+
+ @Override
+ public void put(String key, ActionCache.Entry entry) {
+ // Encode record. Note that both methods may create new mappings in the indexer.
+ int index = indexer.getOrCreateIndex(key);
+ byte[] content = encode(indexer, entry);
+
+ // Update validation record.
+ ByteBuffer buffer = ByteBuffer.allocate(4); // size of int in bytes
+ int indexSize = indexer.size();
+ buffer.asIntBuffer().put(indexSize);
+
+ // Note the benign race condition here in which two threads might race on
+ // updating the VALIDATION_KEY. If the most recent update loses the race,
+ // a value lower than the indexer size will remain in the validation record.
+ // This will still pass the integrity check.
+ synchronized (this) {
+ map.put(VALIDATION_KEY, buffer.array());
+ // Now update record itself.
+ map.put(index, content);
+ }
+ }
+
+ @Override
+ public synchronized void remove(String key) {
+ map.remove(indexer.getIndex(key));
+ }
+
+ @Override
+ public synchronized long save() throws IOException {
+ long indexSize = indexer.save();
+ long mapSize = map.save();
+ return indexSize + mapSize;
+ }
+
+ @Override
+ public synchronized String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Action cache (" + map.size() + " records):\n");
+ for (Map.Entry<Integer, byte[]> entry: map.entrySet()) {
+ if (entry.getKey() == VALIDATION_KEY) { continue; }
+ String content;
+ try {
+ content = decode(indexer, entry.getValue()).toString();
+ } catch (IOException e) {
+ content = e.toString() + "\n";
+ }
+ builder.append("-> ").append(indexer.getStringForIndex(entry.getKey())).append("\n")
+ .append(content).append(" packed_len = ").append(entry.getValue().length).append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Dumps action cache content.
+ */
+ @Override
+ public synchronized void dump(PrintStream out) {
+ out.println("String indexer content:\n");
+ out.println(indexer.toString());
+ out.println("Action cache (" + map.size() + " records):\n");
+ for (Map.Entry<Integer, byte[]> entry: map.entrySet()) {
+ if (entry.getKey() == VALIDATION_KEY) { continue; }
+ String content;
+ try {
+ content = CompactPersistentActionCache.decode(indexer, entry.getValue()).toString();
+ } catch (IOException e) {
+ content = e.toString() + "\n";
+ }
+ out.println(entry.getKey() + ", " + indexer.getStringForIndex(entry.getKey()) + ":\n"
+ + content + "\n packed_len = " + entry.getValue().length + "\n");
+ }
+ }
+
+ /**
+ * @return action data encoded as a byte[] array.
+ */
+ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) {
+ Preconditions.checkState(!entry.isCorrupted());
+
+ try {
+ byte[] actionKeyBytes = entry.getActionKey().getBytes(ISO_8859_1);
+ Collection<String> files = entry.getPaths();
+
+ // Estimate the size of the buffer:
+ // 5 bytes max for the actionKey length
+ // + the actionKey itself
+ // + 16 bytes for the digest
+ // + 5 bytes max for the file list length
+ // + 5 bytes max for each file id
+ int maxSize = VarInt.MAX_VARINT_SIZE + actionKeyBytes.length + Digest.MD5_SIZE
+ + VarInt.MAX_VARINT_SIZE + files.size() * VarInt.MAX_VARINT_SIZE;
+ ByteArrayOutputStream sink = new ByteArrayOutputStream(maxSize);
+
+ VarInt.putVarInt(actionKeyBytes.length, sink);
+ sink.write(actionKeyBytes);
+
+ entry.getFileDigest().write(sink);
+
+ VarInt.putVarInt(files.size(), sink);
+ for (String file : files) {
+ VarInt.putVarInt(indexer.getOrCreateIndex(file), sink);
+ }
+ return sink.toByteArray();
+ } catch (IOException e) {
+ // This Exception can never be thrown by ByteArrayOutputStream.
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * Creates new action cache entry using given compressed entry data. Data
+ * will stay in the compressed format until entry is actually used by the
+ * dependency checker.
+ */
+ private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) throws IOException {
+ try {
+ ByteBuffer source = ByteBuffer.wrap(data);
+
+ byte[] actionKeyBytes = new byte[VarInt.getVarInt(source)];
+ source.get(actionKeyBytes);
+ String actionKey = new String(actionKeyBytes, ISO_8859_1);
+
+ Digest digest = Digest.read(source);
+
+ int count = VarInt.getVarInt(source);
+ ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
+ for (int i = 0; i < count; i++) {
+ int id = VarInt.getVarInt(source);
+ String filename = (id >= 0 ? indexer.getStringForIndex(id) : null);
+ if (filename == null) {
+ throw new IOException("Corrupted file index");
+ }
+ builder.add(filename);
+ }
+ if (source.remaining() > 0) {
+ throw new IOException("serialized entry data has not been fully decoded");
+ }
+ return new Entry(actionKey, builder.build(), digest);
+ } catch (BufferUnderflowException e) {
+ throw new IOException("encoded entry data is incomplete", e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java
new file mode 100644
index 0000000..f278507
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java
@@ -0,0 +1,142 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.VarInt;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * A value class for capturing and comparing MD5-based digests.
+ *
+ * <p>Note that this class is responsible for digesting file metadata in an
+ * order-independent manner. Care must be taken to do this properly. The
+ * digest must be a function of the set of (path, metadata) tuples. While the
+ * order of these pairs must not matter, it would <b>not</b> be safe to make
+ * the digest be a function of the set of paths and the set of metadata.
+ *
+ * <p>Note that the (path, metadata) tuples must be unique, otherwise the
+ * XOR-based approach will fail.
+ */
+public class Digest {
+
+ static final int MD5_SIZE = 16;
+
+ private final byte[] digest;
+
+ /**
+ * Construct the digest from the given bytes.
+ * @param digest an MD5 digest. Must be sized properly.
+ */
+ @VisibleForTesting
+ Digest(byte[] digest) {
+ Preconditions.checkState(digest.length == MD5_SIZE);
+ this.digest = Arrays.copyOf(digest, digest.length);
+ }
+
+ /**
+ * @param source the byte buffer source.
+ * @return the digest from the given buffer.
+ * @throws IOException if the byte buffer is incorrectly formatted.
+ */
+ public static Digest read(ByteBuffer source) throws IOException {
+ int size = VarInt.getVarInt(source);
+ if (size != MD5_SIZE) {
+ throw new IOException("Unexpected digest length: " + size);
+ }
+ byte[] bytes = new byte[size];
+ source.get(bytes);
+ return new Digest(bytes);
+ }
+
+ /**
+ * Write the digest to the output stream.
+ */
+ public void write(OutputStream sink) throws IOException {
+ VarInt.putVarInt(digest.length, sink);
+ sink.write(digest);
+ }
+
+ /**
+ * @param mdMap A collection of (execPath, Metadata) pairs.
+ * Values may be null.
+ * @return an <b>order-independent</b> digest from the given "set" of
+ * (path, metadata) pairs.
+ */
+ public static Digest fromMetadata(Map<String, Metadata> mdMap) {
+ byte[] result = new byte[MD5_SIZE];
+ // Profiling showed that MD5 engine instantiation was a hotspot, so create one instance for
+ // this computation to amortize its cost.
+ Fingerprint fp = new Fingerprint();
+ for (Map.Entry<String, Metadata> entry : mdMap.entrySet()) {
+ xorWith(result, getDigest(fp, entry.getKey(), entry.getValue()));
+ fp.reset();
+ }
+ return new Digest(result);
+ }
+
+ /**
+ * @return this Digest as a Metadata with no mtime.
+ */
+ public Metadata asMetadata() {
+ return new Metadata(digest);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(digest);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return (obj instanceof Digest) && Arrays.equals(digest, ((Digest) obj).digest);
+ }
+
+ @Override
+ public String toString() {
+ return Fingerprint.hexDigest(digest);
+ }
+
+ private static byte[] getDigest(Fingerprint fp, String execPath, Metadata md) {
+ fp.addString(execPath);
+
+ if (md == null) {
+ // Move along, nothing to see here.
+ } else if (md.digest == null) {
+ // Use the timestamp if the digest is not present, but not both.
+ // Modifying a timestamp while keeping the contents of a file the
+ // same should not cause rebuilds.
+ fp.addLong(md.mtime);
+ } else {
+ fp.addBytes(md.digest);
+ }
+ return fp.digestAndReset();
+ }
+
+ /**
+ * Compute lhs ^= rhs bitwise operation of the arrays.
+ */
+ private static void xorWith(byte[] lhs, byte[] rhs) {
+ for (int i = 0; i < lhs.length; i++) {
+ lhs[i] ^= rhs[i];
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java
new file mode 100644
index 0000000..7295fb5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java
@@ -0,0 +1,130 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.logging.Level;
+
+import javax.annotation.Nullable;
+
+/**
+ * Utility class for getting md5 digests of files.
+ */
+public class DigestUtils {
+ // Object to synchronize on when serializing large file reads.
+ private static final Object MD5_LOCK = new Object();
+
+ /** Private constructor to prevent instantiation of utility class. */
+ private DigestUtils() {}
+
+ /**
+ * Returns true iff using MD5 digests is appropriate for an artifact.
+ *
+ * @param artifact Artifact in question.
+ * @param isFile whether or not Artifact is a file versus a directory, isFile() on its stat.
+ * @param size size of Artifact on filesystem in bytes, getSize() on its stat.
+ */
+ public static boolean useFileDigest(Artifact artifact, boolean isFile, long size) {
+ // Use timestamps for directories. Use digests for everything else.
+ return isFile && size != 0;
+ }
+
+ /**
+ * Obtain file's MD5 metadata using synchronized method, ensuring that system
+ * is not overloaded in case when multiple threads are requesting MD5
+ * calculations and underlying file system cannot provide it via extended
+ * attribute.
+ */
+ private static byte[] getDigestInExclusiveMode(Path path) throws IOException {
+ long startTime = BlazeClock.nanoTime();
+ synchronized (MD5_LOCK) {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, path.getPathString());
+ return getDigestInternal(path);
+ }
+ }
+
+ private static byte[] getDigestInternal(Path path) throws IOException {
+ long startTime = BlazeClock.nanoTime();
+ byte[] md5bin = path.getMD5Digest();
+
+ long millis = (BlazeClock.nanoTime() - startTime) / 1000000;
+ if (millis > 5000L) {
+ System.err.println("Slow read: a " + path.getFileSize() + "-byte read from " + path
+ + " took " + millis + "ms.");
+ }
+ return md5bin;
+ }
+
+ private static boolean binaryDigestWellFormed(byte[] digest) {
+ Preconditions.checkNotNull(digest);
+ return digest.length == 16;
+ }
+
+ /**
+ * Returns the the fast md5 digest of the file, or null if not available.
+ */
+ @Nullable
+ public static byte[] getFastDigest(Path path) throws IOException {
+ return path.getFastDigestFunctionType().equals("MD5") ? path.getFastDigest() : null;
+ }
+
+ /**
+ * Get the md5 digest of {@code path}, using a constant-time xattr call if the filesystem supports
+ * it, and calculating the digest manually otherwise.
+ *
+ * @param path Path of the file.
+ * @param fileSize size of the file. Used to determine if digest calculation should be done
+ * serially or in parallel. Files larger than a certain threshold will be read serially, in order
+ * to avoid excessive disk seeks.
+ */
+ public static byte[] getDigestOrFail(Path path, long fileSize) throws IOException {
+ // TODO(bazel-team): the action cache currently only works with md5 digests but it ought to
+ // work with any opaque digest.
+ byte[] md5bin = null;
+ if (Objects.equals(path.getFastDigestFunctionType(), "MD5")) {
+ md5bin = getFastDigest(path);
+ }
+ if (md5bin != null && !binaryDigestWellFormed(md5bin)) {
+ // Fail-soft in cases where md5bin is non-null, but not a valid digest.
+ String msg = String.format("Malformed digest '%s' for file %s",
+ BaseEncoding.base16().lowerCase().encode(md5bin),
+ path);
+ LoggingUtil.logToRemote(Level.SEVERE, msg, new IllegalStateException(msg));
+ md5bin = null;
+ }
+ if (md5bin != null) {
+ return md5bin;
+ } else if (fileSize > 4096) {
+ // We'll have to read file content in order to calculate the digest. In that case
+ // it would be beneficial to serialize those calculations since there is a high
+ // probability that MD5 will be requested for multiple output files simultaneously.
+ // Exception is made for small (<=4K) files since they will not likely to introduce
+ // significant delays (at worst they will result in two extra disk seeks by
+ // interrupting other reads).
+ return getDigestInExclusiveMode(path);
+ } else {
+ return getDigestInternal(path);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java b/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java
new file mode 100644
index 0000000..9764cd8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.devtools.build.lib.vfs.FileStatus;
+
+/**
+ * A FileStatus corresponding to a file that is not determined by querying the file system.
+ */
+public class InjectedStat implements FileStatus {
+
+ private final long mtime;
+ private final long size;
+ private final long nodeId;
+
+ public InjectedStat(long mtime, long size, long nodeId) {
+ this.mtime = mtime;
+ this.size = size;
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public boolean isFile() {
+ return true;
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return false;
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return false;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public long getLastModifiedTime() {
+ return mtime;
+ }
+
+ @Override
+ public long getLastChangeTime() {
+ return getLastModifiedTime();
+ }
+
+ @Override
+ public long getNodeId() {
+ return nodeId;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
new file mode 100644
index 0000000..36c52b9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
@@ -0,0 +1,92 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.util.Arrays;
+import java.util.Date;
+
+/**
+ * A class to represent file metadata.
+ * ActionCacheChecker may assume that, for a given file, equal
+ * metadata at different moments implies equal file-contents,
+ * where metadata equality is computed using Metadata.equals().
+ * <p>
+ * NB! Several other parts of Blaze are relying on the fact that metadata
+ * uses mtime and not ctime. If metadata is ever changed
+ * to use ctime, all uses of Metadata must be carefully examined.
+ */
+@Immutable @ThreadSafe
+public final class Metadata {
+ public final long mtime;
+ public final byte[] digest;
+
+ // Convenience object for use with volatile files that we do not want checked
+ // (e.g. the build-changelist.txt)
+ public static final Metadata CONSTANT_METADATA = new Metadata(-1);
+
+ public Metadata(long mtime) {
+ this.mtime = mtime;
+ this.digest = null;
+ }
+
+ public Metadata(byte[] digest) {
+ this.mtime = 0L;
+ this.digest = Preconditions.checkNotNull(digest);
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 0;
+ if (digest != null) {
+ // We are already dealing with the digest so we can just use portion of it
+ // as a hash code.
+ hash += digest[0] + (digest[1] << 8) + (digest[2] << 16) + (digest[3] << 24);
+ } else {
+ // Inlined hashCode for Long, so we don't
+ // have to construct an Object, just to compute
+ // a 32-bit hash out of a 64 bit value.
+ hash = (int) (mtime ^ (mtime >>> 32));
+ }
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+ if (!(that instanceof Metadata)) {
+ return false;
+ }
+ // Do a strict comparison - both digest and mtime should match
+ return Arrays.equals(this.digest, ((Metadata) that).digest)
+ && this.mtime == ((Metadata) that).mtime;
+ }
+
+ @Override
+ public String toString() {
+ if (digest != null) {
+ return "MD5 " + BaseEncoding.base16().lowerCase().encode(digest);
+ } else if (mtime > 0) {
+ return "timestamp " + new Date(mtime);
+ }
+ return "no metadata";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
new file mode 100644
index 0000000..9d288db
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.vfs.FileStatus;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/** Retrieves {@link Metadata} of {@link Artifact}s, and inserts virtual metadata as well. */
+public interface MetadataHandler {
+ /**
+ * Returns metadata for the given artifact or null if it does not exist.
+ *
+ * @param artifact artifact
+ *
+ * @return metadata instance or null if metadata cannot be obtained.
+ */
+ Metadata getMetadataMaybe(Artifact artifact);
+ /**
+ * Returns metadata for the given artifact or throws an exception if the
+ * metadata could not be obtained.
+ *
+ * @return metadata instance
+ *
+ * @throws IOException if metadata could not be obtained.
+ */
+ Metadata getMetadata(Artifact artifact) throws IOException;
+
+ /** Sets digest for virtual artifacts (e.g. middlemen). {@code digest} must not be null. */
+ void setDigestForVirtualArtifact(Artifact artifact, Digest digest);
+
+ /**
+ * Injects provided digest into the metadata handler, simultaneously caching lstat() data as well.
+ */
+ void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest);
+
+ /** Returns true iff artifact exists. */
+ boolean artifactExists(Artifact artifact);
+ /** Returns true iff artifact is a regular file. */
+ boolean isRegularFile(Artifact artifact);
+
+ /**
+ * @return Whether the artifact's data was injected.
+ * @throws IOException if implementation tried to stat artifact which threw an exception.
+ * Technically, this means that the artifact could not have been injected, but by throwing
+ * here we save the caller trying to stat this file on their own and throwing the same
+ * exception. Implementations are not guaranteed to throw in this case if they are able to
+ * determine that the artifact is not injected without statting it.
+ */
+ boolean isInjected(Artifact artifact) throws IOException;
+
+ /** Discards all metadata for the given artifacts, presumably because they will be modified. */
+ void discardMetadata(Collection<Artifact> artifactList);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java
new file mode 100644
index 0000000..0975150
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions.cache;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * A no-op action cache that never caches anything.
+ */
+public final class NullActionCache implements ActionCache {
+
+ @Override
+ public void put(String key, Entry entry) {
+ }
+
+ @Override
+ public Entry get(String key) {
+ return null;
+ }
+
+ @Override
+ public void remove(String key) {
+ }
+
+ @Override
+ public Entry createEntry(String key) {
+ return new ActionCache.Entry(key);
+ }
+
+ @Override
+ public long save() throws IOException {
+ return 0;
+ }
+
+ @Override
+ public void dump(PrintStream out) {
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java
new file mode 100644
index 0000000..bd98b2b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java
@@ -0,0 +1,161 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.common.collect.MapMaker;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe;
+import com.google.devtools.build.lib.util.CanonicalStringIndexer;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.PersistentMap;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * Persistent version of the CanonicalStringIndexer.
+ *
+ * <p>This class is backed by a PersistentMap that holds one direction of the
+ * canonicalization mapping. The other direction is handled purely in memory
+ * and reconstituted at load-time.
+ *
+ * <p>Thread-safety is ensured by locking on all mutating operations from the
+ * superclass. Read-only operations are not locked, but rather backed by
+ * ConcurrentMaps.
+ */
+@ConditionallyThreadSafe // condition: each instance must instantiated with
+ // different dataFile.
+final class PersistentStringIndexer extends CanonicalStringIndexer {
+
+ /**
+ * Persistent metadata map. Used as a backing map to provide a persistent
+ * implementation of the metadata cache.
+ */
+ private static final class PersistentIndexMap extends PersistentMap<String, Integer> {
+ private static final int VERSION = 0x01;
+ private static final long SAVE_INTERVAL_NS = 3L * 1000 * 1000 * 1000;
+
+ private final Clock clock;
+ private long nextUpdate;
+
+ public PersistentIndexMap(Path mapFile, Path journalFile, Clock clock) throws IOException {
+ super(VERSION, PersistentStringIndexer.<String, Integer>newConcurrentMap(INITIAL_ENTRIES),
+ mapFile, journalFile);
+ this.clock = clock;
+ nextUpdate = clock.nanoTime();
+ load(/*throwOnLoadFailure=*/true);
+ }
+
+ @Override
+ protected boolean updateJournal() {
+ long time = clock.nanoTime();
+ if (SAVE_INTERVAL_NS == 0 || time > nextUpdate) {
+ nextUpdate = time + SAVE_INTERVAL_NS;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public Integer remove(Object object) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ public void flush() {
+ super.forceFlush();
+ }
+
+ @Override
+ protected String readKey(DataInputStream in) throws IOException {
+ int length = in.readInt();
+ if (length < 0) {
+ throw new IOException("corrupt key length: " + length);
+ }
+ byte[] content = new byte[length];
+ in.readFully(content);
+ return StringCanonicalizer.intern(bytes2string(content));
+ }
+
+ @Override
+ protected Integer readValue(DataInputStream in) throws IOException {
+ return in.readInt();
+ }
+
+ @Override
+ protected void writeKey(String key, DataOutputStream out) throws IOException {
+ byte[] content = string2bytes(key);
+ out.writeInt(content.length);
+ out.write(content);
+ }
+
+ @Override
+ protected void writeValue(Integer value, DataOutputStream out) throws IOException {
+ out.writeInt(value);
+ }
+ }
+
+ private final PersistentIndexMap persistentIndexMap;
+ private static final int INITIAL_ENTRIES = 10000;
+
+ /**
+ * Instantiates and loads instance of the persistent string indexer.
+ */
+ static PersistentStringIndexer newPersistentStringIndexer(Path dataPath,
+ Clock clock) throws IOException {
+ PersistentIndexMap persistentIndexMap = new PersistentIndexMap(dataPath,
+ FileSystemUtils.replaceExtension(dataPath, ".journal"), clock);
+ Map<Integer, String> reverseMapping = newConcurrentMap(INITIAL_ENTRIES);
+ for (Map.Entry<String, Integer> entry : persistentIndexMap.entrySet()) {
+ if (reverseMapping.put(entry.getValue(), entry.getKey()) != null) {
+ throw new IOException("Corrupted filename index has duplicate entry: " + entry.getKey());
+ }
+ }
+ return new PersistentStringIndexer(persistentIndexMap, reverseMapping);
+ }
+
+ private PersistentStringIndexer(PersistentIndexMap stringToInt,
+ Map<Integer, String> intToString) {
+ super(stringToInt, intToString);
+ this.persistentIndexMap = stringToInt;
+ }
+
+ /**
+ * Saves index data to the file.
+ */
+ synchronized long save() throws IOException {
+ return persistentIndexMap.save();
+ }
+
+ /**
+ * Flushes the journal.
+ */
+ synchronized void flush() {
+ persistentIndexMap.flush();
+ }
+
+ private static <K, V> ConcurrentMap<K, V> newConcurrentMap(int expectedCapacity) {
+ return new MapMaker().initialCapacity(expectedCapacity).makeMap();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java
new file mode 100644
index 0000000..debb8ea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import com.google.devtools.build.lib.actions.ActionInput;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An ActionInput that does not actually exist on the filesystem, but can still be written to an
+ * OutputStream.
+ */
+public interface VirtualActionInput extends ActionInput {
+ /**
+ * Writes the the fake file to an OutputStream. MUST be deterministic, in that multiple calls
+ * to write the same VirtualActionInput must write identical bytes.
+ */
+ void writeTo(OutputStream out) throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AbstractConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/AbstractConfiguredTarget.java
new file mode 100644
index 0000000..e574978
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AbstractConfiguredTarget.java
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+
+/**
+ * An abstract implementation of ConfiguredTarget in which all properties are
+ * assigned trivial default values.
+ */
+public abstract class AbstractConfiguredTarget
+ implements ConfiguredTarget, VisibilityProvider, ClassObject {
+ private final Target target;
+ private final BuildConfiguration configuration;
+
+ private final NestedSet<PackageSpecification> visibility;
+
+ AbstractConfiguredTarget(Target target,
+ BuildConfiguration configuration) {
+ this.target = target;
+ this.configuration = configuration;
+ this.visibility = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ AbstractConfiguredTarget(TargetContext targetContext) {
+ this.target = targetContext.getTarget();
+ this.configuration = targetContext.getConfiguration();
+ this.visibility = targetContext.getVisibility();
+ }
+
+ @Override
+ public final NestedSet<PackageSpecification> getVisibility() {
+ return visibility;
+ }
+
+ @Override
+ public Target getTarget() {
+ return target;
+ }
+
+ @Override
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public Label getLabel() {
+ return getTarget().getLabel();
+ }
+
+ @Override
+ public String toString() {
+ return "ConfiguredTarget(" + getTarget().getLabel() + ", " + getConfiguration() + ")";
+ }
+
+ @Override
+ public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) {
+ AnalysisUtils.checkProvider(provider);
+ if (provider.isAssignableFrom(getClass())) {
+ return provider.cast(this);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Object getValue(String name) {
+ if (name.equals("label")) {
+ return getLabel();
+ } else if (name.equals("files")) {
+ // A shortcut for files to build in Skylark. FileConfiguredTarget and RunleConfiguredTarget
+ // always has FileProvider and Error- and PackageGroupConfiguredTarget-s shouldn't be
+ // accessible in Skylark.
+ return SkylarkNestedSet.of(Artifact.class, getProvider(FileProvider.class).getFilesToBuild());
+ }
+ return get(name);
+ }
+
+ @Override
+ public String errorMessage(String name) {
+ return null;
+ }
+
+ @Override
+ public ImmutableCollection<String> getKeys() {
+ return ImmutableList.<String>builder().add("label").add("files").build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AlwaysBuiltArtifactsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/AlwaysBuiltArtifactsProvider.java
new file mode 100644
index 0000000..e4d40fc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AlwaysBuiltArtifactsProvider.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Artifacts that should be built when a target is mentioned in the command line, but are neither in
+ * the {@code filesToBuild} nor in the runfiles.
+ *
+ * <p>
+ * Link actions, may not run a link for their transitive dependencies, so it does not force the
+ * source files in the transitive closure to be built by default. However, users expect builds to
+ * fail when there is an error in a dependent library, so we use this mechanism to force their
+ * compilation.
+ */
+@Immutable
+public final class AlwaysBuiltArtifactsProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> artifactsToAlwaysBuild;
+
+ public AlwaysBuiltArtifactsProvider(NestedSet<Artifact> artifactsToAlwaysBuild) {
+ this.artifactsToAlwaysBuild = artifactsToAlwaysBuild;
+ }
+
+ /**
+ * Returns the collection of artifacts to be built.
+ */
+ public NestedSet<Artifact> getArtifactsToAlwaysBuild() {
+ return artifactsToAlwaysBuild;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
new file mode 100644
index 0000000..0bccc72
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
@@ -0,0 +1,128 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionRegistry;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MiddlemanFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+
+/**
+ * The set of services that are provided to {@link ConfiguredTarget} objects
+ * during initialization.
+ */
+public interface AnalysisEnvironment extends ActionRegistry {
+ /**
+ * Returns a callback to be used in this build for reporting analysis errors.
+ */
+ EventHandler getEventHandler();
+
+ /**
+ * Returns whether any errors were reported to this instance.
+ */
+ boolean hasErrors();
+
+ /**
+ * Returns the artifact for the derived file {@code rootRelativePath}.
+ *
+ * <p>Creates the artifact if necessary and sets the root of that artifact to {@code root}.
+ */
+ Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root);
+
+ /**
+ * Returns an artifact for the derived file {@code rootRelativePath} whose changes do not cause
+ * a rebuild.
+ *
+ * <p>Creates the artifact if necessary and sets the root of that artifact to {@code root}.
+ *
+ * <p>This is useful for files that store data that changes very frequently (e.g. current time)
+ * but does not substantially affect the result of the build.
+ */
+ Artifact getConstantMetadataArtifact(PathFragment rootRelativePath,
+ Root root);
+
+ /**
+ * Returns the artifact for the derived file {@code rootRelativePath},
+ * creating it if necessary, and setting the root of that artifact to
+ * {@code root}. The artifact will represent the output directory of a {@code Fileset}.
+ */
+ Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root);
+
+ /**
+ * Returns the artifact for the specified tool.
+ */
+ Artifact getEmbeddedToolArtifact(String embeddedPath);
+
+ /**
+ * Returns the middleman factory associated with the build.
+ */
+ // TODO(bazel-team): remove this method and replace it with delegate methods.
+ MiddlemanFactory getMiddlemanFactory();
+
+ /**
+ * Returns the generating action for the given local artifact.
+ *
+ * If the artifact was created in another analysis environment (e.g. by a different configured
+ * target instance) or the artifact is a source artifact, it returns null.
+ */
+ Action getLocalGeneratingAction(Artifact artifact);
+
+ /**
+ * Returns the actions that were registered so far with this analysis environment, that is, all
+ * the actions that were created by the current target being analyzed.
+ */
+ Iterable<Action> getRegisteredActions();
+
+ /**
+ * Returns the Skyframe SkyFunction.Environment if available. Otherwise, null.
+ *
+ * <p>If you need to use this for something other than genquery, please think long and hard
+ * about that.
+ */
+ SkyFunction.Environment getSkyframeEnv();
+
+ /**
+ * Returns the Artifact that is used to hold the non-volatile workspace status for the current
+ * build request.
+ */
+ Artifact getStableWorkspaceStatusArtifact();
+
+ /**
+ * Returns the Artifact that is used to hold the volatile workspace status (e.g. build
+ * changelist) for the current build request.
+ */
+ Artifact getVolatileWorkspaceStatusArtifact();
+
+ /**
+ * Returns the Artifacts that contain the workspace status for the current build request.
+ *
+ * @param ruleContext the rule to use for error reporting and to determine the
+ * configuration
+ */
+ ImmutableList<Artifact> getBuildInfo(RuleContext ruleContext, BuildInfoKey key);
+
+ /**
+ * Returns the set of orphan Artifacts (i.e. Artifacts without generating action). Should only be
+ * called after the ConfiguredTarget is created.
+ */
+ ImmutableSet<Artifact> getOrphanArtifacts();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisFailureEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisFailureEvent.java
new file mode 100644
index 0000000..5064163
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisFailureEvent.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * This event is fired during the build, when it becomes known that the analysis
+ * of a target cannot be completed because of an error in one of its
+ * dependencies.
+ */
+public class AnalysisFailureEvent {
+ private final LabelAndConfiguration failedTarget;
+ private final Label failureReason;
+
+ public AnalysisFailureEvent(LabelAndConfiguration failedTarget, Label failureReason) {
+ this.failedTarget = failedTarget;
+ this.failureReason = failureReason;
+ }
+
+ public LabelAndConfiguration getFailedTarget() {
+ return failedTarget;
+ }
+
+ public Label getFailureReason() {
+ return failureReason;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisHooks.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisHooks.java
new file mode 100644
index 0000000..82c4485
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisHooks.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+
+/**
+ * This interface resolves target - configuration pairs to {@link ConfiguredTarget} instances.
+ *
+ * <p>This interface is used to provide analysis phase functionality to actions that need it in
+ * the execution phase.
+ */
+public interface AnalysisHooks {
+ /**
+ * Returns the package manager used during the analysis phase.
+ */
+ PackageManager getPackageManager();
+
+ /**
+ * Resolves an existing configured target. Returns null if it is not in the cache.
+ */
+ ConfiguredTarget getExistingConfiguredTarget(Target target, BuildConfiguration configuration);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java
new file mode 100644
index 0000000..0d1e565
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Collection;
+
+/**
+ * This event is fired after the analysis phase is complete.
+ */
+public class AnalysisPhaseCompleteEvent {
+
+ private final Collection<ConfiguredTarget> targets;
+ private final long timeInMs;
+ private int targetsVisited;
+
+ /**
+ * Construct the event.
+ * @param targets The set of active targets that remain.
+ */
+ public AnalysisPhaseCompleteEvent(Collection<? extends ConfiguredTarget> targets,
+ int targetsVisited, long timeInMs) {
+ this.timeInMs = timeInMs;
+ this.targets = ImmutableList.copyOf(targets);
+ this.targetsVisited = targetsVisited;
+ }
+
+ /**
+ * @return The set of active targets remaining, which is a subset
+ * of the targets we attempted to analyze.
+ */
+ public Collection<ConfiguredTarget> getTargets() {
+ return targets;
+ }
+
+ /**
+ * @return The number of targets freshly visited during analysis
+ */
+ public int getTargetsVisited() {
+ return targetsVisited;
+ }
+
+ public long getTimeInMs() {
+ return timeInMs;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseStartedEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseStartedEvent.java
new file mode 100644
index 0000000..fc97c60
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseStartedEvent.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+
+/**
+ * This event is fired before the analysis phase is started.
+ */
+public class AnalysisPhaseStartedEvent {
+
+ private final Iterable<Label> labels;
+
+ /**
+ * Construct the event.
+ * @param targets The set of active targets that remain.
+ */
+ public AnalysisPhaseStartedEvent(Collection<Target> targets) {
+ this.labels = Iterables.transform(targets, new Function<Target, Label>() {
+ @Override
+ public Label apply(Target input) {
+ return input.getLabel();
+ }
+ });
+ }
+
+ /**
+ * @return The set of active targets remaining, which is a subset
+ * of the targets we attempted to load.
+ */
+ public Iterable<Label> getLabels() {
+ return labels;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java
new file mode 100644
index 0000000..2e4c251
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.TriState;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * Utility functions for use during analysis.
+ */
+public final class AnalysisUtils {
+
+ private AnalysisUtils() {
+ throw new IllegalStateException(); // utility class
+ }
+
+ /**
+ * Returns whether link stamping is enabled for a rule.
+ *
+ * <p>This returns false for unstampable rule classes and for rules in the
+ * host configuration. Otherwise it returns the value of the stamp attribute,
+ * or of the stamp option if the attribute value is -1.
+ */
+ public static boolean isStampingEnabled(RuleContext ruleContext) {
+ BuildConfiguration config = ruleContext.getConfiguration();
+ Rule rule = ruleContext.getRule();
+ if (config.isHostConfiguration()
+ || !rule.getRuleClassObject().hasAttr("stamp", Type.TRISTATE)) {
+ return false;
+ }
+ TriState stamp = ruleContext.attributes().get("stamp", Type.TRISTATE);
+ return stamp == TriState.YES || (stamp == TriState.AUTO && config.stampBinaries());
+ }
+
+ // TODO(bazel-team): These need Iterable<? extends TransitiveInfoCollection> because they need to
+ // be called with Iterable<ConfiguredTarget>. Once the configured target lockdown is complete, we
+ // can eliminate the "extends" clauses.
+ /**
+ * Returns the list of providers of the specified type from a set of transitive info
+ * collections.
+ */
+ public static <C extends TransitiveInfoProvider> Iterable<C> getProviders(
+ Iterable<? extends TransitiveInfoCollection> prerequisites, Class<C> provider) {
+ Collection<C> result = new ArrayList<>();
+ for (TransitiveInfoCollection prerequisite : prerequisites) {
+ C prerequisiteProvider = prerequisite.getProvider(provider);
+ if (prerequisiteProvider != null) {
+ result.add(prerequisiteProvider);
+ }
+ }
+ return ImmutableList.copyOf(result);
+ }
+
+ /**
+ * Returns the iterable of collections that have the specified provider.
+ */
+ public static <S extends TransitiveInfoCollection, C extends TransitiveInfoProvider> Iterable<S>
+ filterByProvider(Iterable<S> prerequisites, final Class<C> provider) {
+ return Iterables.filter(prerequisites, new Predicate<S>() {
+ @Override
+ public boolean apply(S target) {
+ return target.getProvider(provider) != null;
+ }
+ });
+ }
+
+ /**
+ * Returns the path of the associated manifest file for the path of a Fileset. Works for both
+ * exec paths and root relative paths.
+ */
+ public static PathFragment getManifestPathFromFilesetPath(PathFragment filesetDir) {
+ PathFragment manifestDir = filesetDir.replaceName("_" + filesetDir.getBaseName());
+ PathFragment outputManifestFrag = manifestDir.getRelative("MANIFEST");
+ return outputManifestFrag;
+ }
+
+ /**
+ * Returns the middleman artifact on the specified attribute of the specified rule, or an empty
+ * set if it does not exist.
+ */
+ public static NestedSet<Artifact> getMiddlemanFor(RuleContext rule, String attribute) {
+ TransitiveInfoCollection prereq = rule.getPrerequisite(attribute, Mode.HOST);
+ if (prereq == null) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+ MiddlemanProvider provider = prereq.getProvider(MiddlemanProvider.class);
+ if (provider == null) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+ return provider.getMiddlemanArtifact();
+ }
+
+ /**
+ * Returns a path fragment qualified by the rule name and unique fragment to
+ * disambiguate artifacts produced from the source file appearing in
+ * multiple rules.
+ *
+ * <p>For example "//pkg:target" -> "pkg/<fragment>/target.
+ */
+ public static PathFragment getUniqueDirectory(Label label, PathFragment fragment) {
+ return label.getPackageFragment().getRelative(fragment)
+ .getRelative(label.getName());
+ }
+
+ /**
+ * Checks that the given provider class either refers to an interface or to a value class.
+ */
+ public static <T extends TransitiveInfoProvider> void checkProvider(Class<T> clazz) {
+ if (!clazz.isInterface()) {
+ Preconditions.checkArgument(Modifier.isFinal(clazz.getModifiers()),
+ clazz.getName() + " has to be final");
+ Preconditions.checkArgument(clazz.isAnnotationPresent(Immutable.class),
+ clazz.getName() + " has to be tagged with @Immutable");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Aspect.java b/src/main/java/com/google/devtools/build/lib/analysis/Aspect.java
new file mode 100644
index 0000000..3f4a06e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/Aspect.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Extra information about a configured target computed on request of a dependent.
+ *
+ * <p>Analogous to {@link ConfiguredTarget}: contains a bunch of transitive info providers, which
+ * are merged with the providers of the associated configured target before they are passed to
+ * the configured target factories that depend on the configured target to which this aspect is
+ * added.
+ *
+ * <p>Aspects are created alongside configured targets on request from dependents.
+ */
+@Immutable
+public final class Aspect implements Iterable<TransitiveInfoProvider> {
+ private final
+ ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers;
+
+ private Aspect(
+ ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers) {
+ this.providers = providers;
+ }
+
+ /**
+ * Returns the providers created by the aspect.
+ */
+ public ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider>
+ getProviders() {
+ return providers;
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ return providers.values().iterator();
+ }
+
+ /**
+ * Builder for {@link Aspect}.
+ */
+ public static class Builder {
+ private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider>
+ providers = new LinkedHashMap<>();
+
+ /**
+ * Adds a provider to the aspect.
+ */
+ public Builder addProvider(
+ Class<? extends TransitiveInfoProvider> key, TransitiveInfoProvider value) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(value);
+ AnalysisUtils.checkProvider(key);
+ Preconditions.checkState(!providers.containsKey(key));
+ providers.put(key, value);
+ return this;
+ }
+
+ public Aspect build() {
+ return new Aspect(ImmutableMap.copyOf(providers));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java b/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java
new file mode 100644
index 0000000..ad5756e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java
@@ -0,0 +1,265 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA;
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS;
+import static com.google.devtools.build.lib.packages.Type.INTEGER;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.analysis.constraints.EnvironmentRule;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabelList;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.List;
+
+/**
+ * Rule class definitions used by (almost) every rule.
+ */
+public class BaseRuleClasses {
+ /**
+ * Label of the pseudo-filegroup that contains all the targets that are needed
+ * for running tests in coverage mode.
+ */
+ private static final Label COVERAGE_SUPPORT_LABEL =
+ Label.parseAbsoluteUnchecked("//tools/defaults:coverage");
+
+ private static final Attribute.ComputedDefault obsoleteDefault =
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.getPackageDefaultObsolete();
+ }
+ };
+
+ private static final Attribute.ComputedDefault testonlyDefault =
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.getPackageDefaultTestOnly();
+ }
+ };
+
+ private static final Attribute.ComputedDefault deprecationDefault =
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.getPackageDefaultDeprecation();
+ }
+ };
+
+ /**
+ * Implementation for the :action_listener attribute.
+ */
+ private static final LateBoundLabelList<BuildConfiguration> ACTION_LISTENER =
+ new LateBoundLabelList<BuildConfiguration>() {
+ @Override
+ public List<Label> getDefault(Rule rule, BuildConfiguration configuration) {
+ // action_listeners are special rules; they tell the build system to add extra_actions to
+ // existing rules. As such they need an edge to every ConfiguredTarget with the limitation
+ // that they only run on the target configuration and should not operate on action_listeners
+ // and extra_actions themselves (to avoid cycles).
+ return configuration.getActionListeners();
+ }
+ };
+
+ private static final LateBoundLabelList<BuildConfiguration> COVERAGE_SUPPORT =
+ new LateBoundLabelList<BuildConfiguration>(ImmutableList.of(COVERAGE_SUPPORT_LABEL)) {
+ @Override
+ public List<Label> getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.isCodeCoverageEnabled()
+ ? ImmutableList.<Label>copyOf(configuration.getCoverageLabels())
+ : ImmutableList.<Label>of();
+ }
+ };
+
+ private static final LateBoundLabelList<BuildConfiguration> COVERAGE_REPORT_GENERATOR =
+ new LateBoundLabelList<BuildConfiguration>(ImmutableList.of(COVERAGE_SUPPORT_LABEL)) {
+ @Override
+ public List<Label> getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.isCodeCoverageEnabled()
+ ? ImmutableList.<Label>copyOf(configuration.getCoverageReportGeneratorLabels())
+ : ImmutableList.<Label>of();
+ }
+ };
+
+ /**
+ * Implementation for the :run_under attribute.
+ */
+ private static final LateBoundLabel<BuildConfiguration> RUN_UNDER =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ RunUnder runUnder = configuration.getRunUnder();
+ return runUnder == null ? null : runUnder.getLabel();
+ }
+ };
+
+ /**
+ * A base rule for all test rules.
+ */
+ @BlazeRule(name = "$test_base_rule",
+ type = RuleClassType.ABSTRACT)
+ public static final class TestBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("size", STRING).value("medium").taggable()
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("timeout", STRING).taggable()
+ .nonconfigurable("policy decision: should be consistent across configurations")
+ .value(new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ TestSize size = TestSize.getTestSize(rule.get("size", Type.STRING));
+ if (size != null) {
+ String timeout = size.getDefaultTimeout().toString();
+ if (timeout != null) {
+ return timeout;
+ }
+ }
+ return "illegal";
+ }
+ }))
+ .add(attr("flaky", BOOLEAN).value(false).taggable()
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("shard_count", INTEGER).value(-1))
+ .add(attr("local", BOOLEAN).value(false).taggable()
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("args", STRING_LIST)
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("$test_runtime", LABEL_LIST).cfg(HOST).value(ImmutableList.of(
+ env.getLabel("//tools/test:runtime"))))
+
+ // TODO(bazel-team): TestActions may need to be run with coverage, so all tests
+ // implicitly depend on crosstool, which provides gcov. We could add gcov to
+ // InstrumentedFilesProvider.getInstrumentationMetadataFiles() (or a new method) for
+ // all the test rules that have C++ in their transitive closure. Then this could go.
+ .add(attr(":coverage_support", LABEL_LIST).cfg(HOST).value(COVERAGE_SUPPORT))
+ .add(attr(":coverage_report_generator", LABEL_LIST).cfg(HOST)
+ .value(COVERAGE_REPORT_GENERATOR))
+
+ // The target itself and run_under both run on the same machine. We use the DATA config
+ // here because the run_under acts like a data dependency (e.g. no LIPO optimization).
+ .add(attr(":run_under", LABEL).cfg(DATA).value(RUN_UNDER))
+ .build();
+ }
+ }
+
+ /**
+ * Share common attributes across both base and Skylark base rules.
+ */
+ public static RuleClass.Builder commonCoreAndSkylarkAttributes(RuleClass.Builder builder) {
+ return builder
+ // The visibility attribute is special: it is a nodep label, and loading the
+ // necessary package groups is handled by {@link LabelVisitor#visitTargetVisibility}.
+ // Package groups always have the null configuration so that they are not duplicated
+ // needlessly.
+ .add(attr("visibility", NODEP_LABEL_LIST).orderIndependent().cfg(HOST)
+ .nonconfigurable("special attribute integrated more deeply into Bazel's core logic"))
+ .add(attr("deprecation", STRING).value(deprecationDefault)
+ .nonconfigurable("Used in core loading phase logic with no access to configs"))
+ .add(attr("tags", STRING_LIST).orderIndependent().taggable()
+ .nonconfigurable("low-level attribute, used in TargetUtils without configurations"))
+ .add(attr("generator_name", STRING).undocumented("internal"))
+ .add(attr("generator_function", STRING).undocumented("internal"))
+ .add(attr("testonly", BOOLEAN).value(testonlyDefault)
+ .nonconfigurable("policy decision: rules testability should be consistent"))
+ .add(attr(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR, LABEL_LIST)
+ .allowedRuleClasses(EnvironmentRule.RULE_NAME)
+ .cfg(Attribute.ConfigurationTransition.HOST)
+ .allowedFileTypes(FileTypeSet.NO_FILE)
+ .undocumented("not yet released"))
+ .add(attr(RuleClass.RESTRICTED_ENVIRONMENT_ATTR, LABEL_LIST)
+ .allowedRuleClasses(EnvironmentRule.RULE_NAME)
+ .cfg(Attribute.ConfigurationTransition.HOST)
+ .allowedFileTypes(FileTypeSet.NO_FILE)
+ .undocumented("not yet released"));
+ }
+
+ /**
+ * Common parts of rules.
+ */
+ @BlazeRule(name = "$base_rule",
+ type = RuleClassType.ABSTRACT)
+ public static final class BaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) {
+ return commonCoreAndSkylarkAttributes(builder)
+ // The name attribute is handled specially, so it does not appear here.
+ //
+ // Aggregates the labels of all {@link ConfigRuleClasses} rules this rule uses (e.g.
+ // keys for configurable attributes). This is specially populated in
+ // {@RuleClass#populateRuleAttributeValues}.
+ //
+ // This attribute is not needed for actual builds. Its main purpose is so query's
+ // proto/XML output includes the labels of config dependencies, so, e.g., depserver
+ // reverse dependency lookups remain accurate. These can't just be added to the
+ // attribute definitions proto/XML queries already output because not all attributes
+ // contain labels.
+ //
+ // Builds and Blaze-interactive queries don't need this because they find dependencies
+ // through direct Rule label visitation, which already factors these in.
+ .add(attr("$config_dependencies", LABEL_LIST)
+ .nonconfigurable("not intended for actual builds"))
+ .add(attr("licenses", LICENSE)
+ .nonconfigurable("Used in core loading phase logic with no access to configs"))
+ .add(attr("distribs", DISTRIBUTIONS)
+ .nonconfigurable("Used in core loading phase logic with no access to configs"))
+ .add(attr("obsolete", BOOLEAN).value(obsoleteDefault)
+ .nonconfigurable("Used in core loading phase logic with no access to configs"))
+ .add(attr(":action_listener", LABEL_LIST).cfg(HOST).value(ACTION_LISTENER))
+ .build();
+ }
+ }
+
+ /**
+ * Common ancestor class for all rules.
+ */
+ @BlazeRule(name = "$rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRule.class })
+ public static final class RuleBase implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("deps", LABEL_LIST).legacyAllowAnyFileType())
+ .add(attr("data", LABEL_LIST).cfg(DATA).allowedFileTypes(FileTypeSet.ANY_FILE))
+ .build();
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BaselineCoverageArtifactsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/BaselineCoverageArtifactsProvider.java
new file mode 100644
index 0000000..ab74581
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BaselineCoverageArtifactsProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A {@link TransitiveInfoProvider} that has baseline coverage artifacts.
+ */
+@Immutable
+public final class BaselineCoverageArtifactsProvider implements TransitiveInfoProvider {
+ private final ImmutableList<Artifact> baselineCoverageArtifacts;
+
+ public BaselineCoverageArtifactsProvider(ImmutableList<Artifact> baselineCoverageArtifacts) {
+ this.baselineCoverageArtifacts = baselineCoverageArtifacts;
+ }
+
+ /**
+ * Returns a set of baseline coverage artifacts for a given set of configured targets.
+ *
+ * <p>These artifacts represent "empty" code coverage data for non-test libraries and binaries and
+ * used to establish correct baseline when calculating code coverage ratios since they would cover
+ * completely non-tested code as well.
+ */
+ public ImmutableList<Artifact> getBaselineCoverageArtifacts() {
+ return baselineCoverageArtifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java b/src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java
new file mode 100644
index 0000000..38a8d41
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java
@@ -0,0 +1,183 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.Serializable;
+
+import javax.annotation.Nullable;
+
+/**
+ * Encapsulation of all of the interesting top-level directories in any Blaze application.
+ *
+ * <p>The <code>installBase</code> is the directory where the Blaze binary has been installed.The
+ * <code>workspace</code> is the top-level directory in the user's client (possibly read-only).The
+ * <code>outputBase</code> is the directory below which Blaze puts all its state. The
+ * <code>execRoot</code> is the working directory for all spawned tools, which is generally below
+ * <code>outputBase</code>.
+ *
+ * <p>There is a 1:1 correspondence between a running Blaze instance and an output base directory;
+ * however, multiple Blaze instances may compile code that's in the same workspace, even on the same
+ * machine. If the user does not qualify an output base directory, the startup code will derive it
+ * deterministically from the workspace. Note also that while the Blaze server process runs with the
+ * workspace directory as its working directory, the client process may have a different working
+ * directory, typically a subdirectory.
+ *
+ * <p>Do not put shortcuts to specific files here!
+ */
+@Immutable
+public final class BlazeDirectories implements Serializable {
+
+ // Output directory name, relative to the execRoot.
+ // TODO(bazel-team): (2011) make this private?
+ public static final String RELATIVE_OUTPUT_PATH = StringCanonicalizer.intern(
+ Constants.PRODUCT_NAME + "-out");
+
+ // Include directory name, relative to execRoot/blaze-out/configuration.
+ public static final String RELATIVE_INCLUDE_DIR = StringCanonicalizer.intern("include");
+
+ private final Path installBase; // Where Blaze gets unpacked
+ private final Path workspace; // Workspace root and server CWD
+ private final Path outputBase; // The root of the temp and output trees
+ private final Path execRoot; // the root of all build actions
+
+ // These two are kept to avoid creating new objects every time they are accessed. This showed up
+ // in a profiler.
+ private final Path outputPath;
+ private final Path localOutputPath;
+
+ public BlazeDirectories(Path installBase, Path outputBase, @Nullable Path workspace) {
+ this.installBase = installBase;
+ this.workspace = workspace;
+ this.outputBase = outputBase;
+ if (this.workspace == null) {
+ // TODO(bazel-team): this should be null, but at the moment there is a lot of code that
+ // depends on it being non-null.
+ this.execRoot = outputBase.getChild("default-exec-root");
+ } else {
+ this.execRoot = outputBase.getChild(workspace.getBaseName());
+ }
+ this.outputPath = execRoot.getRelative(RELATIVE_OUTPUT_PATH);
+ Preconditions.checkState(this.workspace == null || outputPath.asFragment().equals(
+ outputPathFromOutputBase(outputBase.asFragment(), workspace.asFragment())));
+ this.localOutputPath = outputBase.getRelative(BlazeDirectories.RELATIVE_OUTPUT_PATH);
+ }
+
+ /**
+ * Returns the Filesystem that all of our directories belong to. Handy for
+ * resolving absolute paths.
+ */
+ public FileSystem getFileSystem() {
+ return installBase.getFileSystem();
+ }
+
+ /**
+ * Returns the installation base directory. Currently used by info command only.
+ */
+ public Path getInstallBase() {
+ return installBase;
+ }
+
+ /**
+ * Returns the workspace directory, which is also the working dir of the server.
+ */
+ public Path getWorkspace() {
+ return workspace;
+ }
+
+ /**
+ * Returns if the workspace directory is a valid workspace.
+ */
+ public boolean inWorkspace() {
+ return this.workspace != null;
+ }
+
+ /**
+ * Returns the base of the output tree, which hosts all build and scratch
+ * output for a user and workspace.
+ */
+ public Path getOutputBase() {
+ return outputBase;
+ }
+
+ /**
+ * Returns the execution root. This is the directory underneath which Blaze builds the source
+ * symlink forest, to represent the merged view of different workspaces specified
+ * with --package_path.
+ */
+ public Path getExecRoot() {
+ return execRoot;
+ }
+
+ /**
+ * Returns the output path used by this Blaze instance.
+ */
+ public Path getOutputPath() {
+ return outputPath;
+ }
+
+ /**
+ * @param outputBase the outputBase as a path fragment.
+ * @param workspace the workspace as a path fragment.
+ * @return the outputPath as a path fragment, given the outputBase.
+ */
+ public static PathFragment outputPathFromOutputBase(
+ PathFragment outputBase, PathFragment workspace) {
+ if (workspace.equals(PathFragment.EMPTY_FRAGMENT)) {
+ return outputBase;
+ }
+ return outputBase.getRelative(workspace.getBaseName() + "/" + RELATIVE_OUTPUT_PATH);
+ }
+
+ /**
+ * Returns the local output path used by this Blaze instance.
+ */
+ public Path getLocalOutputPath() {
+ return localOutputPath;
+ }
+
+ /**
+ * Returns the directory where the stdout/stderr for actions can be stored
+ * temporarily for a build. If the directory already exists, the directory
+ * is cleaned.
+ */
+ public Path getActionConsoleOutputDirectory() {
+ return getOutputBase().getRelative("action_outs");
+ }
+
+ /**
+ * Returns the installed embedded binaries directory, under the shared
+ * installBase location.
+ */
+ public Path getEmbeddedBinariesRoot() {
+ return installBase.getChild("_embedded_binaries");
+ }
+
+ /**
+ * Returns the configuration-independent root where the build-data should be placed, given the
+ * {@link BlazeDirectories} of this server instance. Nothing else should be placed here.
+ */
+ public Root getBuildDataDirectory() {
+ return Root.asDerivedRoot(getExecRoot(), getOutputPath());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BlazeRule.java b/src/main/java/com/google/devtools/build/lib/analysis/BlazeRule.java
new file mode 100644
index 0000000..c349e65
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BlazeRule.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation for rule classes.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface BlazeRule {
+ /**
+ * The name of the rule, as it appears in the BUILD file. If it starts with
+ * '$', the rule will be hidden from users and will only be usable from
+ * inside Blaze.
+ */
+ String name();
+
+ /**
+ * The type of the rule. It can be an abstract rule, a normal rule or a test
+ * rule. If the rule type is abstract, the configured class must not be set.
+ */
+ RuleClassType type() default RuleClassType.NORMAL;
+
+ /**
+ * The {@link RuleConfiguredTargetFactory} class that implements this rule. If the rule is
+ * abstract, this must not be set.
+ */
+ Class<? extends RuleConfiguredTargetFactory> factoryClass()
+ default RuleConfiguredTargetFactory.class;
+
+ /**
+ * The list of other rule classes this rule inherits from.
+ */
+ Class<? extends RuleDefinition>[] ancestors() default {};
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BlazeVersionInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/BlazeVersionInfo.java
new file mode 100644
index 0000000..d5b5f94
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BlazeVersionInfo.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.util.StringUtilities;
+
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * Determines the version information of the current process.
+ *
+ * <p>The version information is a dictionary mapping from string keys to string values. For
+ * build stamping, it should have the key "Build label", which contains among others a
+ * XXXXXXX-YYYY.MM.DD string to indicate the version of the release. If no data is available
+ * (eg. when running non-released version), {@link #isAvailable()} returns false.
+ */
+public class BlazeVersionInfo {
+ private final Map<String, String> buildData = Maps.newTreeMap();
+ private static BlazeVersionInfo instance = null;
+ private static final String BUILD_LABEL = "Build label";
+
+ private static final Logger LOG = Logger.getLogger(BlazeVersionInfo.class.getName());
+
+ public BlazeVersionInfo(Map<String, String> info) {
+ buildData.putAll(info);
+ }
+
+ /**
+ * Accessor method for BlazeVersionInfo singleton.
+ *
+ * <p>If setBuildInfo was not called, returns an empty BlazeVersionInfo instance, which should
+ * not be persisted.
+ */
+ public static synchronized BlazeVersionInfo instance() {
+ if (instance == null) {
+ return new BlazeVersionInfo(ImmutableMap.<String, String>of());
+ }
+ return instance;
+ }
+
+ private static void logVersionInfo(BlazeVersionInfo info) {
+ if (info.getSummary() == null) {
+ LOG.warning("Blaze release version information not available");
+ } else {
+ LOG.info("Blaze version info: " + info.getSummary());
+ }
+ }
+
+ /**
+ * Sets build info.
+ *
+ * <p>This should be called once in the program execution, as early soon as possible, so we
+ * can have the version information even before modules are initialized.
+ */
+ public static synchronized void setBuildInfo(Map<String, String> info) {
+ if (instance != null) {
+ throw new IllegalStateException("setBuildInfo called twice.");
+ }
+ instance = new BlazeVersionInfo(info);
+ logVersionInfo(instance);
+ }
+
+ /**
+ * Indicates whether version information is available.
+ */
+ public boolean isAvailable() {
+ return !buildData.isEmpty();
+ }
+
+ /**
+ * Returns the summary which gets displayed in the 'version' command.
+ * The summary is a list of formatted key / value pairs.
+ */
+ public String getSummary() {
+ if (buildData.isEmpty()) {
+ return null;
+ }
+ return StringUtilities.layoutTable(buildData);
+ }
+
+ /**
+ * Returns true iff this binary is released--that is, a
+ * binary built with a release label.
+ */
+ public boolean isReleasedBlaze() {
+ String buildLabel = buildData.get(BUILD_LABEL);
+ return buildLabel != null && buildLabel.length() > 0;
+ }
+
+ /**
+ * Returns the release label, if any, or "development version".
+ */
+ public String getReleaseName() {
+ String buildLabel = buildData.get(BUILD_LABEL);
+ return (buildLabel != null && buildLabel.length() > 0)
+ ? "release " + buildLabel
+ : "development version";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java
new file mode 100644
index 0000000..59d1514
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * This event is fired once build info data is available.
+ */
+public final class BuildInfoEvent {
+ private final Map<String, String> buildInfoMap;
+
+ /**
+ * Construct the event from a map.
+ */
+ public BuildInfoEvent(Map<String, String> buildInfo) {
+ buildInfoMap = ImmutableMap.copyOf(buildInfo);
+ }
+
+ /**
+ * Return immutable map populated with build info key/value pairs.
+ */
+ public Map<String, String> getBuildInfoMap() {
+ return buildInfoMap;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoHelper.java
new file mode 100644
index 0000000..755df9f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoHelper.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.AbstractActionOwner;
+import com.google.devtools.build.lib.actions.ActionOwner;
+
+// TODO(bazel-team): move BUILD_INFO_ACTION_OWNER somewhere else and remove this class.
+/**
+ * Helper class for the CompatibleWriteBuildInfoAction, which holds the
+ * methods for generating build information.
+ * Abstracted away to allow non-action code to also generate build info under
+ * --nobuild or --check_up_to_date.
+ */
+public abstract class BuildInfoHelper {
+ /** ActionOwner for BuildInfoActions. */
+ public static final ActionOwner BUILD_INFO_ACTION_OWNER =
+ AbstractActionOwner.SYSTEM_ACTION_OWNER;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
new file mode 100644
index 0000000..052d0a2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
@@ -0,0 +1,1056 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.PackageRootResolver;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider.ExtraArtifactSet;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.DelegatingEventHandler;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.events.WarningsAsErrorsEventHandler;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
+import com.google.devtools.build.lib.skyframe.CoverageReportValue;
+import com.google.devtools.build.lib.skyframe.SkyframeBuildView;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * <p>The BuildView presents a semantically-consistent and transitively-closed
+ * dependency graph for some set of packages.
+ *
+ * <h2>Package design</h2>
+ *
+ * <p>This package contains the Blaze dependency analysis framework (aka
+ * "analysis phase"). The goal of this code is to perform semantic analysis of
+ * all of the build targets required for a given build, to report
+ * errors/warnings for any problems in the input, and to construct an "action
+ * graph" (see {@code lib.actions} package) correctly representing the work to
+ * be done during the execution phase of the build.
+ *
+ * <p><b>Configurations</b> the inputs to a build come from two sources: the
+ * intrinsic inputs, specified in the BUILD file, are called <em>targets</em>.
+ * The environmental inputs, coming from the build tool, the command-line, or
+ * configuration files, are called the <em>configuration</em>. Only when a
+ * target and a configuration are combined is there sufficient information to
+ * perform a build. </p>
+ *
+ * <p>Targets are implemented by the {@link Target} hierarchy in the {@code
+ * lib.packages} code. Configurations are implemented by {@link
+ * BuildConfiguration}. The pair of these together is represented by an
+ * instance of class {@link ConfiguredTarget}; this is the root of a hierarchy
+ * with different implementations for each kind of target: source file, derived
+ * file, rules, etc.
+ *
+ * <p>The framework code in this package (as opposed to its subpackages) is
+ * responsible for constructing the {@code ConfiguredTarget} graph for a given
+ * target and configuration, taking care of such issues as:
+ * <ul>
+ * <li>caching common subgraphs.
+ * <li>detecting and reporting cycles.
+ * <li>correct propagation of errors through the graph.
+ * <li>reporting universal errors, such as dependencies from production code
+ * to tests, or to experimental branches.
+ * <li>capturing and replaying errors.
+ * <li>maintaining the graph from one build to the next to
+ * avoid unnecessary recomputation.
+ * <li>checking software licenses.
+ * </ul>
+ *
+ * <p>See also {@link ConfiguredTarget} which documents some important
+ * invariants.
+ */
+public class BuildView {
+
+ /**
+ * Options that affect the <i>mechanism</i> of analysis. These are distinct from {@link
+ * com.google.devtools.build.lib.analysis.config.BuildOptions}, which affect the <i>value</i>
+ * of a BuildConfiguration.
+ */
+ public static class Options extends OptionsBase {
+
+ @Option(name = "keep_going",
+ abbrev = 'k',
+ defaultValue = "false",
+ category = "strategy",
+ help = "Continue as much as possible after an error. While the "
+ + "target that failed, and those that depend on it, cannot be "
+ + "analyzed (or built), the other prerequisites of these "
+ + "targets can be analyzed (or built) all the same.")
+ public boolean keepGoing;
+
+ @Option(name = "analysis_warnings_as_errors",
+ defaultValue = "false",
+ category = "strategy",
+ help = "Treat visible analysis warnings as errors.")
+ public boolean analysisWarningsAsErrors;
+
+ @Option(name = "discard_analysis_cache",
+ defaultValue = "false",
+ category = "strategy",
+ help = "Discard the analysis cache immediately after the analysis phase completes. "
+ + "Reduces memory usage by ~10%, but makes further incremental builds slower.")
+ public boolean discardAnalysisCache;
+
+ @Option(name = "keep_forward_graph",
+ deprecationWarning = "keep_forward_graph is now a no-op and will be removed in an "
+ + "upcoming Blaze release",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Cache the forward action graph across builds for faster "
+ + "incremental rebuilds. May slightly increase memory while Blaze "
+ + "server is idle."
+ )
+ public boolean keepForwardGraph;
+
+ @Option(name = "experimental_extra_action_filter",
+ defaultValue = "",
+ category = "experimental",
+ converter = RegexFilter.RegexFilterConverter.class,
+ help = "Filters set of targets to schedule extra_actions for.")
+ public RegexFilter extraActionFilter;
+
+ @Option(name = "experimental_extra_action_top_level_only",
+ defaultValue = "false",
+ category = "experimental",
+ help = "Only schedules extra_actions for top level targets.")
+ public boolean extraActionTopLevelOnly;
+
+ @Option(name = "version_window_for_dirty_node_gc",
+ defaultValue = "0",
+ category = "undocumented",
+ help = "Nodes that have been dirty for more than this many versions will be deleted"
+ + " from the graph upon the next update. Values must be non-negative long integers,"
+ + " or -1 indicating the maximum possible window.")
+ public long versionWindowForDirtyNodeGc;
+ }
+
+ private static Logger LOG = Logger.getLogger(BuildView.class.getName());
+
+ private final BlazeDirectories directories;
+
+ private final SkyframeExecutor skyframeExecutor;
+ private final SkyframeBuildView skyframeBuildView;
+
+ private final PackageManager packageManager;
+
+ private final BinTools binTools;
+
+ private BuildConfigurationCollection configurations;
+
+ private final ConfiguredRuleClassProvider ruleClassProvider;
+
+ private final ArtifactFactory artifactFactory;
+
+ /**
+ * A factory class to create the coverage report action. May be null.
+ */
+ @Nullable private final CoverageReportActionFactory coverageReportActionFactory;
+
+ /**
+ * A union of package roots of all previous incremental analysis results. This is used to detect
+ * changes of package roots between incremental analysis instances.
+ */
+ private final Map<PackageIdentifier, Path> cumulativePackageRoots = new HashMap<>();
+
+ /**
+ * Used only for testing that we clear Skyframe caches correctly.
+ * TODO(bazel-team): Remove this once we get rid of legacy Skyframe synchronization.
+ */
+ private boolean skyframeCacheWasInvalidated = false;
+
+ /**
+ * If the last build was executed with {@code Options#discard_analysis_cache} and we are not
+ * running Skyframe full, we should clear the legacy data since it is out-of-sync.
+ */
+ private boolean skyframeAnalysisWasDiscarded = false;
+
+ @VisibleForTesting
+ public Set<SkyKey> getSkyframeEvaluatedTargetKeysForTesting() {
+ return skyframeBuildView.getEvaluatedTargetKeys();
+ }
+
+ /** The number of targets freshly evaluated in the last analysis run. */
+ public int getTargetsVisited() {
+ return skyframeBuildView.getEvaluatedTargetKeys().size();
+ }
+
+ /**
+ * Returns true iff Skyframe was invalidated during the analysis phase.
+ * TODO(bazel-team): Remove this once we do not need to keep legacy in sync with Skyframe.
+ */
+ @VisibleForTesting
+ boolean wasSkyframeCacheInvalidatedDuringAnalysis() {
+ return skyframeCacheWasInvalidated;
+ }
+
+ public BuildView(BlazeDirectories directories, PackageManager packageManager,
+ ConfiguredRuleClassProvider ruleClassProvider,
+ SkyframeExecutor skyframeExecutor,
+ BinTools binTools, CoverageReportActionFactory coverageReportActionFactory) {
+ this.directories = directories;
+ this.packageManager = packageManager;
+ this.binTools = binTools;
+ this.coverageReportActionFactory = coverageReportActionFactory;
+ this.artifactFactory = new ArtifactFactory(directories.getExecRoot());
+ this.ruleClassProvider = ruleClassProvider;
+ this.skyframeExecutor = Preconditions.checkNotNull(skyframeExecutor);
+ this.skyframeBuildView =
+ new SkyframeBuildView(
+ new ConfiguredTargetFactory(ruleClassProvider),
+ artifactFactory,
+ skyframeExecutor,
+ new Runnable() {
+ @Override
+ public void run() {
+ clear();
+ }
+ },
+ binTools);
+ skyframeExecutor.setSkyframeBuildView(skyframeBuildView);
+ }
+
+ /** Returns the action graph. */
+ public ActionGraph getActionGraph() {
+ return new ActionGraph() {
+ @Override
+ public Action getGeneratingAction(Artifact artifact) {
+ return skyframeExecutor.getGeneratingAction(artifact);
+ }
+ };
+ }
+
+ /**
+ * Returns whether the given configured target has errors.
+ */
+ @VisibleForTesting
+ public boolean hasErrors(ConfiguredTarget configuredTarget) {
+ return configuredTarget == null;
+ }
+
+ /**
+ * Sets the configurations. Not thread-safe. DO NOT CALL except from tests!
+ */
+ @VisibleForTesting
+ void setConfigurationsForTesting(BuildConfigurationCollection configurations) {
+ this.configurations = configurations;
+ }
+
+ public BuildConfigurationCollection getConfigurationCollection() {
+ return configurations;
+ }
+
+ /**
+ * Clear the graphs of ConfiguredTargets and Artifacts.
+ */
+ @VisibleForTesting
+ public void clear() {
+ cumulativePackageRoots.clear();
+ artifactFactory.clear();
+ }
+
+ public ArtifactFactory getArtifactFactory() {
+ return artifactFactory;
+ }
+
+ @VisibleForTesting
+ WorkspaceStatusAction getLastWorkspaceBuildInfoActionForTesting() {
+ return skyframeExecutor.getLastWorkspaceStatusActionForTesting();
+ }
+
+ /**
+ * Returns a corresponding ConfiguredTarget, if one exists; otherwise throws an {@link
+ * NoSuchConfiguredTargetException}.
+ */
+ @ThreadSafe
+ private ConfiguredTarget getConfiguredTarget(Target target, BuildConfiguration config)
+ throws NoSuchConfiguredTargetException {
+ ConfiguredTarget result =
+ getExistingConfiguredTarget(target.getLabel(), config);
+ if (result == null) {
+ throw new NoSuchConfiguredTargetException(target.getLabel(), config);
+ }
+ return result;
+ }
+
+ /**
+ * Obtains a {@link ConfiguredTarget} given a {@code label}, by delegating
+ * to the package cache and
+ * {@link #getConfiguredTarget(Target, BuildConfiguration)}.
+ */
+ public ConfiguredTarget getConfiguredTarget(Label label, BuildConfiguration config)
+ throws NoSuchPackageException, NoSuchTargetException, NoSuchConfiguredTargetException {
+ return getConfiguredTarget(packageManager.getLoadedTarget(label), config);
+ }
+
+ public Iterable<ConfiguredTarget> getDirectPrerequisites(ConfiguredTarget ct) {
+ return getDirectPrerequisites(ct, null);
+ }
+
+ public Iterable<ConfiguredTarget> getDirectPrerequisites(ConfiguredTarget ct,
+ @Nullable final LoadingCache<Label, Target> targetCache) {
+ if (!(ct.getTarget() instanceof Rule)) {
+ return ImmutableList.of();
+ }
+
+ class SilentDependencyResolver extends DependencyResolver {
+ @Override
+ protected void invalidVisibilityReferenceHook(TargetAndConfiguration node, Label label) {
+ // The error must have been reported already during analysis.
+ }
+
+ @Override
+ protected void invalidPackageGroupReferenceHook(TargetAndConfiguration node, Label label) {
+ // The error must have been reported already during analysis.
+ }
+
+ @Override
+ protected Target getTarget(Label label) throws NoSuchThingException {
+ if (targetCache == null) {
+ return packageManager.getLoadedTarget(label);
+ }
+
+ try {
+ return targetCache.get(label);
+ } catch (ExecutionException e) {
+ // All lookups should succeed because we should not be looking up any targets in error.
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
+ DependencyResolver dependencyResolver = new SilentDependencyResolver();
+ TargetAndConfiguration ctgNode =
+ new TargetAndConfiguration(ct.getTarget(), ct.getConfiguration());
+ return skyframeExecutor.getConfiguredTargets(
+ dependencyResolver.dependentNodes(ctgNode, getConfigurableAttributeKeys(ctgNode)));
+ }
+
+ /**
+ * Returns ConfigMatchingProvider instances corresponding to the configurable attribute keys
+ * present in this rule's attributes.
+ */
+ private Set<ConfigMatchingProvider> getConfigurableAttributeKeys(TargetAndConfiguration ctg) {
+ if (!(ctg.getTarget() instanceof Rule)) {
+ return ImmutableSet.of();
+ }
+ Rule rule = (Rule) ctg.getTarget();
+ ImmutableSet.Builder<ConfigMatchingProvider> keys = ImmutableSet.builder();
+ RawAttributeMapper mapper = RawAttributeMapper.of(rule);
+ for (Attribute attribute : rule.getAttributes()) {
+ for (Label label : mapper.getConfigurabilityKeys(attribute.getName(), attribute.getType())) {
+ if (Type.Selector.isReservedLabel(label)) {
+ continue;
+ }
+ try {
+ ConfiguredTarget ct = getConfiguredTarget(label, ctg.getConfiguration());
+ keys.add(Preconditions.checkNotNull(ct.getProvider(ConfigMatchingProvider.class)));
+ } catch (NoSuchPackageException e) {
+ // All lookups should succeed because we should not be looking up any targets in error.
+ throw new IllegalStateException(e);
+ } catch (NoSuchTargetException e) {
+ // All lookups should succeed because we should not be looking up any targets in error.
+ throw new IllegalStateException(e);
+ } catch (NoSuchConfiguredTargetException e) {
+ // All lookups should succeed because we should not be looking up any targets in error.
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+ return keys.build();
+ }
+
+ public TransitiveInfoCollection getGeneratingRule(OutputFileConfiguredTarget target) {
+ return target.getGeneratingRule();
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException(); // avoid nondeterminism
+ }
+
+ /**
+ * Return value for {@link BuildView#update} and {@code BuildTool.prepareToBuild}.
+ */
+ public static final class AnalysisResult {
+
+ public static final AnalysisResult EMPTY = new AnalysisResult(
+ ImmutableList.<ConfiguredTarget>of(), null, null, null,
+ ImmutableList.<Artifact>of(),
+ ImmutableList.<ConfiguredTarget>of(),
+ ImmutableList.<ConfiguredTarget>of(),
+ null);
+
+ private final ImmutableList<ConfiguredTarget> targetsToBuild;
+ @Nullable private final ImmutableList<ConfiguredTarget> targetsToTest;
+ @Nullable private final String error;
+ private final ActionGraph actionGraph;
+ private final ImmutableSet<Artifact> artifactsToBuild;
+ private final ImmutableSet<ConfiguredTarget> parallelTests;
+ private final ImmutableSet<ConfiguredTarget> exclusiveTests;
+ @Nullable private final TopLevelArtifactContext topLevelContext;
+
+ private AnalysisResult(
+ Collection<ConfiguredTarget> targetsToBuild, Collection<ConfiguredTarget> targetsToTest,
+ @Nullable String error, ActionGraph actionGraph,
+ Collection<Artifact> artifactsToBuild, Collection<ConfiguredTarget> parallelTests,
+ Collection<ConfiguredTarget> exclusiveTests, TopLevelArtifactContext topLevelContext) {
+ this.targetsToBuild = ImmutableList.copyOf(targetsToBuild);
+ this.targetsToTest = targetsToTest == null ? null : ImmutableList.copyOf(targetsToTest);
+ this.error = error;
+ this.actionGraph = actionGraph;
+ this.artifactsToBuild = ImmutableSet.copyOf(artifactsToBuild);
+ this.parallelTests = ImmutableSet.copyOf(parallelTests);
+ this.exclusiveTests = ImmutableSet.copyOf(exclusiveTests);
+ this.topLevelContext = topLevelContext;
+ }
+
+ /**
+ * Returns configured targets to build.
+ */
+ public Collection<ConfiguredTarget> getTargetsToBuild() {
+ return targetsToBuild;
+ }
+
+ /**
+ * Returns the configured targets to run as tests, or {@code null} if testing was not
+ * requested (e.g. "build" command rather than "test" command).
+ */
+ @Nullable
+ public Collection<ConfiguredTarget> getTargetsToTest() {
+ return targetsToTest;
+ }
+
+ public ImmutableSet<Artifact> getAdditionalArtifactsToBuild() {
+ return artifactsToBuild;
+ }
+
+ public ImmutableSet<ConfiguredTarget> getExclusiveTests() {
+ return exclusiveTests;
+ }
+
+ public ImmutableSet<ConfiguredTarget> getParallelTests() {
+ return parallelTests;
+ }
+
+ /**
+ * Returns an error description (if any).
+ */
+ @Nullable public String getError() {
+ return error;
+ }
+
+ /**
+ * Returns the action graph.
+ */
+ public ActionGraph getActionGraph() {
+ return actionGraph;
+ }
+
+ public TopLevelArtifactContext getTopLevelContext() {
+ return topLevelContext;
+ }
+ }
+
+
+ /**
+ * Returns the collection of configured targets corresponding to any of the provided targets.
+ */
+ @VisibleForTesting
+ static Iterable<? extends ConfiguredTarget> filterTestsByTargets(
+ Collection<? extends ConfiguredTarget> targets,
+ final Set<? extends Target> allowedTargets) {
+ return Iterables.filter(targets,
+ new Predicate<ConfiguredTarget>() {
+ @Override
+ public boolean apply(ConfiguredTarget rule) {
+ return allowedTargets.contains(rule.getTarget());
+ }
+ });
+ }
+
+ private void prepareToBuild(PackageRootResolver resolver) throws ViewCreationFailedException {
+ for (BuildConfiguration config : configurations.getTargetConfigurations()) {
+ config.prepareToBuild(directories.getExecRoot(), getArtifactFactory(), resolver);
+ }
+ }
+
+ @ThreadCompatible
+ public AnalysisResult update(LoadingResult loadingResult,
+ BuildConfigurationCollection configurations, Options viewOptions,
+ TopLevelArtifactContext topLevelOptions, EventHandler eventHandler, EventBus eventBus)
+ throws ViewCreationFailedException, InterruptedException {
+
+ // Detect errors during analysis and don't attempt a build.
+ //
+ // (Errors reported during the previous step, package loading, that do
+ // not cause the visitation of the transitive closure to abort, are
+ // recoverable. For example, an error encountered while evaluating an
+ // irrelevant rule in a visited package causes an error to be reported,
+ // but visitation still succeeds.)
+ ErrorCollector errorCollector = null;
+ if (!viewOptions.keepGoing) {
+ eventHandler = errorCollector = new ErrorCollector(eventHandler);
+ }
+
+ // Treat analysis warnings as errors, to enable strict builds.
+ //
+ // Warnings reported during analysis are converted to errors, ultimately
+ // triggering failure. This check needs to be added after the keep-going check
+ // above so that it is invoked first (FIFO eventHandler chain). This way, detected
+ // warnings are converted to errors first, and then the proper error handling
+ // logic is invoked.
+ WarningsAsErrorsEventHandler warningsHandler = null;
+ if (viewOptions.analysisWarningsAsErrors) {
+ eventHandler = warningsHandler = new WarningsAsErrorsEventHandler(eventHandler);
+ }
+
+ skyframeBuildView.setWarningListener(eventHandler);
+ skyframeExecutor.setErrorEventListener(eventHandler);
+
+ LOG.info("Starting analysis");
+ pollInterruptedStatus();
+
+ skyframeBuildView.resetEvaluatedConfiguredTargetKeysSet();
+
+ Collection<Target> targets = loadingResult.getTargets();
+ eventBus.post(new AnalysisPhaseStartedEvent(targets));
+
+ skyframeCacheWasInvalidated = false;
+ // Clear all cached ConfiguredTargets on configuration change. We need to do this explicitly
+ // because we need to make sure that the legacy action graph does not contain multiple actions
+ // with different versions of the same (target/host/etc.) configuration.
+ // In the future the action graph will be probably be keyed by configurations, which should
+ // obviate the need for this workaround.
+ //
+ // Also if --discard_analysis_cache was used in the last build we want to clear the legacy
+ // data.
+ if ((this.configurations != null && !configurations.equals(this.configurations))
+ || skyframeAnalysisWasDiscarded) {
+ skyframeExecutor.dropConfiguredTargets();
+ skyframeCacheWasInvalidated = true;
+ clear();
+ }
+ skyframeAnalysisWasDiscarded = false;
+ ImmutableMap<PackageIdentifier, Path> packageRoots = loadingResult.getPackageRoots();
+
+ if (buildHasIncompatiblePackageRoots(packageRoots)) {
+ // When a package root changes source artifacts with the new root will be created, but we
+ // cannot be sure that there are no references remaining to the corresponding artifacts
+ // with the old root. To avoid that scenario, the analysis cache is simply dropped when
+ // a package root change is detected.
+ LOG.info("Discarding analysis cache: package roots have changed.");
+
+ skyframeExecutor.dropConfiguredTargets();
+ skyframeCacheWasInvalidated = true;
+ clear();
+ }
+ cumulativePackageRoots.putAll(packageRoots);
+ this.configurations = configurations;
+ setArtifactRoots(packageRoots);
+
+ // Determine the configurations.
+ List<TargetAndConfiguration> nodes = nodesForTargets(targets);
+
+ List<ConfiguredTargetKey> targetSpecs =
+ Lists.transform(nodes, new Function<TargetAndConfiguration, ConfiguredTargetKey>() {
+ @Override
+ public ConfiguredTargetKey apply(TargetAndConfiguration node) {
+ return new ConfiguredTargetKey(node.getLabel(), node.getConfiguration());
+ }
+ });
+
+ prepareToBuild(new SkyframePackageRootResolver(skyframeExecutor));
+ skyframeBuildView.setWarningListener(warningsHandler);
+ skyframeExecutor.injectWorkspaceStatusData();
+ Collection<ConfiguredTarget> configuredTargets;
+ try {
+ configuredTargets = skyframeBuildView.configureTargets(
+ targetSpecs, eventBus, viewOptions.keepGoing);
+ } finally {
+ skyframeBuildView.clearInvalidatedConfiguredTargets();
+ }
+
+ int numTargetsToAnalyze = nodes.size();
+ int numSuccessful = configuredTargets.size();
+ boolean analysisSuccessful = (numSuccessful == numTargetsToAnalyze);
+ if (0 < numSuccessful && numSuccessful < numTargetsToAnalyze) {
+ String msg = String.format("Analysis succeeded for only %d of %d top-level targets",
+ numSuccessful, numTargetsToAnalyze);
+ eventHandler.handle(Event.info(msg));
+ LOG.info(msg);
+ }
+
+ postUpdateValidation(errorCollector, warningsHandler);
+
+ AnalysisResult result = createResult(loadingResult, topLevelOptions,
+ viewOptions, configuredTargets, analysisSuccessful);
+ LOG.info("Finished analysis");
+ return result;
+ }
+
+ // Validates that the update has been done correctly
+ private void postUpdateValidation(ErrorCollector errorCollector,
+ WarningsAsErrorsEventHandler warningsHandler) throws ViewCreationFailedException {
+ if (warningsHandler != null && warningsHandler.warningsEncountered()) {
+ throw new ViewCreationFailedException("Warnings being treated as errors");
+ }
+
+ if (errorCollector != null && !errorCollector.getEvents().isEmpty()) {
+ // This assertion ensures that if any errors were reported during the
+ // initialization phase, the call to configureTargets will fail with a
+ // ViewCreationFailedException. Violation of this invariant leads to
+ // incorrect builds, because the fact that errors were encountered is not
+ // properly recorded in the view (i.e. the graph of configured targets).
+ // Rule errors must be reported via RuleConfiguredTarget.reportError,
+ // which causes the rule's hasErrors() flag to be set, and thus the
+ // hasErrors() flag of anything that depends on it transitively. If the
+ // toplevel rule hasErrors, then analysis is aborted and we do not
+ // proceed to the execution phase of a build.
+ //
+ // Reporting errors directly through the Reporter does not set the error
+ // flag, so analysis may succeed spuriously, allowing the execution
+ // phase to begin with unpredictable consequences.
+ //
+ // The use of errorCollector (rather than an ErrorSensor) makes the
+ // assertion failure messages more informative.
+ // Note we tolerate errors iff --keep-going, because some of the
+ // requested targets may have had problems during analysis, but that's ok.
+ StringBuilder message = new StringBuilder("Unexpected errors reported during analysis:");
+ for (Event event : errorCollector.getEvents()) {
+ message.append('\n').append(event);
+ }
+ throw new IllegalStateException(message.toString());
+ }
+ }
+
+ /**
+ * Skyframe implementation of {@link PackageRootResolver}.
+ *
+ * <p> Note: you should not use this class inside of any SkyFunction.
+ */
+ @VisibleForTesting
+ public static final class SkyframePackageRootResolver implements PackageRootResolver {
+ private final SkyframeExecutor executor;
+
+ public SkyframePackageRootResolver(SkyframeExecutor executor) {
+ this.executor = executor;
+ }
+
+ @Override
+ public Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths) {
+ return executor.getArtifactRoots(execPaths);
+ }
+ }
+
+ private AnalysisResult createResult(LoadingResult loadingResult,
+ TopLevelArtifactContext topLevelOptions, BuildView.Options viewOptions,
+ Collection<ConfiguredTarget> configuredTargets, boolean analysisSuccessful)
+ throws InterruptedException {
+ Collection<Target> testsToRun = loadingResult.getTestsToRun();
+ Collection<ConfiguredTarget> allTargetsToTest = null;
+ if (testsToRun != null) {
+ // Determine the subset of configured targets that are meant to be run as tests.
+ allTargetsToTest = Lists.newArrayList(
+ filterTestsByTargets(configuredTargets, Sets.newHashSet(testsToRun)));
+ }
+
+ skyframeExecutor.injectTopLevelContext(topLevelOptions);
+
+ Set<Artifact> artifactsToBuild = new HashSet<>();
+ Set<ConfiguredTarget> parallelTests = new HashSet<>();
+ Set<ConfiguredTarget> exclusiveTests = new HashSet<>();
+ Collection<Artifact> buildInfoArtifacts;
+ buildInfoArtifacts = skyframeExecutor.getWorkspaceStatusArtifacts();
+ // build-info and build-changelist.
+ Preconditions.checkState(buildInfoArtifacts.size() == 2, buildInfoArtifacts);
+ artifactsToBuild.addAll(buildInfoArtifacts);
+ addExtraActionsIfRequested(viewOptions, artifactsToBuild, configuredTargets);
+ if (coverageReportActionFactory != null) {
+ Action action = coverageReportActionFactory.createCoverageReportAction(
+ allTargetsToTest,
+ getBaselineCoverageArtifacts(configuredTargets),
+ artifactFactory,
+ CoverageReportValue.ARTIFACT_OWNER);
+ if (action != null) {
+ skyframeExecutor.injectCoverageReportData(action);
+ artifactsToBuild.addAll(action.getOutputs());
+ }
+ }
+
+ // Note that this must come last, so that the tests are scheduled after all artifacts are built.
+ scheduleTestsIfRequested(parallelTests, exclusiveTests, topLevelOptions, allTargetsToTest);
+
+ String error = !loadingResult.hasLoadingError()
+ ? (analysisSuccessful
+ ? null
+ : "execution phase succeeded, but not all targets were analyzed")
+ : "execution phase succeeded, but there were loading phase errors";
+ return new AnalysisResult(configuredTargets, allTargetsToTest, error, getActionGraph(),
+ artifactsToBuild, parallelTests, exclusiveTests, topLevelOptions);
+ }
+
+ private static ImmutableSet<Artifact> getBaselineCoverageArtifacts(
+ Collection<ConfiguredTarget> configuredTargets) {
+ Set<Artifact> baselineCoverageArtifacts = Sets.newHashSet();
+ for (ConfiguredTarget target : configuredTargets) {
+ BaselineCoverageArtifactsProvider provider =
+ target.getProvider(BaselineCoverageArtifactsProvider.class);
+ if (provider != null) {
+ baselineCoverageArtifacts.addAll(provider.getBaselineCoverageArtifacts());
+ }
+ }
+ return ImmutableSet.copyOf(baselineCoverageArtifacts);
+ }
+
+ private void addExtraActionsIfRequested(BuildView.Options viewOptions,
+ Set<Artifact> artifactsToBuild, Iterable<ConfiguredTarget> topLevelTargets) {
+ NestedSetBuilder<ExtraArtifactSet> builder = NestedSetBuilder.stableOrder();
+ for (ConfiguredTarget topLevel : topLevelTargets) {
+ ExtraActionArtifactsProvider provider = topLevel.getProvider(
+ ExtraActionArtifactsProvider.class);
+ if (provider != null) {
+ if (viewOptions.extraActionTopLevelOnly) {
+ builder.add(ExtraArtifactSet.of(topLevel.getLabel(), provider.getExtraActionArtifacts()));
+ } else {
+ builder.addTransitive(provider.getTransitiveExtraActionArtifacts());
+ }
+ }
+ }
+
+ RegexFilter filter = viewOptions.extraActionFilter;
+ for (ExtraArtifactSet set : builder.build()) {
+ boolean filterMatches = filter == null || filter.isIncluded(set.getLabel().toString());
+ if (filterMatches) {
+ Iterables.addAll(artifactsToBuild, set.getArtifacts());
+ }
+ }
+ }
+
+ private static void scheduleTestsIfRequested(Collection<ConfiguredTarget> targetsToTest,
+ Collection<ConfiguredTarget> targetsToTestExclusive, TopLevelArtifactContext topLevelOptions,
+ Collection<ConfiguredTarget> allTestTargets) {
+ if (!topLevelOptions.compileOnly() && !topLevelOptions.compilationPrerequisitesOnly()
+ && allTestTargets != null) {
+ scheduleTests(targetsToTest, targetsToTestExclusive, allTestTargets,
+ topLevelOptions.runTestsExclusively());
+ }
+ }
+
+
+ /**
+ * Returns set of artifacts representing test results, writing into targetsToTest and
+ * targetsToTestExclusive.
+ */
+ private static void scheduleTests(Collection<ConfiguredTarget> targetsToTest,
+ Collection<ConfiguredTarget> targetsToTestExclusive,
+ Collection<ConfiguredTarget> allTestTargets,
+ boolean isExclusive) {
+ for (ConfiguredTarget target : allTestTargets) {
+ if (target.getTarget() instanceof Rule) {
+ boolean exclusive =
+ isExclusive || TargetUtils.isExclusiveTestRule((Rule) target.getTarget());
+ Collection<ConfiguredTarget> testCollection = exclusive
+ ? targetsToTestExclusive
+ : targetsToTest;
+ testCollection.add(target);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ List<TargetAndConfiguration> nodesForTargets(Collection<Target> targets) {
+ // We use a hash set here to remove duplicate nodes; this can happen for input files and package
+ // groups.
+ LinkedHashSet<TargetAndConfiguration> nodes = new LinkedHashSet<>(targets.size());
+ for (BuildConfiguration config : configurations.getTargetConfigurations()) {
+ for (Target target : targets) {
+ nodes.add(new TargetAndConfiguration(target,
+ BuildConfigurationCollection.configureTopLevelTarget(config, target)));
+ }
+ }
+ return ImmutableList.copyOf(nodes);
+ }
+
+ /**
+ * Detects when a package root changes between instances of incremental analysis.
+ *
+ * <p>This case is currently problematic for incremental analysis because when a package root
+ * changes, source artifacts with the new root will be created, but we can not be sure that there
+ * are no references remaining to the corresponding artifacts with the old root.
+ */
+ private boolean buildHasIncompatiblePackageRoots(Map<PackageIdentifier, Path> packageRoots) {
+ for (Map.Entry<PackageIdentifier, Path> entry : packageRoots.entrySet()) {
+ Path prevRoot = cumulativePackageRoots.get(entry.getKey());
+ if (prevRoot != null && !entry.getValue().equals(prevRoot)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns an existing ConfiguredTarget for the specified target and
+ * configuration, or null if none exists. No validity check is done.
+ */
+ @ThreadSafe
+ public ConfiguredTarget getExistingConfiguredTarget(Target target, BuildConfiguration config) {
+ return getExistingConfiguredTarget(target.getLabel(), config);
+ }
+
+ /**
+ * Returns an existing ConfiguredTarget for the specified node, or null if none exists. No
+ * validity check is done.
+ */
+ @ThreadSafe
+ private ConfiguredTarget getExistingConfiguredTarget(
+ Label label, BuildConfiguration configuration) {
+ return Iterables.getFirst(
+ skyframeExecutor.getConfiguredTargets(
+ ImmutableList.of(new Dependency(label, configuration))),
+ null);
+ }
+
+ @VisibleForTesting
+ ListMultimap<Attribute, ConfiguredTarget> getPrerequisiteMapForTesting(ConfiguredTarget target) {
+ DependencyResolver resolver = new DependencyResolver() {
+ @Override
+ protected void invalidVisibilityReferenceHook(TargetAndConfiguration node, Label label) {
+ throw new RuntimeException("bad visibility on " + label + " during testing unexpected");
+ }
+
+ @Override
+ protected void invalidPackageGroupReferenceHook(TargetAndConfiguration node, Label label) {
+ throw new RuntimeException("bad package group on " + label + " during testing unexpected");
+ }
+
+ @Override
+ protected Target getTarget(Label label) throws NoSuchThingException {
+ return packageManager.getLoadedTarget(label);
+ }
+ };
+ TargetAndConfiguration ctNode = new TargetAndConfiguration(target);
+ ListMultimap<Attribute, Dependency> depNodeNames;
+ try {
+ depNodeNames = resolver.dependentNodeMap(ctNode, null, getConfigurableAttributeKeys(ctNode));
+ } catch (EvalException e) {
+ throw new IllegalStateException(e);
+ }
+
+ final Map<LabelAndConfiguration, ConfiguredTarget> depMap = new HashMap<>();
+ for (ConfiguredTarget dep : skyframeExecutor.getConfiguredTargets(depNodeNames.values())) {
+ depMap.put(LabelAndConfiguration.of(dep.getLabel(), dep.getConfiguration()), dep);
+ }
+
+ return Multimaps.transformValues(depNodeNames, new Function<Dependency, ConfiguredTarget>() {
+ @Override
+ public ConfiguredTarget apply(Dependency depName) {
+ return depMap.get(LabelAndConfiguration.of(depName.getLabel(),
+ depName.getConfiguration()));
+ }
+ });
+ }
+
+ /**
+ * Sets the possible artifact roots in the artifact factory. This allows the
+ * factory to resolve paths with unknown roots to artifacts.
+ * <p>
+ * <em>Note: This must be called before any call to
+ * {@link #getConfiguredTarget(Label, BuildConfiguration)}
+ * </em>
+ */
+ @VisibleForTesting // for BuildViewTestCase
+ void setArtifactRoots(ImmutableMap<PackageIdentifier, Path> packageRoots) {
+ Map<Path, Root> rootMap = new HashMap<>();
+ Map<PackageIdentifier, Root> realPackageRoots = new HashMap<>();
+ for (Map.Entry<PackageIdentifier, Path> entry : packageRoots.entrySet()) {
+ Root root = rootMap.get(entry.getValue());
+ if (root == null) {
+ root = Root.asSourceRoot(entry.getValue());
+ rootMap.put(entry.getValue(), root);
+ }
+ realPackageRoots.put(entry.getKey(), root);
+ }
+ // Source Artifact roots:
+ artifactFactory.setPackageRoots(realPackageRoots);
+
+ // Derived Artifact roots:
+ ImmutableList.Builder<Root> roots = ImmutableList.builder();
+
+ // build-info.txt and friends; this root is not configuration specific.
+ roots.add(directories.getBuildDataDirectory());
+
+ // The roots for each configuration - duplicates are automatically removed in the call below.
+ for (BuildConfiguration cfg : configurations.getAllConfigurations()) {
+ roots.addAll(cfg.getRoots());
+ }
+
+ artifactFactory.setDerivedArtifactRoots(roots.build());
+ }
+
+ /**
+ * Returns a configured target for the specified target and configuration.
+ * This should only be called from test cases, and is needed, because
+ * plain {@link #getConfiguredTarget(Target, BuildConfiguration)} does not
+ * construct the configured target graph, and would thus fail if called from
+ * outside an update.
+ */
+ @VisibleForTesting
+ public ConfiguredTarget getConfiguredTargetForTesting(Label label, BuildConfiguration config)
+ throws NoSuchPackageException, NoSuchTargetException {
+ return getConfiguredTargetForTesting(packageManager.getLoadedTarget(label), config);
+ }
+
+ @VisibleForTesting
+ public ConfiguredTarget getConfiguredTargetForTesting(Target target, BuildConfiguration config) {
+ return skyframeExecutor.getConfiguredTargetForTesting(target.getLabel(), config);
+ }
+
+ /**
+ * Returns a RuleContext which is the same as the original RuleContext of the target parameter.
+ */
+ @VisibleForTesting
+ public RuleContext getRuleContextForTesting(ConfiguredTarget target,
+ StoredEventHandler eventHandler) {
+ BuildConfiguration config = target.getConfiguration();
+ CachingAnalysisEnvironment analysisEnvironment =
+ new CachingAnalysisEnvironment(artifactFactory,
+ new ConfiguredTargetKey(target.getLabel(), config),
+ /*isSystemEnv=*/false, config.extendedSanityChecks(), eventHandler,
+ /*skyframeEnv=*/null, config.isActionsEnabled(), binTools);
+ RuleContext ruleContext = new RuleContext.Builder(analysisEnvironment,
+ (Rule) target.getTarget(), config, ruleClassProvider.getPrerequisiteValidator())
+ .setVisibility(NestedSetBuilder.<PackageSpecification>create(
+ Order.STABLE_ORDER, PackageSpecification.EVERYTHING))
+ .setPrerequisites(getPrerequisiteMapForTesting(target))
+ .setConfigConditions(ImmutableSet.<ConfigMatchingProvider>of())
+ .build();
+ return ruleContext;
+ }
+
+ /**
+ * Tests and clears the current thread's pending "interrupted" status, and
+ * throws InterruptedException iff it was set.
+ */
+ protected final void pollInterruptedStatus() throws InterruptedException {
+ if (Thread.interrupted()) {
+ throw new InterruptedException();
+ }
+ }
+
+ /**
+ * Drops the analysis cache. If building with Skyframe, targets in {@code topLevelTargets} may
+ * remain in the cache for use during the execution phase.
+ *
+ * @see BuildView.Options#discardAnalysisCache
+ */
+ public void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) {
+ // TODO(bazel-team): Consider clearing packages too to save more memory.
+ skyframeAnalysisWasDiscarded = true;
+ skyframeExecutor.clearAnalysisCache(topLevelTargets);
+ }
+
+ /********************************************************************
+ * *
+ * 'blaze dump' related functions *
+ * *
+ ********************************************************************/
+
+ /**
+ * Collects and stores error events while also forwarding them to another eventHandler.
+ */
+ public static class ErrorCollector extends DelegatingEventHandler {
+ private final List<Event> events;
+
+ public ErrorCollector(EventHandler delegate) {
+ super(delegate);
+ this.events = Lists.newArrayList();
+ }
+
+ public List<Event> getEvents() {
+ return events;
+ }
+
+ @Override
+ public void handle(Event e) {
+ super.handle(e);
+ if (e.getKind() == EventKind.ERROR) {
+ events.add(e);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
new file mode 100644
index 0000000..bc45ba3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
@@ -0,0 +1,303 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.MiddlemanFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue;
+import com.google.devtools.build.lib.skyframe.WorkspaceStatusValue;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * The implementation of AnalysisEnvironment used for analysis. It tracks metadata for each
+ * configured target, such as the errors and warnings emitted by that target. It is intended that
+ * a separate instance is used for each configured target, so that these don't mix up.
+ */
+public class CachingAnalysisEnvironment implements AnalysisEnvironment {
+ private final ArtifactFactory artifactFactory;
+
+ private final ArtifactOwner owner;
+ /**
+ * If this is the system analysis environment, then errors and warnings are directly reported
+ * to the global reporter, rather than stored, i.e., we don't track here whether there are any
+ * errors.
+ */
+ private final boolean isSystemEnv;
+ private final boolean extendedSanityChecks;
+
+ /**
+ * If false, no actions will be registered, they'll all be just dropped.
+ *
+ * <p>Usually, an analysis environment should register all actions. However, in some scenarios we
+ * analyze some targets twice, but the first one only serves the purpose of collecting information
+ * for the second analysis. In this case we don't register actions created by the first pass in
+ * order to avoid action conflicts.
+ */
+ private final boolean allowRegisteringActions;
+
+ private boolean enabled = true;
+ private MiddlemanFactory middlemanFactory;
+ private EventHandler errorEventListener;
+ private SkyFunction.Environment skyframeEnv;
+ private Map<Artifact, String> artifacts;
+ private final BinTools binTools;
+
+ /**
+ * The list of actions registered by the configured target this analysis environment is
+ * responsible for. May get cleared out at the end of the analysis of said target.
+ */
+ final List<Action> actions = new ArrayList<>();
+
+ public CachingAnalysisEnvironment(ArtifactFactory artifactFactory,
+ ArtifactOwner owner, boolean isSystemEnv, boolean extendedSanityChecks,
+ EventHandler errorEventListener, SkyFunction.Environment env, boolean allowRegisteringActions,
+ BinTools binTools) {
+ this.artifactFactory = artifactFactory;
+ this.owner = Preconditions.checkNotNull(owner);
+ this.isSystemEnv = isSystemEnv;
+ this.extendedSanityChecks = extendedSanityChecks;
+ this.errorEventListener = errorEventListener;
+ this.skyframeEnv = env;
+ this.allowRegisteringActions = allowRegisteringActions;
+ this.binTools = binTools;
+ middlemanFactory = new MiddlemanFactory(artifactFactory, this);
+ artifacts = new HashMap<>();
+ }
+
+ public void disable(Target target) {
+ if (!hasErrors() && allowRegisteringActions) {
+ verifyGeneratedArtifactHaveActions(target);
+ }
+ artifacts = null;
+ middlemanFactory = null;
+ enabled = false;
+ errorEventListener = null;
+ skyframeEnv = null;
+ }
+
+ private static StringBuilder shortDescription(Action action) {
+ if (action == null) {
+ return new StringBuilder("null Action");
+ }
+ return new StringBuilder()
+ .append(action.getClass().getName())
+ .append(' ')
+ .append(action.getMnemonic());
+ }
+
+ /**
+ * Sanity checks that all generated artifacts have a generating action.
+ * @param target for error reporting
+ */
+ public void verifyGeneratedArtifactHaveActions(Target target) {
+ Collection<String> orphanArtifacts = getOrphanArtifactMap().values();
+ List<String> checkedActions = null;
+ if (!orphanArtifacts.isEmpty()) {
+ checkedActions = Lists.newArrayListWithCapacity(actions.size());
+ for (Action action : actions) {
+ StringBuilder sb = shortDescription(action);
+ for (Artifact o : action.getOutputs()) {
+ sb.append("\n ");
+ sb.append(o.getExecPathString());
+ }
+ checkedActions.add(sb.toString());
+ }
+ throw new IllegalStateException(
+ String.format(
+ "%s %s : These artifacts miss a generating action:\n%s\n"
+ + "These actions we checked:\n%s\n",
+ target.getTargetKind(), target.getLabel(),
+ Joiner.on('\n').join(orphanArtifacts), Joiner.on('\n').join(checkedActions)));
+ }
+ }
+
+ @Override
+ public ImmutableSet<Artifact> getOrphanArtifacts() {
+ return ImmutableSet.copyOf(getOrphanArtifactMap().keySet());
+ }
+
+ private Map<Artifact, String> getOrphanArtifactMap() {
+ // Construct this set to avoid poor performance under large --runs_per_test.
+ Set<Artifact> artifactsWithActions = new HashSet<>();
+ for (Action action : actions) {
+ // Don't bother checking that every Artifact only appears once; that test is performed
+ // elsewhere (see #testNonUniqueOutputs in ActionListenerIntegrationTest).
+ artifactsWithActions.addAll(action.getOutputs());
+ }
+ // The order of the artifacts.entrySet iteration is unspecified - we use a TreeMap here to
+ // guarantee that the return value of this method is deterministic.
+ Map<Artifact, String> orphanArtifacts = new TreeMap<>();
+ for (Map.Entry<Artifact, String> entry : artifacts.entrySet()) {
+ Artifact a = entry.getKey();
+ if (!a.isSourceArtifact() && !artifactsWithActions.contains(a)) {
+ orphanArtifacts.put(a, String.format("%s\n%s",
+ a.getExecPathString(), // uncovered artifact
+ entry.getValue())); // origin of creation
+ }
+ }
+ return orphanArtifacts;
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ return errorEventListener;
+ }
+
+ @Override
+ public boolean hasErrors() {
+ // The system analysis environment never has errors.
+ if (isSystemEnv) {
+ return false;
+ }
+ Preconditions.checkState(enabled);
+ return ((StoredEventHandler) errorEventListener).hasErrors();
+ }
+
+ @Override
+ public MiddlemanFactory getMiddlemanFactory() {
+ Preconditions.checkState(enabled);
+ return middlemanFactory;
+ }
+
+ /**
+ * Keeps track of artifacts. We check that all of them have an owner when the environment is
+ * sealed (disable()). For performance reasons we only track the originating stacktrace when
+ * running with --experimental_extended_sanity_checks.
+ */
+ private Artifact trackArtifactAndOrigin(Artifact a, @Nullable Throwable e) {
+ if ((e != null) && !artifacts.containsKey(a)) {
+ StringWriter sw = new StringWriter();
+ e.printStackTrace(new PrintWriter(sw));
+ artifacts.put(a, sw.toString());
+ } else {
+ artifacts.put(a, "No origin, run with --experimental_extended_sanity_checks");
+ }
+ return a;
+ }
+
+ @Override
+ public Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root) {
+ Preconditions.checkState(enabled);
+ return trackArtifactAndOrigin(
+ artifactFactory.getDerivedArtifact(rootRelativePath, root, getOwner()),
+ extendedSanityChecks ? new Throwable() : null);
+ }
+
+ @Override
+ public Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root) {
+ Preconditions.checkState(enabled);
+ return trackArtifactAndOrigin(
+ artifactFactory.getFilesetArtifact(rootRelativePath, root, getOwner()),
+ extendedSanityChecks ? new Throwable() : null);
+ }
+
+ @Override
+ public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root) {
+ return artifactFactory.getConstantMetadataArtifact(rootRelativePath, root, getOwner());
+ }
+
+ @Override
+ public Artifact getEmbeddedToolArtifact(String embeddedPath) {
+ Preconditions.checkState(enabled);
+ return binTools.getEmbeddedArtifact(embeddedPath, artifactFactory);
+ }
+
+ @Override
+ public void registerAction(Action... actions) {
+ Preconditions.checkState(enabled);
+ if (allowRegisteringActions) {
+ for (Action action : actions) {
+ this.actions.add(action);
+ }
+ }
+ }
+
+ @Override
+ public Action getLocalGeneratingAction(Artifact artifact) {
+ Preconditions.checkState(allowRegisteringActions);
+ for (Action action : actions) {
+ if (action.getOutputs().contains(artifact)) {
+ return action;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Collection<Action> getRegisteredActions() {
+ return Collections.unmodifiableCollection(actions);
+ }
+
+ @Override
+ public SkyFunction.Environment getSkyframeEnv() {
+ return skyframeEnv;
+ }
+
+ @Override
+ public Artifact getStableWorkspaceStatusArtifact() {
+ return ((WorkspaceStatusValue) skyframeEnv.getValue(WorkspaceStatusValue.SKY_KEY))
+ .getStableArtifact();
+ }
+
+ @Override
+ public Artifact getVolatileWorkspaceStatusArtifact() {
+ return ((WorkspaceStatusValue) skyframeEnv.getValue(WorkspaceStatusValue.SKY_KEY))
+ .getVolatileArtifact();
+ }
+
+ @Override
+ public ImmutableList<Artifact> getBuildInfo(RuleContext ruleContext, BuildInfoKey key) {
+ boolean stamp = AnalysisUtils.isStampingEnabled(ruleContext);
+ BuildInfoCollection collection =
+ ((BuildInfoCollectionValue) skyframeEnv.getValue(BuildInfoCollectionValue.key(
+ new BuildInfoCollectionValue.BuildInfoKeyAndConfig(key, ruleContext.getConfiguration()))))
+ .getCollection();
+ return stamp ? collection.getStampedBuildInfo() : collection.getRedactedBuildInfo();
+ }
+
+ @Override
+ public ArtifactOwner getOwner() {
+ return owner;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java
new file mode 100644
index 0000000..2c05f0f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java
@@ -0,0 +1,307 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Provides shared functionality for parameterized command-line launching
+ * e.g. {@link com.google.devtools.build.lib.view.genrule.GenRule}
+ * Also used by {@link com.google.devtools.build.lib.rules.extra.ExtraActionFactory}.
+ *
+ * Two largely independent separate sets of functionality are provided:
+ * 1- string interpolation for {@code $(location[s] ...)} and {@code $(MakeVariable)}
+ * 2- a utility to build potentially large command lines (presumably made of multiple commands),
+ * that if presumed too large for the kernel's taste can be dumped into a shell script
+ * that will contain the same commands,
+ * at which point the shell script is added to the list of inputs.
+ */
+@SkylarkModule(name = "command_helper", doc = "A helper class to create shell commands.")
+public final class CommandHelper {
+
+ /**
+ * Maximum total command-line length, in bytes, not counting "/bin/bash -c ".
+ * If the command is very long, then we write the command to a script file,
+ * to avoid overflowing any limits on command-line length.
+ * For short commands, we just use /bin/bash -c command.
+ */
+ @VisibleForTesting
+ public static int maxCommandLength = 64000;
+
+ /**
+ * A map of remote path prefixes and corresponding runfiles manifests for tools
+ * used by this rule.
+ */
+ private final ImmutableMap<PathFragment, Artifact> remoteRunfileManifestMap;
+
+ /**
+ * Use labelMap for heuristically expanding labels (does not include "outs")
+ * This is similar to heuristic location expansion in LocationExpander
+ * and should be kept in sync.
+ */
+ private final ImmutableMap<Label, ImmutableCollection<Artifact>> labelMap;
+
+ /**
+ * The ruleContext this helper works on
+ */
+ private final RuleContext ruleContext;
+
+ /**
+ * Output executable files from the 'tools' attribute.
+ */
+ private final ImmutableList<Artifact> resolvedTools;
+
+ /**
+ * Creates an {@link CommandHelper}.
+ *
+ * @param tools - Resolves set of tools into set of executable binaries. Populates manifests,
+ * remoteRunfiles and label map where required.
+ * @param labelMap - Adds files to set of known files of label. Used for resolving $(location)
+ * variables.
+ */
+ public CommandHelper(RuleContext ruleContext,
+ Iterable<FilesToRunProvider> tools,
+ ImmutableMap<Label, Iterable<Artifact>> labelMap) {
+ this.ruleContext = ruleContext;
+
+ ImmutableList.Builder<Artifact> resolvedToolsBuilder = ImmutableList.builder();
+ ImmutableMap.Builder<PathFragment, Artifact> remoteRunfileManifestBuilder =
+ ImmutableMap.builder();
+ Map<Label, Collection<Artifact>> tempLabelMap = new HashMap<>();
+
+ for (Map.Entry<Label, Iterable<Artifact>> entry : labelMap.entrySet()) {
+ Iterables.addAll(mapGet(tempLabelMap, entry.getKey()), entry.getValue());
+ }
+
+ for (FilesToRunProvider tool : tools) { // (Note: host configuration)
+ Label label = tool.getLabel();
+ Collection<Artifact> files = tool.getFilesToRun();
+ resolvedToolsBuilder.addAll(files);
+ Artifact executableArtifact = tool.getExecutable();
+ // If the label has an executable artifact add that to the multimaps.
+ if (executableArtifact != null) {
+ mapGet(tempLabelMap, label).add(executableArtifact);
+ // Also send the runfiles when running remotely.
+ Artifact runfilesManifest = tool.getRunfilesManifest();
+ if (runfilesManifest != null) {
+ remoteRunfileManifestBuilder.put(
+ BaseSpawn.runfilesForFragment(executableArtifact.getExecPath()), runfilesManifest);
+ }
+ } else {
+ // Map all depArtifacts to the respective label using the multimaps.
+ Iterables.addAll(mapGet(tempLabelMap, label), files);
+ }
+ }
+
+ this.resolvedTools = resolvedToolsBuilder.build();
+ this.remoteRunfileManifestMap = remoteRunfileManifestBuilder.build();
+ ImmutableMap.Builder<Label, ImmutableCollection<Artifact>> labelMapBuilder =
+ ImmutableMap.builder();
+ for (Entry<Label, Collection<Artifact>> entry : tempLabelMap.entrySet()) {
+ labelMapBuilder.put(entry.getKey(), ImmutableList.copyOf(entry.getValue()));
+ }
+ this.labelMap = labelMapBuilder.build();
+ }
+
+ @SkylarkCallable(name = "resolved_tools", doc = "", structField = true)
+ public List<Artifact> getResolvedTools() {
+ return resolvedTools;
+ }
+
+ @SkylarkCallable(name = "runfiles_manifests", doc = "", structField = true)
+ public ImmutableMap<PathFragment, Artifact> getRemoteRunfileManifestMap() {
+ return remoteRunfileManifestMap;
+ }
+
+ // Returns the value in the specified corresponding to 'key', creating and
+ // inserting an empty container if absent. We use Map not Multimap because
+ // we need to distinguish the cases of "empty value" and "absent key".
+ private static Collection<Artifact> mapGet(Map<Label, Collection<Artifact>> map, Label key) {
+ Collection<Artifact> values = map.get(key);
+ if (values == null) {
+ // We use sets not lists, because it's conceivable that the same artifact
+ // could appear twice, e.g. in "srcs" and "deps".
+ values = Sets.newHashSet();
+ map.put(key, values);
+ }
+ return values;
+ }
+
+ /**
+ * Resolves the 'cmd' attribute, and expands known locations for $(location)
+ * variables.
+ */
+ @SkylarkCallable(doc = "")
+ public String resolveCommandAndExpandLabels(Boolean supportLegacyExpansion,
+ Boolean allowDataInLabel) {
+ String command = ruleContext.attributes().get("cmd", Type.STRING);
+ command = new LocationExpander(ruleContext, allowDataInLabel).expand("cmd", command);
+
+ if (supportLegacyExpansion) {
+ command = expandLabels(command, labelMap);
+ }
+ return command;
+ }
+
+ /**
+ * Expands labels occurring in the string "expr" in the rule 'cmd'.
+ * Each label must be valid, be a declared prerequisite, and expand to a
+ * unique path.
+ *
+ * <p>If the expansion fails, an attribute error is reported and the original
+ * expression is returned.
+ */
+ private <T extends Iterable<Artifact>> String expandLabels(String expr, Map<Label, T> labelMap) {
+ try {
+ return LabelExpander.expand(expr, labelMap, ruleContext.getLabel());
+ } catch (LabelExpander.NotUniqueExpansionException nuee) {
+ ruleContext.attributeError("cmd", nuee.getMessage());
+ return expr;
+ }
+ }
+
+ private static Pair<List<String>, Artifact> buildCommandLineMaybeWithScriptFile(
+ RuleContext ruleContext, String command, String scriptPostFix) {
+ List<String> argv;
+ Artifact scriptFileArtifact = null;
+ if (command.length() <= maxCommandLength) {
+ argv = buildCommandLineSimpleArgv(ruleContext, command);
+ } else {
+ // Use script file.
+ scriptFileArtifact = buildCommandLineArtifact(ruleContext, command, scriptPostFix);
+ argv = buildCommandLineArgvWithArtifact(ruleContext, scriptFileArtifact);
+ }
+ return Pair.of(argv, scriptFileArtifact);
+
+ }
+
+ /**
+ * Builds the set of command-line arguments. Creates a bash script if the
+ * command line is longer than the allowed maximum {@link
+ * #maxCommandLength}. Fixes up the input artifact list with the
+ * created bash script when required.
+ * TODO(bazel-team): do away with the side effect on inputs (ugh).
+ */
+ public static List<String> buildCommandLine(RuleContext ruleContext,
+ String command, NestedSetBuilder<Artifact> inputs, String scriptPostFix) {
+ Pair<List<String>, Artifact> argvAndScriptFile =
+ buildCommandLineMaybeWithScriptFile(ruleContext, command, scriptPostFix);
+ if (argvAndScriptFile.second != null) {
+ inputs.add(argvAndScriptFile.second);
+ }
+ return argvAndScriptFile.first;
+ }
+
+ /**
+ * Builds the set of command-line arguments. Creates a bash script if the
+ * command line is longer than the allowed maximum {@link
+ * #maxCommandLength}. Fixes up the input artifact list with the
+ * created bash script when required.
+ * TODO(bazel-team): do away with the side effect on inputs (ugh).
+ */
+ public static List<String> buildCommandLine(RuleContext ruleContext,
+ String command, List<Artifact> inputs, String scriptPostFix) {
+ Pair<List<String>, Artifact> argvAndScriptFile =
+ buildCommandLineMaybeWithScriptFile(ruleContext, command, scriptPostFix);
+ if (argvAndScriptFile.second != null) {
+ inputs.add(argvAndScriptFile.second);
+ }
+ return argvAndScriptFile.first;
+ }
+
+ /**
+ * Builds the set of command-line arguments. Creates a bash script if the
+ * command line is longer than the allowed maximum {@link
+ * #maxCommandLength}. Fixes up the input artifact list with the
+ * created bash script when required.
+ * TODO(bazel-team): do away with the side effect on inputs (ugh).
+ */
+ public static List<String> buildCommandLine(RuleContext ruleContext,
+ String command, ImmutableSet.Builder<Artifact> inputs, String scriptPostFix) {
+ Pair<List<String>, Artifact> argvAndScriptFile =
+ buildCommandLineMaybeWithScriptFile(ruleContext, command, scriptPostFix);
+ if (argvAndScriptFile.second != null) {
+ inputs.add(argvAndScriptFile.second);
+ }
+ return argvAndScriptFile.first;
+ }
+
+ private static ImmutableList<String> buildCommandLineArgvWithArtifact(RuleContext ruleContext,
+ Artifact scriptFileArtifact) {
+ return ImmutableList.of(
+ ruleContext.getConfiguration().getShExecutable().getPathString(),
+ scriptFileArtifact.getExecPathString());
+ }
+
+ private static Artifact buildCommandLineArtifact(RuleContext ruleContext, String command,
+ String scriptPostFix) {
+ String scriptFileName = ruleContext.getTarget().getName() + scriptPostFix;
+ String scriptFileContents = "#!/bin/bash\n" + command;
+ Artifact scriptFileArtifact = FileWriteAction.createFile(
+ ruleContext, scriptFileName, scriptFileContents, /*executable=*/true);
+ return scriptFileArtifact;
+ }
+
+ private static ImmutableList<String> buildCommandLineSimpleArgv(RuleContext ruleContext,
+ String command) {
+ return ImmutableList.of(
+ ruleContext.getConfiguration().getShExecutable().getPathString(), "-c", command);
+ }
+
+ /**
+ * Builds the set of command-line arguments. Creates a bash script if the
+ * command line is longer than the allowed maximum {@link
+ * #maxCommandLength}. Fixes up the input artifact list with the
+ * created bash script when required.
+ */
+ public List<String> buildCommandLine(
+ String command, NestedSetBuilder<Artifact> inputs, String scriptPostFix) {
+ return buildCommandLine(ruleContext, command, inputs, scriptPostFix);
+ }
+
+ /**
+ * Builds the set of command-line arguments. Creates a bash script if the
+ * command line is longer than the allowed maximum {@link
+ * #maxCommandLength}. Fixes up the input artifact list with the
+ * created bash script when required.
+ */
+ @SkylarkCallable(doc = "")
+ public List<String> buildCommandLine(
+ String command, List<Artifact> inputs, String scriptPostFix) {
+ return buildCommandLine(ruleContext, command, inputs, scriptPostFix);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CompilationHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/CompilationHelper.java
new file mode 100644
index 0000000..23dd8d4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/CompilationHelper.java
@@ -0,0 +1,92 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MiddlemanFactory;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+
+import java.util.List;
+
+/**
+ * A helper class for compilation helpers.
+ */
+public final class CompilationHelper {
+ /**
+ * Returns a middleman for all files to build for the given configured target.
+ * If multiple calls are made, then it returns the same artifact for
+ * configurations with the same internal directory.
+ *
+ * <p>The resulting middleman only aggregates the inputs and must be expanded
+ * before an action using it can be sent to a distributed execution-system.
+ */
+ public static NestedSet<Artifact> getAggregatingMiddleman(
+ RuleContext ruleContext, String purpose, NestedSet<Artifact> filesToBuild) {
+ return NestedSetBuilder.wrap(Order.STABLE_ORDER, getMiddlemanInternal(
+ ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose,
+ filesToBuild));
+ }
+
+ /**
+ * Internal implementation for getAggregatingMiddleman / getAggregatingMiddlemanWithSolibSymlinks.
+ */
+ private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env,
+ RuleContext ruleContext, ActionOwner actionOwner, String purpose,
+ NestedSet<Artifact> filesToBuild) {
+ if (filesToBuild == null) {
+ return ImmutableList.of();
+ }
+ MiddlemanFactory factory = env.getMiddlemanFactory();
+ return ImmutableList.of(factory.createMiddlemanAllowMultiple(
+ env, actionOwner, purpose, filesToBuild,
+ ruleContext.getConfiguration().getMiddlemanDirectory()));
+ }
+
+ // TODO(bazel-team): remove this duplicated code after the ConfiguredTarget migration
+ /**
+ * Returns a middleman for all files to build for the given configured target.
+ * If multiple calls are made, then it returns the same artifact for
+ * configurations with the same internal directory.
+ *
+ * <p>The resulting middleman only aggregates the inputs and must be expanded
+ * before an action using it can be sent to a distributed execution-system.
+ */
+ public static NestedSet<Artifact> getAggregatingMiddleman(
+ RuleContext ruleContext, String purpose, TransitiveInfoCollection dep) {
+ return NestedSetBuilder.wrap(Order.STABLE_ORDER, getMiddlemanInternal(
+ ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose,
+ dep));
+ }
+
+ /**
+ * Internal implementation for getAggregatingMiddleman / getAggregatingMiddlemanWithSolibSymlinks.
+ */
+ private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env,
+ RuleContext ruleContext, ActionOwner actionOwner, String purpose,
+ TransitiveInfoCollection dep) {
+ if (dep == null) {
+ return ImmutableList.of();
+ }
+ MiddlemanFactory factory = env.getMiddlemanFactory();
+ Iterable<Artifact> artifacts = dep.getProvider(FileProvider.class).getFilesToBuild();
+ return ImmutableList.of(factory.createMiddlemanAllowMultiple(
+ env, actionOwner, purpose, artifacts,
+ ruleContext.getConfiguration().getMiddlemanDirectory()));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CompilationPrerequisitesProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/CompilationPrerequisitesProvider.java
new file mode 100644
index 0000000..8c3a6ce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/CompilationPrerequisitesProvider.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A provider for compilation prerequisites of a given target.
+ */
+@Immutable
+public final class CompilationPrerequisitesProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> compilationPrerequisites;
+
+ public CompilationPrerequisitesProvider(NestedSet<Artifact> compilationPrerequisites) {
+ this.compilationPrerequisites = compilationPrerequisites;
+ }
+
+ /**
+ * Returns compilation prerequisites (e.g., generated sources and headers) for a rule.
+ */
+ public NestedSet<Artifact> getCompilationPrerequisites() {
+ return compilationPrerequisites;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationCollectionFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationCollectionFactory.java
new file mode 100644
index 0000000..8ffac43
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationCollectionFactory.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.events.EventHandler;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A factory for configuration collection creation.
+ */
+public interface ConfigurationCollectionFactory {
+ /**
+ * Creates the top-level configuration for a build.
+ *
+ * <p>Also it may create a set of BuildConfigurations and define a transition table over them.
+ * All configurations during a build should be accessible from this top-level configuration
+ * via configuration transitions.
+ * @param loadedPackageProvider the package provider
+ * @param buildOptions top-level build options representing the command-line
+ * @param clientEnv the system environment
+ * @param errorEventListener the event listener for errors
+ * @param performSanityCheck flag to signal about performing sanity check. Can be false only for
+ * tests in skyframe. Legacy mode uses loadedPackageProvider == null condition for this.
+ * @return the top-level configuration
+ * @throws InvalidConfigurationException
+ */
+ @Nullable
+ public BuildConfiguration createConfigurations(
+ ConfigurationFactory configurationFactory,
+ PackageProviderForConfigurations loadedPackageProvider,
+ BuildOptions buildOptions,
+ Map<String, String> clientEnv,
+ EventHandler errorEventListener,
+ boolean performSanityCheck) throws InvalidConfigurationException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationMakeVariableContext.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationMakeVariableContext.java
new file mode 100644
index 0000000..8c98f5f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationMakeVariableContext.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Package;
+
+import java.util.Map;
+
+/**
+ * Implements make variable expansion for make variables that depend on the
+ * configuration and the target (not on behavior of the
+ * {@link ConfiguredTarget} implementation)
+ */
+public class ConfigurationMakeVariableContext implements MakeVariableExpander.Context {
+ private final Package pkg;
+ private final Map<String, String> commandLineEnv;
+ private final Map<String, String> globalEnv;
+ private final String platform;
+
+ public ConfigurationMakeVariableContext(Package pkg, BuildConfiguration configuration) {
+ this.pkg = pkg;
+ commandLineEnv = configuration.getCommandLineDefines();
+ globalEnv = configuration.getGlobalMakeEnvironment();
+ platform = configuration.getPlatformName();
+ }
+
+ @Override
+ public String lookupMakeVariable(String var) throws ExpansionException {
+ String value = commandLineEnv.get(var);
+ if (value == null) {
+ value = pkg.lookupMakeVariable(var, platform);
+ }
+ if (value == null) {
+ value = globalEnv.get(var);
+ }
+ if (value == null) {
+ throw new MakeVariableExpander.ExpansionException("$(" + var + ") not defined");
+ }
+
+ return value;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCreatedEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCreatedEvent.java
new file mode 100644
index 0000000..8b0621a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCreatedEvent.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+
+/**
+ * This event is fired when the build configurations are created.
+ */
+public class ConfigurationsCreatedEvent {
+
+ private final BuildConfigurationCollection configurations;
+
+ /**
+ * Construct the ConfigurationsCreatedEvent.
+ *
+ * @param configurations the build configuration collection
+ */
+ public ConfigurationsCreatedEvent(BuildConfigurationCollection configurations) {
+ this.configurations = configurations;
+ }
+
+ public BuildConfigurationCollection getBuildConfigurationCollection() {
+ return configurations;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java
new file mode 100644
index 0000000..955241a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.packages.AspectFactory;
+
+/**
+ * Instantiation of {@link AspectFactory} with the actual types.
+ *
+ * <p>This is needed because {@link AspectFactory} is needed in the {@code packages} package to
+ * do loading phase things properly and to be able to specify them on attributes, but the actual
+ * classes are in the {@code view} package, which is not available there.
+ */
+public interface ConfiguredAspectFactory
+ extends AspectFactory<ConfiguredTarget, RuleContext, Aspect> {
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAttributeMapper.java
new file mode 100644
index 0000000..84029b2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAttributeMapper.java
@@ -0,0 +1,158 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.packages.AbstractAttributeMapper;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * {@link AttributeMap} implementation that binds a rule's attribute as follows:
+ *
+ * <ol>
+ * <li>If the attribute is selectable (i.e. its BUILD declaration is of the form
+ * "attr = { config1: "value1", "config2: "value2", ... }", returns the subset of values
+ * chosen by the current configuration in accordance with Bazel's documented policy on
+ * configurable attribute selection.
+ * <li>If the attribute is not selectable (i.e. its value is static), returns that value with
+ * no additional processing.
+ * </ol>
+ *
+ * <p>Example usage:
+ * <pre>
+ * Label fooLabel = ConfiguredAttributeMapper.of(ruleConfiguredTarget).get("foo", Type.LABEL);
+ * </pre>
+ */
+public class ConfiguredAttributeMapper extends AbstractAttributeMapper {
+
+ private final Map<Label, ConfigMatchingProvider> configConditions;
+ private Rule rule;
+
+ private ConfiguredAttributeMapper(Rule rule, Set<ConfigMatchingProvider> configConditions) {
+ super(Preconditions.checkNotNull(rule).getPackage(), rule.getRuleClassObject(), rule.getLabel(),
+ rule.getAttributeContainer());
+ ImmutableMap.Builder<Label, ConfigMatchingProvider> builder = ImmutableMap.builder();
+ for (ConfigMatchingProvider configCondition : configConditions) {
+ builder.put(configCondition.label(), configCondition);
+ }
+ this.configConditions = builder.build();
+ this.rule = rule;
+ }
+
+ /**
+ * "Do-it-all" constructor that just needs a {@link RuleConfiguredTarget}.
+ */
+ public static ConfiguredAttributeMapper of(RuleConfiguredTarget ct) {
+ return new ConfiguredAttributeMapper(ct.getTarget(), ct.getConfigConditions());
+ }
+
+ /**
+ * "Manual" constructor that requires the caller to pass the set of configurability conditions
+ * that trigger this rule's configurable attributes.
+ *
+ * <p>If you don't know how to do this, you really want to use one of the "do-it-all"
+ * constructors.
+ */
+ static ConfiguredAttributeMapper of(Rule rule, Set<ConfigMatchingProvider> configConditions) {
+ return new ConfiguredAttributeMapper(rule, configConditions);
+ }
+
+ /**
+ * Checks that all attributes can be mapped to their configured values. This is
+ * useful for checking that the configuration space in a configured attribute doesn't
+ * contain unresolvable contradictions.
+ *
+ * @throws EvalException if any attribute's value can't be resolved under this mapper
+ */
+ public void validateAttributes() throws EvalException {
+ for (String attrName : getAttributeNames()) {
+ getAndValidate(attrName, getAttributeType(attrName));
+ }
+ }
+
+ /**
+ * Variation of {@link #get} that throws an informative exception if the attribute
+ * can't be resolved due to intrinsic contradictions in the configuration.
+ */
+ private <T> T getAndValidate(String attributeName, Type<T> type) throws EvalException {
+ Type.Selector<T> selector = getSelector(attributeName, type);
+ if (selector == null) {
+ // This is a normal attribute.
+ return super.get(attributeName, type);
+ }
+
+ // We expect exactly one of this attribute's conditions to match (including the default
+ // condition, if specified). Throw an exception if our expectations aren't met.
+ Label matchingCondition = null;
+ T matchingValue = null;
+
+ // Find the matching condition and record its value (checking for duplicates).
+ for (Map.Entry<Label, T> entry : selector.getEntries().entrySet()) {
+ Label curCondition = entry.getKey();
+ if (Type.Selector.isReservedLabel(curCondition)) {
+ continue;
+ } else if (Preconditions.checkNotNull(configConditions.get(curCondition)).matches()) {
+ if (matchingCondition != null) {
+ throw new EvalException(rule.getAttributeLocation(attributeName),
+ "Both " + matchingCondition.toString() + " and " + curCondition.toString()
+ + " match configurable attribute \"" + attributeName + "\" in " + getLabel()
+ + ". At most one match is allowed");
+ }
+ matchingCondition = curCondition;
+ matchingValue = entry.getValue();
+ }
+ }
+
+ // If nothing matched, choose the default condition.
+ if (matchingCondition == null) {
+ if (!selector.hasDefault()) {
+ throw new EvalException(rule.getAttributeLocation(attributeName),
+ "Configurable attribute \"" + attributeName + "\" doesn't match this "
+ + "configuration (would a default condition help?)");
+ }
+ matchingValue = selector.getDefault();
+ }
+
+ return matchingValue;
+ }
+
+ @Override
+ public <T> T get(String attributeName, Type<T> type) {
+ try {
+ return getAndValidate(attributeName, type);
+ } catch (EvalException e) {
+ // Callers that reach this branch should explicitly validate the attribute through an
+ // appropriate call and handle the exception directly. This method assumes
+ // pre-validated attributes.
+ throw new IllegalStateException(
+ "lookup failed on attribute " + attributeName + ": " + e.getMessage());
+ }
+ }
+
+ @Override
+ protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) {
+ T value = get(attributeName, type);
+ return value == null ? ImmutableList.<T>of() : ImmutableList.of(value);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java
new file mode 100644
index 0000000..e84f9aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java
@@ -0,0 +1,376 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import static com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType.ABSTRACT;
+import static com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType.TEST;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
+import com.google.devtools.build.lib.analysis.config.FragmentOptions;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.graph.Digraph;
+import com.google.devtools.build.lib.graph.Node;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.SkylarkModules;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.syntax.SkylarkType;
+import com.google.devtools.build.lib.syntax.ValidationEnvironment;
+import com.google.devtools.common.options.OptionsClassProvider;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Knows about every rule Blaze supports and the associated configuration options.
+ *
+ * <p>This class is initialized on server startup and the set of rules, build info factories
+ * and configuration options is guarantees not to change over the life time of the Blaze server.
+ */
+public class ConfiguredRuleClassProvider implements RuleClassProvider {
+ /**
+ * Custom dependency validation logic.
+ */
+ public static interface PrerequisiteValidator {
+ /**
+ * Checks whether the rule in {@code contextBuilder} is allowed to depend on
+ * {@code prerequisite} through the attribute {@code attribute}.
+ *
+ * <p>Can be used for enforcing any organization-specific policies about the layout of the
+ * workspace.
+ */
+ void validate(
+ RuleContext.Builder contextBuilder, ConfiguredTarget prerequisite, Attribute attribute);
+ }
+
+ /**
+ * Builder for {@link ConfiguredRuleClassProvider}.
+ */
+ public static class Builder implements RuleDefinitionEnvironment {
+ private final List<ConfigurationFragmentFactory> configurationFragments = new ArrayList<>();
+ private final List<BuildInfoFactory> buildInfoFactories = new ArrayList<>();
+ private final List<Class<? extends FragmentOptions>> configurationOptions = new ArrayList<>();
+
+ private final Map<String, RuleClass> ruleClassMap = new HashMap<>();
+ private final Map<String, Class<? extends RuleDefinition>> ruleDefinitionMap =
+ new HashMap<>();
+ private final Map<Class<? extends RuleDefinition>, RuleClass> ruleMap = new HashMap<>();
+ private final Digraph<Class<? extends RuleDefinition>> dependencyGraph =
+ new Digraph<>();
+ private ConfigurationCollectionFactory configurationCollectionFactory;
+ private PrerequisiteValidator prerequisiteValidator;
+ private ImmutableMap<String, SkylarkType> skylarkAccessibleJavaClasses = ImmutableMap.of();
+
+ public Builder setPrerequisiteValidator(PrerequisiteValidator prerequisiteValidator) {
+ this.prerequisiteValidator = prerequisiteValidator;
+ return this;
+ }
+
+ public Builder addBuildInfoFactory(BuildInfoFactory factory) {
+ buildInfoFactories.add(factory);
+ return this;
+ }
+
+ public Builder addRuleDefinition(Class<? extends RuleDefinition> ruleDefinition) {
+ dependencyGraph.createNode(ruleDefinition);
+ BlazeRule annotation = ruleDefinition.getAnnotation(BlazeRule.class);
+ for (Class<? extends RuleDefinition> ancestor : annotation.ancestors()) {
+ dependencyGraph.addEdge(ancestor, ruleDefinition);
+ }
+
+ return this;
+ }
+
+ public Builder addConfigurationOptions(Class<? extends FragmentOptions> configurationOptions) {
+ this.configurationOptions.add(configurationOptions);
+ return this;
+ }
+
+ public Builder addConfigurationFragment(ConfigurationFragmentFactory factory) {
+ configurationFragments.add(factory);
+ return this;
+ }
+
+ public Builder setConfigurationCollectionFactory(ConfigurationCollectionFactory factory) {
+ this.configurationCollectionFactory = factory;
+ return this;
+ }
+
+ public Builder setSkylarkAccessibleJavaClasses(ImmutableMap<String, SkylarkType> objects) {
+ this.skylarkAccessibleJavaClasses = objects;
+ return this;
+ }
+
+ private RuleConfiguredTargetFactory createFactory(
+ Class<? extends RuleConfiguredTargetFactory> factoryClass) {
+ try {
+ Constructor<? extends RuleConfiguredTargetFactory> ctor = factoryClass.getConstructor();
+ return ctor.newInstance();
+ } catch (NoSuchMethodException | IllegalAccessException | InstantiationException
+ | InvocationTargetException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private RuleClass commitRuleDefinition(Class<? extends RuleDefinition> definitionClass) {
+ BlazeRule annotation = definitionClass.getAnnotation(BlazeRule.class);
+ Preconditions.checkArgument(ruleClassMap.get(annotation.name()) == null, annotation.name());
+
+ Preconditions.checkArgument(
+ annotation.type() == ABSTRACT ^
+ annotation.factoryClass() != RuleConfiguredTargetFactory.class);
+ Preconditions.checkArgument(
+ (annotation.type() != TEST) ||
+ Arrays.asList(annotation.ancestors()).contains(
+ BaseRuleClasses.TestBaseRule.class));
+
+ RuleDefinition instance;
+ try {
+ instance = definitionClass.newInstance();
+ } catch (IllegalAccessException | InstantiationException e) {
+ throw new IllegalStateException(e);
+ }
+ RuleClass[] ancestorClasses = new RuleClass[annotation.ancestors().length];
+ for (int i = 0; i < annotation.ancestors().length; i++) {
+ ancestorClasses[i] = ruleMap.get(annotation.ancestors()[i]);
+ if (ancestorClasses[i] == null) {
+ // Ancestors should have been initialized by now
+ throw new IllegalStateException("Ancestor " + annotation.ancestors()[i] + " of "
+ + annotation.name() + " is not initialized");
+ }
+ }
+
+ RuleConfiguredTargetFactory factory = null;
+ if (annotation.type() != ABSTRACT) {
+ factory = createFactory(annotation.factoryClass());
+ }
+
+ RuleClass.Builder builder = new RuleClass.Builder(
+ annotation.name(), annotation.type(), false, ancestorClasses);
+ builder.factory(factory);
+ RuleClass ruleClass = instance.build(builder, this);
+ ruleMap.put(definitionClass, ruleClass);
+ ruleClassMap.put(ruleClass.getName(), ruleClass);
+ ruleDefinitionMap.put(ruleClass.getName(), definitionClass);
+
+ return ruleClass;
+ }
+
+ public ConfiguredRuleClassProvider build() {
+ for (Node<Class<? extends RuleDefinition>> ruleDefinition :
+ dependencyGraph.getTopologicalOrder()) {
+ commitRuleDefinition(ruleDefinition.getLabel());
+ }
+
+ return new ConfiguredRuleClassProvider(
+ ImmutableMap.copyOf(ruleClassMap),
+ ImmutableMap.copyOf(ruleDefinitionMap),
+ ImmutableList.copyOf(buildInfoFactories),
+ ImmutableList.copyOf(configurationOptions),
+ ImmutableList.copyOf(configurationFragments),
+ configurationCollectionFactory,
+ prerequisiteValidator,
+ skylarkAccessibleJavaClasses);
+ }
+
+ @Override
+ public Label getLabel(String labelValue) {
+ return LABELS.getUnchecked(labelValue);
+ }
+ }
+
+ /**
+ * Used to make the label instances unique, so that we don't create a new
+ * instance for every rule.
+ */
+ private static LoadingCache<String, Label> LABELS = CacheBuilder.newBuilder().build(
+ new CacheLoader<String, Label>() {
+ @Override
+ public Label load(String from) {
+ try {
+ return Label.parseAbsolute(from);
+ } catch (Label.SyntaxException e) {
+ throw new IllegalArgumentException(from);
+ }
+ }
+ });
+
+ /**
+ * Maps rule class name to the metaclass instance for that rule.
+ */
+ private final ImmutableMap<String, RuleClass> ruleClassMap;
+
+ /**
+ * Maps rule class name to the rule definition metaclasses.
+ */
+ private final ImmutableMap<String, Class<? extends RuleDefinition>> ruleDefinitionMap;
+
+ /**
+ * The configuration options that affect the behavior of the rules.
+ */
+ private final ImmutableList<Class<? extends FragmentOptions>> configurationOptions;
+
+ /**
+ * The set of configuration fragment factories.
+ */
+ private final ImmutableList<ConfigurationFragmentFactory> configurationFragments;
+
+ /**
+ * The factory that creates the configuration collection.
+ */
+ private final ConfigurationCollectionFactory configurationCollectionFactory;
+
+ private final ImmutableList<BuildInfoFactory> buildInfoFactories;
+
+ private final PrerequisiteValidator prerequisiteValidator;
+
+ private final ImmutableMap<String, SkylarkType> skylarkAccessibleJavaClasses;
+
+ private final ValidationEnvironment skylarkValidationEnvironment;
+
+ public ConfiguredRuleClassProvider(
+ ImmutableMap<String, RuleClass> ruleClassMap,
+ ImmutableMap<String, Class<? extends RuleDefinition>> ruleDefinitionMap,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ ImmutableList<Class<? extends FragmentOptions>> configurationOptions,
+ ImmutableList<ConfigurationFragmentFactory> configurationFragments,
+ ConfigurationCollectionFactory configurationCollectionFactory,
+ PrerequisiteValidator prerequisiteValidator,
+ ImmutableMap<String, SkylarkType> skylarkAccessibleJavaClasses) {
+
+ this.ruleClassMap = ruleClassMap;
+ this.ruleDefinitionMap = ruleDefinitionMap;
+ this.buildInfoFactories = buildInfoFactories;
+ this.configurationOptions = configurationOptions;
+ this.configurationFragments = configurationFragments;
+ this.configurationCollectionFactory = configurationCollectionFactory;
+ this.prerequisiteValidator = prerequisiteValidator;
+ this.skylarkAccessibleJavaClasses = skylarkAccessibleJavaClasses;
+ this.skylarkValidationEnvironment = SkylarkModules.getValidationEnvironment(
+ ImmutableMap.<String, SkylarkType>builder()
+ .putAll(skylarkAccessibleJavaClasses)
+ .put("native", SkylarkType.of(NativeModule.class))
+ .build());
+ }
+
+ public PrerequisiteValidator getPrerequisiteValidator() {
+ return prerequisiteValidator;
+ }
+
+ @Override
+ public Map<String, RuleClass> getRuleClassMap() {
+ return ruleClassMap;
+ }
+
+ /**
+ * Returns a list of build info factories that are needed for the supported languages.
+ */
+ public ImmutableList<BuildInfoFactory> getBuildInfoFactories() {
+ return buildInfoFactories;
+ }
+
+ /**
+ * Returns the set of configuration fragments provided by this module.
+ */
+ public ImmutableList<ConfigurationFragmentFactory> getConfigurationFragments() {
+ return configurationFragments;
+ }
+
+ /**
+ * Returns the set of configuration options that are supported in this module.
+ */
+ public ImmutableList<Class<? extends FragmentOptions>> getConfigurationOptions() {
+ return configurationOptions;
+ }
+
+ /**
+ * Returns the definition of the rule class definition with the specified name.
+ */
+ public Class<? extends RuleDefinition> getRuleClassDefinition(String ruleClassName) {
+ return ruleDefinitionMap.get(ruleClassName);
+ }
+
+ /**
+ * Returns the configuration collection creator.
+ */
+ public ConfigurationCollectionFactory getConfigurationCollectionFactory() {
+ return configurationCollectionFactory;
+ }
+
+ /**
+ * Returns the defaults package for the default settings.
+ */
+ public String getDefaultsPackageContent() {
+ return DefaultsPackage.getDefaultsPackageContent(configurationOptions);
+ }
+
+ /**
+ * Returns the defaults package for the given options taken from an optionsProvider.
+ */
+ public String getDefaultsPackageContent(OptionsClassProvider optionsProvider) {
+ return DefaultsPackage.getDefaultsPackageContent(
+ BuildOptions.of(configurationOptions, optionsProvider));
+ }
+
+ /**
+ * Creates a BuildOptions class for the given options taken from an optionsProvider.
+ */
+ public BuildOptions createBuildOptions(OptionsClassProvider optionsProvider) {
+ return BuildOptions.of(configurationOptions, optionsProvider);
+ }
+
+ @SkylarkModule(name = "native", namespace = true, onlyLoadingPhase = true,
+ doc = "Module for native rules.")
+ private static final class NativeModule {}
+
+ public static final NativeModule nativeModule = new NativeModule();
+
+ @Override
+ public SkylarkEnvironment createSkylarkRuleClassEnvironment(
+ EventHandler eventHandler, String astFileContentHashCode) {
+ SkylarkEnvironment env = SkylarkModules.getNewEnvironment(eventHandler, astFileContentHashCode);
+ for (Map.Entry<String, SkylarkType> entry : skylarkAccessibleJavaClasses.entrySet()) {
+ env.update(entry.getKey(), entry.getValue().getType());
+ }
+ return env;
+ }
+
+ @Override
+ public ValidationEnvironment getSkylarkValidationEnvironment() {
+ return skylarkValidationEnvironment;
+ }
+
+ @Override
+ public Object getNativeModule() {
+ return nativeModule;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTarget.java
new file mode 100644
index 0000000..7aad367
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTarget.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Target;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link ConfiguredTarget} is conceptually a {@link TransitiveInfoCollection} coupled
+ * with the {@link Target} and {@link BuildConfiguration} objects it was created from.
+ *
+ * <p>This interface is supposed to only be used in {@link BuildView} and above. In particular,
+ * rule implementations should not be able to access the {@link ConfiguredTarget} objects
+ * associated with their direct dependencies, only the corresponding
+ * {@link TransitiveInfoCollection}s. Also, {@link ConfiguredTarget} objects should not be
+ * accessible from the action graph.
+ */
+public interface ConfiguredTarget extends TransitiveInfoCollection {
+ /**
+ * Returns the Target with which this {@link ConfiguredTarget} is associated.
+ */
+ Target getTarget();
+
+ /**
+ * <p>Returns the {@link BuildConfiguration} for which this {@link ConfiguredTarget} is
+ * defined. Configuration is defined for all configured targets with exception
+ * of the {@link InputFileConfiguredTarget} for which it is always
+ * <b>null</b>.</p>
+ */
+ @Override
+ @Nullable
+ BuildConfiguration getConfiguration();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTargetFactory.java
new file mode 100644
index 0000000..d265533
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTargetFactory.java
@@ -0,0 +1,302 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.FailAction;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.PackageGroupsRuleVisibility;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.rules.SkylarkRuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * This class creates {@link ConfiguredTarget} instances using a given {@link
+ * ConfiguredRuleClassProvider}.
+ */
+@ThreadSafe
+public final class ConfiguredTargetFactory {
+ // This class is not meant to be outside of the analysis phase machinery and is only public
+ // in order to be accessible from the .view.skyframe package.
+
+ private final ConfiguredRuleClassProvider ruleClassProvider;
+
+ public ConfiguredTargetFactory(ConfiguredRuleClassProvider ruleClassProvider) {
+ this.ruleClassProvider = ruleClassProvider;
+ }
+
+ /**
+ * Returns the visibility of the given target. Errors during package group resolution are reported
+ * to the {@code AnalysisEnvironment}.
+ */
+ private NestedSet<PackageSpecification> convertVisibility(
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, EventHandler reporter,
+ Target target, BuildConfiguration packageGroupConfiguration) {
+ RuleVisibility ruleVisibility = target.getVisibility();
+ if (ruleVisibility instanceof ConstantRuleVisibility) {
+ return ((ConstantRuleVisibility) ruleVisibility).isPubliclyVisible()
+ ? NestedSetBuilder.<PackageSpecification>create(
+ Order.STABLE_ORDER, PackageSpecification.EVERYTHING)
+ : NestedSetBuilder.<PackageSpecification>emptySet(Order.STABLE_ORDER);
+ } else if (ruleVisibility instanceof PackageGroupsRuleVisibility) {
+ PackageGroupsRuleVisibility packageGroupsVisibility =
+ (PackageGroupsRuleVisibility) ruleVisibility;
+
+ NestedSetBuilder<PackageSpecification> packageSpecifications =
+ NestedSetBuilder.stableOrder();
+ for (Label groupLabel : packageGroupsVisibility.getPackageGroups()) {
+ // PackageGroupsConfiguredTargets are always in the package-group configuration.
+ ConfiguredTarget group =
+ findPrerequisite(prerequisiteMap, groupLabel, packageGroupConfiguration);
+ PackageSpecificationProvider provider = null;
+ // group == null can only happen if the package group list comes
+ // from a default_visibility attribute, because in every other case,
+ // this missing link is caught during transitive closure visitation or
+ // if the RuleConfiguredTargetGraph threw out a visibility edge
+ // because if would have caused a cycle. The filtering should be done
+ // in a single place, ConfiguredTargetGraph, but for now, this is the
+ // minimally invasive way of providing a sane error message in case a
+ // cycle is created by a visibility attribute.
+ if (group != null) {
+ provider = group.getProvider(PackageSpecificationProvider.class);
+ }
+ if (provider != null) {
+ packageSpecifications.addTransitive(provider.getPackageSpecifications());
+ } else {
+ reporter.handle(Event.error(target.getLocation(),
+ String.format("Label '%s' does not refer to a package group", groupLabel)));
+ }
+ }
+
+ packageSpecifications.addAll(packageGroupsVisibility.getDirectPackages());
+ return packageSpecifications.build();
+ } else {
+ throw new IllegalStateException("unknown visibility");
+ }
+ }
+
+ private ConfiguredTarget findPrerequisite(
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, Label label,
+ BuildConfiguration config) {
+ for (ConfiguredTarget prerequisite : prerequisiteMap.get(null)) {
+ if (prerequisite.getLabel().equals(label) && (prerequisite.getConfiguration() == config)) {
+ return prerequisite;
+ }
+ }
+ return null;
+ }
+
+ private Artifact getOutputArtifact(OutputFile outputFile, BuildConfiguration configuration,
+ boolean isFileset, ArtifactFactory artifactFactory) {
+ Rule rule = outputFile.getAssociatedRule();
+ Root root = rule.hasBinaryOutput()
+ ? configuration.getBinDirectory()
+ : configuration.getGenfilesDirectory();
+ ArtifactOwner owner =
+ new ConfiguredTargetKey(rule.getLabel(), configuration.getArtifactOwnerConfiguration());
+ PathFragment rootRelativePath = Util.getWorkspaceRelativePath(outputFile);
+ Artifact result = isFileset
+ ? artifactFactory.getFilesetArtifact(rootRelativePath, root, owner)
+ : artifactFactory.getDerivedArtifact(rootRelativePath, root, owner);
+ // The associated rule should have created the artifact.
+ Preconditions.checkNotNull(result, "no artifact for %s", rootRelativePath);
+ return result;
+ }
+
+ /**
+ * Invokes the appropriate constructor to create a {@link ConfiguredTarget} instance.
+ * <p>For use in {@code ConfiguredTargetFunction}.
+ *
+ * <p>Returns null if Skyframe deps are missing or upon certain errors.
+ */
+ @Nullable
+ public final ConfiguredTarget createConfiguredTarget(AnalysisEnvironment analysisEnvironment,
+ ArtifactFactory artifactFactory, Target target, BuildConfiguration config,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions)
+ throws InterruptedException {
+ if (target instanceof Rule) {
+ return createRule(
+ analysisEnvironment, (Rule) target, config, prerequisiteMap, configConditions);
+ }
+
+ // Visibility, like all package groups, doesn't have a configuration
+ NestedSet<PackageSpecification> visibility = convertVisibility(
+ prerequisiteMap, analysisEnvironment.getEventHandler(), target, null);
+ TargetContext targetContext = new TargetContext(analysisEnvironment, target, config,
+ prerequisiteMap.get(null), visibility);
+ if (target instanceof OutputFile) {
+ OutputFile outputFile = (OutputFile) target;
+ boolean isFileset = outputFile.getGeneratingRule().getRuleClass().equals("Fileset");
+ Artifact artifact = getOutputArtifact(outputFile, config, isFileset, artifactFactory);
+ TransitiveInfoCollection rule = targetContext.findDirectPrerequisite(
+ outputFile.getGeneratingRule().getLabel(), config);
+ if (isFileset) {
+ return new FilesetOutputConfiguredTarget(targetContext, outputFile, rule, artifact);
+ } else {
+ return new OutputFileConfiguredTarget(targetContext, outputFile, rule, artifact);
+ }
+ } else if (target instanceof InputFile) {
+ InputFile inputFile = (InputFile) target;
+ Artifact artifact = artifactFactory.getSourceArtifact(
+ inputFile.getExecPath(),
+ Root.asSourceRoot(inputFile.getPackage().getSourceRoot()),
+ new ConfiguredTargetKey(target.getLabel(), config));
+
+ return new InputFileConfiguredTarget(targetContext, inputFile, artifact);
+ } else if (target instanceof PackageGroup) {
+ PackageGroup packageGroup = (PackageGroup) target;
+ return new PackageGroupConfiguredTarget(targetContext, packageGroup);
+ } else {
+ throw new AssertionError("Unexpected target class: " + target.getClass().getName());
+ }
+ }
+
+ /**
+ * Factory method: constructs a RuleConfiguredTarget of the appropriate class, based on the rule
+ * class. May return null if an error occurred.
+ */
+ @Nullable
+ private ConfiguredTarget createRule(
+ AnalysisEnvironment env, Rule rule, BuildConfiguration configuration,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions) throws InterruptedException {
+ // Visibility computation and checking is done for every rule.
+ RuleContext ruleContext = new RuleContext.Builder(env, rule, configuration,
+ ruleClassProvider.getPrerequisiteValidator())
+ .setVisibility(convertVisibility(prerequisiteMap, env.getEventHandler(), rule, null))
+ .setPrerequisites(prerequisiteMap)
+ .setConfigConditions(configConditions)
+ .build();
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+ if (!rule.getRuleClassObject().getRequiredConfigurationFragments().isEmpty()) {
+ if (!configuration.hasAllFragments(
+ rule.getRuleClassObject().getRequiredConfigurationFragments())) {
+ if (rule.getRuleClassObject().failIfMissingConfigurationFragment()) {
+ ruleContext.ruleError(missingFragmentError(ruleContext));
+ return null;
+ }
+ return createFailConfiguredTarget(ruleContext);
+ }
+ }
+ if (rule.getRuleClassObject().isSkylarkExecutable()) {
+ // TODO(bazel-team): maybe merge with RuleConfiguredTargetBuilder?
+ return SkylarkRuleConfiguredTargetBuilder.buildRule(
+ ruleContext, rule.getRuleClassObject().getConfiguredTargetFunction());
+ } else {
+ RuleClass.ConfiguredTargetFactory<ConfiguredTarget, RuleContext> factory =
+ rule.getRuleClassObject().<ConfiguredTarget, RuleContext>getConfiguredTargetFactory();
+ Preconditions.checkNotNull(factory, rule.getRuleClassObject());
+ return factory.create(ruleContext);
+ }
+ }
+
+ private String missingFragmentError(RuleContext ruleContext) {
+ RuleClass ruleClass = ruleContext.getRule().getRuleClassObject();
+ Set<Class<?>> missingFragments = new LinkedHashSet<>();
+ for (Class<?> fragment : ruleClass.getRequiredConfigurationFragments()) {
+ if (!ruleContext.getConfiguration().hasFragment(fragment.asSubclass(Fragment.class))) {
+ missingFragments.add(fragment);
+ }
+ }
+ Preconditions.checkState(!missingFragments.isEmpty());
+ StringBuilder result = new StringBuilder();
+ result.append("all rules of type " + ruleClass.getName() + " require the presence of ");
+ List<String> names = new ArrayList<>();
+ for (Class<?> fragment : missingFragments) {
+ // TODO(bazel-team): Using getSimpleName here is sub-optimal, but we don't have anything
+ // better right now.
+ names.add(fragment.getSimpleName());
+ }
+ result.append("all of [");
+ result.append(Joiner.on(",").join(names));
+ result.append("], but these were all disabled");
+ return result.toString();
+ }
+
+ /**
+ * Constructs an {@link Aspect}. Returns null if an error occurs; in that case,
+ * {@code aspectFactory} should call one of the error reporting methods of {@link RuleContext}.
+ */
+ public Aspect createAspect(
+ AnalysisEnvironment env, RuleConfiguredTarget associatedTarget,
+ ConfiguredAspectFactory aspectFactory,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions) {
+ RuleContext.Builder builder = new RuleContext.Builder(env,
+ associatedTarget.getTarget(),
+ associatedTarget.getConfiguration(),
+ ruleClassProvider.getPrerequisiteValidator());
+ RuleContext ruleContext = builder
+ .setVisibility(convertVisibility(
+ prerequisiteMap, env.getEventHandler(), associatedTarget.getTarget(), null))
+ .setPrerequisites(prerequisiteMap)
+ .setConfigConditions(configConditions)
+ .build();
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ return aspectFactory.create(associatedTarget, ruleContext);
+ }
+
+ /**
+ * A pseudo-implementation for configured targets that creates fail actions for all declared
+ * outputs, both implicit and explicit.
+ */
+ private static ConfiguredTarget createFailConfiguredTarget(RuleContext ruleContext) {
+ RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext);
+ if (!ruleContext.getOutputArtifacts().isEmpty()) {
+ ruleContext.registerAction(new FailAction(ruleContext.getActionOwner(),
+ ruleContext.getOutputArtifacts(), "Can't build this"));
+ }
+ builder.add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY));
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java b/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java
new file mode 100644
index 0000000..64cddb1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java
@@ -0,0 +1,573 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap;
+import com.google.devtools.build.lib.packages.AspectDefinition;
+import com.google.devtools.build.lib.packages.AspectFactory;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundDefault;
+import com.google.devtools.build.lib.packages.Attribute.SplitTransition;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.EnvironmentGroup;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import javax.annotation.Nullable;
+
+/**
+ * Resolver for dependencies between configured targets.
+ *
+ * <p>Includes logic to derive the right configurations depending on transition type.
+ */
+public abstract class DependencyResolver {
+ /**
+ * A dependency of a configured target through a label.
+ *
+ * <p>Includes the target and the configuration of the dependency configured target and any
+ * aspects that may be required.
+ *
+ * <p>Note that the presence of an aspect here does not necessarily mean that it will be created.
+ * They will be filtered based on the {@link TransitiveInfoProvider} instances their associated
+ * configured targets have (we cannot do that here because the configured targets are not
+ * available yet). No error or warning is reported in this case, because it is expected that rules
+ * sometimes over-approximate the providers they supply in their definitions.
+ */
+ public static final class Dependency {
+
+ /**
+ * Returns the {@link ConfiguredTargetKey} for a direct dependency.
+ *
+ * <p>Essentially the same information as {@link Dependency} minus the aspects.
+ */
+ public static final Function<Dependency, ConfiguredTargetKey>
+ TO_CONFIGURED_TARGET_KEY = new Function<Dependency, ConfiguredTargetKey>() {
+ @Override
+ public ConfiguredTargetKey apply(Dependency input) {
+ return new ConfiguredTargetKey(input.getLabel(), input.getConfiguration());
+ }
+ };
+
+ private final Label label;
+ private final BuildConfiguration configuration;
+ private final ImmutableSet<Class<? extends ConfiguredAspectFactory>> aspects;
+
+ public Dependency(Label label, BuildConfiguration configuration,
+ ImmutableSet<Class<? extends ConfiguredAspectFactory>> aspects) {
+ this.label = label;
+ this.configuration = configuration;
+ this.aspects = aspects;
+ }
+
+ public Dependency(Label label, BuildConfiguration configuration) {
+ this(label, configuration, ImmutableSet.<Class<? extends ConfiguredAspectFactory>>of());
+ }
+
+ public Label getLabel() {
+ return label;
+ }
+
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public ImmutableSet<Class<? extends ConfiguredAspectFactory>> getAspects() {
+ return aspects;
+ }
+ }
+
+ protected DependencyResolver() {
+ }
+
+ /**
+ * Returns ids for dependent nodes of a given node, sorted by attribute. Note that some
+ * dependencies do not have a corresponding attribute here, and we use the null attribute to
+ * represent those edges. Visibility attributes are only visited if {@code visitVisibility} is
+ * {@code true}.
+ *
+ * <p>If {@code aspect} is null, returns the dependent nodes of the configured target node
+ * representing the given target and configuration, otherwise that of the aspect node accompanying
+ * the aforementioned configured target node for the specified aspect.
+ *
+ * <p>The values are not simply labels because this also implements the first step of applying
+ * configuration transitions, namely, split transitions. This needs to be done before the labels
+ * are resolved because late bound attributes depend on the configuration. A good example for this
+ * is @{code :cc_toolchain}.
+ *
+ * <p>The long-term goal is that most configuration transitions be applied here. However, in order
+ * to do that, we first have to eliminate transitions that depend on the rule class of the
+ * dependency.
+ */
+ public final ListMultimap<Attribute, Dependency> dependentNodeMap(
+ TargetAndConfiguration node, AspectDefinition aspect,
+ Set<ConfigMatchingProvider> configConditions)
+ throws EvalException {
+ Target target = node.getTarget();
+ BuildConfiguration config = node.getConfiguration();
+ ListMultimap<Attribute, Dependency> outgoingEdges = ArrayListMultimap.create();
+ if (target instanceof OutputFile) {
+ Preconditions.checkNotNull(config);
+ visitTargetVisibility(node, outgoingEdges.get(null));
+ Rule rule = ((OutputFile) target).getGeneratingRule();
+ outgoingEdges.get(null).add(new Dependency(rule.getLabel(), config));
+ } else if (target instanceof InputFile) {
+ visitTargetVisibility(node, outgoingEdges.get(null));
+ } else if (target instanceof EnvironmentGroup) {
+ visitTargetVisibility(node, outgoingEdges.get(null));
+ } else if (target instanceof Rule) {
+ Preconditions.checkNotNull(config);
+ visitTargetVisibility(node, outgoingEdges.get(null));
+ Rule rule = (Rule) target;
+ ListMultimap<Attribute, LabelAndConfiguration> labelMap =
+ resolveAttributes(rule, aspect, config, configConditions);
+ visitRule(rule, aspect, labelMap, outgoingEdges);
+ } else if (target instanceof PackageGroup) {
+ visitPackageGroup(node, (PackageGroup) target, outgoingEdges.get(null));
+ } else {
+ throw new IllegalStateException(target.getLabel().toString());
+ }
+ return outgoingEdges;
+ }
+
+ private ListMultimap<Attribute, LabelAndConfiguration> resolveAttributes(
+ Rule rule, AspectDefinition aspect, BuildConfiguration configuration,
+ Set<ConfigMatchingProvider> configConditions)
+ throws EvalException {
+ ConfiguredAttributeMapper attributeMap = ConfiguredAttributeMapper.of(rule, configConditions);
+ attributeMap.validateAttributes();
+ List<Attribute> attributes;
+ if (aspect == null) {
+ attributes = rule.getRuleClassObject().getAttributes();
+ } else {
+ attributes = new ArrayList<>();
+ attributes.addAll(rule.getRuleClassObject().getAttributes());
+ if (aspect != null) {
+ attributes.addAll(aspect.getAttributes().values());
+ }
+ }
+
+ ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> result =
+ ImmutableSortedKeyListMultimap.builder();
+
+ resolveExplicitAttributes(rule, configuration, attributeMap, result);
+ resolveImplicitAttributes(rule, configuration, attributeMap, attributes, result);
+ resolveLateBoundAttributes(rule, configuration, attributeMap, attributes, result);
+
+ // Add the rule's visibility labels (which may come from the rule or from package defaults).
+ addExplicitDeps(result, rule, "visibility", rule.getVisibility().getDependencyLabels(),
+ configuration);
+
+ // Add package default constraints when the rule doesn't explicitly declare them.
+ //
+ // Note that this can have subtle implications for constraint semantics. For example: say that
+ // package defaults declare compatibility with ':foo' and rule R declares compatibility with
+ // ':bar'. Does that mean that R is compatible with [':foo', ':bar'] or just [':bar']? In other
+ // words, did R's author intend to add additional compatibility to the package defaults or to
+ // override them? More severely, what if package defaults "restrict" support to just [':baz']?
+ // Should R's declaration signify [':baz'] + ['bar'], [ORIGINAL_DEFAULTS] + ['bar'], or
+ // something else?
+ //
+ // Rather than try to answer these questions with possibly confusing logic, we take the
+ // simple approach of assigning the rule's "restriction" attribute to the rule-declared value if
+ // it exists, else the package defaults value (and likewise for "compatibility"). This may not
+ // always provide what users want, but it makes it easy for them to understand how rule
+ // declarations and package defaults intermix (and how to refactor them to get what they want).
+ //
+ // An alternative model would be to apply the "rule declaration" / "rule class defaults"
+ // relationship, i.e. the rule class' "compatibility" and "restriction" declarations are merged
+ // to generate a set of default environments, then the rule's declarations are independently
+ // processed on top of that. This protects against obscure coupling behavior between
+ // declarations from wildly different places (e.g. it offers clear answers to the examples posed
+ // above). But within the scope of a single package it seems better to keep the model simple and
+ // make the user responsible for resolving ambiguities.
+ if (!rule.isAttributeValueExplicitlySpecified(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR)) {
+ addExplicitDeps(result, rule, RuleClass.COMPATIBLE_ENVIRONMENT_ATTR,
+ rule.getPackage().getDefaultCompatibleWith(), configuration);
+ }
+ if (!rule.isAttributeValueExplicitlySpecified(RuleClass.RESTRICTED_ENVIRONMENT_ATTR)) {
+ addExplicitDeps(result, rule, RuleClass.RESTRICTED_ENVIRONMENT_ATTR,
+ rule.getPackage().getDefaultRestrictedTo(), configuration);
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Adds new dependencies to the given rule under the given attribute name
+ *
+ * @param result the builder for the attribute --> dependency labels map
+ * @param rule the rule being evaluated
+ * @param attrName the name of the attribute to add dependency labels to
+ * @param labels the dependencies to add
+ * @param configuration the configuration to apply to those dependencies
+ */
+ private void addExplicitDeps(
+ ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> result, Rule rule,
+ String attrName, Iterable<Label> labels, BuildConfiguration configuration) {
+ if (!rule.isAttrDefined(attrName, Type.LABEL_LIST)
+ && !rule.isAttrDefined(attrName, Type.NODEP_LABEL_LIST)) {
+ return;
+ }
+ Attribute attribute = rule.getRuleClassObject().getAttributeByName(attrName);
+ for (Label label : labels) {
+ // The configuration must be the configuration after the first transition step (applying
+ // split configurations). The proper configuration (null) for package groups will be set
+ // later.
+ result.put(attribute, LabelAndConfiguration.of(label, configuration));
+ }
+ }
+
+ private void resolveExplicitAttributes(Rule rule, final BuildConfiguration configuration,
+ AttributeMap attributes,
+ final ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> builder) {
+ attributes.visitLabels(
+ new AttributeMap.AcceptsLabelAttribute() {
+ @Override
+ public void acceptLabelAttribute(Label label, Attribute attribute) {
+ String attributeName = attribute.getName();
+ if (attributeName.equals("abi_deps")) {
+ // abi_deps is handled specially: we visit only the branch that
+ // needs to be taken based on the configuration.
+ return;
+ }
+
+ if (attribute.getType() == Type.NODEP_LABEL) {
+ return;
+ }
+
+ if (attribute.isImplicit() || attribute.isLateBound()) {
+ return;
+ }
+
+ builder.put(attribute, LabelAndConfiguration.of(label, configuration));
+ }
+ });
+
+ // TODO(bazel-team): Remove this in favor of the new configurable attributes.
+ if (attributes.getAttributeDefinition("abi_deps") != null) {
+ Attribute depsAttribute = attributes.getAttributeDefinition("deps");
+ MakeVariableExpander.Context context = new ConfigurationMakeVariableContext(
+ rule.getPackage(), configuration);
+ String abi = null;
+ try {
+ abi = MakeVariableExpander.expand(attributes.get("abi", Type.STRING), context);
+ } catch (MakeVariableExpander.ExpansionException e) {
+ // Ignore this. It will be handled during the analysis phase.
+ }
+
+ if (abi != null) {
+ for (Map.Entry<String, List<Label>> entry
+ : attributes.get("abi_deps", Type.LABEL_LIST_DICT).entrySet()) {
+ try {
+ if (Pattern.matches(entry.getKey(), abi)) {
+ for (Label label : entry.getValue()) {
+ builder.put(depsAttribute, LabelAndConfiguration.of(label, configuration));
+ }
+ }
+ } catch (PatternSyntaxException e) {
+ // Ignore this. It will be handled during the analysis phase.
+ }
+ }
+ }
+ }
+ }
+
+ private void resolveImplicitAttributes(Rule rule, BuildConfiguration configuration,
+ AttributeMap attributeMap, Iterable<Attribute> attributes,
+ ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> builder) {
+ // Since the attributes that come from aspects do not appear in attributeMap, we have to get
+ // their values from somewhere else. This incidentally means that aspects attributes are not
+ // configurable. It would be nice if that wasn't the case, but we'd have to revamp how
+ // attribute mapping works, which is a large chunk of work.
+ ImmutableSet<String> mappedAttributes = ImmutableSet.copyOf(attributeMap.getAttributeNames());
+ for (Attribute attribute : attributes) {
+ if (!attribute.isImplicit() || !attribute.getCondition().apply(attributeMap)) {
+ continue;
+ }
+
+ if (attribute.getType() == Type.LABEL) {
+ Label label = mappedAttributes.contains(attribute.getName())
+ ? attributeMap.get(attribute.getName(), Type.LABEL)
+ : Type.LABEL.cast(attribute.getDefaultValue(rule));
+
+ if (label != null) {
+ builder.put(attribute, LabelAndConfiguration.of(label, configuration));
+ }
+ } else if (attribute.getType() == Type.LABEL_LIST) {
+ List<Label> labelList = mappedAttributes.contains(attribute.getName())
+ ? attributeMap.get(attribute.getName(), Type.LABEL_LIST)
+ : Type.LABEL_LIST.cast(attribute.getDefaultValue(rule));
+
+ for (Label label : labelList) {
+ builder.put(attribute, LabelAndConfiguration.of(label, configuration));
+ }
+ }
+ }
+ }
+
+ private void resolveLateBoundAttributes(Rule rule, BuildConfiguration configuration,
+ AttributeMap attributeMap, Iterable<Attribute> attributes,
+ ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> builder)
+ throws EvalException {
+ for (Attribute attribute : attributes) {
+ if (!attribute.isLateBound() || !attribute.getCondition().apply(attributeMap)) {
+ continue;
+ }
+
+ List<BuildConfiguration> actualConfigurations = ImmutableList.of(configuration);
+ if (attribute.getConfigurationTransition() instanceof SplitTransition<?>) {
+ Preconditions.checkState(attribute.getConfigurator() == null);
+ // TODO(bazel-team): This ends up applying the split transition twice, both here and in the
+ // visitRule method below - this is not currently a problem, because the configuration graph
+ // never contains nested split transitions, so the second application is idempotent.
+ actualConfigurations = configuration.getSplitConfigurations(
+ (SplitTransition<?>) attribute.getConfigurationTransition());
+ }
+
+ for (BuildConfiguration actualConfig : actualConfigurations) {
+ @SuppressWarnings("unchecked")
+ LateBoundDefault<BuildConfiguration> lateBoundDefault =
+ (LateBoundDefault<BuildConfiguration>) attribute.getLateBoundDefault();
+ if (lateBoundDefault.useHostConfiguration()) {
+ actualConfig =
+ actualConfig.getConfiguration(ConfigurationTransition.HOST);
+ }
+ // TODO(bazel-team): This might be too expensive - can we cache this somehow?
+ if (!lateBoundDefault.getRequiredConfigurationFragments().isEmpty()) {
+ if (!actualConfig.hasAllFragments(lateBoundDefault.getRequiredConfigurationFragments())) {
+ continue;
+ }
+ }
+
+ // TODO(bazel-team): We should check if the implementation tries to access an undeclared
+ // fragment.
+ Object actualValue = lateBoundDefault.getDefault(rule, actualConfig);
+ if (attribute.getType() == Type.LABEL) {
+ Label label;
+ label = Type.LABEL.cast(actualValue);
+ if (label != null) {
+ builder.put(attribute, LabelAndConfiguration.of(label, actualConfig));
+ }
+ } else if (attribute.getType() == Type.LABEL_LIST) {
+ for (Label label : Type.LABEL_LIST.cast(actualValue)) {
+ builder.put(attribute, LabelAndConfiguration.of(label, actualConfig));
+ }
+ } else {
+ throw new IllegalStateException(String.format(
+ "Late bound attribute '%s' is not a label or a label list", attribute.getName()));
+ }
+ }
+ }
+ }
+
+ /**
+ * A variant of {@link #dependentNodeMap} that only returns the values of the resulting map, and
+ * also converts any internally thrown {@link EvalException} instances into {@link
+ * IllegalStateException}.
+ */
+ public final Collection<Dependency> dependentNodes(
+ TargetAndConfiguration node, Set<ConfigMatchingProvider> configConditions) {
+ try {
+ return dependentNodeMap(node, null, configConditions).values();
+ } catch (EvalException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Converts the given multimap of attributes to labels into a multi map of attributes to
+ * {@link Dependency} objects using the proper configuration transition for each attribute.
+ *
+ * @throws IllegalArgumentException if the {@code node} does not refer to a {@link Rule} instance
+ */
+ public final Collection<Dependency> resolveRuleLabels(
+ TargetAndConfiguration node, AspectDefinition aspect, ListMultimap<Attribute,
+ LabelAndConfiguration> labelMap) {
+ Preconditions.checkArgument(node.getTarget() instanceof Rule);
+ Rule rule = (Rule) node.getTarget();
+ ListMultimap<Attribute, Dependency> outgoingEdges = ArrayListMultimap.create();
+ visitRule(rule, aspect, labelMap, outgoingEdges);
+ return outgoingEdges.values();
+ }
+
+ private void visitPackageGroup(TargetAndConfiguration node, PackageGroup packageGroup,
+ Collection<Dependency> outgoingEdges) {
+ for (Label label : packageGroup.getIncludes()) {
+ try {
+ Target target = getTarget(label);
+ if (target == null) {
+ return;
+ }
+ if (!(target instanceof PackageGroup)) {
+ // Note that this error could also be caught in PackageGroupConfiguredTarget, but since
+ // these have the null configuration, visiting the corresponding target would trigger an
+ // analysis of a rule with a null configuration, which doesn't work.
+ invalidPackageGroupReferenceHook(node, label);
+ continue;
+ }
+
+ outgoingEdges.add(new Dependency(label, node.getConfiguration()));
+ } catch (NoSuchThingException e) {
+ // Don't visit targets that don't exist (--keep_going)
+ }
+ }
+ }
+
+ private ImmutableSet<Class<? extends ConfiguredAspectFactory>> requiredAspects(
+ AspectDefinition aspect, Attribute attribute, Target target) {
+ if (!(target instanceof Rule)) {
+ return ImmutableSet.of();
+ }
+
+ RuleClass ruleClass = ((Rule) target).getRuleClassObject();
+
+ // The order of this set will be deterministic. This is necessary because this order eventually
+ // influences the order in which aspects are merged into the main configured target, which in
+ // turn influences which aspect takes precedence if two emit the same provider (maybe this
+ // should be an error)
+ Set<Class<? extends AspectFactory<?, ?, ?>>> aspectCandidates = new LinkedHashSet<>();
+ aspectCandidates.addAll(attribute.getAspects());
+ if (aspect != null) {
+ aspectCandidates.addAll(aspect.getAttributeAspects().get(attribute.getName()));
+ }
+
+ ImmutableSet.Builder<Class<? extends ConfiguredAspectFactory>> result = ImmutableSet.builder();
+ for (Class<? extends AspectFactory<?, ?, ?>> candidateClass : aspectCandidates) {
+ ConfiguredAspectFactory candidate =
+ (ConfiguredAspectFactory) AspectFactory.Util.create(candidateClass);
+ if (Sets.difference(
+ candidate.getDefinition().getRequiredProviders(),
+ ruleClass.getAdvertisedProviders()).isEmpty()) {
+ result.add(candidateClass.asSubclass(ConfiguredAspectFactory.class));
+ }
+ }
+
+ return result.build();
+ }
+
+ private void visitRule(Rule rule, AspectDefinition aspect,
+ ListMultimap<Attribute, LabelAndConfiguration> labelMap,
+ ListMultimap<Attribute, Dependency> outgoingEdges) {
+ Preconditions.checkNotNull(labelMap);
+ for (Map.Entry<Attribute, Collection<LabelAndConfiguration>> entry :
+ labelMap.asMap().entrySet()) {
+ Attribute attribute = entry.getKey();
+ for (LabelAndConfiguration dep : entry.getValue()) {
+ Label label = dep.getLabel();
+ BuildConfiguration config = dep.getConfiguration();
+
+ Target toTarget;
+ try {
+ toTarget = getTarget(label);
+ } catch (NoSuchThingException e) {
+ throw new IllegalStateException("not found: " + label + " from " + rule + " in "
+ + attribute.getName());
+ }
+ if (toTarget == null) {
+ continue;
+ }
+ Iterable<BuildConfiguration> toConfigurations = config.evaluateTransition(
+ rule, attribute, toTarget);
+ for (BuildConfiguration toConfiguration : toConfigurations) {
+ outgoingEdges.get(entry.getKey()).add(new Dependency(
+ label, toConfiguration, requiredAspects(aspect, attribute, toTarget)));
+ }
+ }
+ }
+ }
+
+ private void visitTargetVisibility(TargetAndConfiguration node,
+ Collection<Dependency> outgoingEdges) {
+ for (Label label : node.getTarget().getVisibility().getDependencyLabels()) {
+ try {
+ Target visibilityTarget = getTarget(label);
+ if (visibilityTarget == null) {
+ return;
+ }
+ if (!(visibilityTarget instanceof PackageGroup)) {
+ // Note that this error could also be caught in
+ // AbstractConfiguredTarget.convertVisibility(), but we have an
+ // opportunity here to avoid dependency cycles that result from
+ // the visibility attribute of a rule referring to a rule that
+ // depends on it (instead of its package)
+ invalidVisibilityReferenceHook(node, label);
+ continue;
+ }
+
+ // Visibility always has null configuration
+ outgoingEdges.add(new Dependency(label, null));
+ } catch (NoSuchThingException e) {
+ // Don't visit targets that don't exist (--keep_going)
+ }
+ }
+ }
+
+ /**
+ * Hook for the error case when an invalid visibility reference is found.
+ *
+ * @param node the node with the visibility attribute
+ * @param label the invalid visibility reference
+ */
+ protected abstract void invalidVisibilityReferenceHook(TargetAndConfiguration node, Label label);
+
+ /**
+ * Hook for the error case when an invalid package group reference is found.
+ *
+ * @param node the package group node with the includes attribute
+ * @param label the invalid reference
+ */
+ protected abstract void invalidPackageGroupReferenceHook(TargetAndConfiguration node,
+ Label label);
+
+ /**
+ * Returns the target by the given label.
+ *
+ * <p>Throws {@link NoSuchThingException} if the target is known not to exist.
+ *
+ * <p>Returns null if the target is not ready to be returned at this moment. If getTarget returns
+ * null once or more during a {@link #dependentNodeMap} call, the results of that call will be
+ * incomplete. For use within Skyframe, where several iterations may be needed to discover
+ * all dependencies.
+ */
+ @Nullable
+ protected abstract Target getTarget(Label label) throws NoSuchThingException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ErrorConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/ErrorConfiguredTarget.java
new file mode 100644
index 0000000..aa07a7d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ErrorConfiguredTarget.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Target;
+
+/**
+ * A configured target that is used instead of a real configured target if there
+ * are cyclic dependencies or if any of the prerequisites has errors. This
+ * avoids accessing state that shouldn't be accessed.
+ */
+final class ErrorConfiguredTarget extends AbstractConfiguredTarget {
+ ErrorConfiguredTarget(Target target, BuildConfiguration configuration) {
+ super(target, configuration);
+ }
+
+ @Override
+ public Object get(String providerKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ throw new IllegalStateException();
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionArtifactsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionArtifactsProvider.java
new file mode 100644
index 0000000..05b5f37
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionArtifactsProvider.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A {@link TransitiveInfoProvider} that creates extra actions.
+ */
+@Immutable
+public final class ExtraActionArtifactsProvider implements TransitiveInfoProvider {
+ public static final ExtraActionArtifactsProvider EMPTY =
+ new ExtraActionArtifactsProvider(
+ ImmutableList.<Artifact>of(),
+ NestedSetBuilder.<ExtraArtifactSet>emptySet(Order.STABLE_ORDER));
+
+ /**
+ * The set of extra artifacts provided by a single configured target.
+ */
+ @Immutable
+ public static final class ExtraArtifactSet {
+ private final Label label;
+ private final ImmutableList<Artifact> artifacts;
+
+ private ExtraArtifactSet(Label label, Iterable<Artifact> artifacts) {
+ this.label = label;
+ this.artifacts = ImmutableList.copyOf(artifacts);
+ }
+
+ public Label getLabel() {
+ return label;
+ }
+
+ public ImmutableList<Artifact> getArtifacts() {
+ return artifacts;
+ }
+
+ public static ExtraArtifactSet of(Label label, Iterable<Artifact> artifacts) {
+ return new ExtraArtifactSet(label, artifacts);
+ }
+
+ @Override
+ public int hashCode() {
+ return label.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof ExtraArtifactSet)) {
+ return false;
+ }
+
+ return label.equals(((ExtraArtifactSet) other).getLabel());
+ }
+ }
+
+ /** The outputs of the extra actions associated with this target. */
+ private ImmutableList<Artifact> extraActionArtifacts = ImmutableList.of();
+ private NestedSet<ExtraArtifactSet> transitiveExtraActionArtifacts =
+ NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+
+ public ExtraActionArtifactsProvider(ImmutableList<Artifact> extraActionArtifacts,
+ NestedSet<ExtraArtifactSet> transitiveExtraActionArtifacts) {
+ this.extraActionArtifacts = extraActionArtifacts;
+ this.transitiveExtraActionArtifacts = transitiveExtraActionArtifacts;
+ }
+
+ /**
+ * The outputs of the extra actions associated with this target.
+ */
+ public ImmutableList<Artifact> getExtraActionArtifacts() {
+ return extraActionArtifacts;
+ }
+
+ /**
+ * The outputs of the extra actions in the whole transitive closure.
+ */
+ public NestedSet<ExtraArtifactSet> getTransitiveExtraActionArtifacts() {
+ return transitiveExtraActionArtifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionsVisitor.java b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionsVisitor.java
new file mode 100644
index 0000000..79ffe80
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionsVisitor.java
@@ -0,0 +1,84 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionGraphVisitor;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.rules.extra.ExtraActionSpec;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A bipartite graph visitor which accumulates extra actions for a target.
+ */
+final class ExtraActionsVisitor extends ActionGraphVisitor {
+ private final RuleContext ruleContext;
+ private final Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap;
+ private final List<Artifact> extraArtifacts;
+ public final Set<Action> actions = Sets.newHashSet();
+
+ /** Creates a new visitor for the extra actions associated with the given target. */
+ public ExtraActionsVisitor(RuleContext ruleContext,
+ Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap) {
+ super(getActionGraph(ruleContext));
+ this.ruleContext = ruleContext;
+ this.mnemonicToExtraActionMap = mnemonicToExtraActionMap;
+ extraArtifacts = Lists.newArrayList();
+ }
+
+ public void addExtraAction(Action original) {
+ Collection<ExtraActionSpec> extraActions = mnemonicToExtraActionMap.get(
+ original.getMnemonic());
+ if (extraActions != null) {
+ for (ExtraActionSpec extraAction : extraActions) {
+ extraArtifacts.addAll(extraAction.addExtraAction(ruleContext, original));
+ }
+ }
+ }
+
+ @Override
+ protected void visitAction(Action action) {
+ actions.add(action);
+ addExtraAction(action);
+ }
+
+ /** Retrieves the collected artifacts since this method was last called and clears the list. */
+ public ImmutableList<Artifact> getAndResetExtraArtifacts() {
+ ImmutableList<Artifact> collected = ImmutableList.copyOf(extraArtifacts);
+ extraArtifacts.clear();
+ return collected;
+ }
+
+ /** Gets an action graph wrapper for the given target through its analysis environment. */
+ private static ActionGraph getActionGraph(final RuleContext ruleContext) {
+ return new ActionGraph() {
+ @Override
+ @Nullable
+ public Action getGeneratingAction(Artifact artifact) {
+ return ruleContext.getAnalysisEnvironment().getLocalGeneratingAction(artifact);
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FileConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/FileConfiguredTarget.java
new file mode 100644
index 0000000..815eea7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/FileConfiguredTarget.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.FileTarget;
+import com.google.devtools.build.lib.rules.fileset.FilesetProvider;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.util.FileType;
+
+/**
+ * A ConfiguredTarget for a source FileTarget. (Generated files use a
+ * subclass, OutputFileConfiguredTarget.)
+ */
+public abstract class FileConfiguredTarget extends AbstractConfiguredTarget
+ implements FileType.HasFilename, LicensesProvider {
+
+ private final Artifact artifact;
+ private final ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider>
+ providers;
+
+ FileConfiguredTarget(TargetContext targetContext, Artifact artifact) {
+ super(targetContext);
+ NestedSet<Artifact> filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, artifact);
+ this.artifact = artifact;
+ Builder<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> builder = ImmutableMap
+ .<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider>builder()
+ .put(VisibilityProvider.class, this)
+ .put(LicensesProvider.class, this)
+ .put(FileProvider.class, new FileProvider(targetContext.getLabel(), filesToBuild))
+ .put(FilesToRunProvider.class, new FilesToRunProvider(targetContext.getLabel(),
+ ImmutableList.copyOf(filesToBuild), null, artifact));
+ if (this instanceof FilesetProvider) {
+ builder.put(FilesetProvider.class, this);
+ }
+ if (this instanceof InstrumentedFilesProvider) {
+ builder.put(InstrumentedFilesProvider.class, this);
+ }
+ this.providers = builder.build();
+ }
+
+ @Override
+ public FileTarget getTarget() {
+ return (FileTarget) super.getTarget();
+ }
+
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ /**
+ * Returns the file type of this file target.
+ */
+ @Override
+ public String getFilename() {
+ return getTarget().getFilename();
+ }
+
+ @Override
+ public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) {
+ AnalysisUtils.checkProvider(provider);
+ return provider.cast(providers.get(provider));
+ }
+
+ @Override
+ public Object get(String providerKey) {
+ return null;
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ return providers.values().iterator();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FileProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/FileProvider.java
new file mode 100644
index 0000000..893f211
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/FileProvider.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+
+import javax.annotation.Nullable;
+
+/**
+ * A representation of the concept "this transitive info provider builds these files".
+ *
+ * <p>Every transitive info collection contains at least this provider.
+ */
+@Immutable
+@SkylarkModule(name = "file_provider", doc = "An interface for rules that provide files.")
+public final class FileProvider implements TransitiveInfoProvider {
+
+ @Nullable private final Label label;
+ private final NestedSet<Artifact> filesToBuild;
+
+ public FileProvider(@Nullable Label label, NestedSet<Artifact> filesToBuild) {
+ this.label = label;
+ this.filesToBuild = filesToBuild;
+ }
+
+ /**
+ * Returns the label that is associated with this piece of information.
+ *
+ * <p>This is usually the label of the target that provides the information.
+ */
+ @SkylarkCallable(name = "label", doc = "", structField = true)
+ public Label getLabel() {
+ if (label == null) {
+ throw new UnsupportedOperationException();
+ }
+ return label;
+ }
+
+ /**
+ * Returns the set of artifacts that are the "output" of this rule.
+ *
+ * <p>The term "output" is somewhat hazily defined; it is vaguely the set of files that are
+ * passed on to dependent rules that list the rule in their {@code srcs} attribute and the
+ * set of files that are built when a rule is mentioned on the command line. It does
+ * <b>not</b> include the runfiles; that is the bailiwick of {@code FilesToRunProvider}.
+ *
+ * <p>Note that the above definition is somewhat imprecise; in particular, when a rule is
+ * mentioned on the command line, some other files are also built
+ * {@code TopLevelArtifactHelper} and dependent rules are free to filter this set of artifacts
+ * e.g. based on their extension.
+ *
+ * <p>Also, some rules may generate artifacts that are not listed here by way of defining other
+ * implicit targets, for example, deploy jars.
+ */
+ @SkylarkCallable(name = "files_to_build", doc = "", structField = true)
+ public NestedSet<Artifact> getFilesToBuild() {
+ return filesToBuild;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FilesToCompileProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/FilesToCompileProvider.java
new file mode 100644
index 0000000..025392c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/FilesToCompileProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A {@link TransitiveInfoProvider} that provides files to be built when the {@code --compile_only}
+ * command line option is in effect. This is to avoid expensive build steps when the user only
+ * wants a quick syntax check.
+ */
+@Immutable
+public final class FilesToCompileProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<Artifact> filesToCompile;
+
+ public FilesToCompileProvider(ImmutableList<Artifact> filesToCompile) {
+ this.filesToCompile = filesToCompile;
+ }
+
+ /**
+ * Returns the list of artifacts to be built when the {@code --compile_only} command line option
+ * is in effect.
+ */
+ public ImmutableList<Artifact> getFilesToCompile() {
+ return filesToCompile;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FilesToRunProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/FilesToRunProvider.java
new file mode 100644
index 0000000..0e024b1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/FilesToRunProvider.java
@@ -0,0 +1,80 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+import javax.annotation.Nullable;
+
+/**
+ * Returns information about executables produced by a target and the files needed to run it.
+ */
+@Immutable
+public final class FilesToRunProvider implements TransitiveInfoProvider {
+
+ private final Label label;
+ private final ImmutableList<Artifact> filesToRun;
+ @Nullable private final RunfilesSupport runfilesSupport;
+ @Nullable private final Artifact executable;
+
+ public FilesToRunProvider(Label label, ImmutableList<Artifact> filesToRun,
+ @Nullable RunfilesSupport runfilesSupport, @Nullable Artifact executable) {
+ this.label = label;
+ this.filesToRun = filesToRun;
+ this.runfilesSupport = runfilesSupport;
+ this.executable = executable;
+ }
+
+ /**
+ * Returns the label that is associated with this piece of information.
+ *
+ * <p>This is usually the label of the target that provides the information.
+ */
+ public Label getLabel() {
+ return label;
+ }
+
+ /**
+ * Returns artifacts needed to run the executable for this target.
+ */
+ public ImmutableList<Artifact> getFilesToRun() {
+ return filesToRun;
+ }
+
+ /**
+ * Returns the {@RunfilesSupport} object associated with the target or null if it does not exist.
+ */
+ @Nullable public RunfilesSupport getRunfilesSupport() {
+ return runfilesSupport;
+ }
+
+ /**
+ * Returns the Executable or null if it does not exist.
+ */
+ @Nullable public Artifact getExecutable() {
+ return executable;
+ }
+
+ /**
+ * Returns the RunfilesManifest or null if it does not exist. It is a shortcut to
+ * getRunfilesSupport().getRunfilesManifest().
+ */
+ @Nullable public Artifact getRunfilesManifest() {
+ return runfilesSupport != null ? runfilesSupport.getRunfilesManifest() : null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FilesetOutputConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/FilesetOutputConfiguredTarget.java
new file mode 100644
index 0000000..860024d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/FilesetOutputConfiguredTarget.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.rules.fileset.FilesetProvider;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A configured target for output files generated by {@code Fileset} rules. They are almost the
+ * same thing as output files except that they implement {@link FilesetProvider} so that
+ * {@code Fileset} can figure out the link tree behind them.
+ *
+ * <p>In an ideal world, this would not be needed: Filesets would depend on other Filesets and not
+ * their output directories. However, sometimes a Fileset depends on the output directory of
+ * another Fileset. Thus, we need this hack.
+ */
+public final class FilesetOutputConfiguredTarget extends OutputFileConfiguredTarget
+ implements FilesetProvider {
+ private final Artifact filesetInputManifest;
+ private final PathFragment filesetLinkDir;
+
+ FilesetOutputConfiguredTarget(TargetContext targetContext, OutputFile outputFile,
+ TransitiveInfoCollection generatingRule, Artifact outputArtifact) {
+ super(targetContext, outputFile, generatingRule, outputArtifact);
+ FilesetProvider provider = generatingRule.getProvider(FilesetProvider.class);
+ Preconditions.checkArgument(provider != null);
+ filesetInputManifest = provider.getFilesetInputManifest();
+ filesetLinkDir = provider.getFilesetLinkDir();
+ }
+
+ @Override
+ public Artifact getFilesetInputManifest() {
+ return filesetInputManifest;
+ }
+
+ @Override
+ public PathFragment getFilesetLinkDir() {
+ return filesetLinkDir;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/InputFileConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/InputFileConfiguredTarget.java
new file mode 100644
index 0000000..9e56033
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/InputFileConfiguredTarget.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.License;
+
+/**
+ * A ConfiguredTarget for an InputFile.
+ *
+ * All InputFiles for the same target are equivalent, so configuration does not
+ * play any role here and is always set to <b>null</b>.
+ */
+public final class InputFileConfiguredTarget extends FileConfiguredTarget {
+ private final Artifact artifact;
+ private final NestedSet<TargetLicense> licenses;
+
+ InputFileConfiguredTarget(TargetContext targetContext, InputFile inputFile, Artifact artifact) {
+ super(targetContext, artifact);
+ Preconditions.checkArgument(targetContext.getTarget() == inputFile, getLabel());
+ Preconditions.checkArgument(getConfiguration() == null, getLabel());
+ this.artifact = artifact;
+
+ if (inputFile.getLicense() != License.NO_LICENSE) {
+ licenses = NestedSetBuilder.create(Order.LINK_ORDER,
+ new TargetLicense(getLabel(), inputFile.getLicense()));
+ } else {
+ licenses = NestedSetBuilder.emptySet(Order.LINK_ORDER);
+ }
+ }
+
+ @Override
+ public InputFile getTarget() {
+ return (InputFile) super.getTarget();
+ }
+
+ @Override
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ @Override
+ public String toString() {
+ return "InputFileConfiguredTarget(" + getTarget().getLabel() + ")";
+ }
+
+ @Override
+ public final NestedSet<TargetLicense> getTransitiveLicenses() {
+ return licenses;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LabelAndConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/LabelAndConfiguration.java
new file mode 100644
index 0000000..66efba3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/LabelAndConfiguration.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+* A (label,configuration) pair.
+*/
+public final class LabelAndConfiguration {
+ private final Label label;
+ @Nullable
+ private final BuildConfiguration configuration;
+
+ private LabelAndConfiguration(Label label, @Nullable BuildConfiguration configuration) {
+ this.label = Preconditions.checkNotNull(label);
+ this.configuration = configuration;
+ }
+
+ public LabelAndConfiguration(ConfiguredTarget rule) {
+ this(rule.getTarget().getLabel(), rule.getConfiguration());
+ }
+
+ public Label getLabel() {
+ return label;
+ }
+
+ @Nullable
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public int hashCode() {
+ int configVal = configuration == null ? 79 : configuration.hashCode();
+ return 31 * label.hashCode() + configVal;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof LabelAndConfiguration)) {
+ return false;
+ }
+ LabelAndConfiguration other = (LabelAndConfiguration) obj;
+ return Objects.equals(label, other.label) && Objects.equals(configuration, other.configuration);
+ }
+
+ public static LabelAndConfiguration of(
+ Label label, @Nullable BuildConfiguration configuration) {
+ return new LabelAndConfiguration(label, configuration);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LabelExpander.java b/src/main/java/com/google/devtools/build/lib/analysis/LabelExpander.java
new file mode 100644
index 0000000..89ce2e7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/LabelExpander.java
@@ -0,0 +1,181 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class encapsulating string scanning state used during "heuristic"
+ * expansion of labels embedded within rules.
+ */
+public final class LabelExpander {
+ /**
+ * An exception that is thrown when a label is expanded to zero or multiple
+ * files during expansion.
+ */
+ public static class NotUniqueExpansionException extends Exception {
+ public NotUniqueExpansionException(int sizeOfResultSet, String labelText) {
+ super("heuristic label expansion found '" + labelText + "', which expands to "
+ + sizeOfResultSet + " files"
+ + (sizeOfResultSet > 1
+ ? ", please use $(locations " + labelText + ") instead"
+ : ""));
+ }
+ }
+
+ // This is a utility class, no need to instantiate.
+ private LabelExpander() {}
+
+ /**
+ * CharMatcher to determine if a given character is valid for labels.
+ *
+ * <p>The Build Concept Reference additionally allows '=' and ',' to appear in labels,
+ * but for the purposes of the heuristic, this function does not, as it would cause
+ * "--foo=:rule1,:rule2" to scan as a single possible label, instead of three
+ * ("--foo", ":rule1", ":rule2").
+ */
+ private static final CharMatcher LABEL_CHAR_MATCHER =
+ CharMatcher.inRange('a', 'z')
+ .or(CharMatcher.inRange('A', 'Z'))
+ .or(CharMatcher.inRange('0', '9'))
+ .or(CharMatcher.anyOf(":/_.-+" + PathFragment.SEPARATOR_CHAR));
+
+ /**
+ * Expands all references to labels embedded within a string using the
+ * provided expansion mapping from labels to artifacts.
+ *
+ * <p>Since this pass is heuristic, references to non-existent labels (such
+ * as arbitrary words) or invalid labels are simply ignored and are unchanged
+ * in the output. However, if the heuristic discovers a label, which
+ * identifies an existing target producing zero or multiple files, an error
+ * is reported.
+ *
+ * @param expression the expression to expand.
+ * @param labelMap the mapping from labels to artifacts, whose relative path
+ * is to be used as the expansion.
+ * @param labelResolver the {@code Label} that can resolve label strings
+ * to {@code Label} objects. The resolved label is either relative to
+ * {@code labelResolver} or is a global label (i.e. starts with "//").
+ * @return the expansion of the string.
+ * @throws NotUniqueExpansionException if a label that is present in the
+ * mapping expands to zero or multiple files.
+ */
+ public static <T extends Iterable<Artifact>> String expand(@Nullable String expression,
+ Map<Label, T> labelMap, Label labelResolver) throws NotUniqueExpansionException {
+ if (Strings.isNullOrEmpty(expression)) {
+ return "";
+ }
+ Preconditions.checkNotNull(labelMap);
+ Preconditions.checkNotNull(labelResolver);
+
+ int offset = 0;
+ StringBuilder result = new StringBuilder();
+ while (offset < expression.length()) {
+ String labelText = scanLabel(expression, offset);
+ if (labelText != null) {
+ offset += labelText.length();
+ result.append(tryResolvingLabelTextToArtifactPath(labelText, labelMap, labelResolver));
+ } else {
+ result.append(expression.charAt(offset));
+ offset++;
+ }
+ }
+ return result.toString();
+ }
+
+ /**
+ * Tries resolving a label text to a full label for the associated {@code
+ * Artifact}, using the provided mapping.
+ *
+ * <p>The method succeeds if the label text can be resolved to a {@code
+ * Label} object, which is present in the {@code labelMap} and maps to
+ * exactly one {@code Artifact}.
+ *
+ * @param labelText the text to resolve.
+ * @param labelMap the mapping from labels to artifacts, whose relative path
+ * is to be used as the expansion.
+ * @param labelResolver the {@code Label} that can resolve label strings
+ * to {@code Label} objects. The resolved label is either relative to
+ * {@code labelResolver} or is a global label (i.e. starts with "//").
+ * @return an absolute label to an {@code Artifact} if the resolving was
+ * successful or the original label text.
+ * @throws NotUniqueExpansionException if a label that is present in the
+ * mapping expands to zero or multiple files.
+ */
+ private static <T extends Iterable<Artifact>> String tryResolvingLabelTextToArtifactPath(
+ String labelText, Map<Label, T> labelMap, Label labelResolver)
+ throws NotUniqueExpansionException {
+ Label resolvedLabel = resolveLabelText(labelText, labelResolver);
+ if (resolvedLabel != null) {
+ Iterable<Artifact> artifacts = labelMap.get(resolvedLabel);
+ if (artifacts != null) { // resolvedLabel identifies an existing target
+ List<String> locations = new ArrayList<>();
+ Artifact.addExecPaths(artifacts, locations);
+ int resultSetSize = locations.size();
+ if (resultSetSize == 1) {
+ return Iterables.getOnlyElement(locations); // success!
+ } else {
+ throw new NotUniqueExpansionException(resultSetSize, labelText);
+ }
+ }
+ }
+ return labelText;
+ }
+
+ /**
+ * Resolves a string to a label text. Uses {@code labelResolver} to do so.
+ * The result is either relative to {@code labelResolver} or is an absolute
+ * label. In case of an invalid label text, the return value is null.
+ */
+ private static Label resolveLabelText(String labelText, Label labelResolver) {
+ try {
+ return labelResolver.getRelative(labelText);
+ } catch (Label.SyntaxException e) {
+ // It's a heuristic, so quietly ignore "errors". Because Label.getRelative never
+ // returns null, we can use null to indicate an error.
+ return null;
+ }
+ }
+
+ /**
+ * Scans the argument string from a given start position until the name of a
+ * potential label has been consumed, then returns the label text. If
+ * the expression contains no possible label starting at the start position,
+ * the return value is null.
+ */
+ private static String scanLabel(String expression, int start) {
+ int offset = start;
+ while (offset < expression.length() && LABEL_CHAR_MATCHER.matches(expression.charAt(offset))) {
+ ++offset;
+ }
+ if (offset > start) {
+ return expression.substring(start, offset);
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LanguageDependentFragment.java b/src/main/java/com/google/devtools/build/lib/analysis/LanguageDependentFragment.java
new file mode 100644
index 0000000..d42d625
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/LanguageDependentFragment.java
@@ -0,0 +1,109 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Set;
+
+/**
+ * Transitive info provider for rules that behave differently when used from
+ * different languages.
+ *
+ * <p>Most rules generate code for a particular language or are totally language independent.
+ * Some rules, however, behave differently when depended upon from different languages.
+ * They might generate different libraries when used from different languages (and with
+ * different API versions). This interface allows code sharing between implementations.
+ *
+ * <p>This provider is not really a roll-up of transitive information.
+ */
+@Immutable
+public final class LanguageDependentFragment implements TransitiveInfoProvider {
+ /**
+ * A language that can be supported by a multi-language configured target.
+ *
+ * <p>Note that no {@code hashCode}/{@code equals} methods are provided, because these
+ * objects are expected to be compared for object identity, which is the default.
+ */
+ public static final class LibraryLanguage {
+ private final String displayName;
+
+ public LibraryLanguage(String displayName) {
+ this.displayName = displayName;
+ }
+
+ @Override
+ public String toString() {
+ return displayName;
+ }
+ }
+
+ private final Label label;
+ private final ImmutableSet<LibraryLanguage> languages;
+
+ public LanguageDependentFragment(Label label, Set<LibraryLanguage> languages) {
+ this.label = label;
+ this.languages = ImmutableSet.copyOf(languages);
+ }
+
+ /**
+ * Returns the label that is associated with this piece of information.
+ *
+ * <p>This is usually the label of the target that provides the information.
+ */
+ public Label getLabel() {
+ return label;
+ }
+
+ /**
+ * Returns a set of the languages the ConfiguredTarget generates output for.
+ * For use only by rules that directly depend on this library via a "deps" attribute.
+ */
+ public ImmutableSet<LibraryLanguage> getSupportedLanguages() {
+ return languages;
+ }
+
+ /**
+ * Routines for verifying that dependency provide the right output.
+ */
+ public static final class Checker {
+ /**
+ * Checks that given dep supports the given language.
+ */
+ public static boolean depSupportsLanguage(
+ RuleContext context, LanguageDependentFragment dep, LibraryLanguage language) {
+ if (dep.getSupportedLanguages().contains(language)) {
+ return true;
+ } else {
+ context.attributeError(
+ "deps", String.format("'%s' does not produce output for %s", dep.getLabel(), language));
+ return false;
+ }
+ }
+
+ /**
+ * Checks that all LanguageDependentFragment support the given language.
+ */
+ public static void depsSupportsLanguage(RuleContext context, LibraryLanguage language) {
+ for (LanguageDependentFragment dep :
+ context.getPrerequisites("deps", Mode.TARGET, LanguageDependentFragment.class)) {
+ depSupportsLanguage(context, dep, language);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LicensesProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProvider.java
new file mode 100644
index 0000000..548a1f2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProvider.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Objects;
+
+/**
+ * A {@link ConfiguredTarget} that has licensed targets in its transitive closure.
+ */
+public interface LicensesProvider extends TransitiveInfoProvider {
+
+ /**
+ * The set of label - license associations in the transitive closure.
+ *
+ * <p>Always returns an empty set if {@link BuildConfiguration#checkLicenses()} is false.
+ */
+ NestedSet<TargetLicense> getTransitiveLicenses();
+
+ /**
+ * License association for a particular target.
+ */
+ public static final class TargetLicense {
+
+ private final Label label;
+ private final License license;
+
+ public TargetLicense(Label label, License license) {
+ Preconditions.checkNotNull(label);
+ Preconditions.checkNotNull(license);
+ this.label = label;
+ this.license = license;
+ }
+
+ /**
+ * Returns the label of the associated target.
+ */
+ public Label getLabel() {
+ return label;
+ }
+
+ /**
+ * Returns the license for the target.
+ */
+ public License getLicense() {
+ return license;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(label, license);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TargetLicense)) {
+ return false;
+ }
+ TargetLicense other = (TargetLicense) obj;
+ return label.equals(other.label) && license.equals(other.license);
+ }
+
+ @Override
+ public String toString() {
+ return label + " => " + license;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LicensesProviderImpl.java b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProviderImpl.java
new file mode 100644
index 0000000..ffdf9fd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProviderImpl.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A {@link ConfiguredTarget} that has licensed targets in its transitive closure.
+ */
+@Immutable
+public final class LicensesProviderImpl implements LicensesProvider {
+ public static final LicensesProvider EMPTY =
+ new LicensesProviderImpl(NestedSetBuilder.<TargetLicense>emptySet(Order.LINK_ORDER));
+
+ private final NestedSet<TargetLicense> transitiveLicenses;
+
+ public LicensesProviderImpl(NestedSet<TargetLicense> transitiveLicenses) {
+ this.transitiveLicenses = transitiveLicenses;
+ }
+
+ @Override
+ public NestedSet<TargetLicense> getTransitiveLicenses() {
+ return transitiveLicenses;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LocationExpander.java b/src/main/java/com/google/devtools/build/lib/analysis/LocationExpander.java
new file mode 100644
index 0000000..8feb28e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/LocationExpander.java
@@ -0,0 +1,260 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Expands $(location) tags inside target attributes.
+ * You can specify something like this in the BUILD file:
+ *
+ * somerule(name='some name',
+ * someopt = [ '$(location //mypackage:myhelper)' ],
+ * ...)
+ *
+ * and location will be substituted with //mypackage:myhelper executable output.
+ * Note that //mypackage:myhelper should have just one output.
+ */
+public class LocationExpander {
+ private static final int MAX_PATHS_SHOWN = 5;
+ private static final String LOCATION = "$(location";
+ private final RuleContext ruleContext;
+ private Map<Label, Collection<Artifact>> locationMap;
+ private boolean allowDataAttributeEntriesInLabel = false;
+
+ /**
+ * Creates location expander helper bound to specific target and with default
+ * location map.
+ *
+ * @param ruleContext BUILD rule
+ */
+ public LocationExpander(RuleContext ruleContext) {
+ this(ruleContext, false);
+ }
+
+ public LocationExpander(RuleContext ruleContext,
+ boolean allowDataAttributeEntriesInLabel) {
+ this.ruleContext = ruleContext;
+ this.allowDataAttributeEntriesInLabel = allowDataAttributeEntriesInLabel;
+ }
+
+ public Map<Label, Collection<Artifact>> getLocationMap() {
+ if (locationMap == null) {
+ locationMap = buildLocationMap(ruleContext, allowDataAttributeEntriesInLabel);
+ }
+ return locationMap;
+ }
+
+ /**
+ * Expands attribute's location and locations tags based on the target and
+ * location map.
+ *
+ * @param attrName name of the attribute
+ * @param attrValue initial value of the attribute
+ * @return attribute value with expanded location tags or original value in
+ * case of errors
+ */
+ public String expand(String attrName, String attrValue) {
+ int restart = 0;
+
+ int attrLength = attrValue.length();
+ StringBuilder result = new StringBuilder(attrValue.length());
+
+ while (true) {
+ // (1) find '$(location ' or '$(locations '
+ String message = "$(location)";
+ boolean multiple = false;
+ int start = attrValue.indexOf(LOCATION, restart);
+ int scannedLength = LOCATION.length();
+ if (start == -1 || start + scannedLength == attrLength) {
+ result.append(attrValue.substring(restart));
+ break;
+ }
+
+ if (attrValue.charAt(start + scannedLength) == 's') {
+ scannedLength++;
+ if (start + scannedLength == attrLength) {
+ result.append(attrValue.substring(restart));
+ break;
+ }
+ message = "$(locations)";
+ multiple = true;
+ }
+
+ if (attrValue.charAt(start + scannedLength) != ' ') {
+ result.append(attrValue.substring(restart, start + scannedLength));
+ restart = start + scannedLength;
+ continue;
+ }
+ scannedLength++;
+
+ int end = attrValue.indexOf(')', start + scannedLength);
+ if (end == -1) {
+ ruleContext.attributeError(attrName, "unterminated " + message + " expression");
+ return attrValue;
+ }
+
+ // (2) parse label
+ String labelText = attrValue.substring(start + scannedLength, end);
+ Label label;
+ try {
+ label = ruleContext.getLabel().getRelative(labelText);
+ } catch (Label.SyntaxException e) {
+ ruleContext.attributeError(attrName,
+ "invalid label in " + message + " expression: " + e.getMessage());
+ return attrValue;
+ }
+
+ // (3) replace with singleton artifact, iff unique.
+ Collection<Artifact> artifacts = getLocationMap().get(label);
+ if (artifacts == null) {
+ ruleContext.attributeError(attrName,
+ "label '" + label + "' in " + message + " expression is not a "
+ + "declared prerequisite of this rule");
+ return attrValue;
+ }
+ List<String> paths = getPaths(artifacts);
+ if (paths.isEmpty()) {
+ ruleContext.attributeError(attrName,
+ "label '" + label + "' in " + message + " expression expands to no "
+ + "files");
+ return attrValue;
+ }
+
+ result.append(attrValue.substring(restart, start));
+ if (multiple) {
+ Collections.sort(paths);
+ Joiner.on(' ').appendTo(result, paths);
+ } else {
+ if (paths.size() > 1) {
+ ruleContext.attributeError(attrName,
+ String.format(
+ "label '%s' in %s expression expands to more than one file, "
+ + "please use $(locations %s) instead. Files (at most %d shown) are: %s",
+ label, message, label,
+ MAX_PATHS_SHOWN, Iterables.limit(paths, MAX_PATHS_SHOWN)));
+ return attrValue;
+ }
+ result.append(Iterables.getOnlyElement(paths));
+ }
+ restart = end + 1;
+ }
+ return result.toString();
+ }
+
+ /**
+ * Extracts all possible target locations from target specification.
+ *
+ * @param ruleContext BUILD target object
+ * @return map of all possible target locations
+ */
+ private static Map<Label, Collection<Artifact>> buildLocationMap(RuleContext ruleContext,
+ boolean allowDataAttributeEntriesInLabel) {
+ Map<Label, Collection<Artifact>> locationMap = new HashMap<>();
+
+ // Add all destination locations.
+ for (OutputFile out : ruleContext.getRule().getOutputFiles()) {
+ mapGet(locationMap, out.getLabel()).add(ruleContext.createOutputArtifact(out));
+ }
+
+ if (ruleContext.getRule().isAttrDefined("srcs", Type.LABEL_LIST)) {
+ for (FileProvider src : ruleContext
+ .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) {
+ Iterables.addAll(mapGet(locationMap, src.getLabel()), src.getFilesToBuild());
+ }
+ }
+
+ // Add all locations associated with dependencies and tools
+ List<FilesToRunProvider> depsDataAndTools = new ArrayList<>();
+ if (ruleContext.getRule().isAttrDefined("deps", Type.LABEL_LIST)) {
+ Iterables.addAll(depsDataAndTools,
+ ruleContext.getPrerequisites("deps", Mode.DONT_CHECK, FilesToRunProvider.class));
+ }
+ if (allowDataAttributeEntriesInLabel
+ && ruleContext.getRule().isAttrDefined("data", Type.LABEL_LIST)) {
+ Iterables.addAll(depsDataAndTools,
+ ruleContext.getPrerequisites("data", Mode.DATA, FilesToRunProvider.class));
+ }
+ if (ruleContext.getRule().isAttrDefined("tools", Type.LABEL_LIST)) {
+ Iterables.addAll(depsDataAndTools,
+ ruleContext.getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class));
+ }
+
+ for (FilesToRunProvider dep : depsDataAndTools) {
+ Label label = dep.getLabel();
+ Artifact executableArtifact = dep.getExecutable();
+
+ // If the label has an executable artifact add that to the multimaps.
+ if (executableArtifact != null) {
+ mapGet(locationMap, label).add(executableArtifact);
+ } else {
+ mapGet(locationMap, label).addAll(dep.getFilesToRun());
+ }
+ }
+ return locationMap;
+ }
+
+ /**
+ * Extracts list of all executables associated with given collection of label
+ * artifacts.
+ *
+ * @param artifacts to get the paths of
+ * @return all associated executable paths
+ */
+ private static List<String> getPaths(Collection<Artifact> artifacts) {
+ List<String> paths = Lists.newArrayListWithCapacity(artifacts.size());
+ for (Artifact artifact : artifacts) {
+ PathFragment execPath = artifact.getExecPath();
+ if (execPath != null) { // omit middlemen etc
+ paths.add(execPath.getPathString());
+ }
+ }
+ return paths;
+ }
+
+ /**
+ * Returns the value in the specified map corresponding to 'key', creating and
+ * inserting an empty container if absent. We use Map not Multimap because
+ * we need to distinguish the cases of "empty value" and "absent key".
+ *
+ * @return the value in the specified map corresponding to 'key'
+ */
+ private static <K, V> Collection<V> mapGet(Map<K, Collection<V>> map, K key) {
+ Collection<V> values = map.get(key);
+ if (values == null) {
+ // We use sets not lists, because it's conceivable that the same label
+ // could appear twice, in "srcs" and "deps".
+ values = Sets.newHashSet();
+ map.put(key, values);
+ }
+ return values;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/MakeEnvironmentEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/MakeEnvironmentEvent.java
new file mode 100644
index 0000000..f4b9ca8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/MakeEnvironmentEvent.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * This event is fired once the global make environment is available.
+ */
+public final class MakeEnvironmentEvent {
+
+ private final Map<String, String> makeEnvMap;
+
+ /**
+ * Construct the event.
+ */
+ public MakeEnvironmentEvent(Map<String, String> makeEnv) {
+ makeEnvMap = ImmutableMap.copyOf(makeEnv);
+ }
+
+ /**
+ * Returns make environment variable names and values as a map.
+ */
+ public Map<String, String> getMakeEnvMap() {
+ return makeEnvMap;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/MakeVariableExpander.java b/src/main/java/com/google/devtools/build/lib/analysis/MakeVariableExpander.java
new file mode 100644
index 0000000..55366da
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/MakeVariableExpander.java
@@ -0,0 +1,201 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+/**
+ * MakeVariableExpander defines a utility method, <code>expand</code>, for
+ * expanding references to "Make" variables embedded within a string. The
+ * caller provides a Context instance which defines the expansion of each
+ * variable.
+ *
+ * <p>Note that neither <code>$(location x)</code> nor Make-isms are treated
+ * specially in any way by this class.
+ */
+public class MakeVariableExpander {
+
+ private final char[] buffer;
+ private final int length;
+ private int offset;
+
+ private MakeVariableExpander(String expression) {
+ buffer = expression.toCharArray();
+ length = buffer.length;
+ offset = 0;
+ }
+
+ /**
+ * Interface to be implemented by callers of MakeVariableExpander which
+ * defines the expansion of each "Make" variable.
+ */
+ public interface Context {
+
+ /**
+ * Returns the expansion of the specified "Make" variable.
+ *
+ * @param var the variable to expand.
+ * @return the expansion of the variable.
+ * @throws ExpansionException if the variable "var" was not defined or
+ * there was any other error while expanding "var".
+ */
+ String lookupMakeVariable(String var) throws ExpansionException;
+ }
+
+ /**
+ * Exception thrown by MakeVariableExpander.Context.expandVariable when an
+ * unknown variable is passed.
+ */
+ public static class ExpansionException extends Exception {
+ public ExpansionException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Expands all references to "Make" variables embedded within string "expr",
+ * using the provided Context instance to expand individual variables.
+ *
+ * @param expression the string to expand.
+ * @param context the context which defines the expansion of each individual
+ * variable.
+ * @return the expansion of "expr".
+ * @throws ExpansionException if "expr" contained undefined or ill-formed
+ * variables references.
+ */
+ public static String expand(String expression, Context context) throws ExpansionException {
+ if (expression.indexOf('$') < 0) {
+ return expression;
+ }
+ return expand(expression, context, 0);
+ }
+
+ /**
+ * If the string contains a single variable, return the expansion of that variable.
+ * Otherwise, return null.
+ */
+ public static String expandSingleVariable(String expression, Context context)
+ throws ExpansionException {
+ String var = new MakeVariableExpander(expression).getSingleVariable();
+ return (var != null) ? context.lookupMakeVariable(var) : null;
+ }
+
+ // Helper method for counting recursion depth.
+ private static String expand(String expression, Context context, int depth)
+ throws ExpansionException {
+ if (depth > 10) { // plenty!
+ throw new ExpansionException("potentially unbounded recursion during "
+ + "expansion of '" + expression + "'");
+ }
+ return new MakeVariableExpander(expression).expand(context, depth);
+ }
+
+ private String expand(Context context, int depth) throws ExpansionException {
+ StringBuilder result = new StringBuilder();
+ while (offset < length) {
+ char c = buffer[offset];
+ if (c == '$') { // variable
+ offset++;
+ if (offset >= length) {
+ throw new ExpansionException("unterminated $");
+ }
+ if (buffer[offset] == '$') {
+ result.append('$');
+ } else {
+ String var = scanVariable();
+ String value = context.lookupMakeVariable(var);
+ // To prevent infinite recursion for the ignored shell variables
+ if (!value.equals(var)) {
+ // recursively expand using Make's ":=" semantics:
+ value = expand(value, context, depth + 1);
+ }
+ result.append(value);
+ }
+ } else {
+ result.append(c);
+ }
+ offset++;
+ }
+ return result.toString();
+ }
+
+ /**
+ * Starting at the current position, scans forward until the name of a Make
+ * variable has been consumed. Returns the variable name and advances the
+ * position. If the variable is a potential shell variable returns the shell
+ * variable expression itself, so that we can let the shell handle the
+ * expansion.
+ *
+ * @return the name of the variable found at the current point.
+ * @throws ExpansionException if the variable reference was ill-formed.
+ */
+ private String scanVariable() throws ExpansionException {
+ char c = buffer[offset];
+ switch (c) {
+ case '(': { // $(SRCS)
+ offset++;
+ int start = offset;
+ while (offset < length && buffer[offset] != ')') {
+ offset++;
+ }
+ if (offset >= length) {
+ throw new ExpansionException("unterminated variable reference");
+ }
+ return new String(buffer, start, offset - start);
+ }
+ case '{': { // ${SRCS}
+ offset++;
+ int start = offset;
+ while (offset < length && buffer[offset] != '}') {
+ offset++;
+ }
+ if (offset >= length) {
+ throw new ExpansionException("unterminated variable reference");
+ }
+ String expr = new String(buffer, start, offset - start);
+ throw new ExpansionException("'${" + expr + "}' syntax is not supported; use '$(" + expr
+ + ")' instead for \"Make\" variables, or escape the '$' as "
+ + "'$$' if you intended this for the shell");
+ }
+ case '@':
+ case '<':
+ case '^':
+ return String.valueOf(c);
+ default: {
+ int start = offset;
+ while (offset + 1 < length && Character.isJavaIdentifierPart(buffer[offset + 1])) {
+ offset++;
+ }
+ String expr = new String(buffer, start, offset + 1 - start);
+ throw new ExpansionException("'$" + expr + "' syntax is not supported; use '$(" + expr
+ + ")' instead for \"Make\" variables, or escape the '$' as "
+ + "'$$' if you intended this for the shell");
+ }
+ }
+ }
+
+ /**
+ * @return the variable name if the variable spans from offset to the end of
+ * the buffer, otherwise return null.
+ * @throws ExpansionException if the variable reference was ill-formed.
+ */
+ public String getSingleVariable() throws ExpansionException {
+ if (buffer[offset] == '$') {
+ offset++;
+ String result = scanVariable();
+ if (offset + 1 == length) {
+ return result;
+ }
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/MiddlemanProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/MiddlemanProvider.java
new file mode 100644
index 0000000..d8425f2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/MiddlemanProvider.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A provider class that supplies an aggregating middleman to the targets that depend on it.
+ */
+@Immutable
+public final class MiddlemanProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> middlemanArtifact;
+
+ public MiddlemanProvider(NestedSet<Artifact> middlemanArtifact) {
+ this.middlemanArtifact = middlemanArtifact;
+ }
+
+ /**
+ * Returns the middleman for the files produced by the transitive info collection.
+ */
+ public NestedSet<Artifact> getMiddlemanArtifact() {
+ return middlemanArtifact;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/NoSuchConfiguredTargetException.java b/src/main/java/com/google/devtools/build/lib/analysis/NoSuchConfiguredTargetException.java
new file mode 100644
index 0000000..2e9bf8c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/NoSuchConfiguredTargetException.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Exception indicating that the required configured target is not in the
+ * analysis cache.
+ */
+public class NoSuchConfiguredTargetException extends NoSuchThingException {
+ public NoSuchConfiguredTargetException(Label label, BuildConfiguration configuration) {
+ super("not in cache: " + label + " " + configuration);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/OutputFileConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/OutputFileConfiguredTarget.java
new file mode 100644
index 0000000..51122e2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/OutputFileConfiguredTarget.java
@@ -0,0 +1,80 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl;
+
+/**
+ * A ConfiguredTarget for an OutputFile.
+ */
+public class OutputFileConfiguredTarget extends FileConfiguredTarget
+ implements InstrumentedFilesProvider {
+
+ private final TransitiveInfoCollection generatingRule;
+
+ OutputFileConfiguredTarget(
+ TargetContext targetContext, OutputFile outputFile,
+ TransitiveInfoCollection generatingRule, Artifact outputArtifact) {
+ super(targetContext, outputArtifact);
+ Preconditions.checkArgument(targetContext.getTarget() == outputFile);
+ this.generatingRule = generatingRule;
+ }
+
+ @Override
+ public OutputFile getTarget() {
+ return (OutputFile) super.getTarget();
+ }
+
+ public TransitiveInfoCollection getGeneratingRule() {
+ return generatingRule;
+ }
+
+ @Override
+ public NestedSet<TargetLicense> getTransitiveLicenses() {
+ return getProvider(LicensesProvider.class, LicensesProviderImpl.EMPTY)
+ .getTransitiveLicenses();
+ }
+
+ @Override
+ public NestedSet<Artifact> getInstrumentedFiles() {
+ return getProvider(InstrumentedFilesProvider.class, InstrumentedFilesProviderImpl.EMPTY)
+ .getInstrumentedFiles();
+ }
+
+ @Override
+ public NestedSet<Artifact> getInstrumentationMetadataFiles() {
+ return getProvider(InstrumentedFilesProvider.class, InstrumentedFilesProviderImpl.EMPTY)
+ .getInstrumentationMetadataFiles();
+ }
+
+ /**
+ * Returns the corresponding provider from the generating rule, if it is non-null, or {@code
+ * defaultValue} otherwise.
+ */
+ private <T extends TransitiveInfoProvider> T getProvider(Class<T> clazz, T defaultValue) {
+ if (generatingRule != null) {
+ T result = generatingRule.getProvider(clazz);
+ if (result != null) {
+ return result;
+ }
+ }
+ return defaultValue;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PackageGroupConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/PackageGroupConfiguredTarget.java
new file mode 100644
index 0000000..75e2981
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/PackageGroupConfiguredTarget.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Dummy ConfiguredTarget for package groups. Contains no functionality, since
+ * package groups are not really first-class Targets.
+ */
+public final class PackageGroupConfiguredTarget extends AbstractConfiguredTarget
+ implements PackageSpecificationProvider {
+ private final NestedSet<PackageSpecification> packageSpecifications;
+
+ PackageGroupConfiguredTarget(TargetContext targetContext, PackageGroup packageGroup) {
+ super(targetContext);
+ Preconditions.checkArgument(targetContext.getConfiguration() == null);
+
+ NestedSetBuilder<PackageSpecification> builder =
+ NestedSetBuilder.stableOrder();
+ for (Label label : packageGroup.getIncludes()) {
+ TransitiveInfoCollection include = targetContext.findDirectPrerequisite(
+ label, targetContext.getConfiguration());
+ PackageSpecificationProvider provider = include == null ? null :
+ include.getProvider(PackageSpecificationProvider.class);
+ if (provider == null) {
+ targetContext.getAnalysisEnvironment().getEventHandler().handle(Event.error(getTarget().getLocation(),
+ String.format("label '%s' does not refer to a package group", label)));
+ continue;
+ }
+
+ builder.addTransitive(provider.getPackageSpecifications());
+ }
+
+ builder.addAll(packageGroup.getPackageSpecifications());
+ packageSpecifications = builder.build();
+ }
+
+ @Override
+ public PackageGroup getTarget() {
+ return (PackageGroup) super.getTarget();
+ }
+
+ @Override
+ public NestedSet<PackageSpecification> getPackageSpecifications() {
+ return packageSpecifications;
+ }
+
+ @Override
+ public Object get(String providerKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ throw new IllegalStateException();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java
new file mode 100644
index 0000000..3f852c7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+
+/**
+ * A {@link TransitiveInfoProvider} that describes a set of transitive package specifications
+ * used in package groups.
+ */
+public interface PackageSpecificationProvider extends TransitiveInfoProvider {
+ NestedSet<PackageSpecification> getPackageSpecifications();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PrerequisiteArtifacts.java b/src/main/java/com/google/devtools/build/lib/analysis/PrerequisiteArtifacts.java
new file mode 100644
index 0000000..8932d5c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/PrerequisiteArtifacts.java
@@ -0,0 +1,106 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Contains a sequence of prerequisite artifacts and supplies methods for filtering and reporting
+ * errors on those artifacts.
+ */
+public final class PrerequisiteArtifacts {
+ private final RuleContext ruleContext;
+ private final String attributeName;
+ private final ImmutableList<Artifact> artifacts;
+
+ private PrerequisiteArtifacts(
+ RuleContext ruleContext, String attributeName, ImmutableList<Artifact> artifacts) {
+ this.ruleContext = Preconditions.checkNotNull(ruleContext);
+ this.attributeName = Preconditions.checkNotNull(attributeName);
+ this.artifacts = Preconditions.checkNotNull(artifacts);
+ }
+
+ static PrerequisiteArtifacts get(RuleContext ruleContext, String attributeName, Mode mode) {
+ Set<Artifact> result = new LinkedHashSet<>();
+ for (FileProvider target :
+ ruleContext.getPrerequisites(attributeName, mode, FileProvider.class)) {
+ Iterables.addAll(result, target.getFilesToBuild());
+ }
+ return new PrerequisiteArtifacts(ruleContext, attributeName, ImmutableList.copyOf(result));
+ }
+
+ /**
+ * Returns the artifacts this instance contains in an {@link ImmutableList}.
+ */
+ public ImmutableList<Artifact> list() {
+ return artifacts;
+ }
+
+ private PrerequisiteArtifacts filter(Predicate<String> fileType, boolean errorsForNonMatching) {
+ ImmutableList.Builder<Artifact> filtered = new ImmutableList.Builder<Artifact>();
+
+ for (Artifact artifact : artifacts) {
+ if (fileType.apply(artifact.getFilename())) {
+ filtered.add(artifact);
+ } else if (errorsForNonMatching) {
+ ruleContext.attributeError(
+ attributeName,
+ String.format("%s does not match expected type: %s", artifact, fileType));
+ }
+ }
+
+ return new PrerequisiteArtifacts(ruleContext, attributeName, filtered.build());
+ }
+
+ /**
+ * Returns an equivalent instance but only containing artifacts of the given type, reporting
+ * errors for non-matching artifacts.
+ */
+ public PrerequisiteArtifacts errorsForNonMatching(FileType fileType) {
+ return filter(fileType, /*errorsForNonMatching=*/true);
+ }
+
+ /**
+ * Returns an equivalent instance but only containing artifacts of the given types, reporting
+ * errors for non-matching artifacts.
+ */
+ public PrerequisiteArtifacts errorsForNonMatching(FileTypeSet fileTypeSet) {
+ return filter(fileTypeSet, /*errorsForNonMatching=*/true);
+ }
+
+ /**
+ * Returns an equivalent instance but only containing artifacts of the given type.
+ */
+ public PrerequisiteArtifacts filter(FileType fileType) {
+ return filter(fileType, /*errorsForNonMatching=*/false);
+ }
+
+ /**
+ * Returns an equivalent instance but only containing artifacts of the given types.
+ */
+ public PrerequisiteArtifacts filter(FileTypeSet fileTypeSet) {
+ return filter(fileTypeSet, /*errorsForNonMatching=*/false);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PrintActionVisitor.java b/src/main/java/com/google/devtools/build/lib/analysis/PrintActionVisitor.java
new file mode 100644
index 0000000..c852734
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/PrintActionVisitor.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionGraphVisitor;
+import com.google.devtools.build.lib.actions.ActionOwner;
+
+import java.util.List;
+
+/**
+ * A bipartite graph visitor which accumulates actions with matching mnemonics for a target.
+ */
+public final class PrintActionVisitor extends ActionGraphVisitor {
+ private final ConfiguredTarget target;
+ private final List<Action> actions;
+ private final Predicate<Action> actionMnemonicMatcher;
+ private final String targetConfigurationKey;
+
+ /**
+ * Creates a new visitor for the actions associated with the given target that have a matching
+ * mnemonic.
+ */
+ public PrintActionVisitor(ActionGraph actionGraph, ConfiguredTarget target,
+ Predicate<Action> actionMnemonicMatcher) {
+ super(actionGraph);
+ this.target = target;
+ this.actionMnemonicMatcher = actionMnemonicMatcher;
+ actions = Lists.newArrayList();
+ targetConfigurationKey = target.getConfiguration().shortCacheKey();
+ }
+
+ @Override
+ protected boolean shouldVisit(Action action) {
+ ActionOwner owner = action.getOwner();
+ return owner != null && target.getLabel().equals(owner.getLabel())
+ && targetConfigurationKey.equals(owner.getConfigurationShortCacheKey());
+ }
+
+ @Override
+ protected void visitAction(Action action) {
+ if (actionMnemonicMatcher.apply(action)) {
+ actions.add(action);
+ }
+ }
+
+ /** Retrieves the collected actions since this method was last called and clears the list. */
+ public ImmutableList<Action> getActions() {
+ return ImmutableList.copyOf(actions);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PseudoAction.java b/src/main/java/com/google/devtools/build/lib/analysis/PseudoAction.java
new file mode 100644
index 0000000..00d43a3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/PseudoAction.java
@@ -0,0 +1,95 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.protobuf.GeneratedMessage.GeneratedExtension;
+import com.google.protobuf.MessageLite;
+
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * An action that is inserted into the build graph only to provide info
+ * about rules to extra_actions.
+ */
+public class PseudoAction<InfoType extends MessageLite> extends AbstractAction {
+
+ private final UUID uuid;
+ private final String mnemonic;
+ private final GeneratedExtension<ExtraActionInfo, InfoType> infoExtension;
+ private final InfoType info;
+
+ public PseudoAction(UUID uuid, ActionOwner owner,
+ Collection<Artifact> inputs, Collection<Artifact> outputs,
+ String mnemonic,
+ GeneratedExtension<ExtraActionInfo, InfoType> infoExtension, InfoType info) {
+ super(owner, inputs, outputs);
+ this.uuid = uuid;
+ this.mnemonic = mnemonic;
+ this.infoExtension = infoExtension;
+ this.info = info;
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return null;
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ throw new ActionExecutionException(
+ mnemonic + "ExtraAction should not be executed.", this, false);
+ }
+
+ @Override
+ public String getMnemonic() {
+ return mnemonic;
+ }
+
+ @Override
+ protected String computeKey() {
+ return new Fingerprint()
+ .addUUID(uuid)
+ .addBytes(info.toByteArray())
+ .hexDigestAndReset();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ public ExtraActionInfo.Builder getExtraActionInfo() {
+ return super.getExtraActionInfo().setExtension(infoExtension, info);
+ }
+
+ public static Artifact getDummyOutput(RuleContext ruleContext) {
+ return ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ ruleContext.getLabel().toPathFragment().replaceName(
+ ruleContext.getLabel().getName() + ".extra_action_dummy"),
+ ruleContext.getConfiguration().getGenfilesDirectory());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RedirectChaser.java b/src/main/java/com/google/devtools/build/lib/analysis/RedirectChaser.java
new file mode 100644
index 0000000..108a577
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RedirectChaser.java
@@ -0,0 +1,114 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.packages.AbstractAttributeMapper;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tool for chasing filegroup redirects. This is mainly intended to be used during
+ * BuildConfiguration creation.
+ */
+public final class RedirectChaser {
+
+ /**
+ * Custom attribute mapper that throws an exception if an attribute's value depends on the
+ * build configuration.
+ */
+ private static class StaticValuedAttributeMapper extends AbstractAttributeMapper {
+ public StaticValuedAttributeMapper(Rule rule) {
+ super(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(),
+ rule.getAttributeContainer());
+ }
+
+ /**
+ * Returns the value of the given attribute.
+ *
+ * @throws InvalidConfigurationException if the value is configuration-dependent
+ */
+ public <T> T getAndValidate(String attributeName, Type<T> type)
+ throws InvalidConfigurationException {
+ if (getSelector(attributeName, type) != null) {
+ throw new InvalidConfigurationException
+ ("The value of '" + attributeName + "' cannot be configuration-dependent");
+ }
+ return super.get(attributeName, type);
+ }
+
+ @Override
+ protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) {
+ throw new IllegalStateException("Attribute visitation not supported redirect resolution");
+ }
+ }
+
+ /**
+ * Follows the 'srcs' attribute of the given label recursively. Keeps repeating as long as the
+ * labels are filegroups with a single srcs entry.
+ *
+ * @param env for loading the packages
+ * @param label the label to start at
+ * @param name user-meaningful description of the content being resolved
+ * @return the label which cannot be further resolved
+ * @throws InvalidConfigurationException if something goes wrong
+ */
+ @Nullable
+ public static Label followRedirects(ConfigurationEnvironment env, Label label, String name)
+ throws InvalidConfigurationException {
+ Set<Label> visitedLabels = new HashSet<>();
+ visitedLabels.add(label);
+ try {
+ while (true) {
+ Target possibleRedirect = env.getTarget(label);
+ if (possibleRedirect == null) {
+ return null;
+ }
+ if ((possibleRedirect instanceof Rule) &&
+ "filegroup".equals(((Rule) possibleRedirect).getRuleClass())) {
+ List<Label> labels = new StaticValuedAttributeMapper((Rule) possibleRedirect)
+ .getAndValidate("srcs", Type.LABEL_LIST);
+ if (labels.size() != 1) {
+ // We can't distinguish redirects from the final filegroup, so we assume this must be
+ // the final one.
+ return label;
+ }
+ label = labels.get(0);
+ if (!visitedLabels.add(label)) {
+ throw new InvalidConfigurationException("The " + name + " points to a filegroup which "
+ + "recursively includes itself. The label " + label + " is part of the loop");
+ }
+ } else {
+ return label;
+ }
+ }
+ } catch (NoSuchPackageException e) {
+ throw new InvalidConfigurationException(e.getMessage(), e);
+ } catch (NoSuchTargetException e) {
+ return label;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java
new file mode 100644
index 0000000..602e949
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java
@@ -0,0 +1,226 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.Rule;
+
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A generic implementation of RuleConfiguredTarget. Do not use directly. Use {@link
+ * RuleConfiguredTargetBuilder} instead.
+ */
+public final class RuleConfiguredTarget extends AbstractConfiguredTarget {
+ /**
+ * The configuration transition for an attribute through which a prerequisite
+ * is requested.
+ */
+ public enum Mode {
+ TARGET,
+ HOST,
+ DATA,
+ SPLIT,
+ DONT_CHECK
+ }
+
+ private final ImmutableMap<Class<? extends TransitiveInfoProvider>, Object> providers;
+ private final ImmutableList<Artifact> mandatoryStampFiles;
+ private final Set<ConfigMatchingProvider> configConditions;
+ private final ImmutableList<Aspect> aspects;
+
+ RuleConfiguredTarget(RuleContext ruleContext,
+ ImmutableList<Artifact> mandatoryStampFiles,
+ ImmutableMap<String, Object> skylarkProviders,
+ Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers) {
+ super(ruleContext);
+ // We don't use ImmutableMap.Builder here to allow augmenting the initial list of 'default'
+ // providers by passing them in.
+ Map<Class<? extends TransitiveInfoProvider>, Object> providerBuilder = new LinkedHashMap<>();
+ providerBuilder.putAll(providers);
+ Preconditions.checkState(providerBuilder.containsKey(RunfilesProvider.class));
+ Preconditions.checkState(providerBuilder.containsKey(FileProvider.class));
+ Preconditions.checkState(providerBuilder.containsKey(FilesToRunProvider.class));
+
+ providerBuilder.put(SkylarkProviders.class, new SkylarkProviders(skylarkProviders));
+
+ this.providers = ImmutableMap.copyOf(providerBuilder);
+ this.mandatoryStampFiles = mandatoryStampFiles;
+ this.configConditions = ruleContext.getConfigConditions();
+ this.aspects = ImmutableList.of();
+
+ // If this rule is the run_under target, then check that we have an executable; note that
+ // run_under is only set in the target configuration, and the target must also be analyzed for
+ // the target configuration.
+ RunUnder runUnder = getConfiguration().getRunUnder();
+ if (runUnder != null && getLabel().equals(runUnder.getLabel())) {
+ if (getProvider(FilesToRunProvider.class).getExecutable() == null) {
+ ruleContext.ruleError("run_under target " + runUnder.getLabel() + " is not executable");
+ }
+ }
+
+ // Make sure that all declared output files are also created as artifacts. The
+ // CachingAnalysisEnvironment makes sure that they all have generating actions.
+ if (!ruleContext.hasErrors()) {
+ for (OutputFile out : ruleContext.getRule().getOutputFiles()) {
+ ruleContext.createOutputArtifact(out);
+ }
+ }
+ }
+
+ /**
+ * Merge a configured target with its associated aspects.
+ *
+ * <p>If aspects are present, the configured target must be created from a rule (instead of e.g.
+ * an input or an output file).
+ */
+ public static ConfiguredTarget mergeAspects(
+ ConfiguredTarget base, Iterable<Aspect> aspects) {
+ if (Iterables.isEmpty(aspects)) {
+ // If there are no aspects, don't bother with creating a proxy object
+ return base;
+ } else {
+ // Aspects can only be attached to rules for now. This invariant is upheld by
+ // DependencyResolver#requiredAspects()
+ return new RuleConfiguredTarget((RuleConfiguredTarget) base, aspects);
+ }
+ }
+
+ /**
+ * Creates an instance based on a configured target and a set of aspects.
+ */
+ private RuleConfiguredTarget(RuleConfiguredTarget base, Iterable<Aspect> aspects) {
+ super(base.getTarget(), base.getConfiguration());
+
+ Set<Class<? extends TransitiveInfoProvider>> providers = new HashSet<>();
+
+ providers.addAll(base.providers.keySet());
+ for (Aspect aspect : aspects) {
+ for (TransitiveInfoProvider aspectProvider : aspect) {
+ if (!providers.add(aspectProvider.getClass())) {
+ throw new IllegalStateException(
+ "Provider " + aspectProvider.getClass() + " provided twice");
+ }
+ }
+ }
+ this.providers = base.providers;
+ this.mandatoryStampFiles = base.mandatoryStampFiles;
+ this.configConditions = base.configConditions;
+ this.aspects = ImmutableList.copyOf(aspects);
+ }
+
+ /**
+ * The configuration conditions that trigger this rule's configurable attributes.
+ */
+ Set<ConfigMatchingProvider> getConfigConditions() {
+ return configConditions;
+ }
+
+ @Override
+ public <P extends TransitiveInfoProvider> P getProvider(Class<P> providerClass) {
+ AnalysisUtils.checkProvider(providerClass);
+ // TODO(bazel-team): Should aspects be allowed to override providers on the configured target
+ // class?
+ Object provider = providers.get(providerClass);
+ if (provider == null) {
+ for (Aspect aspect : aspects) {
+ provider = aspect.getProviders().get(providerClass);
+ if (provider != null) {
+ break;
+ }
+ }
+ }
+
+ return providerClass.cast(provider);
+ }
+
+ /**
+ * Returns a value provided by this target. Only meant to use from Skylark.
+ */
+ @Override
+ public Object get(String providerKey) {
+ return getProvider(SkylarkProviders.class).skylarkProviders.get(providerKey);
+ }
+
+ public ImmutableList<Artifact> getMandatoryStampFiles() {
+ return mandatoryStampFiles;
+ }
+
+ @Override
+ public final Rule getTarget() {
+ return (Rule) super.getTarget();
+ }
+
+ /**
+ * A helper class for transitive infos provided by Skylark rule implementations.
+ */
+ @Immutable
+ public static final class SkylarkProviders implements TransitiveInfoProvider {
+ private final ImmutableMap<String, Object> skylarkProviders;
+
+ private SkylarkProviders(ImmutableMap<String, Object> skylarkProviders) {
+ Preconditions.checkNotNull(skylarkProviders);
+ this.skylarkProviders = skylarkProviders;
+ }
+
+ /**
+ * Returns the keys for the Skylark providers.
+ */
+ public ImmutableCollection<String> getKeys() {
+ return skylarkProviders.keySet();
+ }
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> allProviders =
+ new LinkedHashMap<>();
+ for (int i = aspects.size() - 1; i >= 0; i++) {
+ for (TransitiveInfoProvider tip : aspects.get(i)) {
+ allProviders.put(tip.getClass(), tip);
+ }
+ }
+
+ for (Map.Entry<Class<? extends TransitiveInfoProvider>, Object> entry : providers.entrySet()) {
+ allProviders.put(entry.getKey(), entry.getKey().cast(entry.getValue()));
+ }
+
+ return ImmutableList.copyOf(allProviders.values()).iterator();
+ }
+
+ @Override
+ public String errorMessage(String name) {
+ return String.format("target (rule class of '%s') doesn't have provider '%s'.",
+ getTarget().getRuleClass(), name);
+ }
+
+ @Override
+ public ImmutableCollection<String> getKeys() {
+ return ImmutableList.<String>builder().addAll(super.getKeys())
+ .addAll(getProvider(SkylarkProviders.class).skylarkProviders.keySet()).build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java
new file mode 100644
index 0000000..b82713f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java
@@ -0,0 +1,423 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider.ExtraArtifactSet;
+import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics;
+import com.google.devtools.build.lib.analysis.constraints.EnvironmentCollection;
+import com.google.devtools.build.lib.analysis.constraints.SupportedEnvironments;
+import com.google.devtools.build.lib.analysis.constraints.SupportedEnvironmentsProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.extra.ExtraActionMapProvider;
+import com.google.devtools.build.lib.rules.extra.ExtraActionSpec;
+import com.google.devtools.build.lib.rules.test.ExecutionInfoProvider;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.rules.test.TestActionBuilder;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.rules.test.TestProvider.TestParams;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Builder class for analyzed rule instances (i.e., instances of {@link ConfiguredTarget}).
+ */
+public final class RuleConfiguredTargetBuilder {
+ private final RuleContext ruleContext;
+ private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers =
+ new LinkedHashMap<>();
+ private final ImmutableMap.Builder<String, Object> skylarkProviders = ImmutableMap.builder();
+
+ /** These are supported by all configured targets and need to be specially handled. */
+ private NestedSet<Artifact> filesToBuild = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ private RunfilesSupport runfilesSupport;
+ private Artifact executable;
+ private ImmutableList<Artifact> mandatoryStampFiles;
+ private ImmutableSet<Action> actionsWithoutExtraAction = ImmutableSet.of();
+
+ public RuleConfiguredTargetBuilder(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ add(LicensesProvider.class, initializeLicensesProvider());
+ add(VisibilityProvider.class, new VisibilityProviderImpl(ruleContext.getVisibility()));
+ }
+
+ /**
+ * Constructs the RuleConfiguredTarget instance based on the values set for this Builder.
+ */
+ public ConfiguredTarget build() {
+ if (ruleContext.getConfiguration().enforceConstraints()) {
+ checkConstraints();
+ }
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ FilesToRunProvider filesToRunProvider = new FilesToRunProvider(ruleContext.getLabel(),
+ RuleContext.getFilesToRun(runfilesSupport, filesToBuild), runfilesSupport, executable);
+ add(FileProvider.class, new FileProvider(ruleContext.getLabel(), filesToBuild));
+ add(FilesToRunProvider.class, filesToRunProvider);
+
+ // Create test action and artifacts if target was successfully initialized
+ // and is a test.
+ if (TargetUtils.isTestRule(ruleContext.getTarget())) {
+ Preconditions.checkState(runfilesSupport != null);
+ add(TestProvider.class, initializeTestProvider(filesToRunProvider));
+ }
+ add(ExtraActionArtifactsProvider.class, initializeExtraActions());
+ return new RuleConfiguredTarget(
+ ruleContext, mandatoryStampFiles, skylarkProviders.build(), providers);
+ }
+
+ /**
+ * Invokes Blaze's constraint enforcement system: checks that this rule's dependencies
+ * support its environments and reports appropriate errors if violations are found. Also
+ * publishes this rule's supported environments for the rules that depend on it.
+ */
+ private void checkConstraints() {
+ if (providers.get(SupportedEnvironmentsProvider.class) == null) {
+ // Note the "environment" rule sets its own SupportedEnvironmentProvider instance, so this
+ // logic is for "normal" rules that just want to apply default semantics.
+ EnvironmentCollection supportedEnvironments =
+ ConstraintSemantics.getSupportedEnvironments(ruleContext);
+ if (supportedEnvironments != null) {
+ add(SupportedEnvironmentsProvider.class, new SupportedEnvironments(supportedEnvironments));
+ ConstraintSemantics.checkConstraints(ruleContext, supportedEnvironments);
+ }
+ }
+ }
+
+ private TestProvider initializeTestProvider(FilesToRunProvider filesToRunProvider) {
+ int explicitShardCount = ruleContext.attributes().get("shard_count", Type.INTEGER);
+ if (explicitShardCount < 0
+ && ruleContext.getRule().isAttributeValueExplicitlySpecified("shard_count")) {
+ ruleContext.attributeError("shard_count", "Must not be negative.");
+ }
+ if (explicitShardCount > 50) {
+ ruleContext.attributeError("shard_count",
+ "Having more than 50 shards is indicative of poor test organization. "
+ + "Please reduce the number of shards.");
+ }
+ final TestParams testParams = new TestActionBuilder(ruleContext)
+ .setFilesToRunProvider(filesToRunProvider)
+ .setInstrumentedFiles(findProvider(InstrumentedFilesProvider.class))
+ .setExecutionRequirements(findProvider(ExecutionInfoProvider.class))
+ .setShardCount(explicitShardCount)
+ .build();
+ final ImmutableList<String> testTags =
+ ImmutableList.copyOf(ruleContext.getRule().getRuleTags());
+ return new TestProvider(testParams, testTags);
+ }
+
+ private LicensesProvider initializeLicensesProvider() {
+ if (!ruleContext.getConfiguration().checkLicenses()) {
+ return LicensesProviderImpl.EMPTY;
+ }
+
+ NestedSetBuilder<TargetLicense> builder = NestedSetBuilder.linkOrder();
+ BuildConfiguration configuration = ruleContext.getConfiguration();
+ Rule rule = ruleContext.getRule();
+ License toolOutputLicense = rule.getToolOutputLicense(ruleContext.attributes());
+ if (configuration.isHostConfiguration() && toolOutputLicense != null) {
+ if (toolOutputLicense != License.NO_LICENSE) {
+ builder.add(new TargetLicense(rule.getLabel(), toolOutputLicense));
+ }
+ } else {
+ if (rule.getLicense() != License.NO_LICENSE) {
+ builder.add(new TargetLicense(rule.getLabel(), rule.getLicense()));
+ }
+
+ for (TransitiveInfoCollection dep : ruleContext.getConfiguredTargetMap().values()) {
+ LicensesProvider provider = dep.getProvider(LicensesProvider.class);
+ if (provider != null) {
+ builder.addTransitive(provider.getTransitiveLicenses());
+ }
+ }
+ }
+
+ return new LicensesProviderImpl(builder.build());
+ }
+
+ /**
+ * Scans {@code action_listeners} associated with this build to see if any
+ * {@code extra_actions} should be added to this configured target. If any
+ * action_listeners are present, a partial visit of the artifact/action graph
+ * is performed (for as long as actions found are owned by this {@link
+ * ConfiguredTarget}). Any actions that match the {@code action_listener}
+ * get an {@code extra_action} associated. The output artifacts of the
+ * extra_action are reported to the {@link AnalysisEnvironment} for
+ * bookkeeping.
+ */
+ private ExtraActionArtifactsProvider initializeExtraActions() {
+ BuildConfiguration configuration = ruleContext.getConfiguration();
+ if (configuration.isHostConfiguration()) {
+ return ExtraActionArtifactsProvider.EMPTY;
+ }
+
+ ImmutableList<Artifact> extraActionArtifacts = ImmutableList.of();
+ NestedSetBuilder<ExtraArtifactSet> builder = NestedSetBuilder.stableOrder();
+
+ List<Label> actionListenerLabels = configuration.getActionListeners();
+ if (!actionListenerLabels.isEmpty()
+ && ruleContext.getRule().getAttributeDefinition(":action_listener") != null) {
+ ExtraActionsVisitor visitor = new ExtraActionsVisitor(ruleContext,
+ computeMnemonicsToExtraActionMap());
+
+ // The action list is modified within the body of the loop by the addExtraAction() call,
+ // thus the copy
+ for (Action action : ImmutableList.copyOf(
+ ruleContext.getAnalysisEnvironment().getRegisteredActions())) {
+ if (!actionsWithoutExtraAction.contains(action)) {
+ visitor.addExtraAction(action);
+ }
+ }
+
+ extraActionArtifacts = visitor.getAndResetExtraArtifacts();
+ if (!extraActionArtifacts.isEmpty()) {
+ builder.add(ExtraArtifactSet.of(ruleContext.getLabel(), extraActionArtifacts));
+ }
+ }
+
+ // Add extra action artifacts from dependencies
+ for (TransitiveInfoCollection dep : ruleContext.getConfiguredTargetMap().values()) {
+ ExtraActionArtifactsProvider provider =
+ dep.getProvider(ExtraActionArtifactsProvider.class);
+ if (provider != null) {
+ builder.addTransitive(provider.getTransitiveExtraActionArtifacts());
+ }
+ }
+
+ if (mandatoryStampFiles != null && !mandatoryStampFiles.isEmpty()) {
+ builder.add(ExtraArtifactSet.of(ruleContext.getLabel(), mandatoryStampFiles));
+ }
+
+ if (extraActionArtifacts.isEmpty() && builder.isEmpty()) {
+ return ExtraActionArtifactsProvider.EMPTY;
+ }
+ return new ExtraActionArtifactsProvider(extraActionArtifacts, builder.build());
+ }
+
+ /**
+ * Populates the configuration specific mnemonicToExtraActionMap
+ * based on all action_listers selected by the user (via the blaze option
+ * --experimental_action_listener=<target>).
+ */
+ private Multimap<String, ExtraActionSpec> computeMnemonicsToExtraActionMap() {
+ // We copy the multimap here every time. This could be expensive.
+ Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap = HashMultimap.create();
+ for (TransitiveInfoCollection actionListener :
+ ruleContext.getPrerequisites(":action_listener", Mode.HOST)) {
+ ExtraActionMapProvider provider = actionListener.getProvider(ExtraActionMapProvider.class);
+ if (provider == null) {
+ ruleContext.ruleError(String.format(
+ "Unable to match experimental_action_listeners to this rule. "
+ + "Specified target %s is not an action_listener rule",
+ actionListener.getLabel().toString()));
+ } else {
+ mnemonicToExtraActionMap.putAll(provider.getExtraActionMap());
+ }
+ }
+ return mnemonicToExtraActionMap;
+ }
+
+ private <T extends TransitiveInfoProvider> T findProvider(Class<T> clazz) {
+ return clazz.cast(providers.get(clazz));
+ }
+
+ /**
+ * Add a specific provider with a given value.
+ */
+ public <T extends TransitiveInfoProvider> RuleConfiguredTargetBuilder add(Class<T> key, T value) {
+ return addProvider(key, value);
+ }
+
+ /**
+ * Add a specific provider with a given value.
+ */
+ public RuleConfiguredTargetBuilder addProvider(
+ Class<? extends TransitiveInfoProvider> key, TransitiveInfoProvider value) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(value);
+ AnalysisUtils.checkProvider(key);
+ providers.put(key, value);
+ return this;
+ }
+
+ /**
+ * Add multiple providers with given values.
+ */
+ public RuleConfiguredTargetBuilder addProviders(
+ Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers) {
+ for (Entry<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> provider :
+ providers.entrySet()) {
+ addProvider(provider.getKey(), provider.getValue());
+ }
+ return this;
+ }
+
+ /**
+ * Add a Skylark transitive info. The provider value must be safe (i.e. a String, a Boolean,
+ * an Integer, an Artifact, a Label, None, a Java TransitiveInfoProvider or something composed
+ * from these in Skylark using lists, sets, structs or dicts). Otherwise an EvalException is
+ * thrown.
+ */
+ public RuleConfiguredTargetBuilder addSkylarkTransitiveInfo(
+ String name, Object value, Location loc) throws EvalException {
+ try {
+ checkSkylarkObjectSafe(value);
+ } catch (IllegalArgumentException e) {
+ throw new EvalException(loc, String.format("Value of provider '%s' is of an illegal type: %s",
+ name, e.getMessage()));
+ }
+ skylarkProviders.put(name, value);
+ return this;
+ }
+
+ /**
+ * Add a Skylark transitive info. The provider value must be safe.
+ */
+ public RuleConfiguredTargetBuilder addSkylarkTransitiveInfo(
+ String name, Object value) {
+ checkSkylarkObjectSafe(value);
+ skylarkProviders.put(name, value);
+ return this;
+ }
+
+ /**
+ * Check if the value provided by a Skylark provider is safe (i.e. can be a
+ * TransitiveInfoProvider value).
+ */
+ private void checkSkylarkObjectSafe(Object value) {
+ if (!isSimpleSkylarkObjectSafe(value.getClass())
+ // Java transitive Info Providers are accessible from Skylark.
+ || value instanceof TransitiveInfoProvider) {
+ checkCompositeSkylarkObjectSafe(value);
+ }
+ }
+
+ private void checkCompositeSkylarkObjectSafe(Object object) {
+ if (object instanceof SkylarkList) {
+ SkylarkList list = (SkylarkList) object;
+ if (list == SkylarkList.EMPTY_LIST || isSimpleSkylarkObjectSafe(list.getGenericType())) {
+ // Try not to iterate over the list if avoidable.
+ return;
+ }
+ // The list can be a tuple or a list of composite items.
+ for (Object listItem : list) {
+ checkSkylarkObjectSafe(listItem);
+ }
+ return;
+ } else if (object instanceof SkylarkNestedSet) {
+ // SkylarkNestedSets cannot have composite items.
+ Class<?> genericType = ((SkylarkNestedSet) object).getGenericType();
+ if (!genericType.equals(Object.class) && !isSimpleSkylarkObjectSafe(genericType)) {
+ throw new IllegalArgumentException(EvalUtils.getDatatypeName(genericType));
+ }
+ return;
+ } else if (object instanceof Map<?, ?>) {
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>) object).entrySet()) {
+ checkSkylarkObjectSafe(entry.getKey());
+ checkSkylarkObjectSafe(entry.getValue());
+ }
+ return;
+ } else if (object instanceof ClassObject) {
+ ClassObject struct = (ClassObject) object;
+ for (String key : struct.getKeys()) {
+ checkSkylarkObjectSafe(struct.getValue(key));
+ }
+ return;
+ }
+ throw new IllegalArgumentException(EvalUtils.getDatatypeName(object));
+ }
+
+ private boolean isSimpleSkylarkObjectSafe(Class<?> type) {
+ return type.equals(String.class)
+ || type.equals(Integer.class)
+ || type.equals(Boolean.class)
+ || Artifact.class.isAssignableFrom(type)
+ || type.equals(Label.class)
+ || type.equals(Environment.NoneType.class);
+ }
+
+ /**
+ * Set the runfiles support for executable targets.
+ */
+ public RuleConfiguredTargetBuilder setRunfilesSupport(
+ RunfilesSupport runfilesSupport, Artifact executable) {
+ this.runfilesSupport = runfilesSupport;
+ this.executable = executable;
+ return this;
+ }
+
+ /**
+ * Set the files to build.
+ */
+ public RuleConfiguredTargetBuilder setFilesToBuild(NestedSet<Artifact> filesToBuild) {
+ this.filesToBuild = filesToBuild;
+ return this;
+ }
+
+ /**
+ * Set the baseline coverage Artifacts.
+ */
+ public RuleConfiguredTargetBuilder setBaselineCoverageArtifacts(
+ Collection<Artifact> artifacts) {
+ return add(BaselineCoverageArtifactsProvider.class,
+ new BaselineCoverageArtifactsProvider(ImmutableList.copyOf(artifacts)));
+ }
+
+ /**
+ * Set the mandatory stamp files.
+ */
+ public RuleConfiguredTargetBuilder setMandatoryStampFiles(ImmutableList<Artifact> files) {
+ this.mandatoryStampFiles = files;
+ return this;
+ }
+
+ /**
+ * Set the extra action pseudo actions.
+ */
+ public RuleConfiguredTargetBuilder setActionsWithoutExtraAction(
+ ImmutableSet<Action> actions) {
+ this.actionsWithoutExtraAction = actions;
+ return this;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
new file mode 100644
index 0000000..9ad7c70
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
@@ -0,0 +1,1391 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.ActionRegistry;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.PrerequisiteValidator;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.Attribute.SplitTransition;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.FileTarget;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleErrorConsumer;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.fileset.FilesetProvider;
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class for rule implementations building and initialization. Objects of this
+ * class are intended to be passed to the builder for the configured target, which then creates the
+ * configured target.
+ */
+public final class RuleContext extends TargetContext
+ implements ActionConstructionContext, ActionRegistry, RuleErrorConsumer {
+
+ /**
+ * The configured version of FilesetEntry.
+ */
+ @Immutable
+ public static final class ConfiguredFilesetEntry {
+ private final FilesetEntry entry;
+ private final TransitiveInfoCollection src;
+ private final ImmutableList<TransitiveInfoCollection> files;
+
+ ConfiguredFilesetEntry(FilesetEntry entry, TransitiveInfoCollection src) {
+ this.entry = entry;
+ this.src = src;
+ this.files = null;
+ }
+
+ ConfiguredFilesetEntry(FilesetEntry entry, ImmutableList<TransitiveInfoCollection> files) {
+ this.entry = entry;
+ this.src = null;
+ this.files = files;
+ }
+
+ public FilesetEntry getEntry() {
+ return entry;
+ }
+
+ public TransitiveInfoCollection getSrc() {
+ return src;
+ }
+
+ /**
+ * Targets from FilesetEntry.files, or null if the user omitted it.
+ */
+ @Nullable
+ public List<TransitiveInfoCollection> getFiles() {
+ return files;
+ }
+ }
+
+ static final String HOST_CONFIGURATION_PROGRESS_TAG = "for host";
+
+ private final Rule rule;
+ private final ListMultimap<String, ConfiguredTarget> targetMap;
+ private final ListMultimap<String, ConfiguredFilesetEntry> filesetEntryMap;
+ private final Set<ConfigMatchingProvider> configConditions;
+ private final AttributeMap attributes;
+ private final ImmutableSet<String> features;
+
+ private ActionOwner actionOwner;
+
+ /* lazily computed cache for Make variables, computed from the above. See get... method */
+ private transient ConfigurationMakeVariableContext configurationMakeVariableContext = null;
+
+ private RuleContext(Builder builder, ListMultimap<String, ConfiguredTarget> targetMap,
+ ListMultimap<String, ConfiguredFilesetEntry> filesetEntryMap,
+ Set<ConfigMatchingProvider> configConditions, ImmutableSet<String> features) {
+ super(builder.env, builder.rule, builder.configuration, builder.prerequisiteMap.get(null),
+ builder.visibility);
+ this.rule = builder.rule;
+ this.targetMap = targetMap;
+ this.filesetEntryMap = filesetEntryMap;
+ this.configConditions = configConditions;
+ this.attributes =
+ ConfiguredAttributeMapper.of(builder.rule, configConditions);
+ this.features = features;
+ }
+
+ @Override
+ public Rule getRule() {
+ return rule;
+ }
+
+ /**
+ * The configuration conditions that trigger this rule's configurable attributes.
+ */
+ Set<ConfigMatchingProvider> getConfigConditions() {
+ return configConditions;
+ }
+
+ /**
+ * Returns the host configuration for this rule; keep in mind that there may be multiple different
+ * host configurations, even during a single build.
+ */
+ public BuildConfiguration getHostConfiguration() {
+ BuildConfiguration configuration = getConfiguration();
+ // Note: the Builder checks that the configuration is non-null.
+ return configuration.getConfiguration(ConfigurationTransition.HOST);
+ }
+
+ /**
+ * Accessor for the Rule's attribute values.
+ */
+ public AttributeMap attributes() {
+ return attributes;
+ }
+
+ /**
+ * Returns whether this instance is known to have errors at this point during analysis. Do not
+ * call this method after the initializationHook has returned.
+ */
+ public boolean hasErrors() {
+ return getAnalysisEnvironment().hasErrors();
+ }
+
+ /**
+ * Returns an immutable map from attribute name to list of configured targets for that attribute.
+ */
+ public ListMultimap<String, ? extends TransitiveInfoCollection> getConfiguredTargetMap() {
+ return targetMap;
+ }
+
+ /**
+ * Returns an immutable map from attribute name to list of fileset entries.
+ */
+ public ListMultimap<String, ConfiguredFilesetEntry> getFilesetEntryMap() {
+ return filesetEntryMap;
+ }
+
+ @Override
+ public ActionOwner getActionOwner() {
+ if (actionOwner == null) {
+ actionOwner = new RuleActionOwner(rule, getConfiguration());
+ }
+ return actionOwner;
+ }
+
+ /**
+ * Returns a configuration fragment for this this target.
+ */
+ @Nullable
+ public <T extends Fragment> T getFragment(Class<T> fragment) {
+ // TODO(bazel-team): The fragments can also be accessed directly through BuildConfiguration.
+ // Can we lock that down somehow?
+ Preconditions.checkArgument(
+ rule.getRuleClassObject().isLegalConfigurationFragment(fragment),
+ "%s does not have access to %s", rule.getRuleClass(), fragment);
+ return getConfiguration().getFragment(fragment);
+ }
+
+ @Override
+ public ArtifactOwner getOwner() {
+ return getAnalysisEnvironment().getOwner();
+ }
+
+ // TODO(bazel-team): This class could be simpler if Rule and BuildConfiguration classes
+ // were immutable. Then we would need to store only references those two.
+ @Immutable
+ private static final class RuleActionOwner implements ActionOwner {
+ private final Label label;
+ private final Location location;
+ private final String configurationName;
+ private final String mnemonic;
+ private final String targetKind;
+ private final String shortCacheKey;
+ private final boolean hostConfiguration;
+
+ private RuleActionOwner(Rule rule, BuildConfiguration configuration) {
+ this.label = rule.getLabel();
+ this.location = rule.getLocation();
+ this.targetKind = rule.getTargetKind();
+ this.configurationName = configuration.getShortName();
+ this.mnemonic = configuration.getMnemonic();
+ this.shortCacheKey = configuration.shortCacheKey();
+ this.hostConfiguration = configuration.isHostConfiguration();
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public String getConfigurationName() {
+ return configurationName;
+ }
+
+ @Override
+ public String getConfigurationMnemonic() {
+ return mnemonic;
+ }
+
+ @Override
+ public String getConfigurationShortCacheKey() {
+ return shortCacheKey;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return targetKind;
+ }
+
+ @Override
+ public String getAdditionalProgressInfo() {
+ return hostConfiguration ? HOST_CONFIGURATION_PROGRESS_TAG : null;
+ }
+ }
+
+ @Override
+ public void registerAction(Action... action) {
+ getAnalysisEnvironment().registerAction(action);
+ }
+
+ /**
+ * Convenience function for subclasses to report non-attribute-specific
+ * errors in the current rule.
+ */
+ @Override
+ public void ruleError(String message) {
+ reportError(rule.getLocation(), prefixRuleMessage(message));
+ }
+
+ /**
+ * Convenience function for subclasses to report non-attribute-specific
+ * warnings in the current rule.
+ */
+ @Override
+ public void ruleWarning(String message) {
+ reportWarning(rule.getLocation(), prefixRuleMessage(message));
+ }
+
+ /**
+ * Convenience function for subclasses to report attribute-specific errors in
+ * the current rule.
+ *
+ * <p>If the name of the attribute starts with <code>$</code>
+ * it is replaced with a string <code>(an implicit dependency)</code>.
+ */
+ @Override
+ public void attributeError(String attrName, String message) {
+ reportError(rule.getAttributeLocation(attrName),
+ prefixAttributeMessage(Attribute.isImplicit(attrName)
+ ? "(an implicit dependency)"
+ : attrName,
+ message));
+ }
+
+ /**
+ * Like attributeError, but does not mark the configured target as errored.
+ *
+ * <p>If the name of the attribute starts with <code>$</code>
+ * it is replaced with a string <code>(an implicit dependency)</code>.
+ */
+ @Override
+ public void attributeWarning(String attrName, String message) {
+ reportWarning(rule.getAttributeLocation(attrName),
+ prefixAttributeMessage(Attribute.isImplicit(attrName)
+ ? "(an implicit dependency)"
+ : attrName,
+ message));
+ }
+
+ private String prefixAttributeMessage(String attrName, String message) {
+ return "in " + attrName + " attribute of "
+ + rule.getRuleClass() + " rule "
+ + getLabel() + ": " + message;
+ }
+
+ private String prefixRuleMessage(String message) {
+ return "in " + rule.getRuleClass() + " rule "
+ + getLabel() + ": " + message;
+ }
+
+ private void reportError(Location location, String message) {
+ getAnalysisEnvironment().getEventHandler().handle(Event.error(location, message));
+ }
+
+ private void reportWarning(Location location, String message) {
+ getAnalysisEnvironment().getEventHandler().handle(Event.warn(location, message));
+ }
+
+ /**
+ * Returns an artifact beneath the root of either the "bin" or "genfiles"
+ * tree, whose path is based on the name of this target and the current
+ * configuration. The choice of which tree to use is based on the rule with
+ * which this target (which must be an OutputFile or a Rule) is associated.
+ */
+ public Artifact createOutputArtifact() {
+ return internalCreateOutputArtifact(getTarget());
+ }
+
+ /**
+ * Returns the output artifact of an {@link OutputFile} of this target.
+ *
+ * @see #createOutputArtifact()
+ */
+ public Artifact createOutputArtifact(OutputFile out) {
+ return internalCreateOutputArtifact(out);
+ }
+
+ /**
+ * Implementation for {@link #createOutputArtifact()} and
+ * {@link #createOutputArtifact(OutputFile)}. This is private so that
+ * {@link #createOutputArtifact(OutputFile)} can have a more specific
+ * signature.
+ */
+ private Artifact internalCreateOutputArtifact(Target target) {
+ Root root = getBinOrGenfilesDirectory();
+ return getAnalysisEnvironment().getDerivedArtifact(Util.getWorkspaceRelativePath(target), root);
+ }
+
+ /**
+ * Returns the root of either the "bin" or "genfiles"
+ * tree, based on this target and the current configuration.
+ * The choice of which tree to use is based on the rule with
+ * which this target (which must be an OutputFile or a Rule) is associated.
+ */
+ public Root getBinOrGenfilesDirectory() {
+ return rule.hasBinaryOutput()
+ ? getConfiguration().getBinDirectory()
+ : getConfiguration().getGenfilesDirectory();
+ }
+
+ /**
+ * Returns the list of transitive info collections that feed into this target through the
+ * specified attribute. Note that you need to specify the correct mode for the attribute,
+ * otherwise an assertion will be raised.
+ */
+ public List<? extends TransitiveInfoCollection> getPrerequisites(String attributeName,
+ Mode mode) {
+ Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName);
+ if ((mode == Mode.TARGET)
+ && (attributeDefinition.getConfigurationTransition() instanceof SplitTransition)) {
+ // TODO(bazel-team): If you request a split-configured attribute in the target configuration,
+ // we return only the list of configured targets for the first architecture; this is for
+ // backwards compatibility with existing code in cases where the call to getPrerequisites is
+ // deeply nested and we can't easily inject the behavior we want. However, we should fix all
+ // such call sites.
+ checkAttribute(attributeName, Mode.SPLIT);
+ Map<String, ? extends List<? extends TransitiveInfoCollection>> map =
+ getSplitPrerequisites(attributeName, /*requireSplit=*/false);
+ return map.isEmpty()
+ ? ImmutableList.<TransitiveInfoCollection>of()
+ : map.entrySet().iterator().next().getValue();
+ }
+
+ checkAttribute(attributeName, mode);
+ return targetMap.get(attributeName);
+ }
+
+ /**
+ * Returns the a prerequisites keyed by the CPU of their configurations; this method throws an
+ * exception if the split transition is not active.
+ */
+ public Map<String, ? extends List<? extends TransitiveInfoCollection>>
+ getSplitPrerequisites(String attributeName) {
+ return getSplitPrerequisites(attributeName, /*requireSplit*/true);
+ }
+
+ private Map<String, ? extends List<? extends TransitiveInfoCollection>>
+ getSplitPrerequisites(String attributeName, boolean requireSplit) {
+ checkAttribute(attributeName, Mode.SPLIT);
+
+ Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName);
+ SplitTransition<?> transition =
+ (SplitTransition<?>) attributeDefinition.getConfigurationTransition();
+ List<BuildConfiguration> configurations =
+ getConfiguration().getTransitions().getSplitConfigurations(transition);
+ if (configurations.size() == 1) {
+ // There are two cases here:
+ // 1. Splitting is enabled, but only one target cpu.
+ // 2. Splitting is disabled, and no --cpu value was provided on the command line.
+ // In the first case, the cpu value is non-null, but in the second case it is null. We only
+ // allow that to proceed if the caller specified that he is going to ignore the cpu value
+ // anyway.
+ String cpu = configurations.get(0).getCpu();
+ if (cpu == null) {
+ Preconditions.checkState(!requireSplit);
+ cpu = "DO_NOT_USE";
+ }
+ return ImmutableMap.of(cpu, targetMap.get(attributeName));
+ }
+
+ Set<String> cpus = new HashSet<>();
+ for (BuildConfiguration config : configurations) {
+ // This method should only be called when the split config is enabled on the command line, in
+ // which case this cpu can't be null.
+ Preconditions.checkNotNull(config.getCpu());
+ cpus.add(config.getCpu());
+ }
+
+ // Use an ImmutableListMultimap.Builder here to preserve ordering.
+ ImmutableListMultimap.Builder<String, TransitiveInfoCollection> result =
+ ImmutableListMultimap.builder();
+ for (TransitiveInfoCollection t : targetMap.get(attributeName)) {
+ if (t.getConfiguration() != null) {
+ result.put(t.getConfiguration().getCpu(), t);
+ } else {
+ // Source files don't have a configuration, so we add them to all architecture entries.
+ for (String cpu : cpus) {
+ result.put(cpu, t);
+ }
+ }
+ }
+ return Multimaps.asMap(result.build());
+ }
+
+ /**
+ * Returns the specified provider of the prerequisite referenced by the attribute in the
+ * argument. Note that you need to specify the correct mode for the attribute, otherwise an
+ * assertion will be raised. If the attribute is empty of it does not support the specified
+ * provider, returns null.
+ */
+ public <C extends TransitiveInfoProvider> C getPrerequisite(
+ String attributeName, Mode mode, Class<C> provider) {
+ TransitiveInfoCollection prerequisite = getPrerequisite(attributeName, mode);
+ return prerequisite == null ? null : prerequisite.getProvider(provider);
+ }
+
+ /**
+ * Returns the transitive info collection that feeds into this target through the specified
+ * attribute. Note that you need to specify the correct mode for the attribute, otherwise an
+ * assertion will be raised. Returns null if the attribute is empty.
+ */
+ public TransitiveInfoCollection getPrerequisite(String attributeName, Mode mode) {
+ checkAttribute(attributeName, mode);
+ List<? extends TransitiveInfoCollection> elements = targetMap.get(attributeName);
+ if (elements.size() > 1) {
+ throw new IllegalStateException(rule.getRuleClass() + " attribute " + attributeName
+ + " produces more then one prerequisites");
+ }
+ return elements.isEmpty() ? null : elements.get(0);
+ }
+
+ /**
+ * Returns all the providers of the specified type that are listed under the specified attribute
+ * of this target in the BUILD file.
+ */
+ public <C extends TransitiveInfoProvider> Iterable<C> getPrerequisites(String attributeName,
+ Mode mode, final Class<C> classType) {
+ AnalysisUtils.checkProvider(classType);
+ return AnalysisUtils.getProviders(getPrerequisites(attributeName, mode), classType);
+ }
+
+ /**
+ * Returns all the providers of the specified type that are listed under the specified attribute
+ * of this target in the BUILD file, and that contain the specified provider.
+ */
+ public <C extends TransitiveInfoProvider> Iterable<? extends TransitiveInfoCollection>
+ getPrerequisitesIf(String attributeName, Mode mode, final Class<C> classType) {
+ AnalysisUtils.checkProvider(classType);
+ return AnalysisUtils.filterByProvider(getPrerequisites(attributeName, mode), classType);
+ }
+
+ /**
+ * Returns the prerequisite referred to by the specified attribute. Also checks whether
+ * the attribute is marked as executable and that the target referred to can actually be
+ * executed.
+ *
+ * <p>The {@code mode} argument must match the configuration transition specified in the
+ * definition of the attribute.
+ *
+ * @param attributeName the name of the attribute
+ * @param mode the configuration transition of the attribute
+ *
+ * @return the {@link FilesToRunProvider} interface of the prerequisite.
+ */
+ public FilesToRunProvider getExecutablePrerequisite(String attributeName, Mode mode) {
+ Attribute ruleDefinition = getRule().getAttributeDefinition(attributeName);
+
+ if (ruleDefinition == null) {
+ throw new IllegalStateException(getRule().getRuleClass() + " attribute " + attributeName
+ + " is not defined");
+ }
+ if (!ruleDefinition.isExecutable()) {
+ throw new IllegalStateException(getRule().getRuleClass() + " attribute " + attributeName
+ + " is not configured to be executable");
+ }
+
+ TransitiveInfoCollection prerequisite = getPrerequisite(attributeName, mode);
+ if (prerequisite == null) {
+ return null;
+ }
+
+ FilesToRunProvider result = prerequisite.getProvider(FilesToRunProvider.class);
+ if (result == null || result.getExecutable() == null) {
+ attributeError(
+ attributeName, prerequisite.getLabel() + " does not refer to a valid executable target");
+ }
+ return result;
+ }
+
+ /**
+ * Gets an attribute of type STRING_LIST expanding Make variables and
+ * tokenizes the result.
+ *
+ * @param attributeName the name of the attribute to process
+ * @return a list of strings containing the expanded and tokenized values for the
+ * attribute
+ */
+ public List<String> getTokenizedStringListAttr(String attributeName) {
+ if (!getRule().isAttrDefined(attributeName, Type.STRING_LIST)) {
+ // TODO(bazel-team): This should be an error.
+ return ImmutableList.of();
+ }
+ List<String> original = attributes().get(attributeName, Type.STRING_LIST);
+ if (original.isEmpty()) {
+ return ImmutableList.of();
+ }
+ List<String> tokens = new ArrayList<>();
+ for (String token : original) {
+ tokenizeAndExpandMakeVars(tokens, attributeName, token);
+ }
+ return ImmutableList.copyOf(tokens);
+ }
+
+ /**
+ * Expands make variables in value and tokenizes the result into tokens.
+ *
+ * <p>This methods should be called only during initialization.
+ */
+ public void tokenizeAndExpandMakeVars(List<String> tokens, String attributeName,
+ String value) {
+ try {
+ ShellUtils.tokenize(tokens, expandMakeVariables(attributeName, value));
+ } catch (ShellUtils.TokenizationException e) {
+ attributeError(attributeName, e.getMessage());
+ }
+ }
+
+ /**
+ * Return a context that maps Make variable names (string) to values (string).
+ *
+ * @return a ConfigurationMakeVariableContext.
+ **/
+ public ConfigurationMakeVariableContext getConfigurationMakeVariableContext() {
+ if (configurationMakeVariableContext == null) {
+ configurationMakeVariableContext = new ConfigurationMakeVariableContext(
+ getRule().getPackage(), getConfiguration());
+ }
+ return configurationMakeVariableContext;
+ }
+
+ /**
+ * Returns the string "expression" after expanding all embedded references to
+ * "Make" variables. If any errors are encountered, they are reported, and
+ * "expression" is returned unchanged.
+ *
+ * @param attributeName the name of the attribute from which "expression" comes;
+ * used for error reporting.
+ * @param expression the string to expand.
+ * @return the expansion of "expression".
+ */
+ public String expandMakeVariables(String attributeName, String expression) {
+ return expandMakeVariables(attributeName, expression, getConfigurationMakeVariableContext());
+ }
+
+ /**
+ * Returns the string "expression" after expanding all embedded references to
+ * "Make" variables. If any errors are encountered, they are reported, and
+ * "expression" is returned unchanged.
+ *
+ * @param attributeName the name of the attribute from which "expression" comes;
+ * used for error reporting.
+ * @param expression the string to expand.
+ * @param context the ConfigurationMakeVariableContext which can have a customized
+ * lookupMakeVariable(String) method.
+ * @return the expansion of "expression".
+ */
+ public String expandMakeVariables(String attributeName, String expression,
+ ConfigurationMakeVariableContext context) {
+ try {
+ return MakeVariableExpander.expand(expression, context);
+ } catch (MakeVariableExpander.ExpansionException e) {
+ attributeError(attributeName, e.getMessage());
+ return expression;
+ }
+ }
+
+ /**
+ * Gets the value of the STRING_LIST attribute expanding all make variables.
+ */
+ public List<String> expandedMakeVariablesList(String attrName) {
+ List<String> variables = new ArrayList<>();
+ for (String variable : attributes().get(attrName, Type.STRING_LIST)) {
+ variables.add(expandMakeVariables(attrName, variable));
+ }
+ return variables;
+ }
+
+ /**
+ * If the string consists of a single variable, returns the expansion of
+ * that variable. Otherwise, returns null. Syntax errors are reported.
+ *
+ * @param attrName the name of the attribute from which "expression" comes;
+ * used for error reporting.
+ * @param expression the string to expand.
+ * @return the expansion of "expression", or null.
+ */
+ public String expandSingleMakeVariable(String attrName, String expression) {
+ try {
+ return MakeVariableExpander.expandSingleVariable(expression,
+ new ConfigurationMakeVariableContext(getRule().getPackage(), getConfiguration()));
+ } catch (MakeVariableExpander.ExpansionException e) {
+ attributeError(attrName, e.getMessage());
+ return expression;
+ }
+ }
+
+ private void checkAttribute(String attributeName, Mode mode) {
+ Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName);
+ if (attributeDefinition == null) {
+ throw new IllegalStateException(getRule().getLocation() + ": " + getRule().getRuleClass()
+ + " attribute " + attributeName + " is not defined");
+ }
+ if (!(attributeDefinition.getType() == Type.LABEL
+ || attributeDefinition.getType() == Type.LABEL_LIST)) {
+ throw new IllegalStateException(rule.getRuleClass() + " attribute " + attributeName
+ + " is not a label type attribute");
+ }
+ if (mode == Mode.HOST) {
+ if (attributeDefinition.getConfigurationTransition() != ConfigurationTransition.HOST) {
+ throw new IllegalStateException(getRule().getLocation() + ": "
+ + getRule().getRuleClass() + " attribute " + attributeName
+ + " is not configured for the host configuration");
+ }
+ } else if (mode == Mode.TARGET) {
+ if (attributeDefinition.getConfigurationTransition() != ConfigurationTransition.NONE) {
+ throw new IllegalStateException(getRule().getLocation() + ": "
+ + getRule().getRuleClass() + " attribute " + attributeName
+ + " is not configured for the target configuration");
+ }
+ } else if (mode == Mode.DATA) {
+ if (attributeDefinition.getConfigurationTransition() != ConfigurationTransition.DATA) {
+ throw new IllegalStateException(getRule().getLocation() + ": "
+ + getRule().getRuleClass() + " attribute " + attributeName
+ + " is not configured for the data configuration");
+ }
+ } else if (mode == Mode.SPLIT) {
+ if (!(attributeDefinition.getConfigurationTransition() instanceof SplitTransition)) {
+ throw new IllegalStateException(getRule().getLocation() + ": "
+ + getRule().getRuleClass() + " attribute " + attributeName
+ + " is not configured for a split transition");
+ }
+ }
+ }
+
+ /**
+ * Returns the Mode for which the attribute is configured.
+ * This is intended for Skylark, where the Mode is implicitly chosen.
+ */
+ public Mode getAttributeMode(String attributeName) {
+ Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName);
+ if (attributeDefinition == null) {
+ throw new IllegalStateException(getRule().getLocation() + ": " + getRule().getRuleClass()
+ + " attribute " + attributeName + " is not defined");
+ }
+ if (!(attributeDefinition.getType() == Type.LABEL
+ || attributeDefinition.getType() == Type.LABEL_LIST)) {
+ throw new IllegalStateException(rule.getRuleClass() + " attribute " + attributeName
+ + " is not a label type attribute");
+ }
+ if (attributeDefinition.getConfigurationTransition() == ConfigurationTransition.HOST) {
+ return Mode.HOST;
+ } else if (attributeDefinition.getConfigurationTransition() == ConfigurationTransition.NONE) {
+ return Mode.TARGET;
+ } else if (attributeDefinition.getConfigurationTransition() == ConfigurationTransition.DATA) {
+ return Mode.DATA;
+ } else if (attributeDefinition.getConfigurationTransition() instanceof SplitTransition) {
+ return Mode.SPLIT;
+ }
+ throw new IllegalStateException(getRule().getLocation() + ": "
+ + getRule().getRuleClass() + " attribute " + attributeName + " is not configured");
+ }
+
+ /**
+ * For the specified attribute "attributeName" (which must be of type
+ * list(label)), resolve all the labels into ConfiguredTargets (for the
+ * configuration appropriate to the attribute) and return their build
+ * artifacts as a {@link PrerequisiteArtifacts} instance.
+ *
+ * @param attributeName the name of the attribute to traverse
+ */
+ public PrerequisiteArtifacts getPrerequisiteArtifacts(String attributeName, Mode mode) {
+ return PrerequisiteArtifacts.get(this, attributeName, mode);
+ }
+
+ /**
+ * For the specified attribute "attributeName" (which must be of type label),
+ * resolves the ConfiguredTarget and returns its single build artifact.
+ *
+ * <p>If the attribute is optional, has no default and was not specified, then
+ * null will be returned. Note also that null is returned (and an attribute
+ * error is raised) if there wasn't exactly one build artifact for the target.
+ */
+ public Artifact getPrerequisiteArtifact(String attributeName, Mode mode) {
+ TransitiveInfoCollection target = getPrerequisite(attributeName, mode);
+ return transitiveInfoCollectionToArtifact(attributeName, target);
+ }
+
+ /**
+ * Equivalent to getPrerequisiteArtifact(), but also asserts that
+ * host-configuration is appropriate for the specified attribute.
+ */
+ public Artifact getHostPrerequisiteArtifact(String attributeName) {
+ TransitiveInfoCollection target = getPrerequisite(attributeName, Mode.HOST);
+ return transitiveInfoCollectionToArtifact(attributeName, target);
+ }
+
+ private Artifact transitiveInfoCollectionToArtifact(
+ String attributeName, TransitiveInfoCollection target) {
+ if (target != null) {
+ Iterable<Artifact> artifacts = target.getProvider(FileProvider.class).getFilesToBuild();
+ if (Iterables.size(artifacts) == 1) {
+ return Iterables.getOnlyElement(artifacts);
+ } else {
+ attributeError(attributeName, target.getLabel() + " expected a single artifact");
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the sole file in the "srcs" attribute. Reports an error and
+ * (possibly) returns null if "srcs" does not identify a single file of the
+ * expected type.
+ */
+ public Artifact getSingleSource(String fileTypeName) {
+ List<Artifact> srcs = PrerequisiteArtifacts.get(this, "srcs", Mode.TARGET).list();
+ switch (srcs.size()) {
+ case 0 : // error already issued by getSrc()
+ return null;
+ case 1 : // ok
+ return Iterables.getOnlyElement(srcs);
+ default :
+ attributeError("srcs", "only a single " + fileTypeName + " is allowed here");
+ return srcs.get(0);
+ }
+ }
+
+ public Artifact getSingleSource() {
+ return getSingleSource(getRule().getRuleClass() + " source file");
+ }
+
+ /**
+ * Returns a path fragment qualified by the rule name and unique fragment to
+ * disambiguate artifacts produced from the source file appearing in
+ * multiple rules.
+ *
+ * <p>For example "pkg/dir/name" -> "pkg/<fragment>/rule/dir/name.
+ */
+ public final PathFragment getUniqueDirectory(String fragment) {
+ return AnalysisUtils.getUniqueDirectory(getLabel(), new PathFragment(fragment));
+ }
+
+ /**
+ * Check that all targets that were specified as sources are from the same
+ * package as this rule. Output a warning or an error for every target that is
+ * imported from a different package.
+ */
+ public void checkSrcsSamePackage(boolean onlyWarn) {
+ PathFragment packageName = getLabel().getPackageFragment();
+ for (Artifact srcItem : PrerequisiteArtifacts.get(this, "srcs", Mode.TARGET).list()) {
+ if (!srcItem.isSourceArtifact()) {
+ // In theory, we should not do this check. However, in practice, we
+ // have a couple of rules that do not obey the "srcs must contain
+ // files and only files" rule. Thus, we are stuck with this hack here :(
+ continue;
+ }
+ Label associatedLabel = srcItem.getOwner();
+ PathFragment itemPackageName = associatedLabel.getPackageFragment();
+ if (!itemPackageName.equals(packageName)) {
+ String message = "please do not import '" + associatedLabel + "' directly. "
+ + "You should either move the file to this package or depend on "
+ + "an appropriate rule there";
+ if (onlyWarn) {
+ attributeWarning("srcs", message);
+ } else {
+ attributeError("srcs", message);
+ }
+ }
+ }
+ }
+
+
+ /**
+ * Returns the label to which the {@code NODEP_LABEL} attribute
+ * {@code attrName} refers, checking that it is a valid label, and that it is
+ * referring to a local target. Reports a warning otherwise.
+ */
+ public Label getLocalNodepLabelAttribute(String attrName) {
+ Label label = attributes().get(attrName, Type.NODEP_LABEL);
+ if (label == null) {
+ return null;
+ }
+
+ if (!getTarget().getLabel().getPackageFragment().equals(label.getPackageFragment())) {
+ attributeWarning(attrName, "does not reference a local rule");
+ }
+
+ return label;
+ }
+
+ /**
+ * Returns the implicit output artifact for a given template function. If multiple or no artifacts
+ * can be found as a result of the template, an exception is thrown.
+ */
+ public Artifact getImplicitOutputArtifact(ImplicitOutputsFunction function) {
+ Iterable<String> result;
+ try {
+ result = function.getImplicitOutputs(RawAttributeMapper.of(rule));
+ } catch (EvalException e) {
+ // It's ok as long as we don't use this method from Skylark.
+ throw new IllegalStateException(e);
+ }
+ return getImplicitOutputArtifact(Iterables.getOnlyElement(result));
+ }
+
+ /**
+ * Only use from Skylark. Returns the implicit output artifact for a given output path.
+ */
+ public Artifact getImplicitOutputArtifact(String path) {
+ Root root = getBinOrGenfilesDirectory();
+ PathFragment packageFragment = getLabel().getPackageFragment();
+ return getAnalysisEnvironment().getDerivedArtifact(packageFragment.getRelative(path), root);
+ }
+
+ /**
+ * Convenience method to return a host configured target for the "compiler"
+ * attribute. Allows caller to decide whether a warning should be printed if
+ * the "compiler" attribute is not set to the default value.
+ *
+ * @param warnIfNotDefault if true, print a warning if the value for the
+ * "compiler" attribute is set to something other than the default
+ * @return a ConfiguredTarget using the host configuration for the "compiler"
+ * attribute
+ */
+ public final FilesToRunProvider getCompiler(boolean warnIfNotDefault) {
+ Label label = attributes().get("compiler", Type.LABEL);
+ if (warnIfNotDefault && !label.equals(getRule().getAttrDefaultValue("compiler"))) {
+ attributeWarning("compiler", "setting the compiler is strongly discouraged");
+ }
+ return getExecutablePrerequisite("compiler", Mode.HOST);
+ }
+
+ /**
+ * Returns the (unmodifiable, ordered) list of artifacts which are the outputs
+ * of this target.
+ *
+ * <p>Each element in this list is associated with a single output, either
+ * declared implicitly (via setImplicitOutputsFunction()) or explicitly
+ * (listed in the 'outs' attribute of our rule).
+ */
+ public final ImmutableList<Artifact> getOutputArtifacts() {
+ ImmutableList.Builder<Artifact> artifacts = ImmutableList.builder();
+ for (OutputFile out : getRule().getOutputFiles()) {
+ artifacts.add(createOutputArtifact(out));
+ }
+ return artifacts.build();
+ }
+
+ /**
+ * Like getFilesToBuild(), except that it also includes the runfiles middleman, if any.
+ * Middlemen are expanded in the SpawnStrategy or by the Distributor.
+ */
+ public static ImmutableList<Artifact> getFilesToRun(
+ RunfilesSupport runfilesSupport, NestedSet<Artifact> filesToBuild) {
+ if (runfilesSupport == null) {
+ return ImmutableList.copyOf(filesToBuild);
+ } else {
+ ImmutableList.Builder<Artifact> allFilesToBuild = ImmutableList.builder();
+ allFilesToBuild.addAll(filesToBuild);
+ allFilesToBuild.add(runfilesSupport.getRunfilesMiddleman());
+ return allFilesToBuild.build();
+ }
+ }
+
+ /**
+ * Like {@link #getOutputArtifacts()} but for a singular output item.
+ * Reports an error if the "out" attribute is not a singleton.
+ *
+ * @return null if the output list is empty, the artifact for the first item
+ * of the output list otherwise
+ */
+ public Artifact getOutputArtifact() {
+ List<Artifact> outs = getOutputArtifacts();
+ if (outs.size() != 1) {
+ attributeError("out", "exactly one output file required");
+ if (outs.isEmpty()) {
+ return null;
+ }
+ }
+ return outs.get(0);
+ }
+
+ /**
+ * Returns an artifact with a given file extension. All other path components
+ * are the same as in {@code pathFragment}.
+ */
+ public final Artifact getRelatedArtifact(PathFragment pathFragment, String extension) {
+ PathFragment file = FileSystemUtils.replaceExtension(pathFragment, extension);
+ return getAnalysisEnvironment().getDerivedArtifact(file, getConfiguration().getBinDirectory());
+ }
+
+ /**
+ * Returns true if runfiles support should create the runfiles tree, or
+ * false if it should just create the manifest.
+ */
+ public boolean shouldCreateRunfilesSymlinks() {
+ // TODO(bazel-team): Ideally we wouldn't need such logic, and we'd
+ // always use the BuildConfiguration#buildRunfiles() to determine
+ // whether to build the runfiles. The problem is that certain build
+ // steps actually consume their runfiles. These include:
+ // a. par files consumes the runfiles directory
+ // We should modify autopar to take a list of files instead.
+ // of the runfiles directory.
+ // b. host tools could potentially use data files, but currently don't
+ // (they're run from the execution root, not a runfiles tree).
+ // Currently hostConfiguration.buildRunfiles() returns true.
+ if (TargetUtils.isTestRule(getTarget())) {
+ // Tests are only executed during testing (duh),
+ // and their runfiles are generated lazily on local
+ // execution (see LocalTestStrategy). Therefore, it
+ // is safe not to build their runfiles.
+ return getConfiguration().buildRunfiles();
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * @return true if {@code rule} is visible from {@code prerequisite}.
+ *
+ * <p>This only computes the logic as implemented by the visibility system. The final decision
+ * whether a dependency is allowed is made by
+ * {@link ConfiguredRuleClassProvider.PrerequisiteValidator}.
+ */
+ public static boolean isVisible(Rule rule, TransitiveInfoCollection prerequisite) {
+ // Check visibility attribute
+ for (PackageSpecification specification :
+ prerequisite.getProvider(VisibilityProvider.class).getVisibility()) {
+ if (specification.containsPackage(rule.getLabel().getPackageFragment())) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * @return the set of features applicable for the current rule's package.
+ */
+ public ImmutableSet<String> getFeatures() {
+ return features;
+ }
+
+ /**
+ * Builder class for a RuleContext.
+ */
+ public static final class Builder {
+ private final AnalysisEnvironment env;
+ private final Rule rule;
+ private final BuildConfiguration configuration;
+ private final PrerequisiteValidator prerequisiteValidator;
+ private ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap;
+ private Set<ConfigMatchingProvider> configConditions;
+ private NestedSet<PackageSpecification> visibility;
+
+ Builder(AnalysisEnvironment env, Rule rule, BuildConfiguration configuration,
+ PrerequisiteValidator prerequisiteValidator) {
+ this.env = Preconditions.checkNotNull(env);
+ this.rule = Preconditions.checkNotNull(rule);
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.prerequisiteValidator = prerequisiteValidator;
+ }
+
+ RuleContext build() {
+ Preconditions.checkNotNull(prerequisiteMap);
+ Preconditions.checkNotNull(configConditions);
+ Preconditions.checkNotNull(visibility);
+ ListMultimap<String, ConfiguredTarget> targetMap = createTargetMap();
+ ListMultimap<String, ConfiguredFilesetEntry> filesetEntryMap =
+ createFilesetEntryMap(rule, configConditions);
+ return new RuleContext(this, targetMap, filesetEntryMap, configConditions,
+ getEnabledFeatures());
+ }
+
+ private ImmutableSet<String> getEnabledFeatures() {
+ Set<String> enabled = new HashSet<>();
+ Set<String> disabled = new HashSet<>();
+ for (String feature : Iterables.concat(getConfiguration().getDefaultFeatures(),
+ getRule().getPackage().getFeatures())) {
+ if (feature.startsWith("-")) {
+ disabled.add(feature.substring(1));
+ } else if (feature.equals("no_layering_check")) {
+ // TODO(bazel-team): Remove once we do not have BUILD files left that contain
+ // 'no_layering_check'.
+ disabled.add(feature.substring(3));
+ } else {
+ enabled.add(feature);
+ }
+ }
+ return Sets.difference(enabled, disabled).immutableCopy();
+ }
+
+ Builder setVisibility(NestedSet<PackageSpecification> visibility) {
+ this.visibility = visibility;
+ return this;
+ }
+
+ /**
+ * Sets the prerequisites and checks their visibility. It also generates appropriate error or
+ * warning messages and sets the error flag as appropriate.
+ */
+ Builder setPrerequisites(ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap) {
+ this.prerequisiteMap = Preconditions.checkNotNull(prerequisiteMap);
+ return this;
+ }
+
+ /**
+ * Sets the configuration conditions needed to determine which paths to follow for this
+ * rule's configurable attributes.
+ */
+ Builder setConfigConditions(Set<ConfigMatchingProvider> configConditions) {
+ this.configConditions = Preconditions.checkNotNull(configConditions);
+ return this;
+ }
+
+ private boolean validateFilesetEntry(FilesetEntry filesetEntry, ConfiguredTarget src) {
+ if (src.getProvider(FilesetProvider.class) != null) {
+ return true;
+ }
+ if (filesetEntry.isSourceFileset()) {
+ return true;
+ }
+
+ Target srcTarget = src.getTarget();
+ if (!(srcTarget instanceof FileTarget)) {
+ attributeError("entries", String.format(
+ "Invalid 'srcdir' target '%s'. Must be another Fileset or package",
+ srcTarget.getLabel()));
+ return false;
+ }
+
+ if (srcTarget instanceof OutputFile) {
+ attributeWarning("entries", String.format("'srcdir' target '%s' is not an input file. "
+ + "This forces the Fileset to be executed unconditionally",
+ srcTarget.getLabel()));
+ }
+
+ return true;
+ }
+
+ /**
+ * Determines and returns a map from attribute name to list of configured fileset entries, based
+ * on a PrerequisiteMap instance.
+ */
+ private ListMultimap<String, ConfiguredFilesetEntry> createFilesetEntryMap(
+ final Rule rule, Set<ConfigMatchingProvider> configConditions) {
+ final ImmutableSortedKeyListMultimap.Builder<String, ConfiguredFilesetEntry> mapBuilder =
+ ImmutableSortedKeyListMultimap.builder();
+ for (Attribute attr : rule.getAttributes()) {
+ if (attr.getType() != Type.FILESET_ENTRY_LIST) {
+ continue;
+ }
+ String attributeName = attr.getName();
+ Map<Label, ConfiguredTarget> ctMap = new HashMap<>();
+ for (ConfiguredTarget prerequisite : prerequisiteMap.get(attr)) {
+ ctMap.put(prerequisite.getLabel(), prerequisite);
+ }
+ List<FilesetEntry> entries = ConfiguredAttributeMapper.of(rule, configConditions)
+ .get(attributeName, Type.FILESET_ENTRY_LIST);
+ for (FilesetEntry entry : entries) {
+ if (entry.getFiles() == null) {
+ Label label = entry.getSrcLabel();
+ ConfiguredTarget src = ctMap.get(label);
+ if (!validateFilesetEntry(entry, src)) {
+ continue;
+ }
+
+ mapBuilder.put(attributeName, new ConfiguredFilesetEntry(entry, src));
+ } else {
+ ImmutableList.Builder<TransitiveInfoCollection> files = ImmutableList.builder();
+ for (Label file : entry.getFiles()) {
+ files.add(ctMap.get(file));
+ }
+ mapBuilder.put(attributeName, new ConfiguredFilesetEntry(entry, files.build()));
+ }
+ }
+ }
+ return mapBuilder.build();
+ }
+
+ /**
+ * Determines and returns a map from attribute name to list of configured targets.
+ */
+ private ImmutableSortedKeyListMultimap<String, ConfiguredTarget> createTargetMap() {
+ ImmutableSortedKeyListMultimap.Builder<String, ConfiguredTarget> mapBuilder =
+ ImmutableSortedKeyListMultimap.builder();
+
+ for (Map.Entry<Attribute, Collection<ConfiguredTarget>> entry :
+ prerequisiteMap.asMap().entrySet()) {
+ Attribute attribute = entry.getKey();
+ if (attribute == null) {
+ continue;
+ }
+ if (attribute.isSilentRuleClassFilter()) {
+ Predicate<RuleClass> filter = attribute.getAllowedRuleClassesPredicate();
+ for (ConfiguredTarget configuredTarget : entry.getValue()) {
+ Target prerequisiteTarget = configuredTarget.getTarget();
+ if ((prerequisiteTarget instanceof Rule)
+ && filter.apply(((Rule) prerequisiteTarget).getRuleClassObject())) {
+ validateDirectPrerequisite(attribute, configuredTarget);
+ mapBuilder.put(attribute.getName(), configuredTarget);
+ }
+ }
+ } else {
+ for (ConfiguredTarget configuredTarget : entry.getValue()) {
+ validateDirectPrerequisite(attribute, configuredTarget);
+ mapBuilder.put(attribute.getName(), configuredTarget);
+ }
+ }
+ }
+
+ // Handle abi_deps+deps error.
+ Attribute abiDepsAttr = rule.getAttributeDefinition("abi_deps");
+ if ((abiDepsAttr != null) && rule.isAttributeValueExplicitlySpecified("abi_deps")
+ && rule.isAttributeValueExplicitlySpecified("deps")) {
+ attributeError("deps", "Only one of deps and abi_deps should be provided");
+ }
+ return mapBuilder.build();
+ }
+
+ private String prefixRuleMessage(String message) {
+ return String.format("in %s rule %s: %s", rule.getRuleClass(), rule.getLabel(), message);
+ }
+
+ private String maskInternalAttributeNames(String name) {
+ return Attribute.isImplicit(name) ? "(an implicit dependency)" : name;
+ }
+
+ private String prefixAttributeMessage(String attrName, String message) {
+ return String.format("in %s attribute of %s rule %s: %s",
+ maskInternalAttributeNames(attrName), rule.getRuleClass(), rule.getLabel(), message);
+ }
+
+ public void reportError(Location location, String message) {
+ env.getEventHandler().handle(Event.error(location, message));
+ }
+
+ public void ruleError(String message) {
+ reportError(rule.getLocation(), prefixRuleMessage(message));
+ }
+
+ public void attributeError(String attrName, String message) {
+ reportError(rule.getAttributeLocation(attrName), prefixAttributeMessage(attrName, message));
+ }
+
+ public void reportWarning(Location location, String message) {
+ env.getEventHandler().handle(Event.warn(location, message));
+ }
+
+ public void ruleWarning(String message) {
+ env.getEventHandler().handle(Event.warn(rule.getLocation(), prefixRuleMessage(message)));
+ }
+
+ public void attributeWarning(String attrName, String message) {
+ reportWarning(rule.getAttributeLocation(attrName), prefixAttributeMessage(attrName, message));
+ }
+
+ private void reportBadPrerequisite(Attribute attribute, String targetKind,
+ Label prerequisiteLabel, String reason, boolean isWarning) {
+ String msgPrefix = targetKind != null ? targetKind + " " : "";
+ String msgReason = reason != null ? " (" + reason + ")" : "";
+ if (isWarning) {
+ attributeWarning(attribute.getName(), String.format(
+ "%s'%s' is unexpected here%s; continuing anyway",
+ msgPrefix, prerequisiteLabel, msgReason));
+ } else {
+ attributeError(attribute.getName(), String.format(
+ "%s'%s' is misplaced here%s", msgPrefix, prerequisiteLabel, msgReason));
+ }
+ }
+
+ private void validateDirectPrerequisiteType(ConfiguredTarget prerequisite,
+ Attribute attribute) {
+ Target prerequisiteTarget = prerequisite.getTarget();
+ Label prerequisiteLabel = prerequisiteTarget.getLabel();
+
+ if (prerequisiteTarget instanceof Rule) {
+ Rule prerequisiteRule = (Rule) prerequisiteTarget;
+
+ String reason = attribute.getValidityPredicate().checkValid(rule, prerequisiteRule);
+ if (reason != null) {
+ reportBadPrerequisite(attribute, prerequisiteTarget.getTargetKind(),
+ prerequisiteLabel, reason, false);
+ }
+ }
+
+ if (attribute.isStrictLabelCheckingEnabled()) {
+ if (prerequisiteTarget instanceof Rule) {
+ RuleClass ruleClass = ((Rule) prerequisiteTarget).getRuleClassObject();
+ if (!attribute.getAllowedRuleClassesPredicate().apply(ruleClass)) {
+ boolean allowedWithWarning = attribute.getAllowedRuleClassesWarningPredicate()
+ .apply(ruleClass);
+ reportBadPrerequisite(attribute, prerequisiteTarget.getTargetKind(), prerequisiteLabel,
+ "expected " + attribute.getAllowedRuleClassesPredicate().toString(),
+ allowedWithWarning);
+ }
+ } else if (prerequisiteTarget instanceof FileTarget) {
+ if (!attribute.getAllowedFileTypesPredicate()
+ .apply(((FileTarget) prerequisiteTarget).getFilename())) {
+ if (prerequisiteTarget instanceof InputFile
+ && !((InputFile) prerequisiteTarget).getPath().exists()) {
+ // Misplaced labels, no corresponding target exists
+ if (attribute.getAllowedFileTypesPredicate().isNone()
+ && !((InputFile) prerequisiteTarget).getFilename().contains(".")) {
+ // There are no allowed files in the attribute but it's not a valid rule,
+ // and the filename doesn't contain a dot --> probably a misspelled rule
+ attributeError(attribute.getName(),
+ "rule '" + prerequisiteLabel + "' does not exist");
+ } else {
+ attributeError(attribute.getName(),
+ "target '" + prerequisiteLabel + "' does not exist");
+ }
+ } else {
+ // The file exists but has a bad extension
+ reportBadPrerequisite(attribute, "file", prerequisiteLabel,
+ "expected " + attribute.getAllowedFileTypesPredicate().toString(), false);
+ }
+ }
+ }
+ }
+ }
+
+ public Rule getRule() {
+ return rule;
+ }
+
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ /**
+ * @return true if {@code rule} is visible from {@code prerequisite}.
+ *
+ * <p>This only computes the logic as implemented by the visibility system. The final decision
+ * whether a dependency is allowed is made by
+ * {@link ConfiguredRuleClassProvider.PrerequisiteValidator}, who is supposed to call this
+ * method to determine whether a dependency is allowed as per visibility rules.
+ */
+ public boolean isVisible(TransitiveInfoCollection prerequisite) {
+ return RuleContext.isVisible(rule, prerequisite);
+ }
+
+ private void validateDirectPrerequisiteFileTypes(ConfiguredTarget prerequisite,
+ Attribute attribute) {
+ if (attribute.isSkipAnalysisTimeFileTypeCheck()) {
+ return;
+ }
+ FileTypeSet allowedFileTypes = attribute.getAllowedFileTypesPredicate();
+ if (allowedFileTypes == FileTypeSet.ANY_FILE && !attribute.isNonEmpty()
+ && !attribute.isSingleArtifact()) {
+ return;
+ }
+
+ // If we allow any file we still need to check if there are actually files generated
+ // Note that this check only runs for ANY_FILE predicates if the attribute is NON_EMPTY
+ // or SINGLE_ARTIFACT
+ // If we performed this check when allowedFileTypes == NO_FILE this would
+ // always throw an error in those cases
+ if (allowedFileTypes != FileTypeSet.NO_FILE) {
+ Iterable<Artifact> artifacts = prerequisite.getProvider(FileProvider.class)
+ .getFilesToBuild();
+ if (attribute.isSingleArtifact() && Iterables.size(artifacts) != 1) {
+ attributeError(attribute.getName(),
+ "'" + prerequisite.getLabel() + "' must produce a single file");
+ return;
+ }
+ for (Artifact sourceArtifact : artifacts) {
+ if (allowedFileTypes.apply(sourceArtifact.getFilename())) {
+ return;
+ }
+ }
+ attributeError(attribute.getName(), "'" + prerequisite.getLabel()
+ + "' does not produce any " + rule.getRuleClass() + " " + attribute.getName()
+ + " files (expected " + allowedFileTypes + ")");
+ }
+ }
+
+ private void validateMandatoryProviders(ConfiguredTarget prerequisite, Attribute attribute) {
+ for (String provider : attribute.getMandatoryProviders()) {
+ if (prerequisite.get(provider) == null) {
+ attributeError(attribute.getName(), "'" + prerequisite.getLabel()
+ + "' does not have mandatory provider '" + provider + "'");
+ }
+ }
+ }
+
+ private void validateDirectPrerequisite(Attribute attribute, ConfiguredTarget prerequisite) {
+ validateDirectPrerequisiteType(prerequisite, attribute);
+ validateDirectPrerequisiteFileTypes(prerequisite, attribute);
+ validateMandatoryProviders(prerequisite, attribute);
+ prerequisiteValidator.validate(this, prerequisite, attribute);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RuleContext(" + getLabel() + ", " + getConfiguration() + ")";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinition.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinition.java
new file mode 100644
index 0000000..c5e32e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinition.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.packages.RuleClass;
+
+/**
+ * This class is a common ancestor for every rule object.
+ *
+ * <p>Implementors are also required to have the {@link BlazeRule} annotation
+ * set.
+ */
+public interface RuleDefinition {
+ /**
+ * This method should return a RuleClass object that represents the rule. The usual pattern is
+ * that various setter methods are called on the builder object passed in as the argument, then
+ * the object that is built by the builder is returned.
+ *
+ * @param builder A {@link com.google.devtools.build.lib.packages.RuleClass.Builder} object
+ * already preloaded with the attributes of the ancestors specified in the {@link
+ * BlazeRule} annotation.
+ * @param environment The services Blaze provides to rule definitions.
+ *
+ * @return the {@link RuleClass} representing the rule.
+ */
+ RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinitionEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinitionEnvironment.java
new file mode 100644
index 0000000..4dcd080
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinitionEnvironment.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Encapsulates the services available for implementors of the {@link RuleDefinition}
+ * interface.
+ */
+public interface RuleDefinitionEnvironment {
+ /**
+ * Parses the given string as a label and returns the label, by calling {@link
+ * Label#parseAbsolute}. Instead of throwing a {@link
+ * com.google.devtools.build.lib.syntax.Label.SyntaxException}, it throws an {@link
+ * IllegalArgumentException}, if the parsing fails.
+ */
+ Label getLabel(String labelValue);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java b/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java
new file mode 100644
index 0000000..a4da4b4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java
@@ -0,0 +1,756 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * An object that encapsulates runfiles. Conceptually, the runfiles are a map of paths to files,
+ * forming a symlink tree.
+ *
+ * <p>In order to reduce memory consumption, this map is not explicitly stored here, but instead as
+ * a combination of four parts: artifacts placed at their root-relative paths, source tree symlinks,
+ * root symlinks (outside of the source tree), and artifacts included as parts of "pruning
+ * manifests" (see {@link PruningManifest}).
+ */
+@Immutable
+public final class Runfiles {
+ private static final Function<Map.Entry<PathFragment, Artifact>, Artifact> TO_ARTIFACT =
+ new Function<Map.Entry<PathFragment, Artifact>, Artifact>() {
+ @Override
+ public Artifact apply(Map.Entry<PathFragment, Artifact> input) {
+ return input.getValue();
+ }
+ };
+
+ private static final Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>>
+ DUMMY_SYMLINK_EXPANDER =
+ new Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>>() {
+ @Override
+ public Map<PathFragment, Artifact> apply(Map<PathFragment, Artifact> input) {
+ return ImmutableMap.of();
+ }
+ };
+
+ // It is important to declare this *after* the DUMMY_SYMLINK_EXPANDER to avoid NPEs
+ public static final Runfiles EMPTY = new Builder().build();
+
+
+ /**
+ * The artifacts that should *always* be present in the runfiles directory. These are
+ * differentiated from the artifacts that may or may not be included by a pruning manifest
+ * (see {@link PruningManifest} below).
+ *
+ * <p>This collection may not include any middlemen. These artifacts will be placed at a location
+ * that corresponds to the root-relative path of each artifact. It's possible for several
+ * artifacts to have the same root-relative path, in which case the last one will win.
+ */
+ private final NestedSet<Artifact> unconditionalArtifacts;
+
+ /**
+ * A map of symlinks that should be present in the runfiles directory. In general, the symlink can
+ * be determined from the artifact by using the root-relative path, so this should only be used
+ * for cases where that isn't possible.
+ *
+ * <p>This may include runfiles symlinks from the root of the runfiles tree.
+ */
+ private final NestedSet<Map.Entry<PathFragment, Artifact>> symlinks;
+
+ /**
+ * A map of symlinks that should be present above the runfiles directory. These are useful for
+ * certain rule types like AppEngine apps which have root level config files outside of the
+ * regular source tree.
+ */
+ private final NestedSet<Map.Entry<PathFragment, Artifact>> rootSymlinks;
+
+ /**
+ * A function to generate extra manifest entries.
+ */
+ private final Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>>
+ manifestExpander;
+
+ /**
+ * Defines a set of artifacts that may or may not be included in the runfiles directory and
+ * a manifest file that makes that determination. These are applied on top of any artifacts
+ * specified in {@link #unconditionalArtifacts}.
+ *
+ * <p>The incentive behind this is to enable execution-phase "pruning" of runfiles. Anything
+ * set in unconditionalArtifacts is hard-set in Blaze's analysis phase, and thus unchangeable in
+ * response to execution phase results. This isn't always convenient. For example, say we have an
+ * action that consumes a set of "possible" runtime dependencies for a source file, parses that
+ * file for "import a.b.c" statements, and outputs a manifest of the actual dependencies that are
+ * referenced and thus really needed. This can reduce the size of the runfiles set, but we can't
+ * use this information until the manifest output is available.
+ *
+ * <p>Only artifacts present in the candidate set AND the manifest output make it into the
+ * runfiles tree. The candidate set requirement guarantees that analysis-time dependencies are a
+ * superset of the pruned dependencies, so undeclared inclusions (which can break build
+ * correctness) aren't possible.
+ */
+ public static class PruningManifest {
+ private final NestedSet<Artifact> candidateRunfiles;
+ private final Artifact manifestFile;
+
+ /**
+ * Creates a new pruning manifest.
+ *
+ * @param candidateRunfiles set of possible artifacts that the manifest file may reference
+ * @param manifestFile the manifest file, expected to be a newline-separated list of
+ * source tree root-relative paths (i.e. "my/package/myfile.txt"). Anything that can't be
+ * resolved back to an entry in candidateRunfiles is ignored and will *not* make it into
+ * the runfiles tree.
+ */
+ public PruningManifest(NestedSet<Artifact> candidateRunfiles, Artifact manifestFile) {
+ this.candidateRunfiles = candidateRunfiles;
+ this.manifestFile = manifestFile;
+ }
+
+ public NestedSet<Artifact> getCandidateRunfiles() {
+ return candidateRunfiles;
+ }
+
+ public Artifact getManifestFile() {
+ return manifestFile;
+ }
+ }
+
+ /**
+ * The pruning manifests that should be applied to these runfiles.
+ */
+ private final NestedSet<PruningManifest> pruningManifests;
+
+ private Runfiles(NestedSet<Artifact> artifacts,
+ NestedSet<Map.Entry<PathFragment, Artifact>> symlinks,
+ NestedSet<Map.Entry<PathFragment, Artifact>> rootSymlinks,
+ NestedSet<PruningManifest> pruningManifests,
+ Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> expander) {
+ this.unconditionalArtifacts = Preconditions.checkNotNull(artifacts);
+ this.symlinks = Preconditions.checkNotNull(symlinks);
+ this.rootSymlinks = Preconditions.checkNotNull(rootSymlinks);
+ this.pruningManifests = Preconditions.checkNotNull(pruningManifests);
+ this.manifestExpander = Preconditions.checkNotNull(expander);
+ }
+
+ /**
+ * Returns the artifacts that are unconditionally included in the runfiles (as opposed to
+ * pruning manifest candidates, which may or may not be included).
+ */
+ @VisibleForTesting
+ public NestedSet<Artifact> getUnconditionalArtifacts() {
+ return unconditionalArtifacts;
+ }
+
+ /**
+ * Returns the artifacts that are unconditionally included in the runfiles (as opposed to
+ * pruning manifest candidates, which may or may not be included). Middleman artifacts are
+ * excluded.
+ */
+ public Iterable<Artifact> getUnconditionalArtifactsWithoutMiddlemen() {
+ return Iterables.filter(unconditionalArtifacts, Artifact.MIDDLEMAN_FILTER);
+ }
+
+ /**
+ * Returns the collection of runfiles as artifacts, including both unconditional artifacts
+ * and pruning manifest candidates.
+ */
+ @VisibleForTesting
+ public NestedSet<Artifact> getArtifacts() {
+ NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder();
+ allArtifacts.addAll(unconditionalArtifacts.toCollection());
+ for (PruningManifest manifest : getPruningManifests()) {
+ allArtifacts.addTransitive(manifest.getCandidateRunfiles());
+ }
+ return allArtifacts.build();
+ }
+
+ /**
+ * Returns the collection of runfiles as artifacts, including both unconditional artifacts
+ * and pruning manifest candidates. Middleman artifacts are excluded.
+ */
+ public Iterable<Artifact> getArtifactsWithoutMiddlemen() {
+ return Iterables.filter(getArtifacts(), Artifact.MIDDLEMAN_FILTER);
+ }
+
+ /**
+ * Returns the symlinks.
+ */
+ public NestedSet<Map.Entry<PathFragment, Artifact>> getSymlinks() {
+ return symlinks;
+ }
+
+ /**
+ * Returns the symlinks as a map from path fragment to artifact.
+ */
+ public Map<PathFragment, Artifact> getSymlinksAsMap() {
+ return entriesToMap(symlinks);
+ }
+
+ /**
+ * @param eventHandler Used for throwing an error if we have an obscuring runlink.
+ * May be null, in which case obscuring symlinks are silently discarded.
+ * @param location Location for reporter. Ignored if reporter is null.
+ * @param workingManifest Manifest to be checked for obscuring symlinks.
+ * @return map of source file names mapped to their location on disk.
+ */
+ public static Map<PathFragment, Artifact> filterListForObscuringSymlinks(
+ EventHandler eventHandler, Location location, Map<PathFragment, Artifact> workingManifest) {
+ Map<PathFragment, Artifact> newManifest = new HashMap<>();
+
+ outer:
+ for (Iterator<Entry<PathFragment, Artifact>> i = workingManifest.entrySet().iterator();
+ i.hasNext(); ) {
+ Entry<PathFragment, Artifact> entry = i.next();
+ PathFragment source = entry.getKey();
+ Artifact symlink = entry.getValue();
+ // drop nested entries; warn if this changes anything
+ int n = source.segmentCount();
+ for (int j = 1; j < n; ++j) {
+ PathFragment prefix = source.subFragment(0, n - j);
+ Artifact ancestor = workingManifest.get(prefix);
+ if (ancestor != null) {
+ // This is an obscuring symlink, so just drop it and move on if there's no reporter.
+ if (eventHandler == null) {
+ continue outer;
+ }
+ PathFragment suffix = source.subFragment(n - j, n);
+ Path viaAncestor = ancestor.getPath().getRelative(suffix);
+ Path expected = symlink.getPath();
+ if (!viaAncestor.equals(expected)) {
+ eventHandler.handle(Event.warn(location, "runfiles symlink " + source + " -> "
+ + expected + " obscured by " + prefix + " -> " + ancestor.getPath()));
+ }
+ continue outer;
+ }
+ }
+ newManifest.put(entry.getKey(), entry.getValue());
+ }
+ return newManifest;
+ }
+
+ /**
+ * Returns the symlinks as a map from PathFragment to Artifact, with PathFragments relativized
+ * and rooted at the specified points.
+ * @param root The root the PathFragment is computed relative to (before it is
+ * rooted again). May be null.
+ * @param eventHandler Used for throwing an error if we have an obscuring runlink.
+ * May be null, in which case obscuring symlinks are silently discarded.
+ * @param location Location for eventHandler warnings. Ignored if eventHandler is null.
+ * @return Pair of Maps from remote path fragment to artifact, the first of normal source tree
+ * entries, the second of any elements that live outside the source tree.
+ */
+ public Pair<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> getRunfilesInputs(
+ PathFragment root, String workspaceSuffix, EventHandler eventHandler, Location location)
+ throws IOException {
+ Map<PathFragment, Artifact> manifest = getSymlinksAsMap();
+ // Add unconditional artifacts (committed to inclusion on construction of runfiles).
+ for (Artifact artifact : getUnconditionalArtifactsWithoutMiddlemen()) {
+ addToManifest(manifest, artifact, root);
+ }
+
+ // Add conditional artifacts (only included if they appear in a pruning manifest).
+ for (Runfiles.PruningManifest pruningManifest : getPruningManifests()) {
+ // This map helps us convert from source tree root-relative paths back to artifacts.
+ Map<String, Artifact> allowedRunfiles = new HashMap<>();
+ for (Artifact artifact : pruningManifest.getCandidateRunfiles()) {
+ allowedRunfiles.put(artifact.getRootRelativePath().getPathString(), artifact);
+ }
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(pruningManifest.getManifestFile().getPath().getInputStream()));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ Artifact artifact = allowedRunfiles.get(line);
+ if (artifact != null) {
+ addToManifest(manifest, artifact, root);
+ }
+ }
+ }
+
+ manifest = filterListForObscuringSymlinks(eventHandler, location, manifest);
+ manifest.putAll(manifestExpander.apply(manifest));
+ PathFragment path = new PathFragment(workspaceSuffix);
+ Map<PathFragment, Artifact> result = new HashMap<>();
+ for (Map.Entry<PathFragment, Artifact> entry : manifest.entrySet()) {
+ result.put(path.getRelative(entry.getKey()), entry.getValue());
+ }
+ return Pair.of(result, (Map<PathFragment, Artifact>) new HashMap<>(getRootSymlinksAsMap()));
+ }
+
+ @VisibleForTesting
+ protected static void addToManifest(Map<PathFragment, Artifact> manifest, Artifact artifact,
+ PathFragment root) {
+ PathFragment rootRelativePath = root != null
+ ? artifact.getRootRelativePath().relativeTo(root)
+ : artifact.getRootRelativePath();
+ manifest.put(rootRelativePath, artifact);
+ }
+
+ /**
+ * Returns the root symlinks.
+ */
+ public NestedSet<Map.Entry<PathFragment, Artifact>> getRootSymlinks() {
+ return rootSymlinks;
+ }
+
+ /**
+ * Returns the root symlinks.
+ */
+ public Map<PathFragment, Artifact> getRootSymlinksAsMap() {
+ return entriesToMap(rootSymlinks);
+ }
+
+ /**
+ * Returns the unified map of path fragments to artifacts, taking both artifacts and symlinks into
+ * account.
+ */
+ public Map<PathFragment, Artifact> asMapWithoutRootSymlinks() {
+ Map<PathFragment, Artifact> result = entriesToMap(symlinks);
+ // If multiple artifacts have the same root-relative path, the last one in the list will win.
+ // That is because the runfiles tree cannot contain the same artifact for different
+ // configurations, because it only uses root-relative paths.
+ for (Artifact artifact : Iterables.filter(unconditionalArtifacts, Artifact.MIDDLEMAN_FILTER)) {
+ result.put(artifact.getRootRelativePath(), artifact);
+ }
+ return result;
+ }
+
+ /**
+ * Returns the pruning manifests specified for this runfiles tree.
+ */
+ public NestedSet<PruningManifest> getPruningManifests() {
+ return pruningManifests;
+ }
+
+ /**
+ * Returns the symlinks expander specified for this runfiles tree.
+ */
+ public Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> getSymlinkExpander() {
+ return manifestExpander;
+ }
+
+ /**
+ * Returns the unified map of path fragments to artifacts, taking into account artifacts,
+ * symlinks, and pruning manifest candidates. The returned set is guaranteed to be a (not
+ * necessarily strict) superset of the actual runfiles tree created at execution time.
+ */
+ public NestedSet<Artifact> getAllArtifacts() {
+ if (isEmpty()) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+ NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder();
+ allArtifacts
+ .addTransitive(unconditionalArtifacts)
+ .addAll(Iterables.transform(symlinks, TO_ARTIFACT))
+ .addAll(Iterables.transform(rootSymlinks, TO_ARTIFACT));
+ for (PruningManifest manifest : getPruningManifests()) {
+ allArtifacts.addTransitive(manifest.getCandidateRunfiles());
+ }
+ return allArtifacts.build();
+ }
+
+ /**
+ * Returns if there are no runfiles.
+ */
+ public boolean isEmpty() {
+ return unconditionalArtifacts.isEmpty() && symlinks.isEmpty() && rootSymlinks.isEmpty() &&
+ pruningManifests.isEmpty();
+ }
+
+ private static <K, V> Map<K, V> entriesToMap(Iterable<Map.Entry<K, V>> entrySet) {
+ Map<K, V> map = new LinkedHashMap<>();
+ for (Map.Entry<K, V> entry : entrySet) {
+ map.put(entry.getKey(), entry.getValue());
+ }
+ return map;
+ }
+
+ /**
+ * Builder for Runfiles objects.
+ */
+ public static final class Builder {
+ /**
+ * This must be COMPILE_ORDER because {@link #asMapWithoutRootSymlinks} overwrites earlier
+ * entries with later ones, so we want a post-order iteration.
+ */
+ private NestedSetBuilder<Artifact> artifactsBuilder =
+ NestedSetBuilder.compileOrder();
+ private NestedSetBuilder<Map.Entry<PathFragment, Artifact>> symlinksBuilder =
+ NestedSetBuilder.stableOrder();
+ private NestedSetBuilder<Map.Entry<PathFragment, Artifact>> rootSymlinksBuilder =
+ NestedSetBuilder.stableOrder();
+ private NestedSetBuilder<PruningManifest> pruningManifestsBuilder =
+ NestedSetBuilder.stableOrder();
+ private Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>>
+ manifestExpander = DUMMY_SYMLINK_EXPANDER;
+
+ /**
+ * Builds a new Runfiles object.
+ */
+ public Runfiles build() {
+ return new Runfiles(artifactsBuilder.build(), symlinksBuilder.build(),
+ rootSymlinksBuilder.build(), pruningManifestsBuilder.build(),
+ manifestExpander);
+ }
+
+ /**
+ * Adds an artifact to the internal collection of artifacts.
+ */
+ public Builder addArtifact(Artifact artifact) {
+ Preconditions.checkNotNull(artifact);
+ artifactsBuilder.add(artifact);
+ return this;
+ }
+
+ /**
+ * Adds several artifacts to the internal collection.
+ */
+ public Builder addArtifacts(Iterable<Artifact> artifacts) {
+ for (Artifact artifact : artifacts) {
+ addArtifact(artifact);
+ }
+ return this;
+ }
+
+
+ /**
+ * Use {@link #addTransitiveArtifacts} instead, to prevent increased memory use.
+ */
+ @Deprecated
+ public Builder addArtifacts(NestedSet<Artifact> artifacts) {
+ // Do not delete this method, or else addArtifacts(Iterable) calls with a NestedSet argument
+ // will not be flagged.
+ Iterable<Artifact> it = artifacts;
+ addArtifacts(it);
+ return this;
+ }
+ /**
+ * Adds a nested set to the internal collection.
+ */
+ public Builder addTransitiveArtifacts(NestedSet<Artifact> artifacts) {
+ artifactsBuilder.addTransitive(artifacts);
+ return this;
+ }
+
+ /**
+ * Adds a symlink.
+ */
+ public Builder addSymlink(PathFragment link, Artifact target) {
+ Preconditions.checkNotNull(link);
+ Preconditions.checkNotNull(target);
+ symlinksBuilder.add(Maps.immutableEntry(link, target));
+ return this;
+ }
+
+ /**
+ * Adds several symlinks.
+ */
+ public Builder addSymlinks(Map<PathFragment, Artifact> symlinks) {
+ symlinksBuilder.addAll(symlinks.entrySet());
+ return this;
+ }
+
+ /**
+ * Adds several symlinks as a NestedSet.
+ */
+ public Builder addSymlinks(NestedSet<Map.Entry<PathFragment, Artifact>> symlinks) {
+ symlinksBuilder.addTransitive(symlinks);
+ return this;
+ }
+
+ /**
+ * Adds several root symlinks.
+ */
+ public Builder addRootSymlinks(Map<PathFragment, Artifact> symlinks) {
+ rootSymlinksBuilder.addAll(symlinks.entrySet());
+ return this;
+ }
+
+ /**
+ * Adds several root symlinks as a NestedSet.
+ */
+ public Builder addRootSymlinks(NestedSet<Map.Entry<PathFragment, Artifact>> symlinks) {
+ rootSymlinksBuilder.addTransitive(symlinks);
+ return this;
+ }
+
+ /**
+ * Adds a pruning manifest. See {@link PruningManifest} for an explanation.
+ */
+ public Builder addPruningManifest(PruningManifest manifest) {
+ pruningManifestsBuilder.add(manifest);
+ return this;
+ }
+
+ /**
+ * Adds several pruning manifests as a NestedSet. See {@link PruningManifest} for an
+ * explanation.
+ */
+ public Builder addPruningManifests(NestedSet<PruningManifest> manifests) {
+ pruningManifestsBuilder.addTransitive(manifests);
+ return this;
+ }
+
+ /**
+ * Specify a function that can create additional manifest entries based on the input entries.
+ */
+ public Builder setManifestExpander(
+ Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> expander) {
+ manifestExpander = Preconditions.checkNotNull(expander);
+ return this;
+ }
+
+ /**
+ * Merges runfiles from a given runfiles support.
+ *
+ * @param runfilesSupport the runfiles support to be merged in
+ */
+ public Builder merge(@Nullable RunfilesSupport runfilesSupport) {
+ return merge(runfilesSupport, null);
+ }
+
+ /**
+ * Merges runfiles from a given runfiles support.
+ *
+ * <p>Sometimes a particular symlink from the runfiles support must not be included in runfiles.
+ * In such cases the path fragment denoting the symlink should be passed in as {@code
+ * ommittedAdditionalSymlink}. The symlink will then be filtered away from the set of additional
+ * symlinks of the target.
+ *
+ * @param runfilesSupport the runfiles support to be merged in
+ * @param omittedAdditionalSymlink the symlink to be omitted, or null if no filtering is needed
+ */
+ public Builder merge(@Nullable RunfilesSupport runfilesSupport,
+ @Nullable final PathFragment omittedAdditionalSymlink) {
+ if (runfilesSupport == null) {
+ return this;
+ }
+ // TODO(bazel-team): We may be able to remove this now.
+ addArtifact(runfilesSupport.getRunfilesMiddleman());
+ Runfiles runfiles = runfilesSupport.getRunfiles();
+ if (omittedAdditionalSymlink == null) {
+ merge(runfiles);
+ } else {
+ artifactsBuilder.addTransitive(runfiles.getUnconditionalArtifacts());
+ symlinksBuilder.addAll(Maps.filterKeys(entriesToMap(runfiles.getSymlinks()),
+ Predicates.not(Predicates.equalTo(omittedAdditionalSymlink))).entrySet());
+ rootSymlinksBuilder.addTransitive(runfiles.getRootSymlinks());
+ pruningManifestsBuilder.addTransitive(runfiles.getPruningManifests());
+ if (manifestExpander == DUMMY_SYMLINK_EXPANDER) {
+ manifestExpander = runfiles.getSymlinkExpander();
+ } else {
+ Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> otherExpander =
+ runfiles.getSymlinkExpander();
+ Preconditions.checkState((otherExpander == DUMMY_SYMLINK_EXPANDER)
+ || manifestExpander.equals(otherExpander));
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Adds the runfiles for a particular target and visits the transitive closure of "srcs",
+ * "deps" and "data", collecting all of their respective runfiles.
+ */
+ public Builder addRunfiles(RuleContext ruleContext,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ Preconditions.checkNotNull(mapping);
+ Preconditions.checkNotNull(ruleContext);
+ addDataDeps(ruleContext);
+ addNonDataDeps(ruleContext, mapping);
+ return this;
+ }
+
+ /**
+ * Adds the files specified by a mapping from the transitive info collection to the runfiles.
+ *
+ * <p>Dependencies in {@code srcs} and {@code deps} are considered.
+ */
+ public Builder add(RuleContext ruleContext,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ Preconditions.checkNotNull(ruleContext);
+ Preconditions.checkNotNull(mapping);
+ for (TransitiveInfoCollection dep : getNonDataDeps(ruleContext)) {
+ Runfiles runfiles = mapping.apply(dep);
+ if (runfiles != null) {
+ merge(runfiles);
+ }
+ }
+
+ return this;
+ }
+
+ /**
+ * Collects runfiles from data dependencies of a target.
+ */
+ public Builder addDataDeps(RuleContext ruleContext) {
+ addTargets(getPrerequisites(ruleContext, "data", Mode.DATA),
+ RunfilesProvider.DATA_RUNFILES);
+ return this;
+ }
+
+ /**
+ * Collects runfiles from "srcs" and "deps" of a target.
+ */
+ public Builder addNonDataDeps(RuleContext ruleContext,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ for (TransitiveInfoCollection target : getNonDataDeps(ruleContext)) {
+ addTargetExceptFileTargets(target, mapping);
+ }
+ return this;
+ }
+
+ public Builder addTargets(Iterable<? extends TransitiveInfoCollection> targets,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ for (TransitiveInfoCollection target : targets) {
+ addTarget(target, mapping);
+ }
+ return this;
+ }
+
+ public Builder addTarget(TransitiveInfoCollection target,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ return addTargetIncludingFileTargets(target, mapping);
+ }
+
+ private Builder addTargetExceptFileTargets(TransitiveInfoCollection target,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ Runfiles runfiles = mapping.apply(target);
+ if (runfiles != null) {
+ merge(runfiles);
+ }
+
+ return this;
+ }
+
+ private Builder addTargetIncludingFileTargets(TransitiveInfoCollection target,
+ Function<TransitiveInfoCollection, Runfiles> mapping) {
+ if (target.getProvider(RunfilesProvider.class) == null
+ && mapping == RunfilesProvider.DATA_RUNFILES) {
+ // RuleConfiguredTarget implements RunfilesProvider, so this will only be called on
+ // FileConfiguredTarget instances.
+ // TODO(bazel-team): This is a terrible hack. We should be able to make this go away
+ // by implementing RunfilesProvider on FileConfiguredTarget. We'd need to be mindful
+ // of the memory use, though, since we have a whole lot of FileConfiguredTarget instances.
+ addTransitiveArtifacts(target.getProvider(FileProvider.class).getFilesToBuild());
+ return this;
+ }
+
+ return addTargetExceptFileTargets(target, mapping);
+ }
+
+ /**
+ * Adds symlinks to given artifacts at their exec paths.
+ */
+ public Builder addSymlinksToArtifacts(Iterable<Artifact> artifacts) {
+ for (Artifact artifact : artifacts) {
+ addSymlink(artifact.getExecPath(), artifact);
+ }
+ return this;
+ }
+
+ /**
+ * Add the other {@link Runfiles} object transitively.
+ */
+ public Builder merge(Runfiles runfiles) {
+ return merge(runfiles, true);
+ }
+
+ /**
+ * Add the other {@link Runfiles} object transitively, but don't merge
+ * pruning manifests.
+ */
+ public Builder mergeExceptPruningManifests(Runfiles runfiles) {
+ return merge(runfiles, false);
+ }
+
+ /**
+ * Add the other {@link Runfiles} object transitively, with the option to include or exclude
+ * pruning manifests in the merge.
+ */
+ private Builder merge(Runfiles runfiles, boolean includePruningManifests) {
+ artifactsBuilder.addTransitive(runfiles.getUnconditionalArtifacts());
+ symlinksBuilder.addTransitive(runfiles.getSymlinks());
+ rootSymlinksBuilder.addTransitive(runfiles.getRootSymlinks());
+ if (includePruningManifests) {
+ pruningManifestsBuilder.addTransitive(runfiles.getPruningManifests());
+ }
+ if (manifestExpander == DUMMY_SYMLINK_EXPANDER) {
+ manifestExpander = runfiles.getSymlinkExpander();
+ } else {
+ Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> otherExpander =
+ runfiles.getSymlinkExpander();
+ Preconditions.checkState((otherExpander == DUMMY_SYMLINK_EXPANDER)
+ || manifestExpander.equals(otherExpander));
+ }
+ return this;
+ }
+
+ private static Iterable<TransitiveInfoCollection> getNonDataDeps(RuleContext ruleContext) {
+ return Iterables.concat(
+ // TODO(bazel-team): This line shouldn't be here. Removing it requires that no rules have
+ // dependent rules in srcs (except for filegroups and such), but always in deps.
+ // TODO(bazel-team): DONT_CHECK is not optimal here. Rules that use split configs need to
+ // be changed not to call into here.
+ getPrerequisites(ruleContext, "srcs", Mode.DONT_CHECK),
+ getPrerequisites(ruleContext, "deps", Mode.DONT_CHECK));
+ }
+
+ /**
+ * For the specified attribute "attributeName" (which must be of type list(label)), resolves all
+ * the labels into ConfiguredTargets (for the same configuration as this one) and returns them
+ * as a list.
+ *
+ * <p>If the rule does not have the specified attribute, returns the empty list.
+ */
+ private static Iterable<? extends TransitiveInfoCollection> getPrerequisites(
+ RuleContext ruleContext, String attributeName, Mode mode) {
+ if (ruleContext.getRule().isAttrDefined(attributeName, Type.LABEL_LIST)) {
+ return ruleContext.getPrerequisites(attributeName, mode);
+ } else {
+ return Collections.emptyList();
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesProvider.java
new file mode 100644
index 0000000..2e26bbc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesProvider.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Runfiles a target contributes to targets that depend on it.
+ *
+ * <p>The set of runfiles contributed can be different if the dependency is through a
+ * <code>data</code> attribute (note that this is just a rough approximation of the reality --
+ * rule implementations are free to request the data runfiles at any time)
+ */
+@Immutable
+public final class RunfilesProvider implements TransitiveInfoProvider {
+ private final Runfiles defaultRunfiles;
+ private final Runfiles dataRunfiles;
+
+ private RunfilesProvider(Runfiles defaultRunfiles, Runfiles dataRunfiles) {
+ this.defaultRunfiles = defaultRunfiles;
+ this.dataRunfiles = dataRunfiles;
+ }
+
+ public Runfiles getDefaultRunfiles() {
+ return defaultRunfiles;
+ }
+
+ public Runfiles getDataRunfiles() {
+ return dataRunfiles;
+ }
+
+ /**
+ * Returns a function that gets the default runfiles from a {@link TransitiveInfoCollection} or
+ * the empty runfiles instance if it does not contain that provider.
+ */
+ public static final Function<TransitiveInfoCollection, Runfiles> DEFAULT_RUNFILES =
+ new Function<TransitiveInfoCollection, Runfiles>() {
+ @Override
+ public Runfiles apply(TransitiveInfoCollection input) {
+ RunfilesProvider provider = input.getProvider(RunfilesProvider.class);
+ if (provider != null) {
+ return provider.getDefaultRunfiles();
+ }
+
+ return Runfiles.EMPTY;
+ }
+ };
+
+ /**
+ * Returns a function that gets the data runfiles from a {@link TransitiveInfoCollection} or the
+ * empty runfiles instance if it does not contain that provider.
+ *
+ * <p>These are usually used if the target is depended on through a {@code data} attribute.
+ */
+ public static final Function<TransitiveInfoCollection, Runfiles> DATA_RUNFILES =
+ new Function<TransitiveInfoCollection, Runfiles>() {
+ @Override
+ public Runfiles apply(TransitiveInfoCollection input) {
+ RunfilesProvider provider = input.getProvider(RunfilesProvider.class);
+ if (provider != null) {
+ return provider.getDataRunfiles();
+ }
+
+ return Runfiles.EMPTY;
+ }
+ };
+
+ public static RunfilesProvider simple(Runfiles defaultRunfiles) {
+ return new RunfilesProvider(defaultRunfiles, defaultRunfiles);
+ }
+
+ public static RunfilesProvider withData(
+ Runfiles defaultRunfiles, Runfiles dataRunfiles) {
+ return new RunfilesProvider(defaultRunfiles, dataRunfiles);
+ }
+
+ public static final RunfilesProvider EMPTY = new RunfilesProvider(
+ Runfiles.EMPTY, Runfiles.EMPTY);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java
new file mode 100644
index 0000000..aa7f429
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java
@@ -0,0 +1,382 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.SourceManifestAction.ManifestType;
+import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.collect.IterablesChain;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class manages the creation of the runfiles symlink farms.
+ *
+ * <p>For executables that might depend on the existence of files at run-time, we create a symlink
+ * farm: a directory which contains symlinks to the right locations for those runfiles.
+ *
+ * <p>The runfiles symlink farm serves two purposes. The first is to allow programs (and
+ * programmers) to refer to files using their workspace-relative paths, regardless of whether the
+ * files were source files or generated files, and regardless of which part of the package path they
+ * came from. The second purpose is to ensure that all run-time dependencies are explicitly declared
+ * in the BUILD files; programs may only use files which the build system knows that they depend on.
+ *
+ * <p>The symlink farm contains a MANIFEST file which describes its contents. The MANIFEST file
+ * lists the names and contents of all of the symlinks in the symlink farm. For efficiency, Blaze's
+ * dependency analysis ignores the actual symlinks and just looks at the MANIFEST file. It is an
+ * invariant that the MANIFEST file should accurately represent the contents of the symlinks
+ * whenever the MANIFEST file is present. build_runfile_links.py preserves this invariant (modulo
+ * bugs - currently it has a bug where it may fail to preserve that invariant if it gets
+ * interrupted). So the Blaze dependency analysis looks only at the MANIFEST file, rather than at
+ * the individual symlinks.
+ *
+ * <p>We create an Artifact for the MANIFEST file and a RunfilesAction Action to create it. This
+ * action does not depend on any other Artifacts.
+ *
+ * <p>When building an executable and running it, there are three things which must be built: the
+ * executable itself, the runfiles symlink farm (represented in the action graph by the Artifact for
+ * its MANIFEST), and the files pointed to by the symlinks in the symlink farm. To avoid redundancy
+ * in the dependency analysis, we create a Middleman Artifact which depends on all of these. Actions
+ * which will run an executable should depend on this Middleman Artifact.
+ */
+public class RunfilesSupport {
+ private static final String RUNFILES_DIR_EXT = ".runfiles";
+
+ private final Runfiles runfiles;
+
+ private final Artifact runfilesInputManifest;
+ private final Artifact runfilesManifest;
+ private final Artifact runfilesMiddleman;
+ private final Artifact sourcesManifest;
+ private final Artifact owningExecutable;
+ private final boolean createSymlinks;
+ private final ImmutableList<String> args;
+
+ /**
+ * Creates the RunfilesSupport helper with the given executable and runfiles.
+ *
+ * @param ruleContext the rule context to create the runfiles support for
+ * @param executable the executable for whose runfiles this runfiles support is responsible, may
+ * be null
+ * @param runfiles the runfiles
+ * @param appendingArgs to be added after the rule's args
+ */
+ private RunfilesSupport(RuleContext ruleContext, Artifact executable, Runfiles runfiles,
+ List<String> appendingArgs, boolean createSymlinks) {
+ owningExecutable = executable;
+ this.createSymlinks = createSymlinks;
+
+ // Adding run_under target to the runfiles manifest so it would become part
+ // of runfiles tree and would be executable everywhere.
+ RunUnder runUnder = ruleContext.getConfiguration().getRunUnder();
+ if (runUnder != null && runUnder.getLabel() != null
+ && TargetUtils.isTestRule(ruleContext.getRule())) {
+ TransitiveInfoCollection runUnderTarget =
+ ruleContext.getPrerequisite(":run_under", Mode.DATA);
+ runfiles = new Runfiles.Builder()
+ .merge(getRunfiles(runUnderTarget))
+ .merge(runfiles)
+ .build();
+ }
+ this.runfiles = runfiles;
+
+ Preconditions.checkState(!runfiles.isEmpty());
+
+ Map<PathFragment, Artifact> symlinks = getRunfilesSymlinks();
+ if (executable != null && !symlinks.values().contains(executable)) {
+ throw new IllegalStateException("main program " + executable + " not included in runfiles");
+ }
+
+ runfilesInputManifest = createRunfilesInputManifestArtifact(ruleContext);
+ this.runfilesManifest = createRunfilesAction(ruleContext, runfiles);
+ this.runfilesMiddleman = createRunfilesMiddleman(ruleContext, runfiles.getAllArtifacts());
+ sourcesManifest = createSourceManifest(ruleContext, runfiles);
+ args = ImmutableList.<String>builder()
+ .addAll(ruleContext.getTokenizedStringListAttr("args"))
+ .addAll(appendingArgs)
+ .build();
+ }
+
+ private RunfilesSupport(Runfiles runfiles, Artifact runfilesInputManifest,
+ Artifact runfilesManifest, Artifact runfilesMiddleman, Artifact sourcesManifest,
+ Artifact owningExecutable, boolean createSymlinks, ImmutableList<String> args) {
+ this.runfiles = runfiles;
+ this.runfilesInputManifest = runfilesInputManifest;
+ this.runfilesManifest = runfilesManifest;
+ this.runfilesMiddleman = runfilesMiddleman;
+ this.sourcesManifest = sourcesManifest;
+ this.owningExecutable = owningExecutable;
+ this.createSymlinks = createSymlinks;
+ this.args = args;
+ }
+
+ /**
+ * Returns the executable owning this RunfilesSupport. Only use from Skylark.
+ */
+ public Artifact getExecutable() {
+ return owningExecutable;
+ }
+
+ /**
+ * Returns the exec path of the directory where the runfiles contained in this
+ * RunfilesSupport are generated. When the owning rule has no executable,
+ * returns null.
+ */
+ public PathFragment getRunfilesDirectoryExecPath() {
+ if (owningExecutable == null) {
+ return null;
+ }
+
+ PathFragment executablePath = owningExecutable.getExecPath();
+ return executablePath.getParentDirectory().getChild(
+ executablePath.getBaseName() + RUNFILES_DIR_EXT);
+ }
+
+ public Runfiles getRunfiles() {
+ return runfiles;
+ }
+
+ /**
+ * For executable programs, the .runfiles_manifest file outside of the
+ * runfiles symlink farm; otherwise, returns null.
+ *
+ * <p>The MANIFEST file represents the contents of all of the symlinks in the
+ * symlink farm. For efficiency, Blaze's dependency analysis ignores the
+ * actual symlinks and just looks at the MANIFEST file. It is an invariant
+ * that the MANIFEST file should accurately represent the contents of the
+ * symlinks whenever the MANIFEST file is present.
+ */
+ public Artifact getRunfilesInputManifest() {
+ return runfilesInputManifest;
+ }
+
+ private Artifact createRunfilesInputManifestArtifact(ActionConstructionContext context) {
+ // The executable may be null for emptyRunfiles
+ PathFragment relativePath = (owningExecutable != null)
+ ? owningExecutable.getRootRelativePath()
+ : Util.getWorkspaceRelativePath(context.getRule());
+ String basename = relativePath.getBaseName();
+ PathFragment inputManifestPath = relativePath.replaceName(basename + ".runfiles_manifest");
+ return context.getAnalysisEnvironment().getDerivedArtifact(inputManifestPath,
+ context.getConfiguration().getBinDirectory());
+ }
+
+ /**
+ * For executable programs, returns the MANIFEST file in the runfiles
+ * symlink farm, if blaze is run with --build_runfile_links; returns
+ * the .runfiles_manifest file outside of the symlink farm, if blaze
+ * is run with --nobuild_runfile_links.
+ * <p>
+ * Beware: In most cases {@link #getRunfilesInputManifest} is the more
+ * appropriate function.
+ */
+ public Artifact getRunfilesManifest() {
+ return runfilesManifest;
+ }
+
+ /**
+ * For executable programs, the root directory of the runfiles symlink farm;
+ * otherwise, returns null.
+ */
+ public Path getRunfilesDirectory() {
+ return FileSystemUtils.replaceExtension(getRunfilesInputManifest().getPath(), RUNFILES_DIR_EXT);
+ }
+
+ /**
+ * Returns the files pointed to by the symlinks in the runfiles symlink farm. This method is slow.
+ */
+ @VisibleForTesting
+ public Collection<Artifact> getRunfilesSymlinkTargets() {
+ return getRunfilesSymlinks().values();
+ }
+
+ /**
+ * Returns the names of the symlinks in the runfiles symlink farm as a Set of PathFragments. This
+ * method is slow.
+ */
+ // We should make this VisibleForTesting, but it is still used by TestHelper
+ public Set<PathFragment> getRunfilesSymlinkNames() {
+ return getRunfilesSymlinks().keySet();
+ }
+
+ /**
+ * Returns the names of the symlinks in the runfiles symlink farm as a Set of PathFragments. This
+ * method is slow.
+ */
+ @VisibleForTesting
+ public Map<PathFragment, Artifact> getRunfilesSymlinks() {
+ return runfiles.asMapWithoutRootSymlinks();
+ }
+
+ /**
+ * Returns both runfiles artifacts and "conditional" artifacts that may be part of a
+ * Runfiles PruningManifest. This means the returned set may be an overapproximation of the
+ * actual set of runfiles (see {@link Runfiles.PruningManifest}).
+ */
+ public Iterable<Artifact> getRunfilesArtifactsWithoutMiddlemen() {
+ return runfiles.getArtifactsWithoutMiddlemen();
+ }
+
+ /**
+ * Returns the middleman artifact that depends on getExecutable(),
+ * getRunfilesManifest(), and getRunfilesSymlinkTargets(). Anything which
+ * needs to actually run the executable should depend on this.
+ */
+ public Artifact getRunfilesMiddleman() {
+ return runfilesMiddleman;
+ }
+
+ /**
+ * Returns the Sources manifest.
+ * This may be null if the owningRule has no executable.
+ */
+ public Artifact getSourceManifest() {
+ return sourcesManifest;
+ }
+
+ private Artifact createRunfilesMiddleman(ActionConstructionContext context,
+ Iterable<Artifact> allRunfilesArtifacts) {
+ Iterable<Artifact> inputs = IterablesChain.<Artifact>builder()
+ .add(allRunfilesArtifacts)
+ .addElement(runfilesManifest)
+ .build();
+ return context.getAnalysisEnvironment().getMiddlemanFactory().createRunfilesMiddleman(
+ context.getActionOwner(), owningExecutable, inputs,
+ context.getConfiguration().getMiddlemanDirectory());
+ }
+
+ /**
+ * Creates a runfiles action for all of the specified files, and returns the
+ * output artifact (the artifact for the MANIFEST file).
+ *
+ * <p>The "runfiles" action creates a symlink farm that links all the runfiles
+ * (which may come from different places, e.g. different package paths,
+ * generated files, etc.) into a single tree, so that programs can access them
+ * using the workspace-relative name.
+ */
+ private Artifact createRunfilesAction(ActionConstructionContext context, Runfiles runfiles) {
+ // Compute the names of the runfiles directory and its MANIFEST file.
+ Artifact inputManifest = getRunfilesInputManifest();
+ context.getAnalysisEnvironment().registerAction(
+ SourceManifestAction.forRunfiles(
+ ManifestType.SOURCE_SYMLINKS, context.getActionOwner(), inputManifest, runfiles));
+
+ if (!createSymlinks) {
+ // Just return the manifest if that's all the build calls for.
+ return inputManifest;
+ }
+
+ PathFragment runfilesDir = FileSystemUtils.replaceExtension(inputManifest.getRootRelativePath(),
+ RUNFILES_DIR_EXT);
+ PathFragment outputManifestPath = runfilesDir.getRelative("MANIFEST");
+
+ BuildConfiguration config = context.getConfiguration();
+ Artifact outputManifest = context.getAnalysisEnvironment().getDerivedArtifact(
+ outputManifestPath, config.getBinDirectory());
+ context.getAnalysisEnvironment().registerAction(new SymlinkTreeAction(
+ context.getActionOwner(), inputManifest, outputManifest, /*filesetTree=*/false));
+ return outputManifest;
+ }
+
+ /**
+ * Creates an Artifact which writes the "sources only" manifest file.
+ *
+ * @param context the owner for the manifest action
+ * @param runfiles the runfiles
+ * @return the Artifact representing the file write action.
+ */
+ private Artifact createSourceManifest(ActionConstructionContext context, Runfiles runfiles) {
+ // Put the sources only manifest next to the MANIFEST file but call it SOURCES.
+ PathFragment runfilesDir = getRunfilesDirectoryExecPath();
+ if (runfilesDir != null) {
+ PathFragment sourcesManifestPath = runfilesDir.getRelative("SOURCES");
+ Artifact sourceOnlyManifest = context.getAnalysisEnvironment().getDerivedArtifact(
+ sourcesManifestPath, context.getConfiguration().getBinDirectory());
+ context.getAnalysisEnvironment().registerAction(
+ SourceManifestAction.forRunfiles(
+ ManifestType.SOURCES_ONLY, context.getActionOwner(), sourceOnlyManifest, runfiles));
+ return sourceOnlyManifest;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Helper method that returns a collection of artifacts that are necessary for the runfiles of the
+ * given target. Note that the runfile symlink tree is never built, so this may include artifacts
+ * that end up not being used (see {@link Runfiles}).
+ *
+ * @return the Runfiles object
+ */
+
+ private static Runfiles getRunfiles(TransitiveInfoCollection target) {
+ RunfilesProvider runfilesProvider = target.getProvider(RunfilesProvider.class);
+ if (runfilesProvider != null) {
+ return runfilesProvider.getDefaultRunfiles();
+ } else {
+ return Runfiles.EMPTY;
+ }
+ }
+
+ /**
+ * Returns the unmodifiable list of expanded and tokenized 'args' attribute
+ * values.
+ */
+ public List<String> getArgs() {
+ return args;
+ }
+
+ /**
+ * Creates and returns a RunfilesSupport object for the given rule and executable. Note that this
+ * method calls back into the passed in rule to obtain the runfiles.
+ */
+ public static RunfilesSupport withExecutable(RuleContext ruleContext, Runfiles runfiles,
+ Artifact executable) {
+ return new RunfilesSupport(ruleContext, executable, runfiles, ImmutableList.<String>of(),
+ ruleContext.shouldCreateRunfilesSymlinks());
+ }
+
+ /**
+ * Creates and returns a RunfilesSupport object for the given rule and executable. Note that this
+ * method calls back into the passed in rule to obtain the runfiles.
+ */
+ public static RunfilesSupport withExecutable(RuleContext ruleContext, Runfiles runfiles,
+ Artifact executable, boolean createSymlinks) {
+ return new RunfilesSupport(ruleContext, executable, runfiles, ImmutableList.<String>of(),
+ createSymlinks);
+ }
+
+ /**
+ * Creates and returns a RunfilesSupport object for the given rule, executable, runfiles and args.
+ */
+ public static RunfilesSupport withExecutable(RuleContext ruleContext, Runfiles runfiles,
+ Artifact executable, List<String> appendingArgs) {
+ return new RunfilesSupport(ruleContext, executable, runfiles,
+ ImmutableList.copyOf(appendingArgs), ruleContext.shouldCreateRunfilesSymlinks());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java
new file mode 100644
index 0000000..5aa3bdc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java
@@ -0,0 +1,404 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Action to create a manifest of input files for processing by a subsequent
+ * build step (e.g. runfiles symlinking or archive building).
+ *
+ * <p>The manifest's format is specifiable by {@link ManifestType}, in
+ * accordance with the needs of the calling functionality.
+ *
+ * <p>Note that this action carefully avoids building the manifest content in
+ * memory.
+ */
+public class SourceManifestAction extends AbstractFileWriteAction {
+ /**
+ * Action context that tells what workspace suffix we should use.
+ */
+ public interface Context extends ActionContext {
+ PathFragment getRunfilesPrefix();
+ }
+
+ private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4";
+
+ /**
+ * Interface for defining manifest formatting and reporting specifics.
+ */
+ @VisibleForTesting
+ interface ManifestWriter {
+
+ /**
+ * Writes a single line of manifest output.
+ *
+ * @param manifestWriter the output stream
+ * @param rootRelativePath path of an entry relative to the manifest's root
+ * @param symlink (optional) symlink that resolves the above path
+ */
+ void writeEntry(Writer manifestWriter, PathFragment rootRelativePath,
+ @Nullable Artifact symlink) throws IOException;
+
+ /**
+ * Fulfills {@link #ActionMetadata.getMnemonic()}
+ */
+ String getMnemonic();
+
+ /**
+ * Fulfills {@link #AbstractAction.getRawProgressMessage()}
+ */
+ String getRawProgressMessage();
+ }
+
+ /**
+ * The strategy we use to write manifest entries.
+ */
+ private final ManifestWriter manifestWriter;
+
+ /**
+ * The runfiles for which to create the symlink tree.
+ */
+ private final Runfiles runfiles;
+
+ /**
+ * If non-null, the paths should be computed relative to this path fragment.
+ */
+ private final PathFragment root;
+
+ /**
+ * Creates a new AbstractSourceManifestAction instance using latin1 encoding
+ * to write the manifest file and with a specified root path for manifest entries.
+ *
+ * @param manifestWriter the strategy to use to write manifest entries
+ * @param owner the action owner
+ * @param output the file to which to write the manifest
+ * @param runfiles runfiles
+ * @param root the artifacts' root-relative path is relativized to this before writing it out
+ */
+ private SourceManifestAction(ManifestWriter manifestWriter, ActionOwner owner, Artifact output,
+ Runfiles runfiles, PathFragment root) {
+ super(owner, getDependencies(runfiles), output, false);
+ this.manifestWriter = manifestWriter;
+ this.runfiles = runfiles;
+ this.root = root;
+ }
+
+ @VisibleForTesting
+ public void writeOutputFile(OutputStream out, EventHandler eventHandler, String workspaceSuffix)
+ throws IOException {
+ writeFile(out, runfiles.getRunfilesInputs(
+ root, workspaceSuffix, eventHandler, getOwner().getLocation()));
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor)
+ throws IOException {
+ final Pair<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> runfilesInputs =
+ runfiles.getRunfilesInputs(root,
+ executor.getContext(Context.class).getRunfilesPrefix().toString(), eventHandler,
+ getOwner().getLocation());
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ writeFile(out, runfilesInputs);
+ }
+ };
+ }
+
+ /**
+ * Returns the input dependencies for this action. Note we don't need to create the symlink
+ * target Artifacts before we write the output manifest, so this Action does not have to
+ * depend on them. The only necessary dependencies are pruning manifests, which must be read
+ * to properly prune the tree.
+ */
+ private static Collection<Artifact> getDependencies(Runfiles runfiles) {
+ ImmutableList.Builder<Artifact> builder = ImmutableList.builder();
+ for (Runfiles.PruningManifest manifest : runfiles.getPruningManifests()) {
+ builder.add(manifest.getManifestFile());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Sort the entries in both the normal and root manifests and write the output
+ * file.
+ *
+ * @param out is the message stream to write errors to.
+ * @param output The actual mapping of the output manifest.
+ * @throws IOException
+ */
+ private void writeFile(OutputStream out,
+ Pair<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> output)
+ throws IOException {
+ Writer manifestFile = new BufferedWriter(new OutputStreamWriter(out, ISO_8859_1));
+
+ Comparator<Map.Entry<PathFragment, Artifact>> fragmentComparator =
+ new Comparator<Map.Entry<PathFragment, Artifact>>() {
+ @Override
+ public int compare(Map.Entry<PathFragment, Artifact> path1,
+ Map.Entry<PathFragment, Artifact> path2) {
+ return path1.getKey().compareTo(path2.getKey());
+ }
+ };
+
+ List<Map.Entry<PathFragment, Artifact>> sortedRootLinks =
+ new ArrayList<>(output.second.entrySet());
+ Collections.sort(sortedRootLinks, fragmentComparator);
+
+ List<Map.Entry<PathFragment, Artifact>> sortedManifest =
+ new ArrayList<>(output.first.entrySet());
+ Collections.sort(sortedManifest, fragmentComparator);
+
+ for (Map.Entry<PathFragment, Artifact> line : sortedRootLinks) {
+ manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue());
+ }
+
+ for (Map.Entry<PathFragment, Artifact> line : sortedManifest) {
+ manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue());
+ }
+ manifestFile.flush();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return manifestWriter.getMnemonic();
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return manifestWriter.getRawProgressMessage() + " for " + getOwner().getLabel();
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ Map<PathFragment, Artifact> symlinks = runfiles.getSymlinksAsMap();
+ f.addInt(symlinks.size());
+ for (Map.Entry<PathFragment, Artifact> symlink : symlinks.entrySet()) {
+ f.addPath(symlink.getKey());
+ f.addPath(symlink.getValue().getPath());
+ }
+ Map<PathFragment, Artifact> rootSymlinks = runfiles.getRootSymlinksAsMap();
+ f.addInt(rootSymlinks.size());
+ for (Map.Entry<PathFragment, Artifact> rootSymlink : rootSymlinks.entrySet()) {
+ f.addPath(rootSymlink.getKey());
+ f.addPath(rootSymlink.getValue().getPath());
+ }
+
+ if (root != null) {
+ for (Artifact artifact : runfiles.getArtifactsWithoutMiddlemen()) {
+ f.addPath(artifact.getRootRelativePath().relativeTo(root));
+ f.addPath(artifact.getPath());
+ }
+ } else {
+ for (Artifact artifact : runfiles.getArtifactsWithoutMiddlemen()) {
+ f.addPath(artifact.getRootRelativePath());
+ f.addPath(artifact.getPath());
+ }
+ }
+ return f.hexDigestAndReset();
+ }
+
+ /**
+ * Supported manifest writing strategies.
+ */
+ public static enum ManifestType implements ManifestWriter {
+
+ /**
+ * Writes each line as:
+ *
+ * [rootRelativePath] [resolvingSymlink]
+ *
+ * <p>This strategy is suitable for creating an input manifest to a source view tree. Its
+ * output is a valid input to {@link com.google.devtools.build.lib.analysis.SymlinkTreeAction}.
+ */
+ SOURCE_SYMLINKS {
+ @Override
+ public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink)
+ throws IOException {
+ manifestWriter.append(rootRelativePath.getPathString());
+ // This trailing whitespace is REQUIRED to process the single entry line correctly.
+ manifestWriter.append(' ');
+ if (symlink != null) {
+ manifestWriter.append(symlink.getPath().getPathString());
+ }
+ manifestWriter.append('\n');
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "SourceSymlinkManifest";
+ }
+
+ @Override
+ public String getRawProgressMessage() {
+ return "Creating source manifest";
+ }
+ },
+
+ /**
+ * Writes each line as:
+ *
+ * [rootRelativePath]
+ *
+ * <p>This strategy is suitable for an input into a packaging system (notably .par) that
+ * consumes a list of all source files but needs that list to be constant with respect to
+ * how the user has their client laid out on local disk.
+ */
+ SOURCES_ONLY {
+ @Override
+ public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink)
+ throws IOException {
+ manifestWriter.append(rootRelativePath.getPathString());
+ manifestWriter.append('\n');
+ manifestWriter.flush();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "PackagingSourcesManifest";
+ }
+
+ @Override
+ public String getRawProgressMessage() {
+ return "Creating file sources list";
+ }
+ }
+ }
+
+ /** Creates an action for the given runfiles. */
+ public static SourceManifestAction forRunfiles(ManifestType manifestType, ActionOwner owner,
+ Artifact output, Runfiles runfiles) {
+ return new SourceManifestAction(manifestType, owner, output, runfiles, null);
+ }
+
+ /**
+ * Builder class to construct {@link SourceManifestAction} instances.
+ */
+ public static final class Builder {
+ private final ManifestWriter manifestWriter;
+ private final ActionOwner owner;
+ private final Artifact output;
+ private PathFragment top;
+ private final Runfiles.Builder runfilesBuilder = new Runfiles.Builder();
+
+ public Builder(ManifestType manifestType, ActionOwner owner, Artifact output) {
+ manifestWriter = manifestType;
+ this.owner = owner;
+ this.output = output;
+ }
+
+ @VisibleForTesting
+ Builder(ManifestWriter manifestWriter, ActionOwner owner, Artifact output) {
+ this.manifestWriter = manifestWriter;
+ this.owner = owner;
+ this.output = output;
+ }
+
+ public SourceManifestAction build() {
+ return new SourceManifestAction(manifestWriter, owner, output, runfilesBuilder.build(), top);
+ }
+
+ /**
+ * Sets the path fragment which is used to relativize the artifacts' root
+ * relative paths further. Most likely, you don't need this.
+ */
+ public Builder setTopLevel(PathFragment top) {
+ this.top = top;
+ return this;
+ }
+
+ /**
+ * Adds a set of symlinks from the artifacts' root-relative paths to the
+ * artifacts themselves.
+ */
+ public Builder addSymlinks(Iterable<Artifact> artifacts) {
+ runfilesBuilder.addArtifacts(artifacts);
+ return this;
+ }
+
+ /**
+ * Adds a map of symlinks.
+ */
+ public Builder addSymlinks(Map<PathFragment, Artifact> symlinks) {
+ runfilesBuilder.addSymlinks(symlinks);
+ return this;
+ }
+
+ /**
+ * Adds a single symlink.
+ */
+ public Builder addSymlink(PathFragment link, Artifact target) {
+ runfilesBuilder.addSymlink(link, target);
+ return this;
+ }
+
+ /**
+ * <p>Adds a mapping of Artifacts to the directory above the normal symlink
+ * forest base.
+ */
+ public Builder addRootSymlinks(Map<PathFragment, Artifact> rootSymlinks) {
+ runfilesBuilder.addRootSymlinks(rootSymlinks);
+ return this;
+ }
+
+ /**
+ * Set an expander function for the symlinks.
+ */
+ @VisibleForTesting
+ Builder setSymlinksExpander(
+ Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> expander) {
+ runfilesBuilder.setManifestExpander(expander);
+ return this;
+ }
+
+ /**
+ * Adds a runfiles pruning manifest.
+ */
+ @VisibleForTesting
+ Builder addPruningManifest(Runfiles.PruningManifest manifest) {
+ runfilesBuilder.addPruningManifest(manifest);
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeAction.java b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeAction.java
new file mode 100644
index 0000000..2dc0d4a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeAction.java
@@ -0,0 +1,109 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+/**
+ * Action responsible for the symlink tree creation.
+ * Used to generate runfiles and fileset symlink farms.
+ */
+public class SymlinkTreeAction extends AbstractAction {
+
+ private static final String GUID = "63412bda-4026-4c8e-a3ad-7deb397728d4";
+
+ private final Artifact inputManifest;
+ private final Artifact outputManifest;
+ private final boolean filesetTree;
+
+ /**
+ * Creates SymlinkTreeAction instance.
+ *
+ * @param owner action owner
+ * @param inputManifest exec path to the input runfiles manifest
+ * @param outputManifest exec path to the generated symlink tree manifest
+ * (must have "MANIFEST" base name). Symlink tree root
+ * will be set to the artifact's parent directory.
+ * @param filesetTree true if this is fileset symlink tree,
+ * false if this is a runfiles symlink tree.
+ */
+ public SymlinkTreeAction(ActionOwner owner, Artifact inputManifest, Artifact outputManifest,
+ boolean filesetTree) {
+ super(owner, ImmutableList.of(inputManifest), ImmutableList.of(outputManifest));
+ Preconditions.checkArgument(outputManifest.getPath().getBaseName().equals("MANIFEST"));
+ this.inputManifest = inputManifest;
+ this.outputManifest = outputManifest;
+ this.filesetTree = filesetTree;
+ }
+
+ public Artifact getInputManifest() {
+ return inputManifest;
+ }
+
+ public Artifact getOutputManifest() {
+ return outputManifest;
+ }
+
+ public boolean isFilesetTree() {
+ return filesetTree;
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "SymlinkTree";
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return (filesetTree ? "Creating Fileset tree " : "Creating runfiles tree ")
+ + outputManifest.getExecPath().getParentDirectory().getPathString();
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addInt(filesetTree ? 1 : 0);
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ // Return null here to indicate that resources would be managed manually
+ // during action execution.
+ return null;
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "local"; // Symlink tree is always generated locally.
+ }
+
+ @Override
+ public void execute(
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ actionExecutionContext.getExecutor().getContext(SymlinkTreeActionContext.class)
+ .createSymlinks(this, actionExecutionContext);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeActionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeActionContext.java
new file mode 100644
index 0000000..fe61056
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeActionContext.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+/**
+ * Action context for symlink tree actions (an action that creates a tree of symlinks).
+ */
+public interface SymlinkTreeActionContext extends ActionContext {
+
+ /**
+ * Creates the symlink tree.
+ */
+ void createSymlinks(SymlinkTreeAction action,
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetAndConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetAndConfiguration.java
new file mode 100644
index 0000000..cc0feb6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetAndConfiguration.java
@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Refers to the pair of a target and a configuration and certain additional information. Not the
+ * same as {@link ConfiguredTarget} -- that also contains the result of the analysis phase.
+ */
+public class TargetAndConfiguration {
+ private final Target target;
+ @Nullable private final BuildConfiguration configuration;
+
+ public TargetAndConfiguration(Target target, @Nullable BuildConfiguration configuration) {
+ this.target = Preconditions.checkNotNull(target);
+ this.configuration = configuration;
+ }
+
+ public TargetAndConfiguration(ConfiguredTarget configuredTarget) {
+ this.target = Preconditions.checkNotNull(configuredTarget).getTarget();
+ this.configuration = configuredTarget.getConfiguration();
+ }
+
+ // The node name in the graph. The name should be unique.
+ // It is not suitable for user display.
+ public String getName() {
+ return target.getLabel() + " "
+ + (configuration == null ? "null" : configuration.shortCacheKey());
+ }
+
+ public static final Function<TargetAndConfiguration, String> NAME_FUNCTION =
+ new Function<TargetAndConfiguration, String>() {
+ @Override
+ public String apply(TargetAndConfiguration node) {
+ return node.getName();
+ }
+ };
+
+ public static final Function<TargetAndConfiguration, ConfiguredTargetKey>
+ TO_LABEL_AND_CONFIGURATION = new Function<TargetAndConfiguration, ConfiguredTargetKey>() {
+ @Override
+ public ConfiguredTargetKey apply(TargetAndConfiguration input) {
+ return new ConfiguredTargetKey(input.getLabel(), input.getConfiguration());
+ }
+ };
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+ if (!(that instanceof TargetAndConfiguration)) {
+ return false;
+ }
+
+ TargetAndConfiguration thatNode = (TargetAndConfiguration) that;
+ return thatNode.target.getLabel().equals(this.target.getLabel()) &&
+ thatNode.configuration == this.configuration;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(target.getLabel(), configuration);
+ }
+
+ @Override
+ public String toString() {
+ return target.getLabel() + " (" + configuration + ")";
+ }
+
+ public Target getTarget() {
+ return target;
+ }
+
+ public Label getLabel() {
+ return target.getLabel();
+ }
+
+ @Nullable
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
new file mode 100644
index 0000000..fcfec55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * This event is fired as soon as a target is either built or fails.
+ */
+public final class TargetCompleteEvent implements SkyValue {
+
+ private final ConfiguredTarget target;
+ private final NestedSet<Label> rootCauses;
+
+ private TargetCompleteEvent(ConfiguredTarget target, NestedSet<Label> rootCauses) {
+ this.target = target;
+ this.rootCauses = (rootCauses == null)
+ ? NestedSetBuilder.<Label>emptySet(Order.STABLE_ORDER)
+ : rootCauses;
+ }
+
+ /**
+ * Construct a successful target completion event.
+ */
+ public static TargetCompleteEvent createSuccessful(ConfiguredTarget ct) {
+ return new TargetCompleteEvent(ct, null);
+ }
+
+ /**
+ * Construct a target completion event for a failed target, with the given non-empty root causes.
+ */
+ public static TargetCompleteEvent createFailed(ConfiguredTarget ct, NestedSet<Label> rootCauses) {
+ Preconditions.checkArgument(!Iterables.isEmpty(rootCauses));
+ return new TargetCompleteEvent(ct, rootCauses);
+ }
+
+ /**
+ * Returns the target associated with the event.
+ */
+ public ConfiguredTarget getTarget() {
+ return target;
+ }
+
+ /**
+ * Determines whether the target has failed or succeeded.
+ */
+ public boolean failed() {
+ return !rootCauses.isEmpty();
+ }
+
+ /**
+ * Get the root causes of the target. May be empty.
+ */
+ public Iterable<Label> getRootCauses() {
+ return rootCauses;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetContext.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetContext.java
new file mode 100644
index 0000000..9c95db3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetContext.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class for building {@link ConfiguredTarget} instances, in particular for non-rule ones.
+ * For {@link RuleConfiguredTarget} instances, use {@link RuleContext} instead,
+ * which is a subclass of this class.
+ *
+ * <p>The class is intended to be sub-classed by RuleContext, in order to share the code. However,
+ * it's not intended for sub-classing beyond that, and the constructor is intentionally package
+ * private to enforce that.
+ */
+public class TargetContext {
+
+ private final AnalysisEnvironment env;
+ private final Target target;
+ private final BuildConfiguration configuration;
+ /**
+ * This list only contains prerequisites that are not declared in rule attributes, with the
+ * exception of visibility (i.e., visibility is represented here, even though it is a rule
+ * attribute in case of a rule). Rule attributes are handled by the {@link RuleContext} subclass.
+ */
+ private final List<ConfiguredTarget> directPrerequisites;
+ private final NestedSet<PackageSpecification> visibility;
+
+ /**
+ * The constructor is intentionally package private.
+ */
+ TargetContext(AnalysisEnvironment env, Target target, BuildConfiguration configuration,
+ List<ConfiguredTarget> directPrerequisites,
+ NestedSet<PackageSpecification> visibility) {
+ this.env = env;
+ this.target = target;
+ this.configuration = configuration;
+ this.directPrerequisites = directPrerequisites;
+ this.visibility = visibility;
+ }
+
+ public AnalysisEnvironment getAnalysisEnvironment() {
+ return env;
+ }
+
+ public Target getTarget() {
+ return target;
+ }
+
+ public Label getLabel() {
+ return target.getLabel();
+ }
+
+ /**
+ * Returns the configuration for this target. This may return null if the target is supposed to be
+ * configuration-independent (like an input file, or a visibility rule). However, this is
+ * guaranteed to be non-null for rules and for output files.
+ */
+ @Nullable
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public NestedSet<PackageSpecification> getVisibility() {
+ return visibility;
+ }
+
+ TransitiveInfoCollection findDirectPrerequisite(Label label, BuildConfiguration config) {
+ for (ConfiguredTarget prerequisite : directPrerequisites) {
+ if (prerequisite.getLabel().equals(label) && (prerequisite.getConfiguration() == config)) {
+ return prerequisite;
+ }
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TempsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TempsProvider.java
new file mode 100644
index 0000000..109992e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TempsProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.Collection;
+
+/**
+ * A {@link TransitiveInfoProvider} for rule classes that save extra files when
+ * {@code --save_temps} is in effect.
+ */
+@Immutable
+public final class TempsProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<Artifact> temps;
+
+ public TempsProvider(ImmutableList<Artifact> temps) {
+ this.temps = temps;
+ }
+
+ /**
+ * Return the extra artifacts to save when {@code --save_temps} is in effect.
+ */
+ public Collection<Artifact> getTemps() {
+ return temps;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactContext.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactContext.java
new file mode 100644
index 0000000..c9b8e51
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactContext.java
@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Contains options which control the set of artifacts to build for top-level targets.
+ */
+@Immutable
+public final class TopLevelArtifactContext {
+
+ public static final TopLevelArtifactContext DEFAULT = new TopLevelArtifactContext(
+ "", /*compileOnly=*/false, /*compilationPrerequisitesOnly*/false,
+ /*runTestsExclusively=*/false, /*outputGroups=*/ImmutableSet.<String>of(),
+ /*shouldRunTests=*/false);
+
+ private final String buildCommand;
+ private final boolean compileOnly;
+ private final boolean compilationPrerequisitesOnly;
+ private final boolean runTestsExclusively;
+ private final ImmutableSet<String> outputGroups;
+ private final boolean shouldRunTests;
+
+ public TopLevelArtifactContext(String buildCommand, boolean compileOnly,
+ boolean compilationPrerequisitesOnly, boolean runTestsExclusively,
+ ImmutableSet<String> outputGroups, boolean shouldRunTests) {
+ this.buildCommand = buildCommand;
+ this.compileOnly = compileOnly;
+ this.compilationPrerequisitesOnly = compilationPrerequisitesOnly;
+ this.runTestsExclusively = runTestsExclusively;
+ this.outputGroups = outputGroups;
+ this.shouldRunTests = shouldRunTests;
+ }
+
+ /** Returns the build command as a string. */
+ public String buildCommand() {
+ return buildCommand;
+ }
+
+ /** Returns the value of the --compile_only flag. */
+ public boolean compileOnly() {
+ return compileOnly;
+ }
+
+ /** Returns the value of the --compilation_prerequisites_only flag. */
+ public boolean compilationPrerequisitesOnly() {
+ return compilationPrerequisitesOnly;
+ }
+
+ /** Whether to run tests in exclusive mode. */
+ public boolean runTestsExclusively() {
+ return runTestsExclusively;
+ }
+
+ /** Returns the value of the --output_groups flag. */
+ public Set<String> outputGroups() {
+ return outputGroups;
+ }
+
+ /** Whether the top-level request command may run tests. */
+ public boolean shouldRunTests() {
+ return shouldRunTests;
+ }
+
+ // TopLevelArtifactContexts are stored in maps in BuildView,
+ // so equals() and hashCode() need to work.
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof TopLevelArtifactContext) {
+ TopLevelArtifactContext otherContext = (TopLevelArtifactContext) other;
+ return buildCommand.equals(otherContext.buildCommand)
+ && compileOnly == otherContext.compileOnly
+ && compilationPrerequisitesOnly == otherContext.compilationPrerequisitesOnly
+ && runTestsExclusively == otherContext.runTestsExclusively
+ && outputGroups.equals(otherContext.outputGroups)
+ && shouldRunTests == otherContext.shouldRunTests;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(buildCommand, compileOnly, compilationPrerequisitesOnly,
+ runTestsExclusively, outputGroups, shouldRunTests);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java
new file mode 100644
index 0000000..3c025f4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java
@@ -0,0 +1,158 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+
+/**
+ * A small static class containing utility methods for handling the inclusion of
+ * extra top-level artifacts into the build.
+ */
+public final class TopLevelArtifactHelper {
+
+ private TopLevelArtifactHelper() {
+ // Prevent instantiation.
+ }
+
+ /** Returns command-specific artifacts which may exist for a given target and build command. */
+ public static final Iterable<Artifact> getCommandArtifacts(TransitiveInfoCollection target,
+ String buildCommand) {
+ TopLevelArtifactProvider provider = target.getProvider(TopLevelArtifactProvider.class);
+ if (provider != null
+ && provider.getCommandsForExtraArtifacts().contains(buildCommand.toLowerCase())) {
+ return provider.getArtifactsForCommand();
+ } else {
+ return ImmutableList.of();
+ }
+ }
+
+ /**
+ * Utility function to form a list of all test output Artifacts of the given targets to test.
+ */
+ public static ImmutableCollection<Artifact> getAllArtifactsToTest(
+ Iterable<? extends TransitiveInfoCollection> targets) {
+ if (targets == null) {
+ return ImmutableList.of();
+ }
+ ImmutableList.Builder<Artifact> allTestArtifacts = ImmutableList.builder();
+ for (TransitiveInfoCollection target : targets) {
+ allTestArtifacts.addAll(TestProvider.getTestStatusArtifacts(target));
+ }
+ return allTestArtifacts.build();
+ }
+
+ /**
+ * Utility function to form a NestedSet of all top-level Artifacts of the given targets.
+ */
+ public static NestedSet<Artifact> getAllArtifactsToBuild(
+ Iterable<? extends TransitiveInfoCollection> targets, TopLevelArtifactContext options) {
+ NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder();
+ for (TransitiveInfoCollection target : targets) {
+ allArtifacts.addTransitive(getAllArtifactsToBuild(target, options));
+ }
+ return allArtifacts.build();
+ }
+
+ /**
+ * Returns all artifacts to build if this target is requested as a top-level target. The resulting
+ * set includes the temps and either the files to compile, if
+ * {@code options.compileOnly() == true}, or the files to run.
+ *
+ * <p>Calls to this method should generally return quickly; however, the runfiles computation can
+ * be lazy, in which case it can be expensive on the first call. Subsequent calls may or may not
+ * return the same {@code Iterable} instance.
+ */
+ public static NestedSet<Artifact> getAllArtifactsToBuild(TransitiveInfoCollection target,
+ TopLevelArtifactContext options) {
+ NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder();
+ TempsProvider tempsProvider = target.getProvider(TempsProvider.class);
+ if (tempsProvider != null) {
+ allArtifacts.addAll(tempsProvider.getTemps());
+ }
+
+ TopLevelArtifactProvider topLevelArtifactProvider =
+ target.getProvider(TopLevelArtifactProvider.class);
+ if (topLevelArtifactProvider != null) {
+ for (String outputGroup : options.outputGroups()) {
+ NestedSet<Artifact> results = topLevelArtifactProvider.getOutputGroup(outputGroup);
+ if (results != null) {
+ allArtifacts.addTransitive(results);
+ }
+ }
+ }
+
+ if (options.compileOnly()) {
+ FilesToCompileProvider provider = target.getProvider(FilesToCompileProvider.class);
+ if (provider != null) {
+ allArtifacts.addAll(provider.getFilesToCompile());
+ }
+ } else if (options.compilationPrerequisitesOnly()) {
+ CompilationPrerequisitesProvider provider =
+ target.getProvider(CompilationPrerequisitesProvider.class);
+ if (provider != null) {
+ allArtifacts.addTransitive(provider.getCompilationPrerequisites());
+ }
+ } else {
+ FilesToRunProvider filesToRunProvider = target.getProvider(FilesToRunProvider.class);
+ boolean hasRunfilesSupport = false;
+ if (filesToRunProvider != null) {
+ allArtifacts.addAll(filesToRunProvider.getFilesToRun());
+ hasRunfilesSupport = filesToRunProvider.getRunfilesSupport() != null;
+ }
+
+ if (!hasRunfilesSupport) {
+ RunfilesProvider runfilesProvider =
+ target.getProvider(RunfilesProvider.class);
+ if (runfilesProvider != null) {
+ allArtifacts.addTransitive(runfilesProvider.getDefaultRunfiles().getAllArtifacts());
+ }
+ }
+
+ AlwaysBuiltArtifactsProvider forcedArtifacts = target.getProvider(
+ AlwaysBuiltArtifactsProvider.class);
+ if (forcedArtifacts != null) {
+ allArtifacts.addTransitive(forcedArtifacts.getArtifactsToAlwaysBuild());
+ }
+ }
+
+ allArtifacts.addAll(getCommandArtifacts(target, options.buildCommand()));
+ allArtifacts.addAll(getCoverageArtifacts(target, options));
+ return allArtifacts.build();
+ }
+
+ private static Iterable<Artifact> getCoverageArtifacts(TransitiveInfoCollection target,
+ TopLevelArtifactContext topLevelOptions) {
+ if (!topLevelOptions.compileOnly() && !topLevelOptions.compilationPrerequisitesOnly()
+ && topLevelOptions.shouldRunTests()) {
+ // Add baseline code coverage artifacts if we are collecting code coverage. We do that only
+ // when running tests.
+ // It might be slightly faster to first check if any configuration has coverage enabled.
+ if (target.getConfiguration() != null
+ && target.getConfiguration().isCodeCoverageEnabled()) {
+ BaselineCoverageArtifactsProvider provider =
+ target.getProvider(BaselineCoverageArtifactsProvider.class);
+ if (provider != null) {
+ return provider.getBaselineCoverageArtifacts();
+ }
+ }
+ }
+ return ImmutableList.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactProvider.java
new file mode 100644
index 0000000..e2a2d57
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactProvider.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * ConfiguredTargets implementing this interface can provide command-specific
+ * and unconditional extra artifacts to the build.
+ */
+@Immutable
+public final class TopLevelArtifactProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<String> commandsForExtraArtifacts;
+ private final ImmutableList<Artifact> artifactsForCommand;
+ private final ImmutableMap<String, NestedSet<Artifact>> outputGroups;
+
+ public TopLevelArtifactProvider(ImmutableList<String> commandsForExtraArtifacts,
+ ImmutableList<Artifact> artifactsForCommand) {
+ this.commandsForExtraArtifacts = commandsForExtraArtifacts;
+ this.artifactsForCommand = artifactsForCommand;
+ this.outputGroups = ImmutableMap.<String, NestedSet<Artifact>>of();
+ }
+
+ public TopLevelArtifactProvider(String key, NestedSet<Artifact> artifactsToBuild) {
+ this.commandsForExtraArtifacts = ImmutableList.of();
+ this.artifactsForCommand = ImmutableList.of();
+ this.outputGroups = ImmutableMap.<String, NestedSet<Artifact>>of(key, artifactsToBuild);
+ }
+
+ /** Returns the commands (in lowercase) that this provider should provide artifacts for. */
+ public ImmutableList<String> getCommandsForExtraArtifacts() {
+ return commandsForExtraArtifacts;
+ }
+
+ /** Returns the extra artifacts for the commands. */
+ public ImmutableList<Artifact> getArtifactsForCommand() {
+ return artifactsForCommand;
+ }
+
+ /** Returns artifacts that are to be built for every command. */
+ public NestedSet<Artifact> getOutputGroup(String outputGroupName) {
+ return outputGroups.get(outputGroupName);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java
new file mode 100644
index 0000000..82396ee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+
+import javax.annotation.Nullable;
+
+/**
+ * Objects that implement this interface bundle multiple {@link TransitiveInfoProvider} interfaces.
+ *
+ * <p>This interface (together with {@link TransitiveInfoProvider} is the cornerstone of the data
+ * model of the analysis phase.
+ *
+ * <p>The computation a configured target does is allowed to depend on the following things:
+ * <ul>
+ * <li>The associated Target (which will usually be a Rule)
+ * <li>Its own configuration (the configured target does not have access to other configurations,
+ * e.g. the host configuration, though)
+ * <li>The transitive info providers and labels of its direct dependencies.
+ * </ul>
+ *
+ * <p>And these are the only inputs. Notably, a configured target is not supposed to access
+ * other configured targets, the transitive info collections of configured targets it does not
+ * directly depend on, the actions created by anyone else or the contents of any input file. We
+ * strive to make it impossible for configured targets to do these things.
+ *
+ * <p>A configured target is expected to produce the following data during its analysis:
+ * <ul>
+ * <li>A number of Artifacts and Actions generating them
+ * <li>A set of {@link TransitiveInfoProvider}s that it passes on to the targets directly dependent
+ * on it
+ * </ul>
+ *
+ * <p>The information that can be passed on to dependent targets by way of
+ * {@link TransitiveInfoProvider} is subject to constraints (which are detailed in the
+ * documentation of that class).
+ *
+ * <p>Configured targets are currently allowed to create artifacts at any exec path. It would be
+ * better if they could be constrained to a subtree based on the label of the configured target,
+ * but this is currently not feasible because multiple rules violate this constraint and the
+ * output format is part of its interface.
+ *
+ * <p>In principle, multiple configured targets should not create actions with conflicting
+ * outputs. There are still a few exceptions to this rule that are slated to be eventually
+ * removed, we have provisions to handle this case (Action instances that share at least one
+ * output file are required to be exactly the same), but this does put some pressure on the design
+ * and we are eventually planning to eliminate this option.
+ *
+ * <p>These restrictions together make it possible to:
+ * <ul>
+ * <li>Correctly cache the analysis phase; by tightly constraining what a configured target is
+ * allowed to access and what it is not, we can know when it needs to invalidate a particular
+ * one and when it can reuse an already existing one.
+ * <li>Serialize / deserialize individual configured targets at will, making it possible for
+ * example to swap out part of the analysis state if there is memory pressure or to move them in
+ * persistent storage so that the state can be reconstructed at a different time or in a
+ * different process. The stretch goal is to eventually facilitate cross-uses caching of this
+ * information.
+ * </ul>
+ *
+ * <p>Implementations of build rules should <b>not</b> hold on to references to the
+ * {@link TransitiveInfoCollection}s representing their direct prerequisites in order to reduce
+ * their memory footprint (otherwise, the referenced object could refer one of its direct
+ * dependencies in turn, thereby making the size of the objects reachable from a single instance
+ * unbounded).
+ *
+ * @see TransitiveInfoProvider
+ */
+@SkylarkModule(name = "target", doc = "A BUILD target.")
+public interface TransitiveInfoCollection extends Iterable<TransitiveInfoProvider> {
+
+ /**
+ * Returns the transitive information provider requested, or null if the provider is not found.
+ * The provider has to be a TransitiveInfoProvider Java class.
+ */
+ @Nullable <P extends TransitiveInfoProvider> P getProvider(Class<P> provider);
+
+ /**
+ * Returns the label associated with this prerequisite.
+ */
+ Label getLabel();
+
+ /**
+ * <p>Returns the {@link BuildConfiguration} for which this transitive info collection is defined.
+ * Configuration is defined for all configured targets with exception of {@link
+ * InputFileConfiguredTarget} and {@link PackageGroupConfiguredTarget} for which it is always
+ * <b>null</b>.</p>
+ */
+ @Nullable BuildConfiguration getConfiguration();
+
+ /**
+ * Returns the transitive information requested or null, if the information is not found.
+ * The transitive information has to have been added using the Skylark framework.
+ */
+ @Nullable Object get(String providerKey);
+
+ /**
+ * Returns an unmodifiable iterator over the transitive info providers in the collections.
+ */
+ @Override
+ UnmodifiableIterator<TransitiveInfoProvider> iterator();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java
new file mode 100644
index 0000000..37ec191
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+/**
+ * This marker interface must be extended by every interface that represents
+ * rolled-up data about the transitive closure of a configured target.
+ *
+ * TransitiveInfoProviders need to be serializable, and for that reason they must conform to
+ * the following restrictions:
+ *
+ * <ul>
+ * <li>The provider interface must directly extend {@code TransitiveInfoProvider}.
+ * <li>Every method must return immutable data.</li>
+ * <li>Every method must return the same object if called multiple times with the same
+ * arguments.</li>
+ * <li>Overloading a method name multiple times is forbidden.</li>
+ * <li>The return type of a method must satisfy one of the following conditions:
+ * <ul>
+ * <li>It must be from the set of {String, Integer, int, Boolean, bool, Label, PathFragment,
+ * Artifact}, OR</li>
+ * <li>it must be an ImmutableList/List/Collection/Iterable of T, where T is either
+ * one of the types above with a default serializer or T implements ValueSerializer), OR</li>
+ * <li>it must be serializable (TBD)</li>
+ * </ul>
+ * <li>If the method takes arguments, it must declare a custom serializer (TBD).</li>
+ * </ul>
+ *
+ * <p>Some typical uses of this interface are:
+ * <ul>
+ * <li>The set of Python source files in the transitive closure of this rule
+ * <li>The set of declared C++ header files in the transitive closure
+ * <li>The files that need to be built when the target is mentioned on the command line
+ * </ul>
+ *
+ * <p>Note that if implemented naively, this would result in the memory requirements
+ * being O(n^2): in a long dependency chain, if every target adds one single artifact, storing the
+ * transitive closures of every rule would take 1+2+3+...+n-1+n = O(n^2) memory.
+ *
+ * <p>In order to avoid this, we introduce the concept of nested sets. A nested set is an immutable
+ * data structure that can contain direct members and other nested sets (recursively). Nested sets
+ * are iterable and can be flattened into ordered sets, where the order depends on which
+ * implementation of NestedSet you pick.
+ *
+ * @see TransitiveInfoCollection
+ */
+public interface TransitiveInfoProvider {
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Util.java b/src/main/java/com/google/devtools/build/lib/analysis/Util.java
new file mode 100644
index 0000000..ee10bf0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/Util.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Utility methods for use by ConfiguredTarget implementations.
+ */
+public abstract class Util {
+
+ private Util() {}
+
+ //---------- Label and Target related methods
+
+ /**
+ * Returns the workspace-relative path of the specified target (file or rule).
+ *
+ * <p>For example, "//foo/bar:wiz" and "//foo:bar/wiz" both result in "foo/bar/wiz".
+ */
+ public static PathFragment getWorkspaceRelativePath(Target target) {
+ return getWorkspaceRelativePath(target.getLabel());
+ }
+
+ /**
+ * Returns the workspace-relative path of the specified target (file or rule).
+ *
+ * <p>For example, "//foo/bar:wiz" and "//foo:bar/wiz" both result in "foo/bar/wiz".
+ */
+ public static PathFragment getWorkspaceRelativePath(Label label) {
+ return label.getPackageFragment().getRelative(label.getName());
+ }
+
+ /**
+ * Returns the workspace-relative path of the specified target (file or rule),
+ * prepending a prefix and appending a suffix.
+ *
+ * <p>For example, "//foo/bar:wiz" and "//foo:bar/wiz" both result in "foo/bar/wiz".
+ */
+ public static PathFragment getWorkspaceRelativePath(Target target, String prefix, String suffix) {
+ return target.getLabel().getPackageFragment().getRelative(prefix + target.getName() + suffix);
+ }
+
+ /**
+ * Checks if a PathFragment contains a '-'.
+ */
+ public static boolean containsHyphen(PathFragment path) {
+ return path.getPathString().indexOf('-') >= 0;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ViewCreationFailedException.java b/src/main/java/com/google/devtools/build/lib/analysis/ViewCreationFailedException.java
new file mode 100644
index 0000000..ae7dfc5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ViewCreationFailedException.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis;
+
+/**
+ * An exception indicating that there was a problem during the view
+ * construction (loading and analysis phases) for one or more targets, that the
+ * configured target graph could not be successfully constructed, and that
+ * a build cannot be started.
+ */
+public class ViewCreationFailedException extends Exception {
+
+ public ViewCreationFailedException(String message) {
+ super(message);
+ }
+
+ public ViewCreationFailedException(String message, Throwable cause) {
+ super(message + ": " + cause.getMessage(), cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProvider.java
new file mode 100644
index 0000000..a438145
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProvider.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+
+/**
+ * Provider class for configured targets that have a visibility.
+ */
+public interface VisibilityProvider extends TransitiveInfoProvider {
+
+ /**
+ * Returns the visibility specification.
+ */
+ NestedSet<PackageSpecification> getVisibility();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProviderImpl.java b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProviderImpl.java
new file mode 100644
index 0000000..01dd06a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProviderImpl.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.packages.PackageSpecification;
+
+/**
+ * Visibility provider implementation.
+ */
+@Immutable
+public final class VisibilityProviderImpl implements VisibilityProvider {
+ private final NestedSet<PackageSpecification> visibility;
+
+ public VisibilityProviderImpl(NestedSet<PackageSpecification> visibility) {
+ this.visibility = visibility;
+ }
+
+ @Override
+ public NestedSet<PackageSpecification> getVisibility() {
+ return visibility;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/WorkspaceStatusAction.java b/src/main/java/com/google/devtools/build/lib/analysis/WorkspaceStatusAction.java
new file mode 100644
index 0000000..5b3bd27
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/WorkspaceStatusAction.java
@@ -0,0 +1,185 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis;
+
+import com.google.common.base.Splitter;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * An action writing the workspace status files.
+ *
+ * <p>These files represent information about the environment the build was run in. They are used
+ * by language-specific build info factories to make the data in them available for individual
+ * languages (e.g. by turning them into .h files for C++)
+ *
+ * <p>The format of these files a list of key-value pairs, one for each line. The key and the value
+ * are separated by a space.
+ *
+ * <p>There are two of these files: volatile and stable. Changes in the volatile file do not
+ * cause rebuilds if no other file is changed. This is useful for frequently-changing information
+ * that does not significantly affect the build, e.g. the current time.
+ */
+public abstract class WorkspaceStatusAction extends AbstractAction {
+
+ /**
+ * The type of a workspace status action key.
+ */
+ public enum KeyType {
+ INTEGER,
+ STRING,
+ VERBATIM,
+ }
+
+ /**
+ * Language for keys that should be present in the build info for every language.
+ */
+ // TODO(bazel-team): Once this is released, migrate the only place in the depot to use
+ // the BUILD_USERNAME, BUILD_HOSTNAME and BUILD_DIRECTORY keys instead of BUILD_INFO. Then
+ // language-specific build info keys can be removed.
+ public static final String ALL_LANGUAGES = "*";
+
+ /**
+ * Action context required by the actions that write language-specific workspace status artifacts.
+ */
+ public static interface Context extends ActionContext {
+ ImmutableMap<String, Key> getStableKeys();
+ ImmutableMap<String, Key> getVolatileKeys();
+ }
+
+ /**
+ * A key in the workspace status info file.
+ */
+ public static class Key {
+ private final KeyType type;
+
+ /**
+ * Should be set to ALL_LANGUAGES if the key should be present in the build info of every
+ * language.
+ */
+ private final String language;
+ private final String defaultValue;
+ private final String redactedValue;
+
+ private Key(KeyType type, String language, String defaultValue, String redactedValue) {
+ this.type = type;
+ this.language = language;
+ this.defaultValue = defaultValue;
+ this.redactedValue = redactedValue;
+ }
+
+ public KeyType getType() {
+ return type;
+ }
+
+ public boolean isInLanguage(String language) {
+ return this.language.equals(ALL_LANGUAGES) || this.language.equals(language);
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+
+ public String getRedactedValue() {
+ return redactedValue;
+ }
+
+ public static Key forLanguage(
+ String language, KeyType type, String defaultValue, String redactedValue) {
+ return new Key(type, language, defaultValue, redactedValue);
+ }
+
+ public static Key of(KeyType type, String defaultValue, String redactedValue) {
+ return new Key(type, ALL_LANGUAGES, defaultValue, redactedValue);
+ }
+ }
+
+ /**
+ * Parses the output of the workspace status action.
+ *
+ * <p>The output is a text file with each line representing a workspace status info key.
+ * The key is the part of the line before the first space and should consist of the characters
+ * [A-Z_] (although this is not checked). Everything after the first space is the value.
+ */
+ public static Map<String, String> parseValues(Path file) throws IOException {
+ HashMap<String, String> result = new HashMap<>();
+ Splitter lineSplitter = Splitter.on(" ").limit(2);
+ for (String line : Splitter.on("\n").split(
+ new String(FileSystemUtils.readContentAsLatin1(file)))) {
+ List<String> items = ImmutableList.copyOf(lineSplitter.split(line));
+ if (items.size() != 2) {
+ continue;
+ }
+
+ result.put(items.get(0), items.get(1));
+ }
+
+ return ImmutableMap.copyOf(result);
+ }
+
+ /**
+ * Factory for {@link WorkspaceStatusAction}.
+ */
+ public interface Factory {
+ /**
+ * Creates the workspace status action.
+ *
+ * <p>If the objects returned for two builds are equals, the workspace status action can be
+ * be reused between them. Note that this only applies to the action object itself (the action
+ * will be unconditionally re-executed on every build)
+ */
+ WorkspaceStatusAction createWorkspaceStatusAction(
+ ArtifactFactory artifactFactory, ArtifactOwner artifactOwner, Supplier<UUID> buildId);
+
+ /**
+ * Creates a dummy workspace status map. Used in cases where the build failed, so that part of
+ * the workspace status is nevertheless available.
+ */
+ Map<String, String> createDummyWorkspaceStatus();
+ }
+
+ protected WorkspaceStatusAction(ActionOwner owner,
+ Iterable<Artifact> inputs,
+ Iterable<Artifact> outputs) {
+ super(owner, inputs, outputs);
+ }
+
+ /**
+ * The volatile status artifact containing items that may change even if nothing changed
+ * between the two builds, e.g. current time.
+ */
+ public abstract Artifact getVolatileStatus();
+
+ /**
+ * The stable status artifact containing items that change only if information relevant to the
+ * build changes, e.g. the name of the user running the build or the hostname.
+ */
+ public abstract Artifact getStableStatus();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java
new file mode 100644
index 0000000..187fc48
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java
@@ -0,0 +1,142 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Abstract Action to write to a file.
+ */
+public abstract class AbstractFileWriteAction extends AbstractAction {
+
+ protected final boolean makeExecutable;
+
+ /**
+ * Creates a new AbstractFileWriteAction instance.
+ *
+ * @param owner the action owner.
+ * @param inputs the Artifacts that this Action depends on
+ * @param output the Artifact that will be created by executing this Action.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ public AbstractFileWriteAction(ActionOwner owner,
+ Iterable<Artifact> inputs, Artifact output, boolean makeExecutable) {
+ // There is only one output, and it is primary.
+ super(owner, inputs, ImmutableList.of(output));
+ this.makeExecutable = makeExecutable;
+ }
+
+ public boolean makeExecutable() {
+ return makeExecutable;
+ }
+
+ @Override
+ public final void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ try {
+ getStrategy(actionExecutionContext.getExecutor()).exec(actionExecutionContext.getExecutor(),
+ this, actionExecutionContext.getFileOutErr(), actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException(
+ "Writing file for rule '" + Label.print(getOwner().getLabel()) + "'",
+ actionExecutionContext.getExecutor().getVerboseFailures(), this);
+ }
+ afterWrite(actionExecutionContext.getExecutor());
+ }
+
+ /**
+ * Produce a DeterministicWriter that can write the file to an OutputStream deterministically.
+ *
+ * @param eventHandler destination for warning messages. (Note that errors should
+ * still be indicated by throwing an exception; reporter.error() will
+ * not cause action execution to fail.)
+ * @param executor the Executor.
+ * @throws IOException if the content cannot be written to the output stream
+ */
+ public abstract DeterministicWriter newDeterministicWriter(EventHandler eventHandler,
+ Executor executor) throws IOException, InterruptedException, ExecException;
+
+ /**
+ * This hook is called after the File has been successfully written to disk.
+ *
+ * @param executor the Executor.
+ */
+ protected void afterWrite(Executor executor) {
+ }
+
+ // We're mainly doing I/O, so estimate very low CPU usage, e.g. 1%. Just a guess.
+ private static final ResourceSet DEFAULT_FILEWRITE_LOCAL_ACTION_RESOURCE_SET =
+ new ResourceSet(/*memoryMb=*/0.0, /*cpuUsage=*/0.01, /*ioUsage=*/0.2);
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return executor.getContext(FileWriteActionContext.class).estimateResourceConsumption(this);
+ }
+
+ public ResourceSet estimateResourceConsumptionLocal() {
+ return DEFAULT_FILEWRITE_LOCAL_ACTION_RESOURCE_SET;
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "FileWrite";
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Writing " + (makeExecutable ? "script " : "file ")
+ + Iterables.getOnlyElement(getOutputs()).prettyPrint();
+ }
+
+ /**
+ * Whether the file write can be generated remotely. If the file is consumed in Blaze
+ * unconditionally, it doesn't make sense to run remotely.
+ */
+ public boolean isRemotable() {
+ return true;
+ }
+
+ @Override
+ public final String describeStrategy(Executor executor) {
+ return executor.getContext(FileWriteActionContext.class).strategyLocality(this);
+ }
+
+ private FileWriteActionContext getStrategy(Executor executor) {
+ return executor.getContext(FileWriteActionContext.class);
+ }
+
+ /**
+ * A deterministic writer writes bytes to an output stream. The same byte stream is written
+ * on every invocation of writeOutputFile().
+ */
+ public interface DeterministicWriter {
+ public void writeOutputFile(OutputStream out) throws IOException;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ActionConstructionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ActionConstructionContext.java
new file mode 100644
index 0000000..b7461e5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ActionConstructionContext.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Rule;
+
+/**
+ * A temporary interface to allow migration from RuleConfiguredTarget to RuleContext. It bundles
+ * the items commonly needed to construct action instances.
+ */
+public interface ActionConstructionContext {
+ /** The rule for which the actions are constructed. */
+ Rule getRule();
+
+ /** Returns the action owner that should be used for actions. */
+ ActionOwner getActionOwner();
+
+ /** Returns the {@link BuildConfiguration} for which the given rule is analyzed. */
+ BuildConfiguration getConfiguration();
+
+ /** The current analysis environment. */
+ AnalysisEnvironment getAnalysisEnvironment();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/BinaryFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/BinaryFileWriteAction.java
new file mode 100644
index 0000000..b9a2ea5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/BinaryFileWriteAction.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Action to write a binary file.
+ */
+public final class BinaryFileWriteAction extends AbstractFileWriteAction {
+
+ private static final String GUID = "eeee07fe-4b40-11e4-82d6-eba0b4f713e2";
+
+ private final ByteSource source;
+
+ /**
+ * Creates a new BinaryFileWriteAction instance without inputs.
+ *
+ * @param owner the action owner.
+ * @param output the Artifact that will be created by executing this Action.
+ * @param source a source of bytes that will be written to the file.
+ * @param makeExecutable iff true will change the output file to be executable.
+ */
+ public BinaryFileWriteAction(
+ ActionOwner owner, Artifact output, ByteSource source, boolean makeExecutable) {
+ super(owner, /*inputs=*/Artifact.NO_ARTIFACTS, output, makeExecutable);
+ this.source = Preconditions.checkNotNull(source);
+ }
+
+ @VisibleForTesting
+ public ByteSource getSource() {
+ return source;
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) {
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ try (InputStream in = source.openStream()) {
+ ByteStreams.copy(in, out);
+ }
+ out.flush();
+ }
+ };
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addString(String.valueOf(makeExecutable));
+
+ try (InputStream in = source.openStream()) {
+ byte[] buffer = new byte[512];
+ int amountRead;
+ while ((amountRead = in.read(buffer)) != -1) {
+ f.addBytes(buffer, 0, amountRead);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return f.hexDigestAndReset();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/CommandLine.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/CommandLine.java
new file mode 100644
index 0000000..ddadc25
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/CommandLine.java
@@ -0,0 +1,123 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+
+/**
+ * A representation of a command line to be executed by a SpawnAction.
+ */
+public abstract class CommandLine {
+ /**
+ * Returns the command line.
+ */
+ public abstract Iterable<String> arguments();
+
+ /**
+ * Returns whether the command line represents a shell command with the given shell executable.
+ * This is used to give better error messages.
+ *
+ * <p>By default, this method returns false.
+ */
+ public boolean isShellCommand() {
+ return false;
+ }
+
+ /**
+ * A default implementation of a command line backed by a copy of the given list of arguments.
+ */
+ static CommandLine ofInternal(Iterable<String> arguments, final boolean isShellCommand) {
+ final Iterable<String> immutableArguments = CollectionUtils.makeImmutable(arguments);
+ return new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return immutableArguments;
+ }
+
+ @Override
+ public boolean isShellCommand() {
+ return isShellCommand;
+ }
+ };
+ }
+
+ /**
+ * Returns a {@link CommandLine} backed by a copy of the given list of arguments.
+ */
+ public static CommandLine of(Iterable<String> arguments, final boolean isShellCommand) {
+ final Iterable<String> immutableArguments = CollectionUtils.makeImmutable(arguments);
+ return new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return immutableArguments;
+ }
+
+ @Override
+ public boolean isShellCommand() {
+ return isShellCommand;
+ }
+ };
+ }
+
+ /**
+ * Returns a {@link CommandLine} that is constructed by prepending the {@code executableArgs} to
+ * {@code commandLine}.
+ */
+ static CommandLine ofMixed(final ImmutableList<String> executableArgs,
+ final CommandLine commandLine, final boolean isShellCommand) {
+ Preconditions.checkState(!executableArgs.isEmpty());
+ return new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return Iterables.concat(executableArgs, commandLine.arguments());
+ }
+
+ @Override
+ public boolean isShellCommand() {
+ return isShellCommand;
+ }
+ };
+ }
+
+ /**
+ * Returns a {@link CommandLine} with {@link CharSequence} arguments. This can be useful to create
+ * memory efficient command lines with {@link com.google.devtools.build.lib.util.LazyString}s.
+ */
+ public static CommandLine ofCharSequences(final ImmutableList<CharSequence> arguments) {
+ return new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (CharSequence arg : arguments) {
+ builder.add(arg.toString());
+ }
+ return builder.build();
+ }
+ };
+ }
+
+ /**
+ * This helps when debugging Blaze code that uses {@link CommandLine}s, as you can see their
+ * content directly in the variable inspector.
+ */
+ @Override
+ public String toString() {
+ return Joiner.on(' ').join(arguments());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/CustomCommandLine.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/CustomCommandLine.java
new file mode 100644
index 0000000..d358f0b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/CustomCommandLine.java
@@ -0,0 +1,358 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A customizable, serializable class for building memory efficient command lines.
+ */
+@Immutable
+public final class CustomCommandLine extends CommandLine {
+
+ private abstract static class ArgvFragment {
+ abstract void eval(ImmutableList.Builder<String> builder);
+ }
+
+ // It's better to avoid anonymous classes if we want to serialize command lines
+
+ private static final class ObjectArg extends ArgvFragment {
+ private final Object arg;
+
+ private ObjectArg(Object arg) {
+ this.arg = arg;
+ }
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ builder.add(arg.toString());
+ }
+ }
+
+ private static final class JoinExecPathsArg extends ArgvFragment {
+
+ private final String delimiter;
+ private final Iterable<Artifact> artifacts;
+
+ private JoinExecPathsArg(String delimiter, Iterable<Artifact> artifacts) {
+ this.delimiter = delimiter;
+ this.artifacts = CollectionUtils.makeImmutable(artifacts);
+ }
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ builder.add(Artifact.joinExecPaths(delimiter, artifacts));
+ }
+ }
+
+ private static final class PathWithTemplateArg extends ArgvFragment {
+
+ private final String template;
+ private final PathFragment[] paths;
+
+ private PathWithTemplateArg(String template, PathFragment... paths) {
+ this.template = template;
+ this.paths = paths;
+ }
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ // PathFragment.toString() uses getPathString()
+ builder.add(String.format(template, (Object[]) paths));
+ }
+ }
+
+ // TODO(bazel-team): CustomArgv and CustomMultiArgv is going to be difficult to expose
+ // in Skylark. Maybe we can get rid of them by refactoring JavaCompileAction. It also
+ // raises immutability / serialization issues.
+ /**
+ * Custom Java code producing a String argument. Usage of this class is discouraged.
+ */
+ public abstract static class CustomArgv extends ArgvFragment {
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ builder.add(argv());
+ }
+
+ public abstract String argv();
+ }
+
+ /**
+ * Custom Java code producing a List of String arguments. Usage of this class is discouraged.
+ */
+ public abstract static class CustomMultiArgv extends ArgvFragment {
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ builder.addAll(argv());
+ }
+
+ public abstract Iterable<String> argv();
+ }
+
+ private static final class JoinPathsArg extends ArgvFragment {
+
+ private final String delimiter;
+ private final Iterable<PathFragment> paths;
+
+ private JoinPathsArg(String delimiter, Iterable<PathFragment> paths) {
+ this.delimiter = delimiter;
+ this.paths = CollectionUtils.makeImmutable(paths);
+ }
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ builder.add(Joiner.on(delimiter).join(paths));
+ }
+ }
+
+ /**
+ * Arguments that intersperse strings between the items in a sequence. There are two forms of
+ * interspersing, and either may be used by this implementation:
+ * <ul>
+ * <li>before each - a string is added before each item in a sequence. e.g.
+ * {@code -f foo -f bar -f baz}
+ * <li>format each - a format string is used to format each item in a sequence. e.g.
+ * {@code -I/foo -I/bar -I/baz} for the format {@code "-I%s"}
+ * </ul>
+ *
+ * <p>This class could be used both with both the "before" and "format" features at the same
+ * time, but this is probably more confusion than it is worth. If you need this functionality,
+ * consider using "before" only but storing the strings pre-formated in a {@link NestedSet}.
+ */
+ private static final class InterspersingArgs extends ArgvFragment {
+ private final Iterable<?> sequence;
+ private final String beforeEach;
+ private final String formatEach;
+
+ /**
+ * Do not call from outside this class because this does not guarantee that {@code sequence} is
+ * immutable.
+ */
+ private InterspersingArgs(Iterable<?> sequence, String beforeEach, String formatEach) {
+ this.sequence = sequence;
+ this.beforeEach = beforeEach;
+ this.formatEach = formatEach;
+ }
+
+ static InterspersingArgs fromStrings(
+ Iterable<?> sequence, String beforeEach, String formatEach) {
+ return new InterspersingArgs(
+ CollectionUtils.makeImmutable(sequence), beforeEach, formatEach);
+ }
+
+ static InterspersingArgs fromExecPaths(
+ Iterable<Artifact> sequence, String beforeEach, String formatEach) {
+ return new InterspersingArgs(
+ Artifact.toExecPaths(CollectionUtils.makeImmutable(sequence)), beforeEach, formatEach);
+ }
+
+ @Override
+ void eval(ImmutableList.Builder<String> builder) {
+ for (Object item : sequence) {
+ if (item == null) {
+ continue;
+ }
+
+ if (beforeEach != null) {
+ builder.add(beforeEach);
+ }
+ String arg = item.toString();
+ if (formatEach != null) {
+ arg = String.format(formatEach, arg);
+ }
+ builder.add(arg);
+ }
+ }
+ }
+
+ /**
+ * A Builder class for CustomCommandLine with the appropriate methods.
+ *
+ * <p>{@link Iterable} instances passed to {@code add*} methods will be stored internally as
+ * collections that are known to be immutable copies. This means that any {@link Iterable} that is
+ * not a {@link NestedSet} or {@link ImmutableList} may be copied.
+ *
+ * <p>{@code addFormatEach*} methods take an {@link Iterable} but use these as arguments to
+ * {@link String#format(String, Object...)} with a certain constant format string. For instance,
+ * if {@code format} is {@code "-I%s"}, then the final arguments may be
+ * {@code -Ifoo -Ibar -Ibaz}
+ *
+ * <p>{@code addBeforeEach*} methods take an {@link Iterable} but insert a certain {@link String}
+ * once before each element in the string, meaning the total number of elements added is twice the
+ * length of the {@link Iterable}. For instance: {@code -f foo -f bar -f baz}
+ */
+ public static final class Builder {
+
+ private final List<ArgvFragment> arguments = new ArrayList<>();
+
+ public Builder add(CharSequence arg) {
+ if (arg != null) {
+ arguments.add(new ObjectArg(arg));
+ }
+ return this;
+ }
+
+ public Builder add(Label arg) {
+ if (arg != null) {
+ arguments.add(new ObjectArg(arg));
+ }
+ return this;
+ }
+
+ public Builder add(String arg, Iterable<String> args) {
+ if (arg != null && args != null) {
+ arguments.add(new ObjectArg(arg));
+ arguments.add(InterspersingArgs.fromStrings(args, /*beforeEach=*/null, "%s"));
+ }
+ return this;
+ }
+
+ public Builder add(Iterable<String> args) {
+ if (args != null) {
+ arguments.add(InterspersingArgs.fromStrings(args, /*beforeEach=*/null, "%s"));
+ }
+ return this;
+ }
+
+ public Builder addExecPath(String arg, Artifact artifact) {
+ if (arg != null && artifact != null) {
+ arguments.add(new ObjectArg(arg));
+ arguments.add(new ObjectArg(artifact.getExecPath()));
+ }
+ return this;
+ }
+
+ public Builder addExecPaths(String arg, Iterable<Artifact> artifacts) {
+ if (arg != null && artifacts != null) {
+ arguments.add(new ObjectArg(arg));
+ arguments.add(InterspersingArgs.fromExecPaths(artifacts, /*beforeEach=*/null, "%s"));
+ }
+ return this;
+ }
+
+ public Builder addExecPaths(Iterable<Artifact> artifacts) {
+ if (artifacts != null) {
+ arguments.add(InterspersingArgs.fromExecPaths(artifacts, /*beforeEach=*/null, "%s"));
+ }
+ return this;
+ }
+
+ public Builder addJoinExecPaths(String arg, String delimiter, Iterable<Artifact> artifacts) {
+ if (arg != null && artifacts != null) {
+ arguments.add(new ObjectArg(arg));
+ arguments.add(new JoinExecPathsArg(delimiter, artifacts));
+ }
+ return this;
+ }
+
+ public Builder addPath(PathFragment path) {
+ if (path != null) {
+ arguments.add(new ObjectArg(path));
+ }
+ return this;
+ }
+
+ public Builder addPaths(String template, PathFragment... path) {
+ if (template != null && path != null) {
+ arguments.add(new PathWithTemplateArg(template, path));
+ }
+ return this;
+ }
+
+ public Builder addJoinPaths(String delimiter, Iterable<PathFragment> paths) {
+ if (delimiter != null && paths != null) {
+ arguments.add(new JoinPathsArg(delimiter, paths));
+ }
+ return this;
+ }
+
+ public Builder addBeforeEachPath(String repeated, Iterable<PathFragment> paths) {
+ if (repeated != null && paths != null) {
+ arguments.add(InterspersingArgs.fromStrings(paths, repeated, "%s"));
+ }
+ return this;
+ }
+
+ public Builder addBeforeEach(String repeated, Iterable<String> strings) {
+ if (repeated != null && strings != null) {
+ arguments.add(InterspersingArgs.fromStrings(strings, repeated, "%s"));
+ }
+ return this;
+ }
+
+ public Builder addBeforeEachExecPath(String repeated, Iterable<Artifact> artifacts) {
+ if (repeated != null && artifacts != null) {
+ arguments.add(InterspersingArgs.fromExecPaths(artifacts, repeated, "%s"));
+ }
+ return this;
+ }
+
+ public Builder addFormatEach(String format, Iterable<String> strings) {
+ if (format != null && strings != null) {
+ arguments.add(InterspersingArgs.fromStrings(strings, /*beforeEach=*/null, format));
+ }
+ return this;
+ }
+
+ public Builder add(CustomArgv arg) {
+ if (arg != null) {
+ arguments.add(arg);
+ }
+ return this;
+ }
+
+ public Builder add(CustomMultiArgv arg) {
+ if (arg != null) {
+ arguments.add(arg);
+ }
+ return this;
+ }
+
+ public CustomCommandLine build() {
+ return new CustomCommandLine(arguments);
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private final ImmutableList<ArgvFragment> arguments;
+
+ private CustomCommandLine(List<ArgvFragment> arguments) {
+ this.arguments = ImmutableList.copyOf(arguments);
+ }
+
+ @Override
+ public Iterable<String> arguments() {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (ArgvFragment arg : arguments) {
+ arg.eval(builder);
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ExecutableSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ExecutableSymlinkAction.java
new file mode 100644
index 0000000..376d9b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ExecutableSymlinkAction.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+
+/**
+ * Action to create an executable symbolic link. It includes additional
+ * validation that symlink target is indeed an executable file.
+ */
+public final class ExecutableSymlinkAction extends SymlinkAction {
+
+ public ExecutableSymlinkAction(ActionOwner owner, Artifact input, Artifact output) {
+ super(owner, input, output, "Symlinking " + owner.getLabel());
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ Path inputPath = actionExecutionContext.getExecutor().getExecRoot().getRelative(
+ getInputPath());
+ try {
+ // Validate that input path is a file with the executable bit is set.
+ if (!inputPath.isFile()) {
+ throw new ActionExecutionException(
+ "'" + Iterables.getOnlyElement(getInputs()).prettyPrint() + "' is not a file", this,
+ false);
+ }
+ if (!inputPath.isExecutable()) {
+ throw new ActionExecutionException("failed to create symbolic link '"
+ + Iterables.getOnlyElement(getOutputs()).prettyPrint()
+ + "': file '" + Iterables.getOnlyElement(getInputs()).prettyPrint()
+ + "' is not executable", this, false);
+ }
+ } catch (IOException e) {
+ throw new ActionExecutionException("failed to create symbolic link '"
+ + Iterables.getOnlyElement(getOutputs()).prettyPrint()
+ + "' to the '" + Iterables.getOnlyElement(getInputs()).prettyPrint()
+ + "' due to I/O error: " + e.getMessage(), e, this, false);
+ }
+
+ super.execute(actionExecutionContext);
+ }
+
+ @Override
+ public String getMnemonic() { return "ExecutableSymlink"; }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
new file mode 100644
index 0000000..1128617
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+
+/**
+ * Action to write to a file.
+ * <p>TODO(bazel-team): Choose a better name to distinguish this class from
+ * {@link BinaryFileWriteAction}.
+ */
+public class FileWriteAction extends AbstractFileWriteAction {
+
+ private static final String GUID = "332877c7-ca9f-4731-b387-54f620408522";
+
+ /**
+ * We keep it as a CharSequence for memory-efficiency reasons. The toString()
+ * method of the object represents the content of the file.
+ *
+ * <p>For example, this allows us to keep a {@code List<Artifact>} wrapped
+ * in a {@code LazyString} instead of the string representation of the concatenation.
+ * This saves memory because the Artifacts are shared objects but the
+ * resulting String is not.
+ */
+ private final CharSequence fileContents;
+
+ /**
+ * Creates a new FileWriteAction instance without inputs using UTF8 encoding.
+ *
+ * @param owner the action owner.
+ * @param output the Artifact that will be created by executing this Action.
+ * @param fileContents the contents to be written to the file.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ public FileWriteAction(ActionOwner owner, Artifact output, CharSequence fileContents,
+ boolean makeExecutable) {
+ this(owner, Artifact.NO_ARTIFACTS, output, fileContents, makeExecutable);
+ }
+
+ /**
+ * Creates a new FileWriteAction instance using UTF8 encoding.
+ *
+ * @param owner the action owner.
+ * @param inputs the Artifacts that this Action depends on
+ * @param output the Artifact that will be created by executing this Action.
+ * @param fileContents the contents to be written to the file.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ public FileWriteAction(ActionOwner owner, Collection<Artifact> inputs,
+ Artifact output, CharSequence fileContents, boolean makeExecutable) {
+ super(owner, inputs, output, makeExecutable);
+ this.fileContents = fileContents;
+ }
+
+ /**
+ * Creates a new FileWriteAction instance using UTF8 encoding.
+ *
+ * @param owner the action owner.
+ * @param inputs the Artifacts that this Action depends on
+ * @param output the Artifact that will be created by executing this Action.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ protected FileWriteAction(ActionOwner owner, Collection<Artifact> inputs,
+ Artifact output, boolean makeExecutable) {
+ this(owner, inputs, output, "", makeExecutable);
+ }
+
+ public String getFileContents() {
+ return fileContents.toString();
+ }
+
+ /**
+ * Create a DeterministicWriter for the content of the output file as provided by
+ * {@link #getFileContents()}.
+ */
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler,
+ Executor executor) {
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ out.write(getFileContents().getBytes(UTF_8));
+ }
+ };
+ }
+
+ /**
+ * Computes the Action key for this action by computing the fingerprint for
+ * the file contents.
+ */
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addString(String.valueOf(makeExecutable));
+ f.addString(getFileContents());
+ return f.hexDigestAndReset();
+ }
+
+ /**
+ * Creates a FileWriteAction to write contents to the resulting artifact
+ * fileName in the genfiles root underneath the package path.
+ *
+ * @param ruleContext the ruleContext that will own the action of creating this file.
+ * @param fileName name of the file to create.
+ * @param contents data to write to file.
+ * @param executable flags that file should be marked executable.
+ * @return Artifact describing the file to create.
+ */
+ public static Artifact createFile(RuleContext ruleContext,
+ String fileName, CharSequence contents, boolean executable) {
+ Artifact scriptFileArtifact = ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ ruleContext.getTarget().getLabel().getPackageFragment().getRelative(fileName),
+ ruleContext.getConfiguration().getGenfilesDirectory());
+ ruleContext.registerAction(new FileWriteAction(
+ ruleContext.getActionOwner(), scriptFileArtifact, contents, executable));
+ return scriptFileArtifact;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteActionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteActionContext.java
new file mode 100644
index 0000000..7e10331
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteActionContext.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+
+/**
+ * The action context for {@link AbstractFileWriteAction} instances (technically instances of
+ * subclasses).
+ */
+public interface FileWriteActionContext extends ActionContext {
+
+ /**
+ * Performs all the setup and then calls back into the action to write the data.
+ */
+ void exec(Executor executor, AbstractFileWriteAction action, FileOutErr outErr,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException;
+
+ /**
+ * Returns the estimated resource consumption of the action.
+ */
+ ResourceSet estimateResourceConsumption(AbstractFileWriteAction action);
+
+ /**
+ * Returns where the action actually runs.
+ */
+ String strategyLocality(AbstractFileWriteAction action);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileHelper.java
new file mode 100644
index 0000000..e7869e4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileHelper.java
@@ -0,0 +1,158 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ParameterFile;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A command-line implementation that wraps another command line and puts the arguments in a
+ * parameter file if necessary
+ *
+ * <p>The Linux kernel has a limit for the command line length, and that can be easily reached
+ * if, for example, a command is listing all its inputs on the command line.
+ */
+@Immutable
+public final class ParamFileHelper {
+
+ /**
+ * Returns a params file artifact or null for a given command description.
+ *
+ * <p>Returns null if parameter files are not to be used according to paramFileInfo, or if the
+ * command line is short enough that a parameter file is not needed.
+ *
+ * <p>Make sure to add the returned artifact (if not null) as an input of the corresponding
+ * action.
+ *
+ * @param executableArgs leading arguments that should never be wrapped in a parameter file
+ * @param arguments arguments to the command (in addition to executableArgs), OR
+ * @param commandLine a {@link CommandLine} that provides the arguments (in addition to
+ * executableArgs)
+ * @param paramFileInfo parameter file information
+ * @param configuration the configuration
+ * @param analysisEnvironment the analysis environment
+ * @param outputs outputs of the action (used to construct a filename for the params file)
+ */
+ public static Artifact getParamsFile(
+ List<String> executableArgs,
+ @Nullable Iterable<String> arguments,
+ @Nullable CommandLine commandLine,
+ @Nullable ParamFileInfo paramFileInfo,
+ BuildConfiguration configuration,
+ AnalysisEnvironment analysisEnvironment,
+ Iterable<Artifact> outputs) {
+ if (paramFileInfo == null ||
+ getParamFileSize(executableArgs, arguments, commandLine)
+ < configuration.getMinParamFileSize()) {
+ return null;
+ }
+
+ PathFragment paramFilePath = ParameterFile.derivePath(
+ Iterables.getFirst(outputs, null).getRootRelativePath());
+ return analysisEnvironment.getDerivedArtifact(paramFilePath, configuration.getBinDirectory());
+ }
+
+ /**
+ * Creates a command line using an external params file.
+ *
+ * <p>Call this with the result of {@link #getParamsFile} if it is not null.
+ *
+ * @param executableArgs leading arguments that should never be wrapped in a parameter file
+ * @param arguments arguments to the command (in addition to executableArgs), OR
+ * @param commandLine a {@link CommandLine} that provides the arguments (in addition to
+ * executableArgs)
+ * @param isShellCommand true if this is a shell command
+ * @param owner owner of the action
+ * @param paramFileInfo parameter file information
+ */
+ public static CommandLine createWithParamsFile(
+ List<String> executableArgs,
+ @Nullable Iterable<String> arguments,
+ @Nullable CommandLine commandLine,
+ boolean isShellCommand,
+ ActionOwner owner,
+ List<Action> requiredActions,
+ ParamFileInfo paramFileInfo,
+ Artifact parameterFile) {
+ Preconditions.checkNotNull(parameterFile);
+ if (commandLine != null && arguments != null && !Iterables.isEmpty(arguments)) {
+ throw new IllegalStateException("must provide either commandLine or arguments: " + arguments);
+ }
+
+ CommandLine paramFileContents =
+ (commandLine != null) ? commandLine : CommandLine.ofInternal(arguments, false);
+ requiredActions.add(new ParameterFileWriteAction(owner, parameterFile, paramFileContents,
+ paramFileInfo.getFileType(), paramFileInfo.getCharset()));
+
+ String pathWithFlag = paramFileInfo.getFlag() + parameterFile.getExecPathString();
+ Iterable<String> commandArgv = Iterables.concat(executableArgs, ImmutableList.of(pathWithFlag));
+ return CommandLine.ofInternal(commandArgv, isShellCommand);
+ }
+
+ /**
+ * Creates a command line without using a params file.
+ *
+ * <p>Call this if {@link #getParamsFile} returns null.
+ *
+ * @param executableArgs leading arguments that should never be wrapped in a parameter file
+ * @param arguments arguments to the command (in addition to executableArgs), OR
+ * @param commandLine a {@link CommandLine} that provides the arguments (in addition to
+ * executableArgs)
+ * @param isShellCommand true if this is a shell command
+ */
+ public static CommandLine createWithoutParamsFile(List<String> executableArgs,
+ Iterable<String> arguments, CommandLine commandLine, boolean isShellCommand) {
+ if (commandLine == null) {
+ Iterable<String> commandArgv = Iterables.concat(executableArgs, arguments);
+ return CommandLine.ofInternal(commandArgv, isShellCommand);
+ }
+
+ if (executableArgs.isEmpty()) {
+ return commandLine;
+ }
+
+ return CommandLine.ofMixed(ImmutableList.copyOf(executableArgs), commandLine, isShellCommand);
+ }
+
+ /**
+ * Estimates the params file size for the given arguments.
+ */
+ private static int getParamFileSize(
+ List<String> executableArgs, Iterable<String> arguments, CommandLine commandLine) {
+ Iterable<String> actualArguments = (commandLine != null) ? commandLine.arguments() : arguments;
+ return getParamFileSize(executableArgs) + getParamFileSize(actualArguments);
+ }
+
+ private static int getParamFileSize(Iterable<String> args) {
+ int size = 0;
+ for (String s : args) {
+ size += s.length() + 1;
+ }
+ return size;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java
new file mode 100644
index 0000000..ae00181
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java
@@ -0,0 +1,79 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+
+import java.nio.charset.Charset;
+import java.util.Objects;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * An object that encapsulates how a params file should be constructed: what is the filetype,
+ * what charset to use and what prefix (typically "@") to use.
+ */
+@Immutable
+public final class ParamFileInfo {
+ private final ParameterFileType fileType;
+ private final Charset charset;
+ private final String flag;
+
+ public ParamFileInfo(ParameterFileType fileType, Charset charset, String flag) {
+ this.fileType = Preconditions.checkNotNull(fileType);
+ this.charset = Preconditions.checkNotNull(charset);
+ this.flag = Preconditions.checkNotNull(flag);
+ }
+
+ /**
+ * Returns the file type.
+ */
+ public ParameterFileType getFileType() {
+ return fileType;
+ }
+
+ /**
+ * Returns the charset.
+ */
+ public Charset getCharset() {
+ return charset;
+ }
+
+ /**
+ * Returns the prefix for the params filename on the command line (typically "@").
+ */
+ public String getFlag() {
+ return flag;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(charset, flag, fileType);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ParamFileInfo)) {
+ return false;
+ }
+ ParamFileInfo other = (ParamFileInfo) obj;
+ return fileType.equals(other.fileType) && charset.equals(other.charset)
+ && flag.equals(other.flag);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParameterFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParameterFileWriteAction.java
new file mode 100644
index 0000000..fd10e2b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParameterFileWriteAction.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.ShellEscaper;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+
+/**
+ * Action to write a parameter file for a {@link CommandLine}.
+ */
+public final class ParameterFileWriteAction extends AbstractFileWriteAction {
+
+ private static final String GUID = "45f678d8-e395-401e-8446-e795ccc6361f";
+
+ private final CommandLine commandLine;
+ private final ParameterFileType type;
+ private final Charset charset;
+
+ /**
+ * Creates a new instance.
+ *
+ * @param owner the action owner
+ * @param output the Artifact that will be created by executing this Action
+ * @param commandLine the contents to be written to the file
+ * @param type the type of the file
+ * @param charset the charset of the file
+ */
+ public ParameterFileWriteAction(ActionOwner owner, Artifact output, CommandLine commandLine,
+ ParameterFileType type, Charset charset) {
+ super(owner, ImmutableList.<Artifact>of(), output, false);
+ this.commandLine = commandLine;
+ this.type = type;
+ this.charset = charset;
+ }
+
+ /**
+ * Returns the list of options written to the parameter file. Don't use this
+ * method outside tests - the list is often huge, resulting in significant
+ * garbage collection overhead.
+ */
+ @VisibleForTesting
+ public Iterable<String> getContents() {
+ return commandLine.arguments();
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) {
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ switch (type) {
+ case SHELL_QUOTED :
+ writeContentQuoted(out);
+ break;
+ case UNQUOTED :
+ writeContentUnquoted(out);
+ break;
+ default :
+ throw new AssertionError();
+ }
+ }
+ };
+ }
+
+ /**
+ * Writes the arguments from the list into the parameter file.
+ */
+ private void writeContentUnquoted(OutputStream outputStream) throws IOException {
+ OutputStreamWriter out = new OutputStreamWriter(outputStream, charset);
+ for (String line : commandLine.arguments()) {
+ out.write(line);
+ out.write('\n');
+ }
+ out.flush();
+ }
+
+ /**
+ * Writes the arguments from the list into the parameter file with shell
+ * quoting (if required).
+ */
+ private void writeContentQuoted(OutputStream outputStream) throws IOException {
+ OutputStreamWriter out = new OutputStreamWriter(outputStream, charset);
+ for (String line : ShellEscaper.escapeAll(commandLine.arguments())) {
+ out.write(line);
+ out.write('\n');
+ }
+ out.flush();
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addString(String.valueOf(makeExecutable));
+ f.addStrings(commandLine.arguments());
+ return f.hexDigestAndReset();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
new file mode 100644
index 0000000..f51c917
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
@@ -0,0 +1,874 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.actions.extra.SpawnInfo;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.IterablesChain;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.protobuf.GeneratedMessage.GeneratedExtension;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.CheckReturnValue;
+
+/**
+ * An Action representing an arbitrary subprocess to be forked and exec'd.
+ */
+public class SpawnAction extends AbstractAction {
+ private static class ExtraActionInfoSupplier<T> {
+ private final GeneratedExtension<ExtraActionInfo, T> extension;
+ private final T value;
+
+ private ExtraActionInfoSupplier(
+ GeneratedExtension<ExtraActionInfo, T> extension,
+ T value) {
+ this.extension = extension;
+ this.value = value;
+ }
+
+ void extend(ExtraActionInfo.Builder builder) {
+ builder.setExtension(extension, value);
+ }
+ }
+
+ private static final String GUID = "ebd6fce3-093e-45ee-adb6-bf513b602f0d";
+
+ private final CommandLine argv;
+
+ private final String progressMessage;
+ private final String mnemonic;
+ // entries are (directory for remote execution, Artifact)
+ private final Map<PathFragment, Artifact> inputManifests;
+
+ private final ResourceSet resourceSet;
+ private final ImmutableMap<String, String> environment;
+ private final ImmutableMap<String, String> executionInfo;
+
+ private final ExtraActionInfoSupplier<?> extraActionInfoSupplier;
+
+ /**
+ * Constructs a SpawnAction using direct initialization arguments.
+ * <p>
+ * All collections provided must not be subsequently modified.
+ *
+ * @param owner the owner of the Action.
+ * @param inputs the set of all files potentially read by this action; must
+ * not be subsequently modified.
+ * @param outputs the set of all files written by this action; must not be
+ * subsequently modified.
+ * @param resourceSet the resources consumed by executing this Action
+ * @param environment the map of environment variables.
+ * @param argv the command line to execute. This is merely a list of options
+ * to the executable, and is uninterpreted by the build tool for the
+ * purposes of dependency checking; typically it may include the names
+ * of input and output files, but this is not necessary.
+ * @param progressMessage the message printed during the progression of the build
+ * @param mnemonic the mnemonic that is reported in the master log.
+ */
+ public SpawnAction(ActionOwner owner,
+ Iterable<Artifact> inputs, Iterable<Artifact> outputs,
+ ResourceSet resourceSet,
+ CommandLine argv,
+ Map<String, String> environment,
+ String progressMessage,
+ String mnemonic) {
+ this(owner, inputs, outputs,
+ resourceSet, argv, ImmutableMap.copyOf(environment),
+ ImmutableMap.<String, String>of(), progressMessage,
+ ImmutableMap.<PathFragment, Artifact>of(), mnemonic, null);
+ }
+
+ /**
+ * Constructs a SpawnAction using direct initialization arguments.
+ *
+ * <p>All collections provided must not be subsequently modified.
+ *
+ * @param owner the owner of the Action.
+ * @param inputs the set of all files potentially read by this action; must
+ * not be subsequently modified.
+ * @param outputs the set of all files written by this action; must not be
+ * subsequently modified.
+ * @param resourceSet the resources consumed by executing this Action
+ * @param environment the map of environment variables.
+ * @param executionInfo out-of-band information for scheduling the spawn.
+ * @param argv the argv array (including argv[0]) of arguments to pass. This
+ * is merely a list of options to the executable, and is uninterpreted
+ * by the build tool for the purposes of dependency checking; typically
+ * it may include the names of input and output files, but this is not
+ * necessary.
+ * @param progressMessage the message printed during the progression of the build
+ * @param inputManifests entries in inputs that are symlink manifest files.
+ * These are passed to remote execution in the environment rather than as inputs.
+ * @param mnemonic the mnemonic that is reported in the master log.
+ */
+ public SpawnAction(ActionOwner owner,
+ Iterable<Artifact> inputs, Iterable<Artifact> outputs,
+ ResourceSet resourceSet,
+ CommandLine argv,
+ ImmutableMap<String, String> environment,
+ ImmutableMap<String, String> executionInfo,
+ String progressMessage,
+ ImmutableMap<PathFragment, Artifact> inputManifests,
+ String mnemonic,
+ ExtraActionInfoSupplier<?> extraActionInfoSupplier) {
+ super(owner, inputs, outputs);
+ this.resourceSet = resourceSet;
+ this.executionInfo = executionInfo;
+ this.environment = environment;
+ this.argv = argv;
+ this.progressMessage = progressMessage;
+ this.inputManifests = inputManifests;
+ this.mnemonic = mnemonic;
+ this.extraActionInfoSupplier = extraActionInfoSupplier;
+ }
+
+ /**
+ * Returns the (immutable) list of all arguments, including the command name, argv[0].
+ */
+ @VisibleForTesting
+ public List<String> getArguments() {
+ return ImmutableList.copyOf(argv.arguments());
+ }
+
+ /**
+ * Returns command argument, argv[0].
+ */
+ @VisibleForTesting
+ public String getCommandFilename() {
+ return Iterables.getFirst(argv.arguments(), null);
+ }
+
+ /**
+ * Returns the (immutable) list of arguments, excluding the command name,
+ * argv[0].
+ */
+ @VisibleForTesting
+ public List<String> getRemainingArguments() {
+ return ImmutableList.copyOf(Iterables.skip(argv.arguments(), 1));
+ }
+
+ @VisibleForTesting
+ public boolean isShellCommand() {
+ return argv.isShellCommand();
+ }
+
+ /**
+ * Executes the action without handling ExecException errors.
+ *
+ * <p>Called by {@link #execute}.
+ */
+ protected void internalExecute(
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
+ getContext(actionExecutionContext.getExecutor()).exec(getSpawn(), actionExecutionContext);
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ try {
+ internalExecute(actionExecutionContext);
+ } catch (ExecException e) {
+ String failMessage = progressMessage;
+ if (isShellCommand()) {
+ // The possible reasons it could fail are: shell executable not found, shell
+ // exited non-zero, or shell died from signal. The first is impossible
+ // and the second two aren't very interesting, so in the interests of
+ // keeping the noise-level down, we don't print a reason why, just the
+ // command that failed.
+ //
+ // 0=shell executable, 1=shell command switch, 2=command
+ failMessage = "error executing shell command: " + "'"
+ + truncate(Iterables.get(argv.arguments(), 2), 200) + "'";
+ }
+ throw e.toActionExecutionException(failMessage, executor.getVerboseFailures(), this);
+ }
+ }
+
+ /**
+ * Returns s, truncated to no more than maxLen characters, appending an
+ * ellipsis if truncation occurred.
+ */
+ private static String truncate(String s, int maxLen) {
+ return s.length() > maxLen
+ ? s.substring(0, maxLen - "...".length()) + "..."
+ : s;
+ }
+
+ /**
+ * Returns a Spawn that is representative of the command that this Action
+ * will execute. This function must not modify any state.
+ */
+ public Spawn getSpawn() {
+ return new ActionSpawn();
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addStrings(argv.arguments());
+ f.addString(getMnemonic());
+ f.addInt(inputManifests.size());
+ for (Map.Entry<PathFragment, Artifact> input : inputManifests.entrySet()) {
+ f.addString(input.getKey().getPathString() + "/");
+ f.addPath(input.getValue().getExecPath());
+ }
+ f.addStringMap(getEnvironment());
+ f.addStringMap(getExecutionInfo());
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public String describeKey() {
+ StringBuilder message = new StringBuilder();
+ message.append(getProgressMessage());
+ message.append('\n');
+ for (Map.Entry<String, String> entry : getEnvironment().entrySet()) {
+ message.append(" Environment variable: ");
+ message.append(ShellEscaper.escapeString(entry.getKey()));
+ message.append('=');
+ message.append(ShellEscaper.escapeString(entry.getValue()));
+ message.append('\n');
+ }
+ for (String argument : ShellEscaper.escapeAll(argv.arguments())) {
+ message.append(" Argument: ");
+ message.append(argument);
+ message.append('\n');
+ }
+ return message.toString();
+ }
+
+ @Override
+ public final String getMnemonic() {
+ return mnemonic;
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return progressMessage;
+ }
+
+ @Override
+ public ExtraActionInfo.Builder getExtraActionInfo() {
+ ExtraActionInfo.Builder builder = super.getExtraActionInfo();
+ if (extraActionInfoSupplier == null) {
+ Spawn spawn = getSpawn();
+ SpawnInfo spawnInfo = spawn.getExtraActionInfo();
+
+ return builder
+ .setExtension(SpawnInfo.spawnInfo, spawnInfo);
+ } else {
+ extraActionInfoSupplier.extend(builder);
+ return builder;
+ }
+ }
+
+ /**
+ * Returns the environment in which to run this action.
+ */
+ public Map<String, String> getEnvironment() {
+ return environment;
+ }
+
+ /**
+ * Returns the out-of-band execution data for this action.
+ */
+ public Map<String, String> getExecutionInfo() {
+ return executionInfo;
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return getContext(executor).strategyLocality(getMnemonic(), isRemotable());
+ }
+
+ protected SpawnActionContext getContext(Executor executor) {
+ return executor.getSpawnActionContext(getMnemonic());
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ SpawnActionContext context = getContext(executor);
+ if (context.isRemotable(getMnemonic(), isRemotable())) {
+ return ResourceSet.ZERO;
+ }
+ return resourceSet;
+ }
+
+ /**
+ * Returns true if this can be run remotely.
+ */
+ public final boolean isRemotable() {
+ // TODO(bazel-team): get rid of this method.
+ return !executionInfo.containsKey("local");
+ }
+
+ /**
+ * The Spawn which this SpawnAction will execute.
+ */
+ private class ActionSpawn extends BaseSpawn {
+
+ private final List<Artifact> filesets = new ArrayList<>();
+
+ public ActionSpawn() {
+ super(ImmutableList.copyOf(argv.arguments()),
+ ImmutableMap.<String, String>of(),
+ executionInfo,
+ inputManifests,
+ SpawnAction.this,
+ resourceSet);
+ for (Artifact input : getInputs()) {
+ if (input.isFileset()) {
+ filesets.add(input);
+ }
+ }
+ }
+
+ @Override
+ public ImmutableMap<String, String> getEnvironment() {
+ return ImmutableMap.copyOf(SpawnAction.this.getEnvironment());
+ }
+
+ @Override
+ public ImmutableList<Artifact> getFilesetManifests() {
+ return ImmutableList.copyOf(filesets);
+ }
+
+ @Override
+ public Iterable<? extends ActionInput> getInputFiles() {
+ // Remove Fileset directories in inputs list. Instead, these are
+ // included as manifests in getEnvironment().
+ List<Artifact> inputs = Lists.newArrayList(getInputs());
+ inputs.removeAll(filesets);
+ inputs.removeAll(inputManifests.values());
+ return inputs;
+ // Also expand middleman artifacts.
+ }
+ }
+
+ /**
+ * Builder class to construct {@link SpawnAction} instances.
+ */
+ public static class Builder {
+
+ private final NestedSetBuilder<Artifact> inputsBuilder =
+ NestedSetBuilder.stableOrder();
+ private final List<Artifact> outputs = new ArrayList<>();
+ private final Map<PathFragment, Artifact> inputManifests = new LinkedHashMap<>();
+ private ResourceSet resourceSet = AbstractAction.DEFAULT_RESOURCE_SET;
+ private ImmutableMap<String, String> environment = ImmutableMap.of();
+ private ImmutableMap<String, String> executionInfo = ImmutableMap.of();
+ private boolean isShellCommand = false;
+ private boolean useDefaultShellEnvironment = false;
+ private PathFragment executable;
+ // executableArgs does not include the executable itself.
+ private List<String> executableArgs;
+ private final IterablesChain.Builder<String> argumentsBuilder = IterablesChain.builder();
+ private CommandLine commandLine;
+
+ private String progressMessage;
+ private ParamFileInfo paramFileInfo = null;
+ private String mnemonic = "Unknown";
+ private ExtraActionInfoSupplier<?> extraActionInfoSupplier = null;
+
+ /**
+ * Creates a SpawnAction builder.
+ */
+ public Builder() {}
+
+ /**
+ * Creates a builder that is a copy of another builder.
+ */
+ public Builder(Builder other) {
+ this.inputsBuilder.addTransitive(other.inputsBuilder.build());
+ this.outputs.addAll(other.outputs);
+ this.inputManifests.putAll(other.inputManifests);
+ this.resourceSet = other.resourceSet;
+ this.environment = other.environment;
+ this.executionInfo = other.executionInfo;
+ this.isShellCommand = other.isShellCommand;
+ this.useDefaultShellEnvironment = other.useDefaultShellEnvironment;
+ this.executable = other.executable;
+ this.executableArgs = (other.executableArgs != null)
+ ? Lists.newArrayList(other.executableArgs)
+ : null;
+ this.argumentsBuilder.add(other.argumentsBuilder.build());
+ this.commandLine = other.commandLine;
+ this.progressMessage = other.progressMessage;
+ this.paramFileInfo = other.paramFileInfo;
+ this.mnemonic = other.mnemonic;
+ }
+
+ /**
+ * Builds the SpawnAction using the passed in action configuration and returns it and all
+ * dependent actions. The first item of the returned array is always the SpawnAction itself.
+ *
+ * <p>This method makes a copy of all the collections, so it is safe to reuse the builder after
+ * this method returns.
+ *
+ * <p>This is annotated with @CheckReturnValue, which causes a compiler error when you call this
+ * method and ignore its return value. This is because some time ago, calling .build() had the
+ * side-effect of registering it with the RuleContext that was passed in to the constructor.
+ * This logic was removed, but if people don't notice and still rely on the side-effect, things
+ * may break.
+ *
+ * @return the SpawnAction and any actions required by it, with the first item always being the
+ * SpawnAction itself.
+ */
+ @CheckReturnValue
+ public Action[] build(ActionConstructionContext context) {
+ return build(context.getActionOwner(), context.getAnalysisEnvironment(),
+ context.getConfiguration());
+ }
+
+ @VisibleForTesting @CheckReturnValue
+ public Action[] build(ActionOwner owner, AnalysisEnvironment analysisEnvironment,
+ BuildConfiguration configuration) {
+ if (isShellCommand && executable == null) {
+ executable = configuration.getShExecutable();
+ }
+ Preconditions.checkNotNull(executable);
+ Preconditions.checkNotNull(executableArgs);
+
+ if (useDefaultShellEnvironment) {
+ this.environment = configuration.getDefaultShellEnvironment();
+ }
+
+ ImmutableList<String> argv = ImmutableList.<String>builder()
+ .add(executable.getPathString())
+ .addAll(executableArgs)
+ .build();
+
+ Iterable<String> arguments = argumentsBuilder.build();
+
+ Artifact paramsFile = ParamFileHelper.getParamsFile(argv, arguments, commandLine,
+ paramFileInfo, configuration, analysisEnvironment, outputs);
+
+ List<Action> actions = new ArrayList<>();
+ CommandLine actualCommandLine;
+ if (paramsFile != null) {
+ actualCommandLine = ParamFileHelper.createWithParamsFile(argv, arguments, commandLine,
+ isShellCommand, owner, actions, paramFileInfo, paramsFile);
+ } else {
+ actualCommandLine = ParamFileHelper.createWithoutParamsFile(argv, arguments, commandLine,
+ isShellCommand);
+ }
+
+ Iterable<Artifact> actualInputs = collectActualInputs(paramsFile);
+
+ actions.add(0, new SpawnAction(owner, actualInputs, ImmutableList.copyOf(outputs),
+ resourceSet, actualCommandLine, environment, executionInfo, progressMessage,
+ ImmutableMap.copyOf(inputManifests), mnemonic, extraActionInfoSupplier));
+ return actions.toArray(new Action[actions.size()]);
+ }
+
+ private Iterable<Artifact> collectActualInputs(Artifact parameterFile) {
+ if (parameterFile != null) {
+ inputsBuilder.add(parameterFile);
+ }
+ return inputsBuilder.build();
+ }
+
+ /**
+ * Adds an input to this action.
+ */
+ public Builder addInput(Artifact artifact) {
+ inputsBuilder.add(artifact);
+ return this;
+ }
+
+ /**
+ * Adds inputs to this action.
+ */
+ public Builder addInputs(Iterable<Artifact> artifacts) {
+ inputsBuilder.addAll(artifacts);
+ return this;
+ }
+
+ /**
+ * Adds transitive inputs to this action.
+ */
+ public Builder addTransitiveInputs(NestedSet<Artifact> artifacts) {
+ inputsBuilder.addTransitive(artifacts);
+ return this;
+ }
+
+ public Builder addInputManifest(Artifact artifact, PathFragment remote) {
+ inputManifests.put(remote, artifact);
+ return this;
+ }
+
+ public Builder addOutput(Artifact artifact) {
+ outputs.add(artifact);
+ return this;
+ }
+
+ public Builder addOutputs(Iterable<Artifact> artifacts) {
+ Iterables.addAll(outputs, artifacts);
+ return this;
+ }
+
+ public Builder setResources(ResourceSet resourceSet) {
+ this.resourceSet = resourceSet;
+ return this;
+ }
+
+ /**
+ * Sets the map of environment variables.
+ */
+ public Builder setEnvironment(Map<String, String> environment) {
+ this.environment = ImmutableMap.copyOf(environment);
+ this.useDefaultShellEnvironment = false;
+ return this;
+ }
+
+ /**
+ * Sets the map of execution info.
+ */
+ public Builder setExecutionInfo(Map<String, String> info) {
+ this.executionInfo = ImmutableMap.copyOf(info);
+ return this;
+ }
+
+ /**
+ * Sets the environment to the configurations default shell environment,
+ * see {@link BuildConfiguration#getDefaultShellEnvironment}.
+ */
+ public Builder useDefaultShellEnvironment() {
+ this.environment = null;
+ this.useDefaultShellEnvironment = true;
+ return this;
+ }
+
+ /**
+ * Sets the executable path; the path is interpreted relative to the
+ * execution root.
+ *
+ * <p>Calling this method overrides any previous values set via calls to
+ * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or
+ * {@link #setShellCommand(String)}.
+ */
+ public Builder setExecutable(PathFragment executable) {
+ this.executable = executable;
+ this.executableArgs = Lists.newArrayList();
+ this.isShellCommand = false;
+ return this;
+ }
+
+ /**
+ * Sets the executable as an artifact.
+ *
+ * <p>Calling this method overrides any previous values set via calls to
+ * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or
+ * {@link #setShellCommand(String)}.
+ */
+ public Builder setExecutable(Artifact executable) {
+ return setExecutable(executable.getExecPath());
+ }
+
+ /**
+ * Sets the executable as a configured target. Automatically adds the files
+ * to run to the inputs and uses the executable of the target as the
+ * executable.
+ *
+ * <p>Calling this method overrides any previous values set via calls to
+ * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or
+ * {@link #setShellCommand(String)}.
+ */
+ public Builder setExecutable(TransitiveInfoCollection executable) {
+ FilesToRunProvider provider = executable.getProvider(FilesToRunProvider.class);
+ Preconditions.checkArgument(provider != null);
+ return setExecutable(provider);
+ }
+
+ /**
+ * Sets the executable as a configured target. Automatically adds the files
+ * to run to the inputs and uses the executable of the target as the
+ * executable.
+ *
+ * <p>Calling this method overrides any previous values set via calls to
+ * {@link #setExecutable}, {@link #setJavaExecutable}, or
+ * {@link #setShellCommand(String)}.
+ */
+ public Builder setExecutable(FilesToRunProvider executableProvider) {
+ Preconditions.checkArgument(executableProvider.getExecutable() != null,
+ "The target does not have an executable");
+ setExecutable(executableProvider.getExecutable().getExecPath());
+ return addTool(executableProvider);
+ }
+
+ private Builder setJavaExecutable(PathFragment javaExecutable, Artifact deployJar,
+ List<String> jvmArgs, String... launchArgs) {
+ this.executable = javaExecutable;
+ this.executableArgs = Lists.newArrayList();
+ executableArgs.add("-Xverify:none");
+ executableArgs.addAll(jvmArgs);
+ for (String arg : launchArgs) {
+ executableArgs.add(arg);
+ }
+ inputsBuilder.add(deployJar);
+ this.isShellCommand = false;
+ return this;
+ }
+
+ /**
+ * Sets the executable to be a java class executed from the given deploy
+ * jar. The deploy jar is automatically added to the action inputs.
+ *
+ * <p>Calling this method overrides any previous values set via calls to
+ * {@link #setExecutable}, {@link #setJavaExecutable}, or
+ * {@link #setShellCommand(String)}.
+ */
+ public Builder setJavaExecutable(PathFragment javaExecutable,
+ Artifact deployJar, String javaMainClass, List<String> jvmArgs) {
+ return setJavaExecutable(javaExecutable, deployJar, jvmArgs, "-cp",
+ deployJar.getExecPathString(), javaMainClass);
+ }
+
+ /**
+ * Sets the executable to be a jar executed from the given deploy jar. The deploy jar is
+ * automatically added to the action inputs.
+ *
+ * <p>This method is similar to {@link #setJavaExecutable} but it assumes that the Jar artifact
+ * declares a main class.
+ *
+ * <p>Calling this method overrides any previous values set via calls to {@link #setExecutable},
+ * {@link #setJavaExecutable}, or {@link #setShellCommand(String)}.
+ */
+ public Builder setJarExecutable(PathFragment javaExecutable,
+ Artifact deployJar, List<String> jvmArgs) {
+ return setJavaExecutable(javaExecutable, deployJar, jvmArgs, "-jar",
+ deployJar.getExecPathString());
+ }
+
+ /**
+ * Sets the executable to be the shell and adds the given command as the
+ * command to be executed.
+ *
+ * <p>Note that this will not clear the arguments, so any arguments will
+ * be passed in addition to the command given here.
+ *
+ * <p>Calling this method overrides any previous values set via calls to
+ * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or
+ * {@link #setShellCommand(String)}.
+ */
+ public Builder setShellCommand(String command) {
+ this.executable = null;
+ // 0=shell command switch, 1=command
+ this.executableArgs = Lists.newArrayList("-c", command);
+ this.isShellCommand = true;
+ return this;
+ }
+
+ /**
+ * Sets the executable to be the shell and adds the given interned commands as the
+ * commands to be executed.
+ */
+ public Builder setShellCommand(Iterable<String> command) {
+ this.executable = new PathFragment(Iterables.getFirst(command, null));
+ // The first item of the commands is the shell executable that should be used.
+ this.executableArgs = ImmutableList.copyOf(Iterables.skip(command, 1));
+ this.isShellCommand = true;
+ return this;
+ }
+
+ /**
+ * Adds an executable and its runfiles, so it can be called from a shell command.
+ */
+ public Builder addTool(FilesToRunProvider tool) {
+ addInputs(tool.getFilesToRun());
+ if (tool.getRunfilesManifest() != null) {
+ addInputManifest(tool.getRunfilesManifest(),
+ BaseSpawn.runfilesForFragment(tool.getExecutable().getExecPath()));
+ }
+ return this;
+ }
+
+ /**
+ * Appends the arguments to the list of executable arguments.
+ */
+ public Builder addExecutableArguments(String... arguments) {
+ Preconditions.checkState(executableArgs != null);
+ executableArgs.addAll(Arrays.asList(arguments));
+ return this;
+ }
+
+ /**
+ * Add multiple arguments in the order they are returned by the collection
+ * to the list of executable arguments.
+ */
+ public Builder addExecutableArguments(Iterable<String> arguments) {
+ Preconditions.checkState(executableArgs != null);
+ Iterables.addAll(executableArgs, arguments);
+ return this;
+ }
+
+ /**
+ * Appends the argument to the list of command-line arguments.
+ */
+ public Builder addArgument(String argument) {
+ Preconditions.checkState(commandLine == null);
+ argumentsBuilder.addElement(argument);
+ return this;
+ }
+
+ /**
+ * Appends the arguments to the list of command-line arguments.
+ */
+ public Builder addArguments(String... arguments) {
+ Preconditions.checkState(commandLine == null);
+ argumentsBuilder.add(ImmutableList.copyOf(arguments));
+ return this;
+ }
+
+ /**
+ * Add multiple arguments in the order they are returned by the collection.
+ */
+ public Builder addArguments(Iterable<String> arguments) {
+ Preconditions.checkState(commandLine == null);
+ argumentsBuilder.add(CollectionUtils.makeImmutable(arguments));
+ return this;
+ }
+
+ /**
+ * Appends the argument both to the inputs and to the list of command-line
+ * arguments.
+ */
+ public Builder addInputArgument(Artifact argument) {
+ Preconditions.checkState(commandLine == null);
+ addInput(argument);
+ addArgument(argument.getExecPathString());
+ return this;
+ }
+
+ /**
+ * Appends the arguments both to the inputs and to the list of command-line
+ * arguments.
+ */
+ public Builder addInputArguments(Iterable<Artifact> arguments) {
+ for (Artifact argument : arguments) {
+ addInputArgument(argument);
+ }
+ return this;
+ }
+
+ /**
+ * Appends the argument both to the ouputs and to the list of command-line
+ * arguments.
+ */
+ public Builder addOutputArgument(Artifact argument) {
+ Preconditions.checkState(commandLine == null);
+ outputs.add(argument);
+ argumentsBuilder.addElement(argument.getExecPathString());
+ return this;
+ }
+
+ /**
+ * Sets a delegate to compute the command line at a later time. This method
+ * cannot be used in conjunction with the {@link #addArgument} or {@link
+ * #addArguments} methods.
+ *
+ * <p>The main intention of this method is to save memory by allowing
+ * client-controlled sharing between actions and configured targets.
+ * Objects passed to this method MUST be immutable.
+ */
+ public Builder setCommandLine(CommandLine commandLine) {
+ Preconditions.checkState(argumentsBuilder.isEmpty());
+ this.commandLine = commandLine;
+ return this;
+ }
+
+ public Builder setProgressMessage(String progressMessage) {
+ this.progressMessage = progressMessage;
+ return this;
+ }
+
+ public Builder setMnemonic(String mnemonic) {
+ Preconditions.checkArgument(
+ !mnemonic.isEmpty() && CharMatcher.JAVA_LETTER_OR_DIGIT.matchesAllOf(mnemonic),
+ "mnemonic must only contain letters and/or digits, and have non-zero length, was: \"%s\"",
+ mnemonic);
+ this.mnemonic = mnemonic;
+ return this;
+ }
+
+ public <T> Builder setExtraActionInfo(
+ GeneratedExtension<ExtraActionInfo, T> extension, T value) {
+ this.extraActionInfoSupplier = new ExtraActionInfoSupplier<T>(extension, value);
+ return this;
+ }
+
+ /**
+ * Enable use of a parameter file and set the encoding to ISO-8859-1 (latin1).
+ *
+ * <p>In order to use parameter files, at least one output artifact must be specified.
+ */
+ public Builder useParameterFile(ParameterFileType parameterFileType) {
+ return useParameterFile(parameterFileType, ISO_8859_1, "@");
+ }
+
+ /**
+ * Enable or disable the use of a parameter file, set the encoding to the given value, and
+ * specify the argument prefix to use in passing the parameter file name to the tool.
+ *
+ * <p>The default argument prefix is "@". In order to use parameter files, at least one output
+ * artifact must be specified.
+ */
+ public Builder useParameterFile(
+ ParameterFileType parameterFileType, Charset charset, String flagPrefix) {
+ paramFileInfo = new ParamFileInfo(parameterFileType, charset, flagPrefix);
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkAction.java
new file mode 100644
index 0000000..2bbb3e2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkAction.java
@@ -0,0 +1,130 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+
+/**
+ * Action to create a symbolic link.
+ */
+public class SymlinkAction extends AbstractAction {
+
+ private static final String GUID = "349675b5-437c-4da8-891a-7fb98fba6ab5";
+
+ private final PathFragment inputPath;
+ private final Artifact output;
+ private final String progressMessage;
+
+ /**
+ * Creates a new SymlinkAction instance.
+ *
+ * @param owner the action owner.
+ * @param input the Artifact that will be the src of the symbolic link.
+ * @param output the Artifact that will be created by executing this Action.
+ * @param progressMessage the progress message.
+ */
+ public SymlinkAction(ActionOwner owner, Artifact input, Artifact output,
+ String progressMessage) {
+ // These actions typically have only one input and one output, which
+ // become the sole and primary in their respective lists.
+ this(owner, input.getExecPath(), input, output, progressMessage);
+ }
+
+ /**
+ * Creates a new SymlinkAction instance, where the inputPath
+ * may be different than that input artifact's path. This is
+ * only useful when dealing with runfiles trees where
+ * link target is a directory.
+ *
+ * @param owner the action owner.
+ * @param inputPath the Path that will be the src of the symbolic link.
+ * @param input the Artifact that is required to build the inputPath.
+ * @param output the Artifact that will be created by executing this Action.
+ * @param progressMessage the progress message.
+ */
+ public SymlinkAction(ActionOwner owner, PathFragment inputPath, Artifact input,
+ Artifact output, String progressMessage) {
+ super(owner, ImmutableList.of(input), ImmutableList.of(output));
+ this.inputPath = Preconditions.checkNotNull(inputPath);
+ this.output = Preconditions.checkNotNull(output);
+ this.progressMessage = progressMessage;
+ }
+
+ public PathFragment getInputPath() {
+ return inputPath;
+ }
+
+ public Path getOutputPath() {
+ return output.getPath();
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ try {
+ getOutputPath().createSymbolicLink(
+ actionExecutionContext.getExecutor().getExecRoot().getRelative(inputPath));
+ } catch (IOException e) {
+ throw new ActionExecutionException("failed to create symbolic link '"
+ + Iterables.getOnlyElement(getOutputs()).prettyPrint()
+ + "' to the '" + Iterables.getOnlyElement(getInputs()).prettyPrint()
+ + "' due to I/O error: " + e.getMessage(), e, this, false);
+ }
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.0);
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ // We don't normally need to add inputs to the key. In this case, however, the inputPath can be
+ // different from the actual input artifact.
+ f.addPath(inputPath);
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "Symlink";
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return progressMessage;
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "local";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/TemplateExpansionAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/TemplateExpansionAction.java
new file mode 100644
index 0000000..b2c83fb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/TemplateExpansionAction.java
@@ -0,0 +1,335 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.actions;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.ResourceFileLoader;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Action to expand a template and write the expanded content to a file.
+ */
+public class TemplateExpansionAction extends AbstractFileWriteAction {
+
+ private static final String GUID = "786c1fe0-dca8-407a-b108-e1ecd6d1bc7f";
+
+ /**
+ * A pair of a string to be substituted and a string to substitute it with.
+ * For simplicity, these are called key and value. All implementations must
+ * be immutable, and always return the identical key. The returned values
+ * must be the same, though they need not be the same object.
+ *
+ * <p>It should be assumed that the {@link #getKey} invocation is cheap, and
+ * that the {@link #getValue} invocation is expensive.
+ */
+ public abstract static class Substitution {
+ private Substitution() {
+ }
+
+ public abstract String getKey();
+ public abstract String getValue();
+
+ /**
+ * Returns an immutable Substitution instance for the given key and value.
+ */
+ public static Substitution of(final String key, final String value) {
+ return new Substitution() {
+ @Override
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+ };
+ }
+
+ /**
+ * Returns an immutable Substitution instance for the key and list of values. The
+ * values will be joined by spaces before substitution.
+ */
+ public static Substitution ofSpaceSeparatedList(final String key, final List<?> value) {
+ return new Substitution() {
+ @Override
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public String getValue() {
+ return Joiner.on(" ").join(value);
+ }
+ };
+ }
+ }
+
+ /**
+ * A substitution with a fixed key, and a computed value. The computed value
+ * must not change over the lifetime of an instance, though the {@link
+ * #getValue} method may return different String objects.
+ *
+ * <p>It should be assumed that the {@link #getKey} invocation is cheap, and
+ * that the {@link #getValue} invocation is expensive.
+ */
+ public abstract static class ComputedSubstitution extends Substitution {
+ private final String key;
+
+ public ComputedSubstitution(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+ }
+
+ /**
+ * A template that contains text content, or alternatively throws an {@link
+ * IOException}.
+ */
+ public abstract static class Template {
+
+ /**
+ * We only allow subclasses in this file.
+ */
+ private Template() {
+ }
+
+ /**
+ * Returns the text content of the template.
+ */
+ protected abstract String getContent() throws IOException;
+
+ /**
+ * Returns a string that is used for the action key. This must change if
+ * the getContent method returns something different, but is not allowed to
+ * throw an exception.
+ */
+ protected abstract String getKey();
+
+ /**
+ * Loads a template from the given resource. The resource is looked up
+ * relative to the given class. If the resource cannot be loaded, the returned
+ * template throws an {@link IOException} when {@link #getContent} is
+ * called. This makes it safe to use this method in a constant initializer.
+ */
+ public static Template forResource(final Class<?> relativeToClass, final String templateName) {
+ try {
+ String content = ResourceFileLoader.loadResource(relativeToClass, templateName);
+ return forString(content);
+ } catch (final IOException e) {
+ return new Template() {
+ @Override
+ protected String getContent() throws IOException {
+ throw new IOException("failed to load resource file '" + templateName
+ + "' due to I/O error: " + e.getMessage(), e);
+ }
+
+ @Override
+ protected String getKey() {
+ return "ERROR: " + e.getMessage();
+ }
+ };
+ }
+ }
+
+ /**
+ * Returns a template for the given text string.
+ */
+ public static Template forString(final String templateText) {
+ return new Template() {
+ @Override
+ protected String getContent() {
+ return templateText;
+ }
+
+ @Override
+ protected String getKey() {
+ return templateText;
+ }
+ };
+ }
+
+ /**
+ * Returns a template that loads the given artifact. It is important that
+ * the artifact is also an input for the action, or this won't work.
+ * Therefore this method is private, and you should use the corresponding
+ * {@link TemplateExpansionAction} constructor.
+ */
+ private static Template forArtifact(final Artifact templateArtifact) {
+ return new Template() {
+ @Override
+ protected String getContent() throws IOException {
+ Path templatePath = templateArtifact.getPath();
+ try {
+ return new String(FileSystemUtils.readContentAsLatin1(templatePath));
+ } catch (IOException e) {
+ throw new IOException("failed to load template file '" + templatePath.getPathString()
+ + "' due to I/O error: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ protected String getKey() {
+ // This isn't strictly necessary, because the action inputs are automatically considered.
+ return "ARTIFACT: " + templateArtifact.getExecPathString();
+ }
+ };
+ }
+ }
+
+ private final Template template;
+ private final List<Substitution> substitutions;
+
+ /**
+ * Creates a new TemplateExpansionAction instance.
+ *
+ * @param owner the action owner.
+ * @param inputs the Artifacts that this Action depends on
+ * @param output the Artifact that will be created by executing this Action.
+ * @param template the template that will be expanded by this Action.
+ * @param substitutions the substitutions that will be applied to the
+ * template. All substitutions will be applied in order.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ private TemplateExpansionAction(ActionOwner owner,
+ Collection<Artifact> inputs,
+ Artifact output,
+ Template template,
+ List<Substitution> substitutions,
+ boolean makeExecutable) {
+ super(owner, inputs, output, makeExecutable);
+ this.template = template;
+ this.substitutions = ImmutableList.copyOf(substitutions);
+ }
+
+ /**
+ * Creates a new TemplateExpansionAction instance for an artifact template.
+ *
+ * @param owner the action owner.
+ * @param templateArtifact the Artifact that will be read as the text template
+ * file
+ * @param output the Artifact that will be created by executing this Action.
+ * @param substitutions the substitutions that will be applied to the
+ * template. All substitutions will be applied in order.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ public TemplateExpansionAction(ActionOwner owner,
+ Artifact templateArtifact,
+ Artifact output,
+ List<Substitution> substitutions,
+ boolean makeExecutable) {
+ this(owner, ImmutableList.of(templateArtifact), output, Template.forArtifact(templateArtifact),
+ substitutions, makeExecutable);
+ }
+
+ /**
+ * Creates a new TemplateExpansionAction instance without inputs.
+ *
+ * @param owner the action owner.
+ * @param output the Artifact that will be created by executing this Action.
+ * @param template the template
+ * @param substitutions the substitutions that will be applied to the
+ * template. All substitutions will be applied in order.
+ * @param makeExecutable iff true will change the output file to be
+ * executable.
+ */
+ public TemplateExpansionAction(ActionOwner owner,
+ Artifact output,
+ Template template,
+ List<Substitution> substitutions,
+ boolean makeExecutable) {
+ this(owner, Artifact.NO_ARTIFACTS, output, template, substitutions, makeExecutable);
+ }
+
+ /**
+ * Expands the template by applying all substitutions.
+ * @param template
+ * @return the expanded text.
+ */
+ private String expandTemplate(String template) {
+ for (Substitution entry : substitutions) {
+ template = StringUtilities.replaceAllLiteral(template, entry.getKey(), entry.getValue());
+ }
+ return template;
+ }
+
+ @VisibleForTesting
+ public String getFileContents() throws IOException {
+ return expandTemplate(template.getContent());
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler,
+ Executor executor) throws IOException {
+ final byte[] bytes = getFileContents().getBytes(UTF_8);
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ out.write(bytes);
+ }
+ };
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addString(String.valueOf(makeExecutable));
+ f.addString(template.getKey());
+ f.addInt(substitutions.size());
+ for (Substitution entry : substitutions) {
+ f.addString(entry.getKey());
+ f.addString(entry.getValue());
+ }
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "TemplateExpand";
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Expanding template " + Iterables.getOnlyElement(getOutputs()).prettyPrint();
+ }
+
+ public List<Substitution> getSubstitutions() {
+ return substitutions;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoCollection.java
new file mode 100644
index 0000000..54067a0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoCollection.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.buildinfo;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+
+import java.util.List;
+
+/**
+ * A collection of build-info files for both stamped and unstamped modes.
+ */
+public final class BuildInfoCollection {
+ private final ImmutableList<Action> actions;
+ private final ImmutableList<Artifact> stampedBuildInfo;
+ private final ImmutableList<Artifact> redactedBuildInfo;
+
+ public BuildInfoCollection(List<? extends Action> actions, List<Artifact> stampedBuildInfo,
+ List<Artifact> redactedBuildInfo) {
+ this.actions = ImmutableList.copyOf(actions);
+ this.stampedBuildInfo = ImmutableList.copyOf(stampedBuildInfo);
+ this.redactedBuildInfo = ImmutableList.copyOf(redactedBuildInfo);
+ }
+
+ public ImmutableList<Action> getActions() {
+ return actions;
+ }
+
+ public ImmutableList<Artifact> getStampedBuildInfo() {
+ return stampedBuildInfo;
+ }
+
+ public ImmutableList<Artifact> getRedactedBuildInfo() {
+ return redactedBuildInfo;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoFactory.java
new file mode 100644
index 0000000..c6ec4d7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoFactory.java
@@ -0,0 +1,99 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.buildinfo;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.Serializable;
+
+/**
+ * A factory for language-specific build-info files. Use this to translate the build-info into
+ * target-independent language-specific files. The generated actions are registered into the action
+ * graph on every build, but only executed if anything depends on them.
+ */
+public interface BuildInfoFactory extends Serializable {
+ /**
+ * Type of the build-data artifact.
+ */
+ public enum BuildInfoType {
+ /**
+ * Ignore changes to this file for the purposes of determining whether an action needs to be
+ * re-executed. I.e., the action is only re-executed if at least one other input has changed.
+ */
+ NO_REBUILD,
+
+ /**
+ * Changes to this file trigger re-execution of actions, similar to source file changes.
+ */
+ FORCE_REBUILD_IF_CHANGED;
+ }
+
+ /**
+ * Context for the creation of build-info artifacts.
+ */
+ public interface BuildInfoContext {
+ Artifact getBuildInfoArtifact(PathFragment rootRelativePath, Root root, BuildInfoType type);
+ Root getBuildDataDirectory();
+ }
+
+ /**
+ * Build-info key for lookup from the {@link
+ * com.google.devtools.build.lib.analysis.AnalysisEnvironment}.
+ */
+ public static final class BuildInfoKey implements Serializable {
+ private final String name;
+
+ public BuildInfoKey(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof BuildInfoKey)) {
+ return false;
+ }
+ return name.equals(((BuildInfoKey) o).name);
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+ }
+
+ /**
+ * Create actions and artifacts for language-specific build-info files.
+ */
+ BuildInfoCollection create(BuildInfoContext context, BuildConfiguration config,
+ Artifact buildInfo, Artifact buildChangelist);
+
+ /**
+ * Returns the key for the information created by this factory.
+ */
+ BuildInfoKey getKey();
+
+ /**
+ * Returns false if this build info factory is disabled based on the configuration (usually by
+ * checking if all required configuration fragments are present).
+ */
+ boolean isEnabled(BuildConfiguration config);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BinTools.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BinTools.java
new file mode 100644
index 0000000..6d2477d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BinTools.java
@@ -0,0 +1,191 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import java.io.IOException;
+
+/**
+ * Initializes the <execRoot>/_bin/ directory that contains auxiliary tools used during action
+ * execution (alarm, etc). The main purpose of this is to make sure that those tools are accessible
+ * using relative paths from the execution root.
+ */
+public final class BinTools {
+ private final BlazeDirectories directories;
+ private final Path binDir; // the working bin directory under execRoot
+ private final ImmutableList<String> embeddedTools;
+
+ private BinTools(BlazeDirectories directories, ImmutableList<String> tools) {
+ this.directories = directories;
+ this.binDir = directories.getExecRoot().getRelative("_bin");
+ this.embeddedTools = tools;
+ }
+
+ /**
+ * Creates an instance with the list of embedded tools obtained from scanning the directory
+ * into which said binaries were extracted by the launcher.
+ */
+ public static BinTools forProduction(BlazeDirectories directories) throws IOException {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ scanDirectoryRecursively(builder, directories.getEmbeddedBinariesRoot(), "");
+ return new BinTools(directories, builder.build());
+ }
+
+ /**
+ * Creates an empty instance for testing.
+ */
+ @VisibleForTesting
+ public static BinTools empty(BlazeDirectories directories) {
+ return new BinTools(directories, ImmutableList.<String>of());
+ }
+
+ /**
+ * Creates an instance for testing without actually symlinking the tools.
+ *
+ * <p>Used for tests that need a set of embedded tools to be present, but not the actual files.
+ */
+ @VisibleForTesting
+ public static BinTools forUnitTesting(BlazeDirectories directories, Iterable<String> tools) {
+ return new BinTools(directories, ImmutableList.copyOf(tools));
+ }
+
+ /**
+ * Populates the _bin directory by symlinking the necessary files from the given
+ * srcDir, and returns the corresponding BinTools.
+ */
+ @VisibleForTesting
+ public static BinTools forIntegrationTesting(
+ BlazeDirectories directories, String srcDir, Iterable<String> tools)
+ throws IOException {
+ Path srcPath = directories.getOutputBase().getFileSystem().getPath(srcDir);
+ for (String embedded : tools) {
+ Path runfilesPath = srcPath.getRelative(embedded);
+ if (!runfilesPath.isFile()) {
+ // The file isn't there - nothing to symlink!
+ //
+ // Note: This path is usually taken by the tests using the in-memory
+ // file system. They can't run the embedded scripts anyhow, so there isn't
+ // much point in creating a symlink to a non-existent binary here.
+ continue;
+ }
+ Path outputPath = directories.getExecRoot().getChild("_bin").getChild(embedded);
+ if (outputPath.exists()) {
+ outputPath.delete();
+ }
+ FileSystemUtils.createDirectoryAndParents(outputPath.getParentDirectory());
+ outputPath.createSymbolicLink(runfilesPath);
+ }
+
+ return new BinTools(directories, ImmutableList.copyOf(tools));
+ }
+
+ private static void scanDirectoryRecursively(
+ ImmutableList.Builder<String> result, Path root, String relative) throws IOException {
+ for (Dirent dirent : root.readdir(Symlinks.NOFOLLOW)) {
+ String childRelative = relative.isEmpty()
+ ? dirent.getName()
+ : relative + "/" + dirent.getName();
+ switch (dirent.getType()) {
+ case FILE:
+ result.add(childRelative);
+ break;
+
+ case DIRECTORY:
+ scanDirectoryRecursively(result, root.getChild(dirent.getName()), childRelative);
+ break;
+
+ default:
+ // Nothing to do here -- we ignore symlinks, since they should not be present in the
+ // embedded binaries tree.
+ break;
+ }
+ }
+ }
+
+ public PathFragment getExecPath(String embedPath) {
+ Preconditions.checkState(embeddedTools.contains(embedPath), "%s not in %s", embedPath,
+ embeddedTools);
+ return new PathFragment("_bin").getRelative(new PathFragment(embedPath).getBaseName());
+ }
+
+ public Artifact getEmbeddedArtifact(String embedPath, ArtifactFactory artifactFactory) {
+ return artifactFactory.getDerivedArtifact(getExecPath(embedPath));
+ }
+
+ public ImmutableList<Artifact> getAllEmbeddedArtifacts(ArtifactFactory artifactFactory) {
+ ImmutableList.Builder<Artifact> builder = ImmutableList.builder();
+ for (String embeddedTool : embeddedTools) {
+ builder.add(getEmbeddedArtifact(embeddedTool, artifactFactory));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Initializes the build tools not available at absolute paths. Note that
+ * these must be constant across all configurations.
+ */
+ public void setupBuildTools() throws ExecException {
+ try {
+ FileSystemUtils.createDirectoryAndParents(binDir);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("could not create directory '" + binDir + "'", e);
+ }
+
+ for (String embeddedPath : embeddedTools) {
+ setupTool(embeddedPath);
+ }
+ }
+
+ private void setupTool(String embeddedPath) throws ExecException {
+ Path sourcePath = directories.getEmbeddedBinariesRoot().getRelative(embeddedPath);
+ Path linkPath = binDir.getRelative(new PathFragment(embeddedPath).getBaseName());
+ linkTool(sourcePath, linkPath);
+ }
+
+ private void linkTool(Path sourcePath, Path linkPath) throws ExecException {
+ if (linkPath.getFileSystem().supportsSymbolicLinks()) {
+ try {
+ if (!linkPath.isSymbolicLink()) {
+ // ensureSymbolicLink() does not handle the case where there is already
+ // a file with the same name, so we need to handle it here.
+ linkPath.delete();
+ }
+ FileSystemUtils.ensureSymbolicLink(linkPath, sourcePath);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("failed to link '" + sourcePath + "'", e);
+ }
+ } else {
+ // For file systems that do not support linking, copy.
+ try {
+ FileSystemUtils.copyTool(sourcePath, linkPath);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("failed to copy '" + sourcePath + "'" , e);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
new file mode 100644
index 0000000..8e80211
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -0,0 +1,1944 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.PackageRootResolver;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection.Transitions;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.Configurator;
+import com.google.devtools.build.lib.packages.Attribute.SplitTransition;
+import com.google.devtools.build.lib.packages.Attribute.Transition;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.rules.test.TestActionBuilder;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.TriState;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Instances of BuildConfiguration represent a collection of context
+ * information which may affect a build (for example: the target platform for
+ * compilation, or whether or not debug tables are required). In fact, all
+ * "environmental" information (e.g. from the tool's command-line, as opposed
+ * to the BUILD file) that can affect the output of any build tool should be
+ * explicitly represented in the BuildConfiguration instance.
+ *
+ * <p>A single build may require building tools to run on a variety of
+ * platforms: when compiling a server application for production, we must build
+ * the build tools (like compilers) to run on the host platform, but cross-compile
+ * the application for the production environment.
+ *
+ * <p>There is always at least one BuildConfiguration instance in any build:
+ * the one representing the host platform. Additional instances may be created,
+ * in a cross-compilation build, for example.
+ *
+ * <p>Instances of BuildConfiguration are canonical:
+ * <pre>c1.equals(c2) <=> c1==c2.</pre>
+ */
+@SkylarkModule(name = "configuration",
+ doc = "Data required for the analysis of a target that comes from targets that "
+ + "depend on it and not targets that it depends on.")
+public final class BuildConfiguration implements Serializable {
+
+ /**
+ * An interface for language-specific configurations.
+ */
+ public abstract static class Fragment implements Serializable {
+ /**
+ * Returns a human-readable name of the configuration fragment.
+ */
+ public abstract String getName();
+
+ /**
+ * Validates the options for this Fragment. Issues warnings for the
+ * use of deprecated options, and warnings or errors for any option settings
+ * that conflict.
+ */
+ @SuppressWarnings("unused")
+ public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
+ }
+
+ /**
+ * Adds mapping of names to values of "Make" variables defined by this configuration.
+ */
+ @SuppressWarnings("unused")
+ public void addGlobalMakeVariables(ImmutableMap.Builder<String, String> globalMakeEnvBuilder) {
+ }
+
+ /**
+ * Collects all labels that should be implicitly loaded from labels that were specified as
+ * options, keyed by the name to be displayed to the user if something goes wrong.
+ * The resulting set only contains labels that were derived from command-line options; the
+ * intention is that it can be used to sanity-check that the command-line options actually
+ * contain these in their transitive closure.
+ */
+ @SuppressWarnings("unused")
+ public void addImplicitLabels(Multimap<String, Label> implicitLabels) {
+ }
+
+ /**
+ * Returns a string that identifies the configuration fragment.
+ */
+ public abstract String cacheKey();
+
+ /**
+ * The fragment may use this hook to perform I/O and read data into memory that is used during
+ * analysis. During the analysis phase disk I/O operations are disallowed.
+ *
+ * <p>This hook is only called for the top-level configuration after the loading phase is
+ * complete.
+ */
+ @SuppressWarnings("unused")
+ public void prepareHook(Path execPath, ArtifactFactory artifactFactory,
+ PathFragment genfilesPath, PackageRootResolver resolver)
+ throws ViewCreationFailedException {
+ }
+
+ /**
+ * Adds all the roots from this fragment.
+ */
+ @SuppressWarnings("unused")
+ public void addRoots(List<Root> roots) {
+ }
+
+ /**
+ * Returns a (key, value) mapping to insert into the subcommand environment for coverage.
+ */
+ public Map<String, String> getCoverageEnvironment() {
+ return ImmutableMap.<String, String>of();
+ }
+
+ /*
+ * Returns the command-line "Make" variable overrides.
+ */
+ public ImmutableMap<String, String> getCommandLineDefines() {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * Returns all the coverage labels for the fragment.
+ */
+ public ImmutableList<Label> getCoverageLabels() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the coverage report generator tool labels.
+ */
+ public ImmutableList<Label> getCoverageReportGeneratorLabels() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns a fragment of the output directory name for this configuration. The output
+ * directory for the whole configuration contains all the short names by all fragments.
+ */
+ @Nullable
+ public String getOutputDirectoryName() {
+ return null;
+ }
+
+ /**
+ * This will be added to the name of the configuration, but not to the output directory name.
+ */
+ @Nullable
+ public String getConfigurationNameSuffix() {
+ return null;
+ }
+
+ /**
+ * The platform name is a concatenation of fragment platform names.
+ */
+ public String getPlatformName() {
+ return "";
+ }
+
+ /**
+ * Return false if incremental build is not possible for some reason.
+ */
+ public boolean supportsIncrementalBuild() {
+ return true;
+ }
+
+ /**
+ * Return true if the fragment performs static linking. This information is needed for
+ * lincence checking.
+ */
+ public boolean performsStaticLink() {
+ return false;
+ }
+
+ /**
+ * Fragments should delete temporary directories they create for their inner mechanisms.
+ * This is only called for target configuration.
+ */
+ @SuppressWarnings("unused")
+ public void prepareForExecutionPhase() throws IOException {
+ }
+
+ /**
+ * Add items to the shell environment.
+ */
+ @SuppressWarnings("unused")
+ public void setupShellEnvironment(ImmutableMap.Builder<String, String> builder) {
+ }
+
+ /**
+ * Add mappings from generally available tool names (like "sh") to their paths
+ * that actions can access.
+ */
+ @SuppressWarnings("unused")
+ public void defineExecutables(ImmutableMap.Builder<String, PathFragment> builder) {
+ }
+
+ /**
+ * Returns { 'option name': 'alternative default' } entries for options where the
+ * "real default" should be something besides the default specified in the {@link Option}
+ * declaration.
+ */
+ public Map<String, Object> lateBoundOptionDefaults() {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * Declares dependencies on any relevant Skyframe values (for example, relevant FileValues).
+ *
+ * @param env the skyframe environment
+ */
+ public void declareSkyframeDependencies(Environment env) {
+ }
+ }
+
+ /**
+ * A converter from strings to Labels.
+ */
+ public static class LabelConverter implements Converter<Label> {
+ @Override
+ public Label convert(String input) throws OptionsParsingException {
+ try {
+ // Check if the input starts with '/'. We don't check for "//" so that
+ // we get a better error message if the user accidentally tries to use
+ // an absolute path (starting with '/') for a label.
+ if (!input.startsWith("/")) {
+ input = "//" + input;
+ }
+ return Label.parseAbsolute(input);
+ } catch (SyntaxException e) {
+ throw new OptionsParsingException(e.getMessage());
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a build target label";
+ }
+ }
+
+ public static class PluginOptionConverter implements Converter<Map.Entry<String, String>> {
+ @Override
+ public Map.Entry<String, String> convert(String input) throws OptionsParsingException {
+ int index = input.indexOf('=');
+ if (index == -1) {
+ throw new OptionsParsingException("Plugin option not in the plugin=option format");
+ }
+ String option = input.substring(0, index);
+ String value = input.substring(index + 1);
+ return Maps.immutableEntry(option, value);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "An option for a plugin";
+ }
+ }
+
+ public static class RunsPerTestConverter extends PerLabelOptions.PerLabelOptionsConverter {
+ @Override
+ public PerLabelOptions convert(String input) throws OptionsParsingException {
+ try {
+ return parseAsInteger(input);
+ } catch (NumberFormatException ignored) {
+ return parseAsRegex(input);
+ }
+ }
+
+ private PerLabelOptions parseAsInteger(String input)
+ throws NumberFormatException, OptionsParsingException {
+ int numericValue = Integer.parseInt(input);
+ if (numericValue <= 0) {
+ throw new OptionsParsingException("'" + input + "' should be >= 1");
+ } else {
+ RegexFilter catchAll = new RegexFilter(Collections.singletonList(".*"),
+ Collections.<String>emptyList());
+ return new PerLabelOptions(catchAll, Collections.singletonList(input));
+ }
+ }
+
+ private PerLabelOptions parseAsRegex(String input) throws OptionsParsingException {
+ PerLabelOptions testRegexps = super.convert(input);
+ if (testRegexps.getOptions().size() != 1) {
+ throw new OptionsParsingException(
+ "'" + input + "' has multiple runs for a single pattern");
+ }
+ String runsPerTest = Iterables.getOnlyElement(testRegexps.getOptions());
+ try {
+ int numericRunsPerTest = Integer.parseInt(runsPerTest);
+ if (numericRunsPerTest <= 0) {
+ throw new OptionsParsingException("'" + input + "' has a value < 1");
+ }
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("'" + input + "' has a non-numeric value", e);
+ }
+ return testRegexps;
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a positive integer or test_regex@runs. This flag may be passed more than once";
+ }
+ }
+
+ /**
+ * Values for the --strict_*_deps option
+ */
+ public static enum StrictDepsMode {
+ /** Silently allow referencing transitive dependencies. */
+ OFF,
+ /** Warn about transitive dependencies being used directly. */
+ WARN,
+ /** Fail the build when transitive dependencies are used directly. */
+ ERROR,
+ /** Transition to strict by default. */
+ STRICT,
+ /** When no flag value is specified on the command line. */
+ DEFAULT
+ }
+
+ /**
+ * Converter for the --strict_*_deps option.
+ */
+ public static class StrictDepsConverter extends EnumConverter<StrictDepsMode> {
+ public StrictDepsConverter() {
+ super(StrictDepsMode.class, "strict dependency checking level");
+ }
+ }
+
+ /**
+ * Options that affect the value of a BuildConfiguration instance.
+ *
+ * <p>(Note: any client that creates a view will also need to declare
+ * BuildView.Options, which affect the <i>mechanism</i> of view construction,
+ * even if they don't affect the value of the BuildConfiguration instances.)
+ *
+ * <p>IMPORTANT: when adding new options, be sure to consider whether those
+ * values should be propagated to the host configuration or not (see
+ * {@link ConfigurationFactory#getConfiguration}.
+ *
+ * <p>ALSO IMPORTANT: all option types MUST define a toString method that
+ * gives identical results for semantically identical option values. The
+ * simplest way to ensure that is to return the input string.
+ */
+ public static class Options extends FragmentOptions implements Cloneable {
+ public String getCpu() {
+ return cpu;
+ }
+
+ @Option(name = "cpu",
+ defaultValue = "null",
+ category = "semantics",
+ help = "The target CPU.")
+ public String cpu;
+
+ @Option(name = "min_param_file_size",
+ defaultValue = "32768",
+ category = "undocumented",
+ help = "Minimum command line length before creating a parameter file.")
+ public int minParamFileSize;
+
+ @Option(name = "experimental_extended_sanity_checks",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Enables internal validation checks to make sure that configured target "
+ + "implementations only access things they should. Causes a performance hit.")
+ public boolean extendedSanityChecks;
+
+ @Option(name = "experimental_allow_runtime_deps_on_neverlink",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Flag to help transition from allowing to disallowing runtime_deps on neverlink"
+ + " Java archives. The depot needs to be cleaned up to roll this out by default.")
+ public boolean allowRuntimeDepsOnNeverLink;
+
+ @Option(name = "strict_filesets",
+ defaultValue = "false",
+ category = "semantics",
+ help = "If this option is enabled, filesets crossing package boundaries are reported "
+ + "as errors. It does not work when check_fileset_dependencies_recursively is "
+ + "disabled.")
+ public boolean strictFilesets;
+
+ // Plugins are build using the host config. To avoid cycles we just don't propagate
+ // this option to the host config. If one day we decide to use plugins when building
+ // host tools, we can improve this by (for example) creating a compiler configuration that is
+ // used only for building plugins.
+ @Option(name = "plugin",
+ converter = LabelConverter.class,
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Plugins to use in the build. Currently works with java_plugin.")
+ public List<Label> pluginList;
+
+ @Option(name = "plugin_copt",
+ converter = PluginOptionConverter.class,
+ allowMultiple = true,
+ category = "flags",
+ defaultValue = ":",
+ help = "Plugin options")
+ public List<Map.Entry<String, String>> pluginCoptList;
+
+ @Option(name = "stamp",
+ defaultValue = "true",
+ category = "semantics",
+ help = "Stamp binaries with the date, username, hostname, workspace information, etc.")
+ public boolean stampBinaries;
+
+ // TODO(bazel-team): delete from OSS tree
+ @Option(name = "instrumentation_filter",
+ converter = RegexFilter.RegexFilterConverter.class,
+ defaultValue = "-javatests,-_test$",
+ category = "semantics",
+ help = "When coverage is enabled, only rules with names included by the "
+ + "specified regex-based filter will be instrumented. Rules prefixed "
+ + "with '-' are excluded instead. By default, rules containing "
+ + "'javatests' or ending with '_test' will not be instrumented.")
+ public RegexFilter instrumentationFilter;
+
+ @Option(name = "show_cached_analysis_results",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Bazel reruns a static analysis only if it detects changes in the analysis "
+ + "or its dependencies. If this option is enabled, Bazel will show the analysis' "
+ + "results, even if it did not rerun the analysis. If this option is disabled, "
+ + "Bazel will show analysis results only if it reran the analysis.")
+ public boolean showCachedAnalysisResults;
+
+ @Option(name = "host_cpu",
+ defaultValue = "null",
+ category = "semantics",
+ help = "The host CPU.")
+ public String hostCpu;
+
+ @Option(name = "compilation_mode",
+ abbrev = 'c',
+ converter = CompilationMode.Converter.class,
+ defaultValue = "fastbuild",
+ category = "semantics", // Should this be "flags"?
+ help = "Specify the mode the binary will be built in. "
+ + "Values: 'fastbuild', 'dbg', 'opt'.")
+ public CompilationMode compilationMode;
+
+ /**
+ * This option is used internally to set the short name (see {@link
+ * #getShortName()}) of the <i>host</i> configuration to a constant, so
+ * that the output files for the host are completely independent of those
+ * for the target, no matter what options are in force (k8/piii, opt/dbg,
+ * etc).
+ */
+ @Option(name = "configuration short name", // (Spaces => can't be specified on command line.)
+ defaultValue = "null",
+ category = "undocumented")
+ public String shortName;
+
+ @Option(name = "platform_suffix",
+ defaultValue = "null",
+ category = "misc",
+ help = "Specifies a suffix to be added to the configuration directory.")
+ public String platformSuffix;
+
+ @Option(name = "test_env",
+ converter = Converters.OptionalAssignmentConverter.class,
+ allowMultiple = true,
+ defaultValue = "",
+ category = "testing",
+ help = "Specifies additional environment variables to be injected into the test runner "
+ + "environment. Variables can be either specified by name, in which case its value "
+ + "will be read from the Bazel client environment, or by the name=value pair. "
+ + "This option can be used multiple times to specify several variables. "
+ + "Used only by the 'bazel test' command."
+ )
+ public List<Map.Entry<String, String>> testEnvironment;
+
+ @Option(name = "collect_code_coverage",
+ defaultValue = "false",
+ category = "testing",
+ help = "If specified, Bazel will instrument code (using offline instrumentation where "
+ + "possible) and will collect coverage information during tests. Only targets that "
+ + " match --instrumentation_filter will be affected. Usually this option should "
+ + " not be specified directly - 'bazel coverage' command should be used instead."
+ )
+ public boolean collectCodeCoverage;
+
+ @Option(name = "microcoverage",
+ defaultValue = "false",
+ category = "testing",
+ help = "If specified with coverage, Blaze will collect microcoverage (per test method "
+ + "coverage) information during tests. Only targets that match "
+ + "--instrumentation_filter will be affected. Usually this option should not be "
+ + "specified directly - 'blaze coverage --microcoverage' command should be used "
+ + "instead."
+ )
+ public boolean collectMicroCoverage;
+
+ @Option(name = "cache_test_results",
+ defaultValue = "auto",
+ category = "testing",
+ abbrev = 't', // it's useful to toggle this on/off quickly
+ help = "If 'auto', Bazel will only rerun a test if any of the following conditions apply: "
+ + "(1) Bazel detects changes in the test or its dependencies "
+ + "(2) the test is marked as external "
+ + "(3) multiple test runs were requested with --runs_per_test"
+ + "(4) the test failed"
+ + "If 'yes', the caching behavior will be the same as 'auto' except that "
+ + "it may cache test failures and test runs with --runs_per_test."
+ + "If 'no', all tests will be always executed.")
+ public TriState cacheTestResults;
+
+ @Deprecated
+ @Option(name = "test_result_expiration",
+ defaultValue = "-1", // No expiration by defualt.
+ category = "testing",
+ help = "This option is deprecated and has no effect.")
+ public int testResultExpiration;
+
+ @Option(name = "test_sharding_strategy",
+ defaultValue = "explicit",
+ category = "testing",
+ converter = TestActionBuilder.ShardingStrategyConverter.class,
+ help = "Specify strategy for test sharding: "
+ + "'explicit' to only use sharding if the 'shard_count' BUILD attribute is present. "
+ + "'disabled' to never use test sharding. "
+ + "'experimental_heuristic' to enable sharding on remotely executed tests without an "
+ + "explicit 'shard_count' attribute which link in a supported framework. Considered "
+ + "experimental.")
+ public TestActionBuilder.TestShardingStrategy testShardingStrategy;
+
+ @Option(name = "runs_per_test",
+ allowMultiple = true,
+ defaultValue = "1",
+ category = "testing",
+ converter = RunsPerTestConverter.class,
+ help = "Specifies number of times to run each test. If any of those attempts "
+ + "fail for any reason, the whole test would be considered failed. "
+ + "Normally the value specified is just an integer. Example: --runs_per_test=3 "
+ + "will run all tests 3 times. "
+ + "Alternate syntax: regex_filter@runs_per_test. Where runs_per_test stands for "
+ + "an integer value and regex_filter stands "
+ + "for a list of include and exclude regular expression patterns (Also see "
+ + "--instrumentation_filter). Example: "
+ + "--runs_per_test=//foo/.*,-//foo/bar/.*@3 runs all tests in //foo/ "
+ + "except those under foo/bar three times. "
+ + "This option can be passed multiple times. ")
+ public List<PerLabelOptions> runsPerTest;
+
+ @Option(name = "build_runfile_links",
+ defaultValue = "true",
+ category = "strategy",
+ help = "If true, build runfiles symlink forests for all targets. "
+ + "If false, write only manifests when possible.")
+ public boolean buildRunfiles;
+
+ @Option(name = "test_arg",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "testing",
+ help = "Specifies additional options and arguments that should be passed to the test "
+ + "executable. Can be used multiple times to specify several arguments. "
+ + "If multiple tests are executed, each of them will receive identical arguments. "
+ + "Used only by the 'bazel test' command."
+ )
+ public List<String> testArguments;
+
+ @Option(name = "test_filter",
+ allowMultiple = false,
+ defaultValue = "null",
+ category = "testing",
+ help = "Specifies a filter to forward to the test framework. Used to limit "
+ + "the tests run. Note that this does not affect which targets are built.")
+ public String testFilter;
+
+ @Option(name = "check_fileset_dependencies_recursively",
+ defaultValue = "true",
+ category = "semantics",
+ help = "If false, fileset targets will, whenever possible, create "
+ + "symlinks to directories instead of creating one symlink for each "
+ + "file inside the directory. Disabling this will significantly "
+ + "speed up fileset builds, but targets that depend on filesets will "
+ + "not be rebuilt if files are added, removed or modified in a "
+ + "subdirectory which has not been traversed.")
+ public boolean checkFilesetDependenciesRecursively;
+
+ @Option(name = "run_under",
+ category = "run",
+ defaultValue = "null",
+ converter = RunUnderConverter.class,
+ help = "Prefix to insert in front of command before running. "
+ + "Examples:\n"
+ + "\t--run_under=valgrind\n"
+ + "\t--run_under=strace\n"
+ + "\t--run_under='strace -c'\n"
+ + "\t--run_under='valgrind --quiet --num-callers=20'\n"
+ + "\t--run_under=//package:target\n"
+ + "\t--run_under='//package:target --options'\n")
+ public RunUnder runUnder;
+
+ @Option(name = "distinct_host_configuration",
+ defaultValue = "true",
+ category = "strategy",
+ help = "Build all the tools used during the build for a distinct configuration from "
+ + "that used for the target program. By default, the same configuration is used "
+ + "for host and target programs, but this may cause undesirable rebuilds of tool "
+ + "such as the protocol compiler (and then everything downstream) whenever a minor "
+ + "change is made to the target configuration, such as setting the linker options. "
+ + "When this flag is specified, a distinct configuration will be used to build the "
+ + "tools, preventing undesired rebuilds. However, certain libraries will then "
+ + "need to be compiled twice, once for each configuration, which may cause some "
+ + "builds to be slower. As a rule of thumb, this option is likely to benefit "
+ + "users that make frequent changes in configuration (e.g. opt/dbg). "
+ + "Please read the user manual for the full explanation.")
+ public boolean useDistinctHostConfiguration;
+
+ @Option(name = "check_visibility",
+ defaultValue = "true",
+ category = "checking",
+ help = "If disabled, visibility errors are demoted to warnings.")
+ public boolean checkVisibility;
+
+ // Moved from viewOptions to here because license information is very expensive to serialize.
+ // Having it here allows us to skip computation of transitive license information completely
+ // when the setting is disabled.
+ @Option(name = "check_licenses",
+ defaultValue = "false",
+ category = "checking",
+ help = "Check that licensing constraints imposed by dependent packages "
+ + "do not conflict with distribution modes of the targets being built. "
+ + "By default, licenses are not checked.")
+ public boolean checkLicenses;
+
+ @Option(name = "experimental_enforce_constraints",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Checks the environments each target is compatible with and reports errors if any "
+ + "target has dependencies that don't support the same environments")
+ public boolean enforceConstraints;
+
+ @Option(name = "experimental_action_listener",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "experimental",
+ converter = LabelConverter.class,
+ help = "Use action_listener to attach an extra_action to existing build actions.")
+ public List<Label> actionListeners;
+
+ @Option(name = "is host configuration",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Shows whether these options are set for host configuration.")
+ public boolean isHost;
+
+ @Option(name = "experimental_proto_header_modules",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Enables compilation of C++ header modules for proto libraries.")
+ public boolean protoHeaderModules;
+
+ @Option(name = "features",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "The given features will be enabled or disabled by default for all packages. "
+ + "Specifying -<feature> will disable the feature globally. "
+ + "Negative features always override positive ones. "
+ + "This flag is used to enable rolling out default feature changes without a "
+ + "Blaze release.")
+ public List<String> defaultFeatures;
+
+ @Override
+ public FragmentOptions getHost(boolean fallback) {
+ Options host = (Options) getDefault();
+
+ host.shortName = "host";
+ host.compilationMode = CompilationMode.OPT;
+ host.isHost = true;
+
+ if (fallback) {
+ // In the fallback case, we have already tried the target options and they didn't work, so
+ // now we try the default options; the hostCpu field has the default value, because we use
+ // getDefault() above.
+ host.cpu = computeHostCpu(host.hostCpu);
+ } else {
+ host.cpu = computeHostCpu(hostCpu);
+ }
+
+ // === Runfiles ===
+ // Ideally we could force this the other way, and skip runfiles construction
+ // for host tools which are never run locally, but that's probably a very
+ // small optimization.
+ host.buildRunfiles = true;
+
+ // === Linkstamping ===
+ // Disable all link stamping for the host configuration, to improve action
+ // cache hit rates for tools.
+ host.stampBinaries = false;
+
+ // === Visibility ===
+ host.checkVisibility = checkVisibility;
+
+ // === Licenses ===
+ host.checkLicenses = checkLicenses;
+
+ // === Allow runtime_deps to depend on neverlink Java libraries.
+ host.allowRuntimeDepsOnNeverLink = allowRuntimeDepsOnNeverLink;
+
+ // === Pass on C++ compiler features.
+ host.defaultFeatures = ImmutableList.copyOf(defaultFeatures);
+
+ return host;
+ }
+
+ private static String computeHostCpu(String explicitHostCpu) {
+ if (explicitHostCpu != null) {
+ return explicitHostCpu;
+ }
+ switch (OS.getCurrent()) {
+ case DARWIN:
+ return "darwin";
+ default:
+ return "k8";
+ }
+ }
+
+ @Override
+ public void addAllLabels(Multimap<String, Label> labelMap) {
+ labelMap.putAll("action_listener", actionListeners);
+ labelMap.putAll("plugins", pluginList);
+ if ((runUnder != null) && (runUnder.getLabel() != null)) {
+ labelMap.put("RunUnder", runUnder.getLabel());
+ }
+ }
+ }
+
+ /**
+ * A list of build configurations that only contains the null element.
+ */
+ private static final List<BuildConfiguration> NULL_LIST =
+ Collections.unmodifiableList(Arrays.asList(new BuildConfiguration[] { null }));
+
+ private final String cacheKey;
+ private final String shortCacheKey;
+
+ private Transitions transitions;
+ private Set<BuildConfiguration> allReachableConfigurations;
+
+ private final ImmutableMap<Class<? extends Fragment>, Fragment> fragments;
+
+ // Directories in the output tree
+ private final Root outputDirectory; // the configuration-specific output directory.
+ private final Root binDirectory;
+ private final Root genfilesDirectory;
+ private final Root coverageMetadataDirectory; // for coverage-related metadata, artifacts, etc.
+ private final Root testLogsDirectory;
+ private final Root includeDirectory;
+ private final Root middlemanDirectory;
+
+ private final PathFragment binFragment;
+ private final PathFragment genfilesFragment;
+
+ // If false, AnalysisEnviroment doesn't register any actions created by the ConfiguredTarget.
+ private final boolean actionsEnabled;
+
+ private final ImmutableSet<Label> coverageLabels;
+ private final ImmutableSet<Label> coverageReportGeneratorLabels;
+
+ // Executables like "perl" or "sh"
+ private final ImmutableMap<String, PathFragment> executables;
+
+ // All the "defglobals" in //tools:GLOBALS for this platform/configuration:
+ private final ImmutableMap<String, String> globalMakeEnv;
+
+ private final ImmutableMap<String, String> defaultShellEnvironment;
+ private final BuildOptions buildOptions;
+ private final Options options;
+
+ private final String shortName;
+ private final String mnemonic;
+ private final String platformName;
+
+ /**
+ * It is not fingerprinted because it should only be used to access
+ * variables that do not break the hermetism of build rules.
+ */
+ private final ImmutableMap<String, String> clientEnvironment;
+
+ /**
+ * Helper container for {@link #transitiveOptionsMap} below.
+ */
+ private static class OptionDetails implements Serializable {
+ private OptionDetails(Class<? extends OptionsBase> optionsClass, Object value,
+ boolean allowsMultiple) {
+ this.optionsClass = optionsClass;
+ this.value = value;
+ this.allowsMultiple = allowsMultiple;
+ }
+
+ /** The {@link FragmentOptions} class that defines this option. */
+ private final Class<? extends OptionsBase> optionsClass;
+
+ /**
+ * The value of the given option (either explicitly defined or default). May be null.
+ */
+ private final Object value;
+
+ /** Whether or not this option supports multiple values. */
+ private final boolean allowsMultiple;
+ }
+
+ /**
+ * Maps option names to the {@link OptionDetails} the option takes for this configuration.
+ *
+ * <p>This can be used to:
+ * <ol>
+ * <li>Find an option's (parsed) value given its command-line name</li>
+ * <li>Parse alternative values for the option.</li>
+ * </ol>
+ *
+ * <p>This map is "transitive" in that it includes *all* options recognizable by this
+ * configuration, including those defined in child fragments.
+ */
+ private final Map<String, OptionDetails> transitiveOptionsMap;
+
+
+ /**
+ * Validates the options for this BuildConfiguration. Issues warnings for the
+ * use of deprecated options, and warnings or errors for any option settings
+ * that conflict.
+ */
+ public void reportInvalidOptions(EventHandler reporter) {
+ for (Fragment fragment : fragments.values()) {
+ fragment.reportInvalidOptions(reporter, this.buildOptions);
+ }
+
+ Set<String> plugins = new HashSet<>();
+ for (Label plugin : options.pluginList) {
+ String name = plugin.getName();
+ if (plugins.contains(name)) {
+ reporter.handle(Event.error("A build cannot have two plugins with the same name"));
+ }
+ plugins.add(name);
+ }
+ for (Map.Entry<String, String> opt : options.pluginCoptList) {
+ if (!plugins.contains(opt.getKey())) {
+ reporter.handle(Event.error("A plugin_copt must refer to an existing plugin"));
+ }
+ }
+
+ if (options.shortName != null) {
+ reporter.handle(Event.error(
+ "The internal '--configuration short name' option cannot be used on the command line"));
+ }
+
+ if (options.testShardingStrategy
+ == TestActionBuilder.TestShardingStrategy.EXPERIMENTAL_HEURISTIC) {
+ reporter.handle(Event.warn(
+ "Heuristic sharding is intended as a one-off experimentation tool for determing the "
+ + "benefit from sharding certain tests. Please don't keep this option in your "
+ + ".blazerc or continuous build"));
+ }
+ }
+
+ private ImmutableMap<String, String> setupShellEnvironment() {
+ ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
+ for (Fragment fragment : fragments.values()) {
+ fragment.setupShellEnvironment(builder);
+ }
+ return builder.build();
+ }
+
+ BuildConfiguration(BlazeDirectories directories,
+ Map<Class<? extends Fragment>, Fragment> fragmentsMap,
+ BuildOptions buildOptions,
+ Map<String, String> clientEnv,
+ boolean actionsDisabled) {
+ this.actionsEnabled = !actionsDisabled;
+ fragments = ImmutableMap.copyOf(fragmentsMap);
+
+ // This is a view that will be updated upon each client command.
+ this.clientEnvironment = ImmutableMap.copyOf(clientEnv);
+
+ this.buildOptions = buildOptions;
+ this.options = buildOptions.get(Options.class);
+
+ this.mnemonic = buildMnemonic();
+ String outputDirName = (options.shortName != null) ? options.shortName : mnemonic;
+ this.shortName = buildShortName(outputDirName);
+
+ this.executables = collectExecutables();
+
+ Path execRoot = directories.getExecRoot();
+ // configuration-specific output tree
+ Path outputDir = directories.getOutputPath().getRelative(outputDirName);
+ this.outputDirectory = Root.asDerivedRoot(execRoot, outputDir);
+
+ // specific subdirs under outputDirectory
+ this.binDirectory = Root.asDerivedRoot(execRoot, outputDir.getRelative("bin"));
+ this.genfilesDirectory = Root.asDerivedRoot(execRoot, outputDir.getRelative("genfiles"));
+ this.coverageMetadataDirectory = Root.asDerivedRoot(execRoot,
+ outputDir.getRelative("coverage-metadata"));
+ this.testLogsDirectory = Root.asDerivedRoot(execRoot, outputDir.getRelative("testlogs"));
+ this.includeDirectory = Root.asDerivedRoot(execRoot,
+ outputDir.getRelative(BlazeDirectories.RELATIVE_INCLUDE_DIR));
+ this.middlemanDirectory = Root.middlemanRoot(execRoot, outputDir);
+
+ // precompute some frequently-used relative paths
+ this.binFragment = getBinDirectory().getExecPath();
+ this.genfilesFragment = getGenfilesDirectory().getExecPath();
+
+ ImmutableSet.Builder<Label> coverageLabelsBuilder = ImmutableSet.builder();
+ ImmutableSet.Builder<Label> coverageReportGeneratorLabelsBuilder = ImmutableSet.builder();
+ for (Fragment fragment : fragments.values()) {
+ coverageLabelsBuilder.addAll(fragment.getCoverageLabels());
+ coverageReportGeneratorLabelsBuilder.addAll(fragment.getCoverageReportGeneratorLabels());
+ }
+ this.coverageLabels = coverageLabelsBuilder.build();
+ this.coverageReportGeneratorLabels = coverageReportGeneratorLabelsBuilder.build();
+
+ // Platform name
+ StringBuilder platformNameBuilder = new StringBuilder();
+ for (Fragment fragment : fragments.values()) {
+ platformNameBuilder.append(fragment.getPlatformName());
+ }
+ this.platformName = platformNameBuilder.toString();
+
+ this.defaultShellEnvironment = setupShellEnvironment();
+
+ this.transitiveOptionsMap = computeOptionsMap(buildOptions, fragments.values());
+
+ ImmutableMap.Builder<String, String> globalMakeEnvBuilder = ImmutableMap.builder();
+ for (Fragment fragment : fragments.values()) {
+ fragment.addGlobalMakeVariables(globalMakeEnvBuilder);
+ }
+
+ // Lots of packages in third_party assume that BINMODE expands to either "-dbg", or "-opt". So
+ // for backwards compatibility we preserve that invariant, setting BINMODE to "-dbg" rather than
+ // "-fastbuild" if the compilation mode is "fastbuild".
+ // We put the real compilation mode in a new variable COMPILATION_MODE.
+ globalMakeEnvBuilder.put("COMPILATION_MODE", options.compilationMode.toString());
+ globalMakeEnvBuilder.put("BINMODE", "-"
+ + ((options.compilationMode == CompilationMode.FASTBUILD)
+ ? "dbg"
+ : options.compilationMode.toString()));
+ /*
+ * Attention! Document these in the build-encyclopedia
+ */
+ // the bin directory and the genfiles directory
+ // These variables will be used on Windows as well, so we need to make sure
+ // that paths use the correct system file-separator.
+ globalMakeEnvBuilder.put("BINDIR", binFragment.getPathString());
+ globalMakeEnvBuilder.put("INCDIR",
+ getIncludeDirectory().getExecPath().getPathString());
+ globalMakeEnvBuilder.put("GENDIR", genfilesFragment.getPathString());
+ globalMakeEnv = globalMakeEnvBuilder.build();
+
+ cacheKey = computeCacheKey(
+ directories, fragmentsMap, this.buildOptions, this.clientEnvironment);
+ shortCacheKey = shortName + "-" + Fingerprint.md5Digest(cacheKey);
+ }
+
+
+ /**
+ * Computes and returns the transitive optionName -> "option info" map for
+ * this configuration.
+ */
+ private static Map<String, OptionDetails> computeOptionsMap(BuildOptions buildOptions,
+ Iterable<Fragment> fragments) {
+ // Collect from our fragments "alternative defaults" for options where the default
+ // should be something other than what's specified in Option.defaultValue.
+ Map<String, Object> lateBoundDefaults = Maps.newHashMap();
+ for (Fragment fragment : fragments) {
+ lateBoundDefaults.putAll(fragment.lateBoundOptionDefaults());
+ }
+
+ ImmutableMap.Builder<String, OptionDetails> map = ImmutableMap.builder();
+ try {
+ for (FragmentOptions options : buildOptions.getOptions()) {
+ for (Field field : options.getClass().getFields()) {
+ if (field.isAnnotationPresent(Option.class)) {
+ Option option = field.getAnnotation(Option.class);
+ Object value = field.get(options);
+ if (value == null) {
+ if (lateBoundDefaults.containsKey(option.name())) {
+ value = lateBoundDefaults.get(option.name());
+ } else if (!option.defaultValue().equals("null")) {
+ // See {@link Option#defaultValue} for an explanation of default "null" strings.
+ value = option.defaultValue();
+ }
+ }
+ map.put(option.name(),
+ new OptionDetails(options.getClass(), value, option.allowMultiple()));
+ }
+ }
+ }
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(
+ "Unexpected illegal access trying to create this configuration's options map: ", e);
+ }
+ return map.build();
+ }
+
+ private String buildShortName(String outputDirName) {
+ ArrayList<String> nameParts = new ArrayList<>(ImmutableList.of(outputDirName));
+ for (Fragment fragment : fragments.values()) {
+ nameParts.add(fragment.getConfigurationNameSuffix());
+ }
+ return Joiner.on('-').skipNulls().join(nameParts);
+ }
+
+ private String buildMnemonic() {
+ // See explanation at getShortName().
+ String platformSuffix = (options.platformSuffix != null) ? options.platformSuffix : "";
+ ArrayList<String> nameParts = new ArrayList<String>();
+ for (Fragment fragment : fragments.values()) {
+ nameParts.add(fragment.getOutputDirectoryName());
+ }
+ nameParts.add(getCompilationMode() + platformSuffix);
+ return Joiner.on('-').join(Iterables.filter(nameParts, Predicates.notNull()));
+ }
+
+ /**
+ * Set the outgoing configuration transitions. During the lifetime of a given build configuration,
+ * this must happen exactly once, shortly after the configuration is created.
+ * TODO(bazel-team): this makes the object mutable, get rid of it.
+ */
+ public void setConfigurationTransitions(Transitions transitions) {
+ Preconditions.checkNotNull(transitions);
+ Preconditions.checkState(this.transitions == null);
+ this.transitions = transitions;
+ }
+
+ public Transitions getTransitions() {
+ Preconditions.checkState(this.transitions != null || isHostConfiguration());
+ return transitions;
+ }
+
+ /**
+ * Returns all configurations that can be reached from this configuration through any kind of
+ * configuration transition.
+ */
+ public synchronized Collection<BuildConfiguration> getAllReachableConfigurations() {
+ if (allReachableConfigurations == null) {
+ // This is needed for every configured target in skyframe m2, so we cache it.
+ // We could alternatively make the corresponding dependencies into a skyframe node.
+ this.allReachableConfigurations = computeAllReachableConfigurations();
+ }
+ return allReachableConfigurations;
+ }
+
+ /**
+ * Returns all configurations that can be reached from this configuration through any kind of
+ * configuration transition.
+ */
+ private Set<BuildConfiguration> computeAllReachableConfigurations() {
+ Set<BuildConfiguration> result = new LinkedHashSet<>();
+ Queue<BuildConfiguration> queue = new LinkedList<>();
+ queue.add(this);
+ while (!queue.isEmpty()) {
+ BuildConfiguration config = queue.remove();
+ if (!result.add(config)) {
+ continue;
+ }
+ config.getTransitions().addDirectlyReachableConfigurations(queue);
+ }
+ return result;
+ }
+
+ /**
+ * Returns the new configuration after traversing a dependency edge with a given configuration
+ * transition.
+ *
+ * @param transition the configuration transition
+ * @return the new configuration
+ * @throws IllegalArgumentException if the transition is a {@link SplitTransition}
+ */
+ public BuildConfiguration getConfiguration(Transition transition) {
+ Preconditions.checkArgument(!(transition instanceof SplitTransition));
+ return transitions.getConfiguration(transition);
+ }
+
+ /**
+ * Returns the new configurations after traversing a dependency edge with a given split
+ * transition.
+ *
+ * @param transition the split configuration transition
+ * @return the new configurations
+ */
+ public List<BuildConfiguration> getSplitConfigurations(SplitTransition<?> transition) {
+ return transitions.getSplitConfigurations(transition);
+ }
+
+ /**
+ * Calculates the configurations of a direct dependency. If a rule in some BUILD file refers
+ * to a target (like another rule or a source file) using a label attribute, that target needs
+ * to have a configuration, too. This method figures out the proper configuration for the
+ * dependency.
+ *
+ * @param fromRule the rule that's depending on some target
+ * @param attribute the attribute using which the rule depends on that target (eg. "srcs")
+ * @param toTarget the target that's dependeded on
+ * @return the configuration that should be associated to {@code toTarget}
+ */
+ public Iterable<BuildConfiguration> evaluateTransition(final Rule fromRule,
+ final Attribute attribute, final Target toTarget) {
+ // Fantastic configurations and where to find them:
+
+ // I. Input files and package groups have no configurations. We don't want to duplicate them.
+ if (toTarget instanceof InputFile || toTarget instanceof PackageGroup) {
+ return NULL_LIST;
+ }
+
+ // II. Host configurations never switch to another. All prerequisites of host targets have the
+ // same host configuration.
+ if (isHostConfiguration()) {
+ return ImmutableList.of(this);
+ }
+
+ // Make sure config_setting dependencies are resolved in the referencing rule's configuration,
+ // unconditionally. For example, given:
+ //
+ // genrule(
+ // name = 'myrule',
+ // tools = select({ '//a:condition': [':sometool'] })
+ //
+ // all labels in "tools" get resolved in the host configuration (since the "tools" attribute
+ // declares a host configuration transition). We want to explicitly exclude configuration labels
+ // from these transitions, since their *purpose* is to do computation on the owning
+ // rule's configuration.
+ // TODO(bazel-team): implement this more elegantly. This is far too hackish. Specifically:
+ // don't reference the rule name explicitly and don't require special-casing here.
+ if (toTarget instanceof Rule && ((Rule) toTarget).getRuleClass().equals("config_setting")) {
+ return ImmutableList.of(this);
+ }
+
+ List<BuildConfiguration> toConfigurations;
+ if (attribute.getConfigurationTransition() instanceof SplitTransition) {
+ Preconditions.checkState(attribute.getConfigurator() == null);
+ toConfigurations = getSplitConfigurations(
+ (SplitTransition<?>) attribute.getConfigurationTransition());
+ } else {
+ // III. Attributes determine configurations. The configuration of a prerequisite is determined
+ // by the attribute.
+ @SuppressWarnings("unchecked")
+ Configurator<BuildConfiguration, Rule> configurator =
+ (Configurator<BuildConfiguration, Rule>) attribute.getConfigurator();
+ toConfigurations = ImmutableList.of((configurator != null)
+ ? configurator.apply(fromRule, this, attribute, toTarget)
+ : getConfiguration(attribute.getConfigurationTransition()));
+ }
+
+ return Iterables.transform(toConfigurations,
+ new Function<BuildConfiguration, BuildConfiguration>() {
+ @Override
+ public BuildConfiguration apply(BuildConfiguration input) {
+ // IV. Allow the transition object to perform an arbitrary switch. Blaze modules can inject
+ // configuration transition logic by extending the Transitions class.
+ BuildConfiguration actual = getTransitions().configurationHook(
+ fromRule, attribute, toTarget, input);
+
+ // V. Allow rule classes to override their own configurations.
+ Rule associatedRule = toTarget.getAssociatedRule();
+ if (associatedRule != null) {
+ @SuppressWarnings("unchecked")
+ RuleClass.Configurator<BuildConfiguration, Rule> func =
+ associatedRule.getRuleClassObject().<BuildConfiguration, Rule>getConfigurator();
+ actual = func.apply(associatedRule, actual);
+ }
+
+ return actual;
+ }
+ });
+ }
+
+ /**
+ * Returns a multimap of all labels that should be implicitly loaded from labels that were
+ * specified as options, keyed by the name to be displayed to the user if something goes wrong.
+ * The returned set only contains labels that were derived from command-line options; the
+ * intention is that it can be used to sanity-check that the command-line options actually contain
+ * these in their transitive closure.
+ */
+ public ListMultimap<String, Label> getImplicitLabels() {
+ ListMultimap<String, Label> implicitLabels = ArrayListMultimap.create();
+ for (Fragment fragment : fragments.values()) {
+ fragment.addImplicitLabels(implicitLabels);
+ }
+ return implicitLabels;
+ }
+
+ /**
+ * For an given environment, returns a subset containing all
+ * variables in the given list if they are defined in the given
+ * environment.
+ */
+ @VisibleForTesting
+ static Map<String, String> getMapping(List<String> variables,
+ Map<String, String> environment) {
+ Map<String, String> result = new HashMap<>();
+ for (String var : variables) {
+ if (environment.containsKey(var)) {
+ result.put(var, environment.get(var));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Avoid this method. The client environment is not part of the configuration's signature, so
+ * calls to this method introduce a non-hermetic access to data that is not visible to Skyframe.
+ *
+ * @return an unmodifiable view of the bazel client's environment
+ * upon its most recent request.
+ */
+ // TODO(bazel-team): Remove this.
+ public Map<String, String> getClientEnv() {
+ return clientEnvironment;
+ }
+
+ /**
+ * Returns the {@link Option} class the defines the given option, null if the
+ * option isn't recognized.
+ *
+ * <p>optionName is the name of the option as it appears on the command line
+ * e.g. {@link Option#name}).
+ */
+ Class<? extends OptionsBase> getOptionClass(String optionName) {
+ OptionDetails optionData = transitiveOptionsMap.get(optionName);
+ return optionData == null ? null : optionData.optionsClass;
+ }
+
+ /**
+ * Returns the value of the specified option for this configuration or null if the
+ * option isn't recognized. Since an option's legitimate value could be null, use
+ * {@link #getOptionClass} to distinguish between that and an unknown option.
+ *
+ * <p>optionName is the name of the option as it appears on the command line
+ * e.g. {@link Option#name}).
+ */
+ Object getOptionValue(String optionName) {
+ OptionDetails optionData = transitiveOptionsMap.get(optionName);
+ return (optionData == null) ? null : optionData.value;
+ }
+
+ /**
+ * Returns whether or not the given option supports multiple values at the command line (e.g.
+ * "--myoption value1 --myOption value2 ..."). Returns false for unrecognized options. Use
+ * {@link #getOptionClass} to distinguish between those and legitimate single-value options.
+ *
+ * <p>As declared in {@link Option#allowMultiple}, multi-value options are expected to be
+ * of type {@code List<T>}.
+ */
+ boolean allowsMultipleValues(String optionName) {
+ OptionDetails optionData = transitiveOptionsMap.get(optionName);
+ return (optionData == null) ? false : optionData.allowsMultiple;
+ }
+
+ /**
+ * The platform string, suitable for use as a key into a MakeEnvironment.
+ */
+ public String getPlatformName() {
+ return platformName;
+ }
+
+ /**
+ * Returns the output directory for this build configuration.
+ */
+ public Root getOutputDirectory() {
+ return outputDirectory;
+ }
+
+ /**
+ * Returns the bin directory for this build configuration.
+ */
+ @SkylarkCallable(name = "bin_dir", structField = true,
+ doc = "The root corresponding to bin directory.")
+ public Root getBinDirectory() {
+ return binDirectory;
+ }
+
+ /**
+ * Returns a relative path to the bin directory at execution time.
+ */
+ public PathFragment getBinFragment() {
+ return binFragment;
+ }
+
+ /**
+ * Returns the include directory for this build configuration.
+ */
+ public Root getIncludeDirectory() {
+ return includeDirectory;
+ }
+
+ /**
+ * Returns the genfiles directory for this build configuration.
+ */
+ @SkylarkCallable(name = "genfiles_dir", structField = true,
+ doc = "The root corresponding to genfiles directory.")
+ public Root getGenfilesDirectory() {
+ return genfilesDirectory;
+ }
+
+ /**
+ * Returns the directory where coverage-related artifacts and metadata files
+ * should be stored. This includes for example uninstrumented class files
+ * needed for Jacoco's coverage reporting tools.
+ */
+ public Root getCoverageMetadataDirectory() {
+ return coverageMetadataDirectory;
+ }
+
+ /**
+ * Returns the testlogs directory for this build configuration.
+ */
+ public Root getTestLogsDirectory() {
+ return testLogsDirectory;
+ }
+
+ /**
+ * Returns a relative path to the genfiles directory at execution time.
+ */
+ public PathFragment getGenfilesFragment() {
+ return genfilesFragment;
+ }
+
+ /**
+ * Returns the path separator for the host platform. This is basically the same as {@link
+ * java.io.File#pathSeparator}, except that that returns the value for this JVM, which may or may
+ * not match the host platform. You should only use this when invoking tools that are known to use
+ * the native path separator, i.e., the path separator for the machine that they run on.
+ */
+ @SkylarkCallable(name = "host_path_separator", structField = true,
+ doc = "Returns the separator for PATH variable, which is ':' on Unix.")
+ public String getHostPathSeparator() {
+ // TODO(bazel-team): This needs to change when we support Windows.
+ return ":";
+ }
+
+ /**
+ * Returns the internal directory (used for middlemen) for this build configuration.
+ */
+ public Root getMiddlemanDirectory() {
+ return middlemanDirectory;
+ }
+
+ public boolean getAllowRuntimeDepsOnNeverLink() {
+ return options.allowRuntimeDepsOnNeverLink;
+ }
+
+ public boolean isStrictFilesets() {
+ return options.strictFilesets;
+ }
+
+ public List<Label> getPlugins() {
+ return options.pluginList;
+ }
+
+ public List<Map.Entry<String, String>> getPluginCopts() {
+ return options.pluginCoptList;
+ }
+
+ /**
+ * Implements a non-injective mapping from BuildConfiguration instances to
+ * strings. The result should identify the aspects of the configuration
+ * that should be reflected in the output file names. Furthermore the
+ * returned string must not contain shell metacharacters.
+ *
+ * <p>The intention here is that we use this string as the directory name
+ * for artifacts of this build.
+ *
+ * <p>For configuration settings which are NOT part of the short name,
+ * rebuilding with a different value of such a setting will build in
+ * the same output directory. This means that any actions whose
+ * keys (see Action.getKey()) have changed will be rerun. That
+ * may result in a lot of recompilation.
+ *
+ * <p>For configuration settings which ARE part of the short name,
+ * rebuilding with a different value of such a setting will rebuild
+ * in a different output directory; this will result in higher disk
+ * usage and more work the _first_ time you rebuild with a different
+ * setting, but will result in less work if you regularly switch
+ * back and forth between different settings.
+ *
+ * <p>With one important exception, it's sound to choose any subset of the
+ * config's components for this string, it just alters the dimensionality
+ * of the cache. In other words, it's a trade-off on the "injectiveness"
+ * scale: at one extreme (shortName is in fact a complete fingerprint, and
+ * thus injective) you get extremely precise caching (no competition for the
+ * same output-file locations) but you have to rebuild for even the
+ * slightest change in configuration. At the other extreme
+ * (PartialFingerprint is a constant) you have very high competition for
+ * output-file locations, but if a slight change in configuration doesn't
+ * affect a particular build step, you're guaranteed not to have to
+ * rebuild it. The important exception has to do with cross-compilation:
+ * the host and target configurations must not map to the same output
+ * directory, because then files would need to get built for the host
+ * and then rebuilt for the target even within a single build, and that
+ * wouldn't work.
+ *
+ * <p>Just to re-iterate: cross-compilation builds (i.e. hostConfig !=
+ * targetConfig) will not work if the two configurations' short names are
+ * equal. This is an important practical case: the mere addition of
+ * a compile flag to the target configuration would cause the build to
+ * fail. In other words, it would break if the host and target
+ * configurations are not identical but are "too close". The current
+ * solution is to set the host configuration equal to the target
+ * configuration if they are "too close"; this may cause the tools to get
+ * rebuild for the new host configuration though.
+ */
+ public String getShortName() {
+ return shortName;
+ }
+
+ /**
+ * Like getShortName(), but always returns a configuration-dependent string even for
+ * the host configuration.
+ */
+ public String getMnemonic() {
+ return mnemonic;
+ }
+
+ @Override
+ public String toString() {
+ return getShortName();
+ }
+
+ /**
+ * Returns the default shell environment
+ */
+ @SkylarkCallable(name = "default_shell_env", structField = true,
+ doc = "A dictionary representing the default environment. It maps variables "
+ + "to their values (strings).")
+ public ImmutableMap<String, String> getDefaultShellEnvironment() {
+ return defaultShellEnvironment;
+ }
+
+ /**
+ * Returns the path to sh.
+ */
+ public PathFragment getShExecutable() {
+ return executables.get("sh");
+ }
+
+ /**
+ * Returns a regex-based instrumentation filter instance that used to match label
+ * names to identify targets to be instrumented in the coverage mode.
+ */
+ public RegexFilter getInstrumentationFilter() {
+ return options.instrumentationFilter;
+ }
+
+ /**
+ * Returns the set of labels for coverage.
+ */
+ public Set<Label> getCoverageLabels() {
+ return coverageLabels;
+ }
+
+ /**
+ * Returns the set of labels for the coverage report generator.
+ */
+ public Set<Label> getCoverageReportGeneratorLabels() {
+ return coverageReportGeneratorLabels;
+ }
+
+ /**
+ * Returns true if bazel should show analyses results, even if it did not
+ * re-run the analysis.
+ */
+ public boolean showCachedAnalysisResults() {
+ return options.showCachedAnalysisResults;
+ }
+
+ /**
+ * Returns a new, unordered mapping of names to values of "Make" variables defined by this
+ * configuration.
+ *
+ * <p>This does *not* include package-defined overrides (e.g. vardef)
+ * and so should not be used by the build logic. This is used only for
+ * the 'info' command.
+ *
+ * <p>Command-line definitions of make enviroments override variables defined by
+ * {@code Fragment.addGlobalMakeVariables()}.
+ */
+ public Map<String, String> getMakeEnvironment() {
+ Map<String, String> makeEnvironment = new HashMap<>();
+ makeEnvironment.putAll(globalMakeEnv);
+ for (Fragment fragment : fragments.values()) {
+ makeEnvironment.putAll(fragment.getCommandLineDefines());
+ }
+ return ImmutableMap.copyOf(makeEnvironment);
+ }
+
+ /**
+ * Returns a new, unordered mapping of names that are set through the command lines.
+ * (Fragments, in particular the Google C++ support, can set variables through the
+ * command line.)
+ */
+ public Map<String, String> getCommandLineDefines() {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ for (Fragment fragment : fragments.values()) {
+ builder.putAll(fragment.getCommandLineDefines());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns the global defaults for this configuration for the Make environment.
+ */
+ public Map<String, String> getGlobalMakeEnvironment() {
+ return globalMakeEnv;
+ }
+
+ /**
+ * Returns a (key, value) mapping to insert into the subcommand environment for coverage
+ * actions.
+ */
+ public Map<String, String> getCoverageEnvironment() {
+ Map<String, String> env = new HashMap<>();
+ for (Fragment fragment : fragments.values()) {
+ env.putAll(fragment.getCoverageEnvironment());
+ }
+ return env;
+ }
+
+ /**
+ * Returns the default value for the specified "Make" variable for this
+ * configuration. Returns null if no value was found.
+ */
+ public String getMakeVariableDefault(String var) {
+ return globalMakeEnv.get(var);
+ }
+
+ /**
+ * Returns a configuration fragment instances of the given class.
+ */
+ @SkylarkCallable(name = "fragment", doc = "Returns a configuration fragment using the key.")
+ public <T extends Fragment> T getFragment(Class<T> clazz) {
+ return clazz.cast(fragments.get(clazz));
+ }
+
+ /**
+ * Returns true if the requested configuration fragment is present.
+ */
+ public <T extends Fragment> boolean hasFragment(Class<T> clazz) {
+ return getFragment(clazz) != null;
+ }
+
+ /**
+ * Returns true if all requested configuration fragment are present (this may be slow).
+ */
+ public boolean hasAllFragments(Set<Class<?>> fragmentClasses) {
+ for (Class<?> fragmentClass : fragmentClasses) {
+ if (!hasFragment(fragmentClass.asSubclass(Fragment.class))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns true if non-functional build stamps are enabled.
+ */
+ public boolean stampBinaries() {
+ return options.stampBinaries;
+ }
+
+ /**
+ * Returns true if extended sanity checks should be enabled.
+ */
+ public boolean extendedSanityChecks() {
+ return options.extendedSanityChecks;
+ }
+
+ /**
+ * Returns true if we are building runfiles symlinks for this configuration.
+ */
+ public boolean buildRunfiles() {
+ return options.buildRunfiles;
+ }
+
+ public boolean getCheckFilesetDependenciesRecursively() {
+ return options.checkFilesetDependenciesRecursively;
+ }
+
+ public List<String> getTestArguments() {
+ return options.testArguments;
+ }
+
+ public String getTestFilter() {
+ return options.testFilter;
+ }
+
+ /**
+ * Returns user-specified test environment variables and their values, as
+ * set by the --test_env options.
+ */
+ public Map<String, String> getTestEnv() {
+ return getTestEnv(options.testEnvironment, clientEnvironment);
+ }
+
+ /**
+ * Returns user-specified test environment variables and their values, as
+ * set by the --test_env options.
+ *
+ * @param envOverrides The --test_env flag values.
+ * @param clientEnvironment The full client environment.
+ */
+ public static Map<String, String> getTestEnv(List<Map.Entry<String, String>> envOverrides,
+ Map<String, String> clientEnvironment) {
+ Map<String, String> testEnv = new HashMap<>();
+ for (Map.Entry<String, String> var : envOverrides) {
+ if (var.getValue() != null) {
+ testEnv.put(var.getKey(), var.getValue());
+ } else {
+ String value = clientEnvironment.get(var.getKey());
+ if (value != null) {
+ testEnv.put(var.getKey(), value);
+ }
+ }
+ }
+ return testEnv;
+ }
+
+ public TriState cacheTestResults() {
+ return options.cacheTestResults;
+ }
+
+ public int getMinParamFileSize() {
+ return options.minParamFileSize;
+ }
+
+ @SkylarkCallable(name = "coverage_enabled", structField = true,
+ doc = "A boolean that tells whether code coverage is enabled.")
+ public boolean isCodeCoverageEnabled() {
+ return options.collectCodeCoverage;
+ }
+
+ public boolean isMicroCoverageEnabled() {
+ return options.collectMicroCoverage;
+ }
+
+ public boolean isActionsEnabled() {
+ return actionsEnabled;
+ }
+
+ public TestActionBuilder.TestShardingStrategy testShardingStrategy() {
+ return options.testShardingStrategy;
+ }
+
+ /**
+ * @return number of times the given test should run.
+ * If the test doesn't match any of the filters, runs it once.
+ */
+ public int getRunsPerTestForLabel(Label label) {
+ for (PerLabelOptions perLabelRuns : options.runsPerTest) {
+ if (perLabelRuns.isIncluded(label)) {
+ return Integer.parseInt(Iterables.getOnlyElement(perLabelRuns.getOptions()));
+ }
+ }
+ return 1;
+ }
+
+ public RunUnder getRunUnder() {
+ return options.runUnder;
+ }
+
+ /**
+ * Returns true if this is a host configuration.
+ */
+ public boolean isHostConfiguration() {
+ return options.isHost;
+ }
+
+ public boolean checkVisibility() {
+ return options.checkVisibility;
+ }
+
+ public boolean checkLicenses() {
+ return options.checkLicenses;
+ }
+
+ public boolean enforceConstraints() {
+ return options.enforceConstraints;
+ }
+
+ public List<Label> getActionListeners() {
+ return actionsEnabled ? options.actionListeners : ImmutableList.<Label>of();
+ }
+
+ /**
+ * Returns compilation mode.
+ */
+ public CompilationMode getCompilationMode() {
+ return options.compilationMode;
+ }
+
+ /**
+ * Helper method to create a key from the BuildConfiguration initialization
+ * parameters and any additional component suppliers.
+ */
+ static String computeCacheKey(BlazeDirectories directories,
+ Map<Class<? extends Fragment>, Fragment> fragments,
+ BuildOptions buildOptions, Map<String, String> clientEnv) {
+
+ // Creates a full fingerprint of all constructor parameters, used for
+ // canonicalization.
+ //
+ // Note the use of each Path's FileSystem field; the test suite creates
+ // many paths of equal name but belonging to distinct filesystems, so we
+ // have to detect this. (Note however that we're relying on the
+ // injectiveness of identityHashCode for FileSystem, which is inelegant,
+ // but only affects the tests, since the production code uses only one
+ // instance.)
+
+ ImmutableList.Builder<String> keys = ImmutableList.builder();
+
+ // NOTE: identityHashCode isn't sound; may cause tests to fail.
+ keys.add(String.valueOf(System.identityHashCode(directories.getOutputBase().getFileSystem())));
+ keys.add(directories.getOutputBase().toString());
+ keys.add(buildOptions.computeCacheKey());
+ // This is needed so that if we have --test_env=VAR, the configuration key is updated if the
+ // environment variable VAR is updated.
+ keys.add(BuildConfiguration.getTestEnv(
+ buildOptions.get(Options.class).testEnvironment, clientEnv).toString());
+ keys.add(directories.getWorkspace().toString());
+
+ for (Fragment fragment : fragments.values()) {
+ keys.add(fragment.cacheKey());
+ }
+
+ // TODO(bazel-team): add hash of the FDO/LIPO profile file to config cache key
+
+ return StringUtilities.combineKeys(keys.build());
+ }
+
+ /**
+ * Returns a string that identifies the configuration.
+ *
+ * <p>The string uniquely identifies the configuration. As a result, it can be rather long and
+ * include spaces and other non-alphanumeric characters. If you need a shorter key, use
+ * {@link #shortCacheKey()}.
+ *
+ * @see #computeCacheKey
+ */
+ public final String cacheKey() {
+ return cacheKey;
+ }
+
+ /**
+ * Returns a (relatively) short key that identifies the configuration.
+ *
+ * <p>The short key is the short name of the configuration concatenated with a hash of the
+ * {@link #cacheKey()}.
+ */
+ public final String shortCacheKey() {
+ return shortCacheKey;
+ }
+
+ /** Returns a copy of the build configuration options for this configuration. */
+ public BuildOptions cloneOptions() {
+ return buildOptions.clone();
+ }
+
+ /**
+ * Prepare the fdo support. It reads data into memory that is used during analysis. The analysis
+ * phase is generally not allowed to perform disk I/O. This code is here because it is
+ * conceptually part of the analysis phase, and it needs to happen when the loading phase is
+ * complete.
+ */
+ public void prepareToBuild(Path execRoot, ArtifactFactory artifactFactory,
+ PackageRootResolver resolver) throws ViewCreationFailedException {
+ for (Fragment fragment : fragments.values()) {
+ fragment.prepareHook(execRoot, artifactFactory, getGenfilesFragment(), resolver);
+ }
+ }
+
+ /**
+ * Declares dependencies on any relevant Skyframe values (for example, relevant FileValues).
+ */
+ public void declareSkyframeDependencies(SkyFunction.Environment env) {
+ for (Fragment fragment : fragments.values()) {
+ fragment.declareSkyframeDependencies(env);
+ }
+ }
+
+ /**
+ * Returns all the roots for this configuration.
+ */
+ public List<Root> getRoots() {
+ List<Root> roots = new ArrayList<>();
+
+ // Configuration-specific roots.
+ roots.add(getBinDirectory());
+ roots.add(getGenfilesDirectory());
+ roots.add(getIncludeDirectory());
+ roots.add(getMiddlemanDirectory());
+ roots.add(getTestLogsDirectory());
+
+ // Fragment-defined roots
+ for (Fragment fragment : fragments.values()) {
+ fragment.addRoots(roots);
+ }
+
+ return ImmutableList.copyOf(roots);
+ }
+
+ public ListMultimap<String, Label> getAllLabels() {
+ return buildOptions.getAllLabels();
+ }
+
+ public String getCpu() {
+ return options.cpu;
+ }
+
+ /**
+ * Returns true is incremental builds are supported with this configuration.
+ */
+ public boolean supportsIncrementalBuild() {
+ for (Fragment fragment : fragments.values()) {
+ if (!fragment.supportsIncrementalBuild()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns true if the configuration performs static linking.
+ */
+ public boolean performsStaticLink() {
+ for (Fragment fragment : fragments.values()) {
+ if (fragment.performsStaticLink()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Deletes temporary directories before execution phase. This is only called for
+ * target configuration.
+ */
+ public void prepareForExecutionPhase() throws IOException {
+ for (Fragment fragment : fragments.values()) {
+ fragment.prepareForExecutionPhase();
+ }
+ }
+
+ /**
+ * Collects executables defined by fragments.
+ */
+ private ImmutableMap<String, PathFragment> collectExecutables() {
+ ImmutableMap.Builder<String, PathFragment> builder = new ImmutableMap.Builder<>();
+ for (Fragment fragment : fragments.values()) {
+ fragment.defineExecutables(builder);
+ }
+ return builder.build();
+ }
+
+ /**
+ * See {@code BuildConfigurationCollection.Transitions.getArtifactOwnerConfiguration()}.
+ */
+ public BuildConfiguration getArtifactOwnerConfiguration() {
+ return transitions.getArtifactOwnerConfiguration();
+ }
+
+ /**
+ * @return whether proto header modules should be built.
+ */
+ public boolean getProtoHeaderModules() {
+ return options.protoHeaderModules;
+ }
+
+ /**
+ * @return the list of default features used for all packages.
+ */
+ public List<String> getDefaultFeatures() {
+ return options.defaultFeatures;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java
new file mode 100644
index 0000000..e36a681
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java
@@ -0,0 +1,276 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.SplitTransition;
+import com.google.devtools.build.lib.packages.Attribute.Transition;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * The primary container for all main {@link BuildConfiguration} instances,
+ * currently "target", "data", and "host".
+ *
+ * <p>The target configuration is used for all targets specified on the command
+ * line. Data dependencies of targets in the target configuration use the data
+ * configuration instead.
+ *
+ * <p>The host configuration is used for tools that are executed during the
+ * build, e. g, compilers.
+ *
+ * <p>The "related" configurations are also contained in this class.
+ */
+@ThreadSafe
+public final class BuildConfigurationCollection implements Serializable {
+ private final ImmutableList<BuildConfiguration> targetConfigurations;
+
+ public BuildConfigurationCollection(List<BuildConfiguration> targetConfigurations)
+ throws InvalidConfigurationException {
+ this.targetConfigurations = ImmutableList.copyOf(targetConfigurations);
+
+ // Except for the host configuration (which may be identical across target configs), the other
+ // configurations must all have different cache keys or we will end up with problems.
+ HashMap<String, BuildConfiguration> cacheKeyConflictDetector = new HashMap<>();
+ for (BuildConfiguration config : getAllConfigurations()) {
+ if (cacheKeyConflictDetector.containsKey(config.cacheKey())) {
+ throw new InvalidConfigurationException("Conflicting configurations: " + config + " & "
+ + cacheKeyConflictDetector.get(config.cacheKey()));
+ }
+ cacheKeyConflictDetector.put(config.cacheKey(), config);
+ }
+ }
+
+ /**
+ * Creates an empty configuration collection which will return null for everything.
+ */
+ public BuildConfigurationCollection() {
+ this.targetConfigurations = ImmutableList.of();
+ }
+
+ public static BuildConfiguration configureTopLevelTarget(BuildConfiguration topLevelConfiguration,
+ Target toTarget) {
+ if (toTarget instanceof InputFile || toTarget instanceof PackageGroup) {
+ return null;
+ }
+ return topLevelConfiguration.getTransitions().toplevelConfigurationHook(toTarget);
+ }
+
+ public ImmutableList<BuildConfiguration> getTargetConfigurations() {
+ return targetConfigurations;
+ }
+
+ /**
+ * Returns all configurations that can be reached from the target configuration through any kind
+ * of configuration transition.
+ */
+ public Collection<BuildConfiguration> getAllConfigurations() {
+ Set<BuildConfiguration> result = new LinkedHashSet<>();
+ for (BuildConfiguration config : targetConfigurations) {
+ result.addAll(config.getAllReachableConfigurations());
+ }
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof BuildConfigurationCollection)) {
+ return false;
+ }
+ BuildConfigurationCollection that = (BuildConfigurationCollection) obj;
+ return this.targetConfigurations.equals(that.targetConfigurations);
+ }
+
+ @Override
+ public int hashCode() {
+ return targetConfigurations.hashCode();
+ }
+
+ /**
+ * Prints the configuration graph in dot format to the given print stream. This is only intended
+ * for debugging.
+ */
+ public void dumpAsDotGraph(PrintStream out) {
+ out.println("digraph g {");
+ out.println(" ratio = 0.3;");
+ for (BuildConfiguration config : getAllConfigurations()) {
+ String from = config.shortCacheKey();
+ for (Map.Entry<? extends Transition, ConfigurationHolder> entry :
+ config.getTransitions().getTransitionTable().entrySet()) {
+ BuildConfiguration toConfig = entry.getValue().getConfiguration();
+ if (toConfig == config) {
+ continue;
+ }
+ String to = toConfig == null ? "ERROR" : toConfig.shortCacheKey();
+ out.println(" \"" + from + "\" -> \"" + to + "\" [label=\"" + entry.getKey() + "\"]");
+ }
+ }
+ out.println("}");
+ }
+
+ /**
+ * The outgoing transitions for a build configuration.
+ */
+ public static class Transitions implements Serializable {
+ protected final BuildConfiguration configuration;
+
+ /**
+ * Look up table for the configuration transitions, i.e., HOST, DATA, etc.
+ */
+ private final Map<? extends Transition, ConfigurationHolder> transitionTable;
+
+ // TODO(bazel-team): Consider merging transitionTable into this.
+ private final ListMultimap<? super SplitTransition<?>, BuildConfiguration> splitTransitionTable;
+
+ public Transitions(BuildConfiguration configuration,
+ Map<? extends Transition, ConfigurationHolder> transitionTable,
+ ListMultimap<? extends SplitTransition<?>, BuildConfiguration> splitTransitionTable) {
+ this.configuration = configuration;
+ this.transitionTable = ImmutableMap.copyOf(transitionTable);
+ this.splitTransitionTable = ImmutableListMultimap.copyOf(splitTransitionTable);
+ }
+
+ public Transitions(BuildConfiguration configuration,
+ Map<? extends Transition, ConfigurationHolder> transitionTable) {
+ this(configuration, transitionTable,
+ ImmutableListMultimap.<SplitTransition<?>, BuildConfiguration>of());
+ }
+
+ public Map<? extends Transition, ConfigurationHolder> getTransitionTable() {
+ return transitionTable;
+ }
+
+ public ListMultimap<? super SplitTransition<?>, BuildConfiguration> getSplitTransitionTable() {
+ return splitTransitionTable;
+ }
+
+ public List<BuildConfiguration> getSplitConfigurations(SplitTransition<?> transition) {
+ if (splitTransitionTable.containsKey(transition)) {
+ return splitTransitionTable.get(transition);
+ } else {
+ Preconditions.checkState(transition.defaultsToSelf());
+ return ImmutableList.of(configuration);
+ }
+ }
+
+ /**
+ * Adds all configurations that are directly reachable from this configuration through
+ * any kind of configuration transition.
+ */
+ public void addDirectlyReachableConfigurations(Collection<BuildConfiguration> queue) {
+ for (ConfigurationHolder holder : transitionTable.values()) {
+ if (holder.configuration != null) {
+ queue.add(holder.configuration);
+ }
+ }
+ queue.addAll(splitTransitionTable.values());
+ }
+
+ /**
+ * Artifacts need an owner in Skyframe. By default it's the same configuration as what
+ * the configured target has, but it can be overridden if necessary.
+ *
+ * @return the artifact owner configuration
+ */
+ public BuildConfiguration getArtifactOwnerConfiguration() {
+ return configuration;
+ }
+
+ /**
+ * Returns the new configuration after traversing a dependency edge with a
+ * given configuration transition.
+ *
+ * @param configurationTransition the configuration transition
+ * @return the new configuration
+ */
+ public BuildConfiguration getConfiguration(Transition configurationTransition) {
+ ConfigurationHolder holder = transitionTable.get(configurationTransition);
+ if (holder == null && configurationTransition.defaultsToSelf()) {
+ return configuration;
+ }
+ return holder.configuration;
+ }
+
+ /**
+ * Arbitrary configuration transitions can be implemented by overriding this hook.
+ */
+ @SuppressWarnings("unused")
+ public BuildConfiguration configurationHook(Rule fromTarget,
+ Attribute attribute, Target toTarget, BuildConfiguration toConfiguration) {
+ return toConfiguration;
+ }
+
+ /**
+ * Associating configurations to top-level targets can be implemented by overriding this hook.
+ */
+ @SuppressWarnings("unused")
+ public BuildConfiguration toplevelConfigurationHook(Target toTarget) {
+ return configuration;
+ }
+ }
+
+ /**
+ * A holder class for {@link BuildConfiguration} instances that allows {@code null} values,
+ * because none of the Table implementations allow them.
+ */
+ public static final class ConfigurationHolder implements Serializable {
+ private final BuildConfiguration configuration;
+
+ public ConfigurationHolder(BuildConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public int hashCode() {
+ return configuration == null ? 0 : configuration.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof ConfigurationHolder)) {
+ return false;
+ }
+ return Objects.equals(configuration, ((ConfigurationHolder) o).configuration);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationKey.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationKey.java
new file mode 100644
index 0000000..e8fcf34
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationKey.java
@@ -0,0 +1,92 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A key for the creation of {@link BuildConfigurationCollection} instances.
+ */
+public final class BuildConfigurationKey {
+
+ private final BuildOptions buildOptions;
+ private final BlazeDirectories directories;
+ private final Map<String, String> clientEnv;
+ private final ImmutableSortedSet<String> multiCpu;
+
+ /**
+ * Creates a key for the creation of {@link BuildConfigurationCollection} instances.
+ *
+ * Note that the BuildConfiguration.Options instance must not contain unresolved relative paths.
+ */
+ public BuildConfigurationKey(BuildOptions buildOptions, BlazeDirectories directories,
+ Map<String, String> clientEnv, Set<String> multiCpu) {
+ this.buildOptions = Preconditions.checkNotNull(buildOptions);
+ this.directories = Preconditions.checkNotNull(directories);
+ this.clientEnv = ImmutableMap.copyOf(clientEnv);
+ this.multiCpu = ImmutableSortedSet.copyOf(multiCpu);
+ }
+
+ public BuildConfigurationKey(BuildOptions buildOptions, BlazeDirectories directories,
+ Map<String, String> clientEnv) {
+ this(buildOptions, directories, clientEnv, ImmutableSet.<String>of());
+ }
+
+ public BuildOptions getBuildOptions() {
+ return buildOptions;
+ }
+
+ public BlazeDirectories getDirectories() {
+ return directories;
+ }
+
+ public Map<String, String> getClientEnv() {
+ return clientEnv;
+ }
+
+ public ImmutableSortedSet<String> getMultiCpu() {
+ return multiCpu;
+ }
+
+ public ListMultimap<String, Label> getLabelsToLoadUnconditionally() {
+ return buildOptions.getAllLabels();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof BuildConfigurationKey)) {
+ return false;
+ }
+ BuildConfigurationKey k = (BuildConfigurationKey) o;
+ return buildOptions.equals(k.buildOptions)
+ && directories.equals(k.directories)
+ && clientEnv.equals(k.clientEnv)
+ && multiCpu.equals(k.multiCpu);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(buildOptions, directories, clientEnv, multiCpu);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java
new file mode 100644
index 0000000..afe408f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java
@@ -0,0 +1,254 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.packages.Attribute.SplitTransition;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/**
+ * This is a collection of command-line options from all configuration fragments. Contains
+ * a single instance for all FragmentOptions classes provided by Blaze language modules.
+ */
+public final class BuildOptions implements Cloneable, Serializable {
+ /**
+ * Creates a BuildOptions object with all options set to its default value.
+ */
+ public static BuildOptions createDefaults(Iterable<Class<? extends FragmentOptions>> options) {
+ Builder builder = builder();
+ for (Class<? extends FragmentOptions> optionsClass : options) {
+ builder.add(Options.getDefaults(optionsClass));
+ }
+ return builder.build();
+ }
+
+ /**
+ * This function creates a new BuildOptions instance for host.
+ *
+ * @param fallback if true, we have already tried the user specified hostCpu options
+ * and it didn't work, so now we try the default options instead.
+ */
+ public BuildOptions createHostOptions(boolean fallback) {
+ Builder builder = builder();
+ for (FragmentOptions options : fragmentOptionsMap.values()) {
+ builder.add(options.getHost(fallback));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns a list of potential split configuration transitions by calling {@link
+ * FragmentOptions#getPotentialSplitTransitions} on all the fragments.
+ */
+ public List<SplitTransition<BuildOptions>> getPotentialSplitTransitions() {
+ List<SplitTransition<BuildOptions>> result = new ArrayList<>();
+ for (FragmentOptions options : fragmentOptionsMap.values()) {
+ result.addAll(options.getPotentialSplitTransitions());
+ }
+ return result;
+ }
+
+ /**
+ * Creates an BuildOptions class by taking the option values from an options provider
+ * (eg. an OptionsParser).
+ */
+ public static BuildOptions of(List<Class<? extends FragmentOptions>> optionsList,
+ OptionsClassProvider provider) {
+ Builder builder = builder();
+ for (Class<? extends FragmentOptions> optionsClass : optionsList) {
+ builder.add(provider.getOptions(optionsClass));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Creates an BuildOptions class by taking the option values from command-line arguments
+ */
+ @VisibleForTesting
+ public static BuildOptions of(List<Class<? extends FragmentOptions>> optionsList, String... args)
+ throws OptionsParsingException {
+ Builder builder = builder();
+ OptionsParser parser = OptionsParser.newOptionsParser(
+ ImmutableList.<Class<? extends OptionsBase>>copyOf(optionsList));
+ parser.parse(args);
+ for (Class<? extends FragmentOptions> optionsClass : optionsList) {
+ builder.add(parser.getOptions(optionsClass));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns the actual instance of a FragmentOptions class.
+ */
+ @SuppressWarnings("unchecked")
+ public <T extends FragmentOptions> T get(Class<T> optionsClass) {
+ FragmentOptions options = fragmentOptionsMap.get(optionsClass);
+ Preconditions.checkNotNull(options);
+ Preconditions.checkArgument(optionsClass.isAssignableFrom(options.getClass()));
+ return (T) options;
+ }
+
+ /**
+ * Returns a multimap of all labels that were specified as options, keyed by the name to be
+ * displayed to the user if something goes wrong. This should be the set of all labels
+ * mentioned in explicit command line options that are not already covered by the
+ * tools/defaults package (see the DefaultsPackage class), and nothing else.
+ */
+ public ListMultimap<String, Label> getAllLabels() {
+ ListMultimap<String, Label> labels = ArrayListMultimap.create();
+ for (FragmentOptions optionsBase : fragmentOptionsMap.values()) {
+ optionsBase.addAllLabels(labels);
+ }
+ return labels;
+ }
+
+ // It would be very convenient to use a Multimap here, but we cannot do that because we need to
+ // support defaults labels that have zero elements.
+ ImmutableMap<String, ImmutableSet<Label>> getDefaultsLabels() {
+ BuildConfiguration.Options opts = get(BuildConfiguration.Options.class);
+ Map<String, Set<Label>> collector = new TreeMap<>();
+ for (FragmentOptions fragment : fragmentOptionsMap.values()) {
+ for (Map.Entry<String, Set<Label>> entry : fragment.getDefaultsLabels(opts).entrySet()) {
+ if (!collector.containsKey(entry.getKey())) {
+ collector.put(entry.getKey(), new TreeSet<Label>());
+ }
+ collector.get(entry.getKey()).addAll(entry.getValue());
+ }
+ }
+
+ ImmutableMap.Builder<String, ImmutableSet<Label>> result = new ImmutableMap.Builder<>();
+ for (Map.Entry<String, Set<Label>> entry : collector.entrySet()) {
+ result.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue()));
+ }
+
+ return result.build();
+ }
+
+ /**
+ * The cache key for the options collection. Recomputes cache key every time it's called.
+ */
+ public String computeCacheKey() {
+ StringBuilder keyBuilder = new StringBuilder();
+ for (FragmentOptions options : fragmentOptionsMap.values()) {
+ keyBuilder.append(options.cacheKey());
+ }
+ return keyBuilder.toString();
+ }
+
+ /**
+ * String representation of build options.
+ */
+ @Override
+ public String toString() {
+ StringBuilder stringBuilder = new StringBuilder();
+ for (FragmentOptions options : fragmentOptionsMap.values()) {
+ stringBuilder.append(options.toString());
+ }
+ return stringBuilder.toString();
+ }
+
+ /**
+ * Returns the options contained in this collection.
+ */
+ public Iterable<FragmentOptions> getOptions() {
+ return fragmentOptionsMap.values();
+ }
+
+ /**
+ * Creates a copy of the BuildOptions object that contains copies of the FragmentOptions.
+ */
+ @Override
+ public BuildOptions clone() {
+ ImmutableMap.Builder<Class<? extends FragmentOptions>, FragmentOptions> builder =
+ ImmutableMap.builder();
+ for (Map.Entry<Class<? extends FragmentOptions>, FragmentOptions> entry :
+ fragmentOptionsMap.entrySet()) {
+ builder.put(entry.getKey(), entry.getValue().clone());
+ }
+ return new BuildOptions(builder.build());
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (this == other) || (other instanceof BuildOptions &&
+ fragmentOptionsMap.equals(((BuildOptions) other).fragmentOptionsMap));
+ }
+
+ @Override
+ public int hashCode() {
+ return fragmentOptionsMap.hashCode();
+ }
+
+ /**
+ * Maps options class definitions to FragmentOptions objects
+ */
+ private final ImmutableMap<Class<? extends FragmentOptions>, FragmentOptions> fragmentOptionsMap;
+
+ private BuildOptions(
+ ImmutableMap<Class<? extends FragmentOptions>, FragmentOptions> fragmentOptionsMap) {
+ this.fragmentOptionsMap = fragmentOptionsMap;
+ }
+
+ /**
+ * Creates a builder object for BuildOptions
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Builder class for BuildOptions.
+ */
+ public static class Builder {
+ /**
+ * Adds a new FragmentOptions instance to the builder. Overrides previous instances of the
+ * exact same subclass of FragmentOptions.
+ */
+ public <T extends FragmentOptions> Builder add(T options) {
+ builderMap.put(options.getClass(), options);
+ return this;
+ }
+
+ public BuildOptions build() {
+ return new BuildOptions(ImmutableMap.copyOf(builderMap));
+ }
+
+ private Map<Class<? extends FragmentOptions>, FragmentOptions> builderMap;
+
+ private Builder() {
+ builderMap = new HashMap<>();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CompilationMode.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CompilationMode.java
new file mode 100644
index 0000000..7f24351
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CompilationMode.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.common.options.EnumConverter;
+
+/**
+ * This class represents the debug/optimization mode the binaries will be
+ * built for.
+ */
+public enum CompilationMode {
+
+ // Fast build mode (-g0).
+ FASTBUILD("fastbuild"),
+ // Debug mode (-g).
+ DBG("dbg"),
+ // Release mode (-g0 -O2 -DNDEBUG).
+ OPT("opt");
+
+ private final String mode;
+
+ private CompilationMode(String mode) {
+ this.mode = mode;
+ }
+
+ @Override
+ public String toString() {
+ return mode;
+ }
+
+ /**
+ * Converts to {@link CompilationMode}.
+ */
+ public static class Converter extends EnumConverter<CompilationMode> {
+ public Converter() {
+ super(CompilationMode.class, "compilation mode");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigMatchingProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigMatchingProvider.java
new file mode 100644
index 0000000..c719191
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigMatchingProvider.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A "configuration target" that asserts whether or not it matches the
+ * configuration it's bound to.
+ *
+ * <p>This can be used, e.g., to declare a BUILD target that defines the
+ * conditions which trigger a configurable attribute branch. In general,
+ * this can be used to trigger for any user-configurable build behavior.
+ */
+@Immutable
+public final class ConfigMatchingProvider implements TransitiveInfoProvider {
+
+ private final Label label;
+ private final boolean matches;
+
+ public ConfigMatchingProvider(Label label, boolean matches) {
+ this.label = label;
+ this.matches = matches;
+ }
+
+ /**
+ * The target's label.
+ */
+ public Label label() {
+ return label;
+ }
+
+ /**
+ * Whether or not the configuration criteria defined by this target match
+ * its actual configuration.
+ */
+ public boolean matches() {
+ return matches;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigRuleClasses.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigRuleClasses.java
new file mode 100644
index 0000000..6ee4cce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigRuleClasses.java
@@ -0,0 +1,204 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Type;
+
+/**
+ * Definitions for rule classes that specify or manipulate configuration settings.
+ *
+ * <p>These are not "traditional" rule classes in that they can't be requested as top-level
+ * targets and don't translate input artifacts into output artifacts. Instead, they affect
+ * how *other* rules work. See individual class comments for details.
+ */
+public class ConfigRuleClasses {
+
+ private static final String NONCONFIGURABLE_ATTRIBUTE_REASON =
+ "part of a rule class that *triggers* configurable behavior";
+
+ /**
+ * Common settings for all configurability rules.
+ */
+ @BlazeRule(name = "$config_base_rule",
+ type = RuleClass.Builder.RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.BaseRule.class })
+ public static final class ConfigBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .override(attr("tags", Type.STRING_LIST)
+ // No need to show up in ":all", etc. target patterns.
+ .value(ImmutableList.of("manual"))
+ .nonconfigurable(NONCONFIGURABLE_ATTRIBUTE_REASON))
+ .build();
+ }
+ }
+
+ /**
+ * A named "partial configuration setting" that specifies a set of command-line
+ * "flag=value" bindings.
+ *
+ * <p>For example:
+ * <pre>
+ * config_setting(
+ * name = 'foo',
+ * values = {
+ * 'flag1': 'aValue'
+ * 'flag2': 'bValue'
+ * })
+ * </pre>
+ *
+ * <p>declares a setting that binds command-line flag <pre>flag1</pre> to value
+ * <pre>aValue</pre> and <pre>flag2</pre> to <pre>bValue</pre>.
+ *
+ * <p>This is used by configurable attributes to determine which branch to
+ * follow based on which <pre>config_setting</pre> instance matches all its
+ * flag values in the configurable attribute owner's configuration.
+ *
+ * <p>This rule isn't accessed through the standard {@link RuleContext#getPrerequisites}
+ * interface. This is because Bazel constructs a rule's configured attribute map *before*
+ * its {@link RuleContext} is created (in fact, the map is an input to the context's
+ * constructor). And the config_settings referenced by the rule's configurable attributes are
+ * themselves inputs to that map. So Bazel has special logic to read and properly apply
+ * config_setting instances. See {@link ConfiguredTargetFunction#getConfigConditions} for details.
+ */
+ @BlazeRule(name = "config_setting",
+ type = RuleClass.Builder.RuleClassType.NORMAL,
+ ancestors = { ConfigBaseRule.class },
+ factoryClass = ConfigSetting.class)
+ public static final class ConfigSettingRule implements RuleDefinition {
+ /**
+ * The name of the attribute that declares flag bindings.
+ */
+ public static final String SETTINGS_ATTRIBUTE = "values";
+
+ @Override
+ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE(config_setting).ATTRIBUTE(values) -->
+ The set of configuration values that match this rule (expressed as Blaze flags)
+
+ <i>(Dictionary mapping flags to expected values, both expressed as strings;
+ mandatory)</i>
+
+ <p>This rule inherits the configuration of the configured target that
+ references it in a <code>select</code> statement. It is considered to
+ "match" a Blaze invocation if, for every entry in the dictionary, its
+ configuration matches the entry's expected value. For example
+ <code>values = {"compilation_mode": "opt"}</code> matches the invocations
+ <code>blaze build --compilation_mode=opt ...</code> and
+ <code>blaze build -c opt ...</code> on target-configured rules.
+ </p>
+
+ <p>For convenience's sake, configuration values are specified as Blaze flags (without
+ the preceding <code>"--"</code>). But keep in mind that the two are not the same. This
+ is because targets can be built in multiple configurations within the same
+ build. For example, a host configuration's "cpu" matches the value of
+ <code>--host_cpu</code>, not <code>--cpu</code>. So different instances of the
+ same <code>config_setting</code> may match the same invocation differently
+ depending on the configuration of the rule using them.
+ </p>
+
+ <p>If a flag is not explicitly set at the command line, its default value is used.
+ If a key appears multiple times in the dictionary, only the last instance is used.
+ If a key references a flag that can be set multiple times on the command line (e.g.
+ <code>blaze build --copt=foo --copt=bar --copt=baz ...</code>), a match occurs if
+ *any* of those settings match.
+ <p>
+
+ <p>This attribute cannot be empty.
+ </p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr(SETTINGS_ATTRIBUTE, STRING_DICT).mandatory()
+ .nonconfigurable(NONCONFIGURABLE_ATTRIBUTE_REASON))
+ .build();
+ }
+ }
+
+/*<!-- #BLAZE_RULE (NAME = config_setting, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>
+ Matches an expected configuration state (expressed as Blaze flags) for the purpose of triggering
+ configurable attributes. See <a href="#select">select</a> for how to consume this rule and
+ <a href="#configurable-attributes">Configurable attributes</a> for an overview of
+ the general feature.
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="config_setting_examples">Examples</h4>
+
+<p>The following matches any Blaze invocation that specifies <code>--compilation_mode=opt</code>
+ or <code>-c opt</code> (either explicitly at the command line or implicitly from .blazerc
+ files, etc.), when applied to a target configuration rule:
+</p>
+
+<pre class="code">
+config_setting(
+ name = "simple",
+ values = {"compilation_mode": "opt"}
+)
+</pre>
+
+<p>The following matches any Blaze invocation that builds for ARM and applies a custom define
+ (e.g. <code>blaze build --cpu=armeabi --define FOO=bar ...</code>), , when applied to a target
+ configuration rule:
+</p>
+
+<pre class="code">
+config_setting(
+ name = "two_conditions",
+ values = {
+ "cpu": "armeabi",
+ "define": "FOO=bar"
+ }
+)
+</pre>
+
+<h4 id="config_setting_notes">Notes</h4>
+
+<p>See <a href="#select">select</a> for policies on what happens depending on how many rules match
+ an invocation.
+</p>
+
+<p>For flags that support shorthand forms (e.g. <code>--compilation_mode</code> vs.
+ <code>-c</code>), <code>values</code> definitions must use the full form. These automatically
+ match invocations using either form.
+</p>
+
+<p>The currently endorsed method for creating custom conditions that can't be expressed through
+ dedicated build flags is through the --define flag. Use this flag with caution: it's not ideal
+ and only endorsed for lack of a currently better workaround. See the
+ <a href="#configurable-attributes">Configurable attributes</a> section for more discussion.
+</p>
+
+<p>Try to consolidate <code>config_setting</code> definitions as much as possible. In other words,
+ define <code>//common/conditions:foo</code> in one common package instead of repeating separate
+ instances in <code>//project1:foo</code>, <code>//project2:foo</code>, etc. that all mean the
+ same thing.
+</p>
+
+<!-- #END_BLAZE_RULE -->*/
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigSetting.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigSetting.java
new file mode 100644
index 0000000..97ad4ff
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigSetting.java
@@ -0,0 +1,170 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Implementation for the config_setting rule.
+ *
+ * <p>This is a "pseudo-rule" in that its purpose isn't to generate output artifacts
+ * from input artifacts. Rather, it provides configuration context to rules that
+ * depend on it.
+ */
+public class ConfigSetting implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ // Get the required flag=value settings for this rule.
+ Map<String, String> settings = NonconfigurableAttributeMapper.of(ruleContext.getRule())
+ .get(ConfigRuleClasses.ConfigSettingRule.SETTINGS_ATTRIBUTE, Type.STRING_DICT);
+ if (settings.isEmpty()) {
+ ruleContext.attributeError(ConfigRuleClasses.ConfigSettingRule.SETTINGS_ATTRIBUTE,
+ "no settings specified");
+ return null;
+ }
+
+ ConfigMatchingProvider configMatcher;
+ try {
+ configMatcher = new ConfigMatchingProvider(ruleContext.getLabel(),
+ matchesConfig(settings, ruleContext.getConfiguration()));
+ } catch (OptionsParsingException e) {
+ ruleContext.attributeError(ConfigRuleClasses.ConfigSettingRule.SETTINGS_ATTRIBUTE,
+ "error while parsing configuration settings: " + e.getMessage());
+ return null;
+ }
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .add(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .add(FileProvider.class, new FileProvider(ruleContext.getLabel(),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER)))
+ .add(FilesToRunProvider.class, new FilesToRunProvider(ruleContext.getLabel(),
+ ImmutableList.<Artifact>of(), null, null))
+ .add(ConfigMatchingProvider.class, configMatcher)
+ .build();
+ }
+
+ /**
+ * Given a list of [flagName, flagValue] pairs, returns true if flagName == flagValue for
+ * every item in the list under this configuration, false otherwise.
+ */
+ private boolean matchesConfig(Map<String, String> expectedSettings, BuildConfiguration config)
+ throws OptionsParsingException {
+ // Rather than returning fast when we find a mismatch, continue looking at the other flags
+ // to check that they're indeed valid flag specifications.
+ boolean foundMismatch = false;
+
+ // Since OptionsParser instantiation involves reflection, let's try to minimize that happening.
+ Map<Class<? extends OptionsBase>, OptionsParser> parserCache = new HashMap<>();
+
+ for (Map.Entry<String, String> setting : expectedSettings.entrySet()) {
+ String optionName = setting.getKey();
+ String expectedRawValue = setting.getValue();
+
+ Class<? extends OptionsBase> optionClass = config.getOptionClass(optionName);
+ if (optionClass == null) {
+ throw new OptionsParsingException("unknown option: '" + optionName + "'");
+ }
+
+ OptionsParser parser = parserCache.get(optionClass);
+ if (parser == null) {
+ parser = OptionsParser.newOptionsParser(optionClass);
+ parserCache.put(optionClass, parser);
+ }
+ parser.parse("--" + optionName + "=" + expectedRawValue);
+ Object expectedParsedValue = parser.getOptions(optionClass).asMap().get(optionName);
+
+ if (!optionMatches(config, optionName, expectedParsedValue)) {
+ foundMismatch = true;
+ }
+ }
+ return !foundMismatch;
+ }
+
+ /**
+ * For single-value options, returns true iff the option's value matches the expected value.
+ *
+ * <p>For multi-value List options, returns true iff any of the option's values matches
+ * the expected value. This means, e.g. "--tool_tag=foo --tool_tag=bar" would match the
+ * expected condition { 'tool_tag': 'bar' }.
+ *
+ * <p>For multi-value Map options, returns true iff the last instance with the same key as the
+ * expected key has the same value. This means, e.g. "--define foo=1 --define bar=2" would
+ * match { 'define': 'foo=1' }, but "--define foo=1 --define bar=2 --define foo=3" would not
+ * match. Note that the definition of --define states that the last instance takes precedence.
+ */
+ private static boolean optionMatches(BuildConfiguration config, String optionName,
+ Object expectedValue) {
+ Object actualValue = config.getOptionValue(optionName);
+ if (actualValue == null) {
+ return expectedValue == null;
+
+ // Single-value case:
+ } else if (!config.allowsMultipleValues(optionName)) {
+ return actualValue.equals(expectedValue);
+ }
+
+ // Multi-value case:
+ Preconditions.checkState(actualValue instanceof List);
+ Preconditions.checkState(expectedValue instanceof List);
+ List<?> actualList = (List<?>) actualValue;
+ List<?> expectedList = (List<?>) expectedValue;
+
+ if (actualList.isEmpty() || expectedList.isEmpty()) {
+ return actualList.isEmpty() && expectedList.isEmpty();
+ }
+
+ // We're expecting a single value of a multi-value type: the options parser still embeds
+ // that single value within a List container. Retrieve it here.
+ Object expectedSingleValue = Iterables.getOnlyElement(expectedList);
+
+ // Multi-value map:
+ if (actualList.get(0) instanceof Map.Entry) {
+ Map.Entry<?, ?> expectedEntry = (Map.Entry<?, ?>) expectedSingleValue;
+ for (Map.Entry<?, ?> actualEntry : Lists.reverse((List<Map.Entry<?, ?>>) actualList)) {
+ if (actualEntry.getKey().equals(expectedEntry.getKey())) {
+ // Found a key match!
+ return actualEntry.getValue().equals(expectedEntry.getValue());
+ }
+ }
+ return false; // Never found any matching key.
+ }
+
+ // Multi-value list:
+ return actualList.contains(expectedSingleValue);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationEnvironment.java
new file mode 100644
index 0000000..89722b5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationEnvironment.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.pkgcache.PackageProvider;
+import com.google.devtools.build.lib.pkgcache.TargetProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+
+import javax.annotation.Nullable;
+
+/**
+ * An environment to support creating BuildConfiguration instances in a hermetic fashion; all
+ * accesses to packages or the file system <b>must</b> go through this interface, so that they can
+ * be recorded for correct caching.
+ */
+public interface ConfigurationEnvironment {
+
+ /**
+ * Returns a target for the given label, loading it if necessary, and throwing an exception if it
+ * does not exist.
+ *
+ * @see TargetProvider#getTarget
+ */
+ Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException;
+
+ /** Returns a path for the given file within the given package. */
+ Path getPath(Package pkg, String fileName);
+
+ /** Returns fragment based on fragment class and build options. */
+ <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType)
+ throws InvalidConfigurationException;
+
+ /** Returns global value of BlazeDirectories. */
+ @Nullable
+ BlazeDirectories getBlazeDirectories();
+
+ /**
+ * An implementation backed by a {@link PackageProvider} instance.
+ */
+ public static final class TargetProviderEnvironment implements ConfigurationEnvironment {
+
+ private final LoadedPackageProvider loadedPackageProvider;
+ private final BlazeDirectories blazeDirectories;
+
+ public TargetProviderEnvironment(LoadedPackageProvider loadedPackageProvider,
+ BlazeDirectories blazeDirectories) {
+ this.loadedPackageProvider = loadedPackageProvider;
+ this.blazeDirectories = blazeDirectories;
+ }
+
+ public TargetProviderEnvironment(LoadedPackageProvider loadedPackageProvider) {
+ this.loadedPackageProvider = loadedPackageProvider;
+ this.blazeDirectories = null;
+ }
+
+ @Override
+ public Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException {
+ return loadedPackageProvider.getLoadedTarget(label);
+ }
+
+ @Override
+ public Path getPath(Package pkg, String fileName) {
+ return pkg.getPackageDirectory().getRelative(fileName);
+ }
+
+ @Override
+ public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BlazeDirectories getBlazeDirectories() {
+ return blazeDirectories;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFactory.java
new file mode 100644
index 0000000..13afb2a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFactory.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.ConfigurationCollectionFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.events.EventHandler;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A factory class for {@link BuildConfiguration} instances. This is unfortunately more complex,
+ * and should be simplified in the future, if
+ * possible. Right now, creating a {@link BuildConfiguration} instance involves
+ * creating the instance itself and the related configurations; the main method
+ * is {@link #createConfiguration}.
+ *
+ * <p>Avoid calling into this class, and instead use the skyframe infrastructure to obtain
+ * configuration instances.
+ *
+ * <p>Blaze currently relies on the fact that all {@link BuildConfiguration}
+ * instances used in a build can be constructed ahead of time by this class.
+ */
+@ThreadCompatible // safe as long as separate instances are used
+public final class ConfigurationFactory {
+ private final List<ConfigurationFragmentFactory> configurationFragmentFactories;
+ private final ConfigurationCollectionFactory configurationCollectionFactory;
+
+ // A cache of key to configuration instances.
+ private final Cache<String, BuildConfiguration> hostConfigCache =
+ CacheBuilder.newBuilder().softValues().build();
+
+ private boolean performSanityCheck = true;
+
+ public ConfigurationFactory(
+ ConfigurationCollectionFactory configurationCollectionFactory,
+ List<ConfigurationFragmentFactory> fragmentFactories) {
+ this.configurationCollectionFactory =
+ Preconditions.checkNotNull(configurationCollectionFactory);
+ this.configurationFragmentFactories = fragmentFactories;
+ }
+
+ @VisibleForTesting
+ public void forbidSanityCheck() {
+ performSanityCheck = false;
+ }
+
+ /** Create the build configurations with the given options. */
+ @Nullable
+ public BuildConfiguration createConfiguration(
+ PackageProviderForConfigurations loadedPackageProvider, BuildOptions buildOptions,
+ BuildConfigurationKey key, EventHandler errorEventListener)
+ throws InvalidConfigurationException {
+ return configurationCollectionFactory.createConfigurations(this,
+ loadedPackageProvider, buildOptions, key.getClientEnv(),
+ errorEventListener, performSanityCheck);
+ }
+
+ /**
+ * Returns a (possibly new) canonical host BuildConfiguration instance based
+ * upon a given request configuration
+ */
+ @Nullable
+ public BuildConfiguration getHostConfiguration(
+ PackageProviderForConfigurations loadedPackageProvider, Map<String, String> clientEnv,
+ BuildOptions buildOptions, boolean fallback) throws InvalidConfigurationException {
+ return getConfiguration(loadedPackageProvider, buildOptions.createHostOptions(fallback),
+ clientEnv, false, hostConfigCache);
+ }
+
+ /**
+ * The core of BuildConfiguration creation. All host and target instances are
+ * constructed and cached here.
+ */
+ @Nullable
+ public BuildConfiguration getConfiguration(PackageProviderForConfigurations loadedPackageProvider,
+ BuildOptions buildOptions, Map<String, String> clientEnv,
+ boolean actionsDisabled, Cache<String, BuildConfiguration> cache)
+ throws InvalidConfigurationException {
+
+ Map<Class<? extends Fragment>, Fragment> fragments = new HashMap<>();
+ // Create configuration fragments
+ for (ConfigurationFragmentFactory factory : configurationFragmentFactories) {
+ Class<? extends Fragment> fragmentType = factory.creates();
+ Fragment fragment = loadedPackageProvider.getFragment(buildOptions, fragmentType);
+ if (fragment != null && fragments.get(fragment) == null) {
+ fragments.put(fragment.getClass(), fragment);
+ }
+ }
+ BlazeDirectories directories = loadedPackageProvider.getDirectories();
+ if (loadedPackageProvider.valuesMissing()) {
+ return null;
+ }
+
+ // Sort the fragments by class name to make sure that the order is stable. Afterwards, copy to
+ // an ImmutableMap, which keeps the order stable, but uses hashing, and drops the reference to
+ // the Comparator object.
+ fragments = ImmutableSortedMap.copyOf(fragments, new Comparator<Class<? extends Fragment>>() {
+ @Override
+ public int compare(Class<? extends Fragment> o1, Class<? extends Fragment> o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ });
+ fragments = ImmutableMap.copyOf(fragments);
+
+ String key = BuildConfiguration.computeCacheKey(
+ directories, fragments, buildOptions, clientEnv);
+ BuildConfiguration configuration = cache.getIfPresent(key);
+ if (configuration == null) {
+ configuration = new BuildConfiguration(directories, fragments, buildOptions,
+ clientEnv, actionsDisabled);
+ cache.put(key, configuration);
+ }
+ return configuration;
+ }
+
+ public List<ConfigurationFragmentFactory> getFactories() {
+ return configurationFragmentFactories;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFragmentFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFragmentFactory.java
new file mode 100644
index 0000000..8ca8f1c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFragmentFactory.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+
+import javax.annotation.Nullable;
+
+/**
+ * A factory that creates configuration fragments.
+ */
+public interface ConfigurationFragmentFactory {
+ /**
+ * Creates a configuration fragment.
+ *
+ * @param env the ConfigurationEnvironment for querying targets and paths
+ * @param buildOptions command-line options (see {@link FragmentOptions})
+ * @return the configuration fragment or null if some required dependencies are missing.
+ */
+ @Nullable
+ BuildConfiguration.Fragment create(ConfigurationEnvironment env, BuildOptions buildOptions)
+ throws InvalidConfigurationException;
+
+ /**
+ * @return the exact type of the fragment this factory creates.
+ */
+ Class<? extends Fragment> creates();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/DefaultsPackage.java b/src/main/java/com/google/devtools/build/lib/analysis/config/DefaultsPackage.java
new file mode 100644
index 0000000..207d49a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/DefaultsPackage.java
@@ -0,0 +1,164 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A helper class to compute and inject a defaults package into the package cache.
+ *
+ * <p>The <code>//tools/defaults</code> package provides a mechanism let tool locations be
+ * specified over the commandline, without requiring any special support in the rule code.
+ * As such, it can be used in genrule <code>$(location)</code> substitutions.
+ *
+ * <p>It works as follows:
+ * <ul>
+ *
+ * <li> SomeLanguage.createCompileAction will refer to a host-configured target for the
+ * compiler by looking for
+ * <code>env.getHostPrerequisiteArtifact("$somelanguage_compiler")</code>.
+ *
+ * <li> the attribute <code>$somelanguage_compiler</code> is defined in the
+ * {@link RuleDefinition} subclass for that language.
+ *
+ * <li> if the attribute cannot be set on the command-line, its value may be a normal label.
+ *
+ * <li> if the attribute can be set on the command-line, its value will be
+ * <code>//tools/defaults:somelanguage_compiler</code>.
+ *
+ * <li> in the latter case, the {@link BuildConfiguration.Fragment} subclass will define the
+ * option (with an existing target, eg. <code>//third_party/somelanguage:compiler</code>), and
+ * return the name in its implementation of {@link FragmentOptions#getDefaultsLabels}.
+ *
+ * <li> On startup, the rule is wired up with <code>//tools/defaults:somelanguage_compiler</code>.
+ *
+ * <li> On starting a build, the <code>//tools/defaults</code> package is synthesized, using
+ * the values as specified on the command-line. The contents of
+ * <code>tools/defaults/BUILD</code> is ignored.
+ *
+ * <li> Hence, changes in the command line values for tools are now handled exactly as if they
+ * were changes in a BUILD file.
+ *
+ * <li> The file <code>tools/defaults/BUILD</code> must exist, so we create a package in that
+ * location.
+ *
+ * <li> The code in {@link DefaultsPackage} can dump the synthesized package as a BUILD file,
+ * so external tooling does not need to understand the intricacies of handling command-line
+ * options.
+ *
+ * </ul>
+ *
+ * <p>For built-in rules (as opposed to genrules), late-bound labels provide an alternative
+ * method of depending on command-line values. These work by declaring attribute default values
+ * to be {@link LateBoundLabel} instances, whose <code>getDefault(Rule rule, T
+ * configuration)</code> method will have access to {@link BuildConfiguration}, which in turn
+ * may depend on command line flag values.
+ */
+public final class DefaultsPackage {
+
+ // The template contents are broken into lines such that the resulting file has no more than 80
+ // characters per line.
+ private static final String HEADER = ""
+ + "# DO NOT EDIT THIS FILE!\n"
+ + "#\n"
+ + "# Bazel does not read this file. Instead, it internally replaces the targets in\n"
+ + "# this package with the correct packages as given on the command line.\n"
+ + "#\n"
+ + "# If these options are not given on the command line, Bazel will use the exact\n"
+ + "# same targets as given here."
+ + "\n"
+ + "package(default_visibility = ['//visibility:public'])\n";
+
+ /**
+ * The map from entries to their values.
+ */
+ private ImmutableMap<String, ImmutableSet<Label>> values;
+
+ private DefaultsPackage(BuildOptions buildOptions) {
+ values = buildOptions.getDefaultsLabels();
+ }
+
+ private String labelsToString(Set<Label> labels) {
+ StringBuffer result = new StringBuffer();
+ for (Label label : labels) {
+ if (result.length() != 0) {
+ result.append(", ");
+ }
+ result.append("'").append(label).append("'");
+ }
+ return result.toString();
+ }
+
+ /**
+ * Returns a string of the defaults package with the given settings.
+ */
+ private String getContent() {
+ Preconditions.checkState(!values.isEmpty());
+ StringBuilder result = new StringBuilder(HEADER);
+ for (Map.Entry<String, ImmutableSet<Label>> entry : values.entrySet()) {
+ result
+ .append("filegroup(name = '")
+ .append(entry.getKey().toLowerCase(Locale.US)).append("',\n")
+ .append(" srcs = [")
+ .append(labelsToString(entry.getValue())).append("])\n");
+ }
+ return result.toString();
+ }
+
+ /**
+ * Returns the defaults package for the default settings.
+ */
+ public static String getDefaultsPackageContent(
+ Iterable<Class<? extends FragmentOptions>> options) {
+ return getDefaultsPackageContent(BuildOptions.createDefaults(options));
+ }
+
+ /**
+ * Returns the defaults package for the given options.
+ */
+ public static String getDefaultsPackageContent(BuildOptions buildOptions) {
+ return new DefaultsPackage(buildOptions).getContent();
+ }
+
+ public static void parseAndAdd(Set<Label> labels, String optionalLabel) {
+ if (optionalLabel != null) {
+ Label label = parseOptionalLabel(optionalLabel);
+ if (label != null) {
+ labels.add(label);
+ }
+ }
+ }
+
+ public static Label parseOptionalLabel(String value) {
+ if (value.startsWith("//")) {
+ try {
+ return Label.parseAbsolute(value);
+ } catch (SyntaxException e) {
+ // We ignore this exception here - it will cause an error message at a later time.
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/FragmentOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/FragmentOptions.java
new file mode 100644
index 0000000..ce4b2d2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/FragmentOptions.java
@@ -0,0 +1,115 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.packages.Attribute.SplitTransition;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Command-line build options for a Blaze module.
+ */
+public abstract class FragmentOptions extends OptionsBase implements Cloneable, Serializable {
+
+ /**
+ * Adds all labels defined by the options to a multimap. See {@code BuildOptions.getAllLabels()}.
+ *
+ * <p>There should generally be no code duplication between this code and DefaultsPackage. Either
+ * the labels are loaded unconditionally using this method, or they are added as magic labels
+ * using the tools/defaults package, but not both.
+ *
+ * @param labelMap a mutable multimap to which the labels of this fragment should be added
+ */
+ public void addAllLabels(Multimap<String, Label> labelMap) {
+ }
+
+ /**
+ * Returns the labels contributed to the defaults package by this fragment.
+ *
+ * <p>The set of keys returned by this function should be constant, however, the values are
+ * allowed to change depending on the value of the options.
+ */
+ @SuppressWarnings("unused")
+ public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * Returns a list of potential split configuration transitions for this fragment. Split
+ * configurations usually need to be explicitly enabled by passing in an option.
+ */
+ public List<SplitTransition<BuildOptions>> getPotentialSplitTransitions() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public FragmentOptions clone() {
+ try {
+ return (FragmentOptions) super.clone();
+ } catch (CloneNotSupportedException e) {
+ // This can't happen.
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Creates a new FragmentOptions instance with all flags set to default.
+ */
+ public FragmentOptions getDefault() {
+ return Options.getDefaults(getClass());
+ }
+
+ /**
+ * Creates a new FragmentOptions instance with flags adjusted to host platform.
+ *
+ * @param fallback see {@code BuildOptions.createHostOptions}
+ */
+ @SuppressWarnings("unused")
+ public FragmentOptions getHost(boolean fallback) {
+ return getDefault();
+ }
+
+ protected void addOptionalLabel(Multimap<String, Label> map, String key, String value) {
+ Label label = parseOptionalLabel(value);
+ if (label != null) {
+ map.put(key, label);
+ }
+ }
+
+ private static Label parseOptionalLabel(String value) {
+ if ((value != null) && value.startsWith("//")) {
+ try {
+ return Label.parseAbsolute(value);
+ } catch (SyntaxException e) {
+ // We ignore this exception here - it will cause an error message at a later time.
+ // TODO(bazel-team): We can use a Converter to check the validity of the crosstoolTop
+ // earlier.
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/InvalidConfigurationException.java b/src/main/java/com/google/devtools/build/lib/analysis/config/InvalidConfigurationException.java
new file mode 100644
index 0000000..c39325d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/InvalidConfigurationException.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+/**
+ * Thrown if the configuration options lead to an invalid configuration, or if any of the
+ * configuration labels cannot be loaded.
+ */
+public class InvalidConfigurationException extends Exception {
+
+ public InvalidConfigurationException(String message) {
+ super(message);
+ }
+
+ public InvalidConfigurationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public InvalidConfigurationException(Throwable cause) {
+ this(cause.getMessage(), cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/PackageProviderForConfigurations.java b/src/main/java/com/google/devtools/build/lib/analysis/config/PackageProviderForConfigurations.java
new file mode 100644
index 0000000..83b4715
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/PackageProviderForConfigurations.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+
+import java.io.IOException;
+
+/**
+ * Extended LoadedPackageProvider which is used during a creation of BuildConfiguration.Fragments.
+ */
+public interface PackageProviderForConfigurations extends LoadedPackageProvider {
+ /**
+ * Adds dependency to fileName if needed. Used only in skyframe, for creating correct dependencies
+ * for {@link com.google.devtools.build.lib.skyframe.ConfigurationCollectionValue}.
+ */
+ void addDependency(Package pkg, String fileName) throws SyntaxException, IOException;
+
+ /**
+ * Returns fragment based on fragment type and build options.
+ */
+ <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType)
+ throws InvalidConfigurationException;
+
+ /**
+ * Returns blaze directories and adds dependency to that value.
+ */
+ BlazeDirectories getDirectories();
+
+ /**
+ * Returns true if any dependency is missing (value of some node hasn't been evaluated yet).
+ */
+ boolean valuesMissing();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/PerLabelOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/PerLabelOptions.java
new file mode 100644
index 0000000..1e921e5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/PerLabelOptions.java
@@ -0,0 +1,128 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.util.RegexFilter.RegexFilterConverter;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Models options that can be added to a command line when a label matches a
+ * given {@link RegexFilter}.
+ */
+public class PerLabelOptions implements Serializable {
+
+ /** The filter used to match labels */
+ private final RegexFilter regexFilter;
+
+ /** The list of options to add when the filter matches a label */
+ private final List<String> optionsList;
+
+ /**
+ * Converts a String to a {@link PerLabelOptions} object. The syntax of the
+ * string is {@code regex_filter@option_1,option_2,...,option_n}. Where
+ * regex_filter stands for the String representation of a {@link RegexFilter},
+ * and {@code option_1} to {@code option_n} stand for arbitrary command line
+ * options. If an option contains a comma it has to be quoted with a
+ * backslash. Options can contain @. Only the first @ is used to split the
+ * string.
+ */
+ public static class PerLabelOptionsConverter implements Converter<PerLabelOptions> {
+
+ @Override
+ public PerLabelOptions convert(String input) throws OptionsParsingException {
+ int atIndex = input.indexOf('@');
+ RegexFilterConverter converter = new RegexFilter.RegexFilterConverter();
+ if (atIndex < 0) {
+ return new PerLabelOptions(converter.convert(input), ImmutableList.<String> of());
+ } else {
+ String filterPiece = input.substring(0, atIndex);
+ String optionsPiece = input.substring(atIndex + 1);
+ List<String> optionsList = new ArrayList<>();
+ for (String option : optionsPiece.split("(?<!\\\\),")) { // Split on ',' but not on '\,'
+ if (option != null && !option.trim().equals("")) {
+ optionsList.add(option.replace("\\,", ","));
+ }
+ }
+ return new PerLabelOptions(converter.convert(filterPiece), optionsList);
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a comma-separated list of regex expressions with prefix '-' specifying"
+ + " excluded paths followed by an @ and a comma separated list of options";
+ }
+ }
+
+ public PerLabelOptions(RegexFilter regexFilter, List<String> optionsList) {
+ this.regexFilter = regexFilter;
+ this.optionsList = optionsList;
+ }
+
+ /**
+ * @return true if the given label is matched by the {@link RegexFilter}.
+ */
+ public boolean isIncluded(Label label) {
+ return regexFilter.isIncluded(label.toString());
+ }
+
+ /**
+ * @return true if the execution path (which includes the base name of the file)
+ * of the given file is matched by the {@link RegexFilter}.
+ */
+ public boolean isIncluded(Artifact artifact) {
+ return regexFilter.isIncluded(artifact.getExecPathString());
+ }
+
+ /**
+ * Returns the list of options to add to a command line.
+ */
+ public List<String> getOptions() {
+ return optionsList;
+ }
+
+ RegexFilter getRegexFilter() {
+ return regexFilter;
+ }
+
+ @Override
+ public String toString() {
+ return regexFilter + " Options: " + optionsList;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ PerLabelOptions otherOptions =
+ other instanceof PerLabelOptions ? (PerLabelOptions) other : null;
+ return this == other || (otherOptions != null &&
+ this.regexFilter.equals(otherOptions.regexFilter) &&
+ this.optionsList.equals(otherOptions.optionsList));
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(regexFilter, optionsList);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnder.java b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnder.java
new file mode 100644
index 0000000..a51ea25
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnder.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Components of --run_under option.
+ */
+public interface RunUnder extends Serializable {
+ /**
+ * @return the whole value passed to --run_under option.
+ */
+ String getValue();
+
+ /**
+ * Returns label corresponding to the first word (according to shell
+ * tokenization) passed to --run_under.
+ *
+ * @return if the first word (according to shell tokenization) passed to
+ * --run_under starts with {@code "//"} returns the label
+ * corresponding to that word otherwise {@code null}
+ */
+ Label getLabel();
+
+ /**
+ * @return if the first word (according to shell tokenization) passed to
+ * --run_under starts with {@code "//"} returns {@code null}
+ * otherwise the first word
+ */
+ String getCommand();
+
+ /**
+ * @return everything except the first word (according to shell
+ * tokenization) passed to --run_under.
+ */
+ List<String> getOptions();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnderConverter.java b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnderConverter.java
new file mode 100644
index 0000000..1f7b660
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnderConverter.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.config;
+
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * --run_under options converter.
+ */
+public class RunUnderConverter implements Converter<RunUnder> {
+ @Override
+ public RunUnder convert(final String input) throws OptionsParsingException {
+ final List<String> runUnderList = new ArrayList<>();
+ try {
+ ShellUtils.tokenize(runUnderList, input);
+ } catch (TokenizationException e) {
+ throw new OptionsParsingException("Not a valid command prefix " + e.getMessage());
+ }
+ if (runUnderList.isEmpty()) {
+ throw new OptionsParsingException("Empty command");
+ }
+ final String runUnderCommand = runUnderList.get(0);
+ if (runUnderCommand.startsWith("//")) {
+ try {
+ final Label runUnderLabel = Label.parseAbsolute(runUnderCommand);
+ return new RunUnderLabel(input, runUnderLabel, runUnderList);
+ } catch (SyntaxException e) {
+ throw new OptionsParsingException("Not a valid label " + e.getMessage());
+ }
+ } else {
+ return new RunUnderCommand(input, runUnderCommand, runUnderList);
+ }
+ }
+
+ private static final class RunUnderLabel implements RunUnder {
+ private final String input;
+ private final Label runUnderLabel;
+ private final List<String> runUnderList;
+
+ public RunUnderLabel(String input, Label runUnderLabel, List<String> runUnderList) {
+ this.input = input;
+ this.runUnderLabel = runUnderLabel;
+ this.runUnderList = new ArrayList<String>(runUnderList.subList(1, runUnderList.size()));
+ }
+
+ @Override public String getValue() { return input; }
+ @Override public Label getLabel() { return runUnderLabel; }
+ @Override public String getCommand() { return null; }
+ @Override public List<String> getOptions() { return runUnderList; }
+ @Override public String toString() { return input; }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (other instanceof RunUnderLabel) {
+ RunUnderLabel otherRunUnderLabel = (RunUnderLabel) other;
+ return Objects.equals(input, otherRunUnderLabel.input)
+ && Objects.equals(runUnderLabel, otherRunUnderLabel.runUnderLabel)
+ && Objects.equals(runUnderList, otherRunUnderLabel.runUnderList);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(input, runUnderLabel, runUnderList);
+ }
+ }
+
+ private static final class RunUnderCommand implements RunUnder {
+ private final String input;
+ private final String runUnderCommand;
+ private final List<String> runUnderList;
+
+ public RunUnderCommand(String input, String runUnderCommand, List<String> runUnderList) {
+ this.input = input;
+ this.runUnderCommand = runUnderCommand;
+ this.runUnderList = new ArrayList<String>(runUnderList.subList(1, runUnderList.size()));
+ }
+
+ @Override public String getValue() { return input; }
+ @Override public Label getLabel() { return null; }
+ @Override public String getCommand() { return runUnderCommand; }
+ @Override public List<String> getOptions() { return runUnderList; }
+ @Override public String toString() { return input; }
+
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (other instanceof RunUnderCommand) {
+ RunUnderCommand otherRunUnderCommand = (RunUnderCommand) other;
+ return Objects.equals(input, otherRunUnderCommand.input)
+ && Objects.equals(runUnderCommand, otherRunUnderCommand.runUnderCommand)
+ && Objects.equals(runUnderList, otherRunUnderCommand.runUnderList);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(input, runUnderCommand, runUnderList);
+ }
+ }
+ @Override
+ public String getTypeDescription() {
+ return "a prefix in front of command";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/ConstraintSemantics.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/ConstraintSemantics.java
new file mode 100644
index 0000000..14ac2bc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/ConstraintSemantics.java
@@ -0,0 +1,473 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.constraints;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.constraints.EnvironmentCollection.EnvironmentWithGroup;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.EnvironmentGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Implementation of the semantics of Bazel's constraint specification and enforcement system.
+ *
+ * <p>This is how the system works:
+ *
+ * <p>All build rules can declare which "environments" they can be built for, where an "environment"
+ * is a label instance of an {@link EnvironmentRule} rule declared in a BUILD file. There are
+ * various ways to do this:
+ *
+ * <ul>
+ * <li>Through a "restricted to" attribute setting
+ * ({@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR}). This is the most direct form of
+ * specification - it declares the exact set of environments the rule supports (for its group -
+ * see precise details below).
+ * <li>Through a "compatible with" attribute setting
+ * ({@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR}. This declares <b>additional</b>
+ * environments a rule supports in addition to "standard" environments that are supported by
+ * default (see below).
+ * <li>Through "default" specifications in {@link EnvironmentGroup} rules. Every environment
+ * belongs to a group of thematically related peers (e.g. "target architectures", "JDK versions",
+ * or "mobile devices"). An environment group's definition includes which of these
+ * environments should be supported "by default" if not otherwise specified by one of the above
+ * mechanisms. In particular, a rule with no environment-related attributes automatically
+ * inherits all defaults.
+ * <li>Through a rule class default ({@link RuleClass.Builder#restrictedTo} and
+ * {@link RuleClass.Builder#compatibleWith}). This overrides global defaults for all instances
+ * of the given rule class. This can be used, for example, to make all *_test rules "testable"
+ * without each instance having to explicitly declare this capability.
+ * </ul>
+ *
+ * <p>Groups exist to model the idea that some environments are related while others have nothing
+ * to do with each other. Say, for example, we want to say a rule works for PowerPC platforms but
+ * not x86. We can do so by setting its "restricted to" attribute to
+ * {@code ['//sample/path:powerpc']}. Because both PowerPC and x86 are in the same
+ * "target architectures" group, this setting removes x86 from the set of supported environments.
+ * But since JDK support belongs to its own group ("JDK versions") it says nothing about which JDK
+ * the rule supports.
+ *
+ * <p>More precisely, if a rule has a "restricted to" value of [A, B, C], this removes support
+ * for all default environments D such that group(D) is in [group(A), group(B), group(C)] AND
+ * D is not in [A, B, C] (in other words, D isn't explicitly opted back in). The rule's full
+ * set of supported environments thus becomes [A, B, C] + all defaults that belong to unrelated
+ * groups.
+ *
+ * <p>If the rule has a "compatible with" value of [E, F, G], these are unconditionally
+ * added to its set of supported environments (in addition to the results from above).
+ *
+ * <p>An environment may not appear in both a rule's "restricted to" and "compatible with" values.
+ * If two environments belong to the same group, they must either both be in "restricted to",
+ * both be in "compatible with", or not explicitly specified.
+ *
+ * <p>Given all the above, constraint enforcement is this: rule A can depend on rule B if, for
+ * every environment A supports, B also supports that environment.
+ */
+public class ConstraintSemantics {
+ private ConstraintSemantics() {
+ }
+
+ /**
+ * Provides a set of default environments for a given environment group.
+ */
+ private interface DefaultsProvider {
+ Collection<Label> getDefaults(EnvironmentGroup group);
+ }
+
+ /**
+ * Provides a group's defaults as specified in the environment group's BUILD declaration.
+ */
+ private static class GroupDefaultsProvider implements DefaultsProvider {
+ @Override
+ public Collection<Label> getDefaults(EnvironmentGroup group) {
+ return group.getDefaults();
+ }
+ }
+
+ /**
+ * Provides a group's defaults, factoring in rule class defaults as specified by
+ * {@link com.google.devtools.build.lib.packages.RuleClass.Builder#compatibleWith}
+ * and {@link com.google.devtools.build.lib.packages.RuleClass.Builder#restrictedTo}.
+ */
+ private static class RuleClassDefaultsProvider implements DefaultsProvider {
+ private final EnvironmentCollection ruleClassDefaults;
+ private final GroupDefaultsProvider groupDefaults;
+
+ RuleClassDefaultsProvider(EnvironmentCollection ruleClassDefaults) {
+ this.ruleClassDefaults = ruleClassDefaults;
+ this.groupDefaults = new GroupDefaultsProvider();
+ }
+
+ @Override
+ public Collection<Label> getDefaults(EnvironmentGroup group) {
+ if (ruleClassDefaults.getGroups().contains(group)) {
+ return ruleClassDefaults.getEnvironments(group);
+ } else {
+ // If there are no rule class defaults for this group, just inherit global defaults.
+ return groupDefaults.getDefaults(group);
+ }
+ }
+ }
+
+ /**
+ * Collects the set of supported environments for a given rule by merging its
+ * restriction-style and compatibility-style environment declarations as specified by
+ * the given attributes. Only includes environments from "known" groups, i.e. the groups
+ * owning the environments explicitly referenced from these attributes.
+ */
+ private static class EnvironmentCollector {
+ private final RuleContext ruleContext;
+ private final String restrictionAttr;
+ private final String compatibilityAttr;
+ private final DefaultsProvider defaultsProvider;
+
+ private final EnvironmentCollection restrictionEnvironments;
+ private final EnvironmentCollection compatibilityEnvironments;
+ private final EnvironmentCollection supportedEnvironments;
+
+ /**
+ * Constructs a new collector on the given attributes.
+ *
+ * @param ruleContext analysis context for the rule
+ * @param restrictionAttr the name of the attribute that declares "restricted to"-style
+ * environments. If the rule doesn't have this attribute, this is considered an
+ * empty declaration.
+ * @param compatibilityAttr the name of the attribute that declares "compatible with"-style
+ * environments. If the rule doesn't have this attribute, this is considered an
+ * empty declaration.
+ * @param defaultsProvider provider for the default environments within a group if not
+ * otherwise overriden by the above attributes
+ */
+ EnvironmentCollector(RuleContext ruleContext, String restrictionAttr, String compatibilityAttr,
+ DefaultsProvider defaultsProvider) {
+ this.ruleContext = ruleContext;
+ this.restrictionAttr = restrictionAttr;
+ this.compatibilityAttr = compatibilityAttr;
+ this.defaultsProvider = defaultsProvider;
+
+ EnvironmentCollection.Builder environmentsBuilder = new EnvironmentCollection.Builder();
+ restrictionEnvironments = collectRestrictionEnvironments(environmentsBuilder);
+ compatibilityEnvironments = collectCompatibilityEnvironments(environmentsBuilder);
+ supportedEnvironments = environmentsBuilder.build();
+ }
+
+ /**
+ * Returns the set of environments supported by this rule, as determined by the
+ * restriction-style attribute, compatibility-style attribute, and group defaults
+ * provider instantiated with this class.
+ */
+ EnvironmentCollection getEnvironments() {
+ return supportedEnvironments;
+ }
+
+ /**
+ * Validity-checks that no group has its environment referenced in both the "compatible with"
+ * and restricted to" attributes. Returns true if all is good, returns false and reports
+ * appropriate errors if there are any problems.
+ */
+ boolean validateEnvironmentSpecifications() {
+ ImmutableCollection<EnvironmentGroup> restrictionGroups = restrictionEnvironments.getGroups();
+ boolean hasErrors = false;
+
+ for (EnvironmentGroup group : compatibilityEnvironments.getGroups()) {
+ if (restrictionGroups.contains(group)) {
+ // To avoid error-spamming the user, when we find a conflict we only report one example
+ // environment from each attribute for that group.
+ Label compatibilityEnv =
+ compatibilityEnvironments.getEnvironments(group).iterator().next();
+ Label restrictionEnv = restrictionEnvironments.getEnvironments(group).iterator().next();
+
+ if (compatibilityEnv.equals(restrictionEnv)) {
+ ruleContext.attributeError(compatibilityAttr, compatibilityEnv
+ + " cannot appear both here and in " + restrictionAttr);
+ } else {
+ ruleContext.attributeError(compatibilityAttr, compatibilityEnv + " and "
+ + restrictionEnv + " belong to the same environment group. They should be declared "
+ + "together either here or in " + restrictionAttr);
+ }
+ hasErrors = true;
+ }
+ }
+
+ return !hasErrors;
+ }
+
+ /**
+ * Adds environments specified in the "restricted to" attribute to the set of supported
+ * environments and returns the environments added.
+ */
+ private EnvironmentCollection collectRestrictionEnvironments(
+ EnvironmentCollection.Builder supportedEnvironments) {
+ return collectEnvironments(restrictionAttr, supportedEnvironments);
+ }
+
+ /**
+ * Adds environments specified in the "compatible with" attribute to the set of supported
+ * environments, along with all defaults from the groups they belong to. Returns these
+ * environments, not including the defaults.
+ */
+ private EnvironmentCollection collectCompatibilityEnvironments(
+ EnvironmentCollection.Builder supportedEnvironments) {
+ EnvironmentCollection compatibilityEnvironments =
+ collectEnvironments(compatibilityAttr, supportedEnvironments);
+ for (EnvironmentGroup group : compatibilityEnvironments.getGroups()) {
+ supportedEnvironments.putAll(group, defaultsProvider.getDefaults(group));
+ }
+ return compatibilityEnvironments;
+ }
+
+ /**
+ * Adds environments specified by the given attribute to the set of supported environments
+ * and returns the environments added.
+ *
+ * <p>If this rule doesn't have the given attributes, returns an empty set.
+ */
+ private EnvironmentCollection collectEnvironments(String attrName,
+ EnvironmentCollection.Builder supportedEnvironments) {
+ if (!ruleContext.getRule().isAttrDefined(attrName, Type.LABEL_LIST)) {
+ return EnvironmentCollection.EMPTY;
+ }
+ EnvironmentCollection.Builder environments = new EnvironmentCollection.Builder();
+ for (TransitiveInfoCollection envTarget :
+ ruleContext.getPrerequisites(attrName, RuleConfiguredTarget.Mode.DONT_CHECK)) {
+ EnvironmentWithGroup envInfo = resolveEnvironment(envTarget);
+ environments.put(envInfo.group(), envInfo.environment());
+ supportedEnvironments.put(envInfo.group(), envInfo.environment());
+ }
+ return environments.build();
+ }
+
+ /**
+ * Returns the environment and its group. An {@link Environment} rule only "supports" one
+ * environment: itself. Extract that from its more generic provider interface and sanity
+ * check that that's in fact what we see.
+ */
+ private static EnvironmentWithGroup resolveEnvironment(TransitiveInfoCollection envRule) {
+ SupportedEnvironmentsProvider prereq =
+ Preconditions.checkNotNull(envRule.getProvider(SupportedEnvironmentsProvider.class));
+ return Iterables.getOnlyElement(prereq.getEnvironments().getGroupedEnvironments());
+ }
+ }
+
+ /**
+ * Returns the set of environments this rule supports, applying the logic described in
+ * {@link ConstraintSemantics}.
+ *
+ * <p>Note this set is <b>not complete</b> - it doesn't include environments from groups we don't
+ * "know about". Environments and groups can be declared in any package. If the rule includes
+ * no references to that package, then it simply doesn't know anything about them. But the
+ * constraint semantics say the rule should support the defaults for that group. We encode this
+ * implicitly: given the returned set, for any group that's not in the set the rule is also
+ * considered to support that group's defaults.
+ *
+ * @param ruleContext analysis context for the rule. A rule error is triggered here if
+ * invalid constraint settings are discovered.
+ * @return the environments this rule supports, not counting defaults "unknown" to this rule
+ * as described above. Returns null if any errors are encountered.
+ */
+ @Nullable
+ public static EnvironmentCollection getSupportedEnvironments(RuleContext ruleContext) {
+ if (!validateAttributes(ruleContext)) {
+ return null;
+ }
+
+ // This rule's rule class defaults (or null if the rule class has no defaults).
+ EnvironmentCollector ruleClassCollector = maybeGetRuleClassDefaults(ruleContext);
+ // Default environments for this rule. If the rule has rule class defaults, this is
+ // those defaults. Otherwise it's the global defaults specified by environment_group
+ // declarations.
+ DefaultsProvider ruleDefaults;
+
+ if (ruleClassCollector != null) {
+ if (!ruleClassCollector.validateEnvironmentSpecifications()) {
+ return null;
+ }
+ ruleDefaults = new RuleClassDefaultsProvider(ruleClassCollector.getEnvironments());
+ } else {
+ ruleDefaults = new GroupDefaultsProvider();
+ }
+
+ EnvironmentCollector ruleCollector = new EnvironmentCollector(ruleContext,
+ RuleClass.RESTRICTED_ENVIRONMENT_ATTR, RuleClass.COMPATIBLE_ENVIRONMENT_ATTR, ruleDefaults);
+ if (!ruleCollector.validateEnvironmentSpecifications()) {
+ return null;
+ }
+
+ EnvironmentCollection supportedEnvironments = ruleCollector.getEnvironments();
+ if (ruleClassCollector != null) {
+ // If we have rule class defaults from groups that aren't referenced from the rule itself,
+ // we need to add them in too to override the global defaults.
+ supportedEnvironments =
+ addUnknownGroupsToCollection(supportedEnvironments, ruleClassCollector.getEnvironments());
+ }
+ return supportedEnvironments;
+ }
+
+ /**
+ * Returns the rule class defaults specified for this rule, or null if there are
+ * no such defaults.
+ */
+ @Nullable
+ private static EnvironmentCollector maybeGetRuleClassDefaults(RuleContext ruleContext) {
+ Rule rule = ruleContext.getRule();
+ String restrictionAttr = RuleClass.DEFAULT_RESTRICTED_ENVIRONMENT_ATTR;
+ String compatibilityAttr = RuleClass.DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR;
+
+ if (rule.isAttrDefined(restrictionAttr, Type.LABEL_LIST)
+ || rule.isAttrDefined(compatibilityAttr, Type.LABEL_LIST)) {
+ return new EnvironmentCollector(ruleContext, restrictionAttr, compatibilityAttr,
+ new GroupDefaultsProvider());
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Adds environments to an {@link EnvironmentCollection} from groups that aren't already
+ * a part of that collection.
+ *
+ * @param environments the collection to add to
+ * @param toAdd the collection to add. All environments in this collection in groups
+ * that aren't represented in {@code environments} are added to {@code environments}.
+ * @return the expanded collection.
+ */
+ private static EnvironmentCollection addUnknownGroupsToCollection(
+ EnvironmentCollection environments, EnvironmentCollection toAdd) {
+ EnvironmentCollection.Builder builder = new EnvironmentCollection.Builder();
+ builder.putAll(environments);
+ for (EnvironmentGroup candidateGroup : toAdd.getGroups()) {
+ if (!environments.getGroups().contains(candidateGroup)) {
+ builder.putAll(candidateGroup, toAdd.getEnvironments(candidateGroup));
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Validity-checks this rule's constraint-related attributes. Returns true if all is good,
+ * returns false and reports appropriate errors if there are any problems.
+ */
+ private static boolean validateAttributes(RuleContext ruleContext) {
+ AttributeMap attributes = ruleContext.attributes();
+
+ // Report an error if "restricted to" is explicitly set to nothing. Even if this made
+ // conceptual sense, we don't know which groups we should apply that to.
+ String restrictionAttr = RuleClass.RESTRICTED_ENVIRONMENT_ATTR;
+ List<? extends TransitiveInfoCollection> restrictionEnvironments = ruleContext
+ .getPrerequisites(restrictionAttr, RuleConfiguredTarget.Mode.DONT_CHECK);
+ if (restrictionEnvironments.isEmpty()
+ && attributes.isAttributeValueExplicitlySpecified(restrictionAttr)) {
+ ruleContext.attributeError(restrictionAttr, "attribute cannot be empty");
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Performs constraint checking on the given rule's dependencies and reports any errors.
+ *
+ * @param ruleContext the rule to analyze
+ * @param supportedEnvironments the rule's supported environments, as defined by the return
+ * value of {@link #getSupportedEnvironments}. In particular, for any environment group that's
+ * not in this collection, the rule is assumed to support the defaults for that group.
+ */
+ public static void checkConstraints(RuleContext ruleContext,
+ EnvironmentCollection supportedEnvironments) {
+
+ Set<EnvironmentGroup> knownGroups = supportedEnvironments.getGroups();
+
+ for (TransitiveInfoCollection dependency : getAllPrerequisites(ruleContext)) {
+ SupportedEnvironmentsProvider depProvider =
+ dependency.getProvider(SupportedEnvironmentsProvider.class);
+ if (depProvider == null) {
+ // Input files (InputFileConfiguredTarget) don't support environments. We may subsequently
+ // opt them into constraint checking, but for now just pass them by.
+ continue;
+ }
+ Collection<Label> depEnvironments = depProvider.getEnvironments().getEnvironments();
+ Set<EnvironmentGroup> groupsKnownToDep = depProvider.getEnvironments().getGroups();
+
+ // Environments we support that the dependency does not support.
+ Set<Label> disallowedEnvironments = new LinkedHashSet<>();
+
+ // For every environment we support, either the dependency must also support it OR it must be
+ // a default for a group the dependency doesn't know about.
+ for (EnvironmentWithGroup supportedEnv : supportedEnvironments.getGroupedEnvironments()) {
+ EnvironmentGroup group = supportedEnv.group();
+ Label environment = supportedEnv.environment();
+ if (!depEnvironments.contains(environment)
+ && (groupsKnownToDep.contains(group) || !group.isDefault(environment))) {
+ disallowedEnvironments.add(environment);
+ }
+ }
+
+ // For any environment group we don't know about, we implicitly support its defaults. Check
+ // that the dep does, too.
+ for (EnvironmentGroup depGroup : groupsKnownToDep) {
+ if (!knownGroups.contains(depGroup)) {
+ for (Label defaultEnv : depGroup.getDefaults()) {
+ if (!depEnvironments.contains(defaultEnv)) {
+ disallowedEnvironments.add(defaultEnv);
+ }
+ }
+ }
+ }
+
+ // Report errors on bad environments.
+ if (!disallowedEnvironments.isEmpty()) {
+ ruleContext.ruleError("dependency " + dependency.getLabel()
+ + " doesn't support expected environment"
+ + (disallowedEnvironments.size() == 1 ? "" : "s")
+ + ": " + Joiner.on(", ").join(disallowedEnvironments));
+ }
+ }
+ }
+
+ /**
+ * Returns all dependencies that should be constraint-checked against the current rule.
+ */
+ private static Iterable<TransitiveInfoCollection> getAllPrerequisites(RuleContext ruleContext) {
+ Set<TransitiveInfoCollection> prerequisites = new LinkedHashSet<>();
+ AttributeMap attributes = ruleContext.attributes();
+
+ for (String attr : attributes.getAttributeNames()) {
+ Type<?> attrType = attributes.getAttributeType(attr);
+ // TODO(bazel-team): support specifying which attributes are subject to constraint checking
+ if ((attrType == Type.LABEL || attrType == Type.LABEL_LIST)
+ && !RuleClass.isConstraintAttribute(attr)
+ && !attr.equals("visibility")) {
+ prerequisites.addAll(
+ ruleContext.getPrerequisites(attr, RuleConfiguredTarget.Mode.DONT_CHECK));
+ }
+ }
+ return prerequisites;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/Environment.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/Environment.java
new file mode 100644
index 0000000..912ed72
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/Environment.java
@@ -0,0 +1,71 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.constraints;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.EnvironmentGroup;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Implementation for the environment rule.
+ */
+public class Environment implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+
+ // The main analysis work to do here is to simply fill in SupportedEnvironmentsProvider to
+ // pass the environment itself to depending rules.
+ //
+ // This will likely expand when we add support for environments fulfilling other environments.
+ Label label = ruleContext.getLabel();
+ Package pkg = ruleContext.getRule().getPackage();
+
+ EnvironmentGroup group = null;
+ for (EnvironmentGroup pkgGroup : pkg.getTargets(EnvironmentGroup.class)) {
+ if (pkgGroup.getEnvironments().contains(label)) {
+ group = pkgGroup;
+ break;
+ }
+ }
+
+ if (group == null) {
+ ruleContext.ruleError("no matching environment group from the same package");
+ return null;
+ }
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .addProvider(SupportedEnvironmentsProvider.class,
+ new SupportedEnvironments(
+ new EnvironmentCollection.Builder().put(group, label).build()))
+ .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .add(FileProvider.class, new FileProvider(ruleContext.getLabel(),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER)))
+ .add(FilesToRunProvider.class, new FilesToRunProvider(ruleContext.getLabel(),
+ ImmutableList.<Artifact>of(), null, null))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentCollection.java
new file mode 100644
index 0000000..1ce5f1c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentCollection.java
@@ -0,0 +1,126 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.constraints;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.packages.EnvironmentGroup;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Map;
+
+/**
+ * Contains a set of {@link Environment} labels and their associated groups.
+ */
+@Immutable
+public class EnvironmentCollection {
+ private final ImmutableMultimap<EnvironmentGroup, Label> map;
+
+ private EnvironmentCollection(ImmutableMultimap<EnvironmentGroup, Label> map) {
+ this.map = map;
+ }
+
+ /**
+ * Stores an environment's build label along with the group it belongs to.
+ */
+ static class EnvironmentWithGroup {
+ private final Label environment;
+ private final EnvironmentGroup group;
+ EnvironmentWithGroup(Label environment, EnvironmentGroup group) {
+ this.environment = environment;
+ this.group = group;
+ }
+ Label environment() { return environment; }
+ EnvironmentGroup group() { return group; }
+ }
+
+ /**
+ * Returns the build labels of each environment in this collection, ordered by
+ * their insertion order in {@link Builder}.
+ */
+ ImmutableCollection<Label> getEnvironments() {
+ return map.values();
+ }
+
+ /**
+ * Returns the set of groups the environments in this collection belong to, ordered by
+ * their insertion order in {@link Builder}
+ */
+ ImmutableSet<EnvironmentGroup> getGroups() {
+ return map.keySet();
+ }
+
+ /**
+ * Returns the build labels of each environment in this collection paired with the
+ * group each environment belongs to, ordered by their insertion order in {@link Builder}.
+ */
+ ImmutableCollection<EnvironmentWithGroup> getGroupedEnvironments() {
+ ImmutableSet.Builder<EnvironmentWithGroup> builder = ImmutableSet.builder();
+ for (Map.Entry<EnvironmentGroup, Label> entry : map.entries()) {
+ builder.add(new EnvironmentWithGroup(entry.getValue(), entry.getKey()));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns the environments in this collection that belong to the given group, ordered by
+ * their insertion order in {@link Builder}. If no environments belong to the given group,
+ * returns an empty collection.
+ */
+ ImmutableCollection<Label> getEnvironments(EnvironmentGroup group) {
+ return map.get(group);
+ }
+
+ /**
+ * An empty collection.
+ */
+ static final EnvironmentCollection EMPTY =
+ new EnvironmentCollection(ImmutableMultimap.<EnvironmentGroup, Label>of());
+
+ static class Builder {
+ private final ImmutableMultimap.Builder<EnvironmentGroup, Label> mapBuilder =
+ ImmutableMultimap.builder();
+
+ /**
+ * Inserts the given environment / owning group pair.
+ */
+ Builder put(EnvironmentGroup group, Label environment) {
+ mapBuilder.put(group, environment);
+ return this;
+ }
+
+ /**
+ * Inserts the given set of environments, all belonging to the specified group.
+ */
+ Builder putAll(EnvironmentGroup group, Iterable<Label> environments) {
+ mapBuilder.putAll(group, environments);
+ return this;
+ }
+
+ /**
+ * Inserts the contents of another {@link EnvironmentCollection} into this one.
+ */
+ Builder putAll(EnvironmentCollection other) {
+ mapBuilder.putAll(other.map);
+ return this;
+ }
+
+ EnvironmentCollection build() {
+ return new EnvironmentCollection(mapBuilder.build());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentRule.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentRule.java
new file mode 100644
index 0000000..0553af0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentRule.java
@@ -0,0 +1,48 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.constraints;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Type;
+
+/**
+ * Rule definition for environment rules (for Bazel's constraint enforcement system).
+ */
+@BlazeRule(name = EnvironmentRule.RULE_NAME,
+ ancestors = { BaseRuleClasses.BaseRule.class },
+ factoryClass = Environment.class)
+public final class EnvironmentRule implements RuleDefinition {
+ public static final String RULE_NAME = "environment";
+
+ @Override
+ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .override(attr("tags", Type.STRING_LIST)
+ // No need to show up in ":all", etc. target patterns.
+ .value(ImmutableList.of("manual"))
+ .nonconfigurable("low-level attribute, used in TargetUtils without configurations"))
+ .removeAttribute(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR)
+ .removeAttribute(RuleClass.RESTRICTED_ENVIRONMENT_ATTR)
+ .setUndocumented()
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironments.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironments.java
new file mode 100644
index 0000000..78c835e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironments.java
@@ -0,0 +1,31 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.constraints;
+
+/**
+ * Standard {@link SupportedEnvironmentsProvider} implementation.
+ */
+public class SupportedEnvironments implements SupportedEnvironmentsProvider {
+ private final EnvironmentCollection supportedEnvironments;
+
+ public SupportedEnvironments(EnvironmentCollection supportedEnvironments) {
+ this.supportedEnvironments = supportedEnvironments;
+ }
+
+ @Override
+ public EnvironmentCollection getEnvironments() {
+ return supportedEnvironments;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironmentsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironmentsProvider.java
new file mode 100644
index 0000000..8200386
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironmentsProvider.java
@@ -0,0 +1,29 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.analysis.constraints;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+
+/**
+ * A provider that advertises which environments the associated target is compatible with
+ * (from the point of view of the constraint enforcement system).
+ */
+public interface SupportedEnvironmentsProvider extends TransitiveInfoProvider {
+
+ /**
+ * Returns the environments the associated target is compatible with.
+ */
+ EnvironmentCollection getEnvironments();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelDiffAwarenessModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelDiffAwarenessModule.java
new file mode 100644
index 0000000..1dad1f5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelDiffAwarenessModule.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.skyframe.DiffAwareness;
+import com.google.devtools.build.lib.skyframe.LocalDiffAwareness;
+
+/**
+ * Provides the {@link DiffAwareness} implementation that uses the Java watch service.
+ */
+public class BazelDiffAwarenessModule extends BlazeModule {
+
+ @Override
+ public Iterable<DiffAwareness.Factory> getDiffAwarenessFactories(boolean watchFS) {
+ ImmutableList.Builder<DiffAwareness.Factory> builder = ImmutableList.builder();
+ if (watchFS) {
+ builder.add(new LocalDiffAwareness.Factory());
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java
new file mode 100644
index 0000000..fd3d000
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+
+import java.util.List;
+
+/**
+ * The main class.
+ */
+public final class BazelMain {
+ private static final List<Class<? extends BlazeModule>> BAZEL_MODULES = ImmutableList.of(
+ com.google.devtools.build.lib.bazel.BazelShutdownLoggerModule.class,
+ com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class,
+ com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class,
+ com.google.devtools.build.lib.bazel.BazelRepositoryModule.class,
+ com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class,
+ com.google.devtools.build.lib.standalone.StandaloneModule.class,
+ com.google.devtools.build.lib.runtime.BuildSummaryStatsModule.class,
+ com.google.devtools.build.lib.webstatusserver.WebStatusServerModule.class
+ );
+
+ public static void main(String[] args) {
+ BlazeRuntime.main(BAZEL_MODULES, args);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
new file mode 100644
index 0000000..483103f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -0,0 +1,100 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.bazel.repository.HttpArchiveFunction;
+import com.google.devtools.build.lib.bazel.repository.HttpJarFunction;
+import com.google.devtools.build.lib.bazel.repository.LocalRepositoryFunction;
+import com.google.devtools.build.lib.bazel.repository.MavenJarFunction;
+import com.google.devtools.build.lib.bazel.repository.NewLocalRepositoryFunction;
+import com.google.devtools.build.lib.bazel.repository.RepositoryDelegatorFunction;
+import com.google.devtools.build.lib.bazel.repository.RepositoryFunction;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.skyframe.SkyFunctions;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Adds support for fetching external code.
+ */
+public class BazelRepositoryModule extends BlazeModule {
+
+ private BlazeDirectories directories;
+ // A map of repository handlers that can be looked up by rule class name.
+ private final ImmutableMap<String, RepositoryFunction> repositoryHandlers;
+
+ public BazelRepositoryModule() {
+ repositoryHandlers = ImmutableMap.of(
+ LocalRepositoryRule.NAME, new LocalRepositoryFunction(),
+ HttpArchiveRule.NAME, new HttpArchiveFunction(),
+ HttpJarRule.NAME, new HttpJarFunction(),
+ MavenJarRule.NAME, new MavenJarFunction(),
+ NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction());
+ }
+
+ @Override
+ public void blazeStartup(OptionsProvider startupOptions,
+ BlazeVersionInfo versionInfo, UUID instanceId, BlazeDirectories directories,
+ Clock clock) {
+ this.directories = directories;
+ for (RepositoryFunction handler : repositoryHandlers.values()) {
+ handler.setDirectories(directories);
+ }
+ }
+
+ @Override
+ public Set<Path> getImmutableDirectories() {
+ return ImmutableSet.of(RepositoryFunction.getExternalRepositoryDirectory(directories));
+ }
+
+ @Override
+ public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
+ for (Entry<String, RepositoryFunction> handler : repositoryHandlers.entrySet()) {
+ builder.addRuleDefinition(handler.getValue().getRuleDefinition());
+ }
+ }
+
+ @Override
+ public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) {
+ ImmutableMap.Builder<SkyFunctionName, SkyFunction> builder = ImmutableMap.builder();
+
+ // Bazel-specific repository downloaders.
+ for (RepositoryFunction handler : repositoryHandlers.values()) {
+ builder.put(handler.getSkyFunctionName(), handler);
+ }
+
+ // Create the delegator everything flows through.
+ builder.put(SkyFunctions.REPOSITORY,
+ new RepositoryDelegatorFunction(repositoryHandlers));
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelShutdownLoggerModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelShutdownLoggerModule.java
new file mode 100644
index 0000000..3c32611
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelShutdownLoggerModule.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel;
+
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.UUID;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+/**
+ * Shutdown log output when Bazel runs in batch mode
+ */
+public class BazelShutdownLoggerModule extends BlazeModule {
+
+ private Logger globalLogger;
+
+ @Override
+ public void blazeStartup(OptionsProvider startupOptions, BlazeVersionInfo versionInfo,
+ UUID instanceId, BlazeDirectories directories, Clock clock) {
+ LogManager.getLogManager().reset();
+ globalLogger = Logger.getGlobal();
+ globalLogger.setLevel(java.util.logging.Level.OFF);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java
new file mode 100644
index 0000000..dda983d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java
@@ -0,0 +1,196 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.BuildInfoHelper;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Key;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.UUID;
+
+/**
+ * Workspace status information for Bazel.
+ *
+ * <p>Currently only a stub.
+ */
+public class BazelWorkspaceStatusModule extends BlazeModule {
+ private static class BazelWorkspaceStatusAction extends WorkspaceStatusAction {
+ private final Artifact stableStatus;
+ private final Artifact volatileStatus;
+
+ private BazelWorkspaceStatusAction(
+ Artifact stableStatus, Artifact volatileStatus) {
+ super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, Artifact.NO_ARTIFACTS,
+ ImmutableList.of(stableStatus, volatileStatus));
+ this.stableStatus = stableStatus;
+ this.volatileStatus = volatileStatus;
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ try {
+ FileSystemUtils.writeContent(stableStatus.getPath(), new byte[] {});
+ FileSystemUtils.writeContent(volatileStatus.getPath(), new byte[] {});
+ } catch (IOException e) {
+ throw new ActionExecutionException(e, this, true);
+ }
+ }
+
+ // TODO(bazel-team): Add test for equals, add hashCode.
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof BazelWorkspaceStatusAction)) {
+ return false;
+ }
+
+ BazelWorkspaceStatusAction that = (BazelWorkspaceStatusAction) o;
+ return this.stableStatus.equals(that.stableStatus)
+ && this.volatileStatus.equals(that.volatileStatus);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(stableStatus, volatileStatus);
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "BazelWorkspaceStatusAction";
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ protected String computeKey() {
+ return "";
+ }
+
+ @Override
+ public Artifact getVolatileStatus() {
+ return volatileStatus;
+ }
+
+ @Override
+ public Artifact getStableStatus() {
+ return stableStatus;
+ }
+ }
+
+ private class BazelStatusActionFactory implements WorkspaceStatusAction.Factory {
+ @Override
+ public Map<String, String> createDummyWorkspaceStatus() {
+ return ImmutableMap.of();
+ }
+
+ @Override
+ public WorkspaceStatusAction createWorkspaceStatusAction(
+ ArtifactFactory factory, ArtifactOwner artifactOwner, Supplier<UUID> buildId) {
+ Root root = runtime.getDirectories().getBuildDataDirectory();
+
+ Artifact stableArtifact = factory.getDerivedArtifact(
+ new PathFragment("stable-status.txt"), root, artifactOwner);
+ Artifact volatileArtifact = factory.getConstantMetadataArtifact(
+ new PathFragment("volatile-status.txt"), root, artifactOwner);
+
+ return new BazelWorkspaceStatusAction(stableArtifact, volatileArtifact);
+ }
+ }
+
+ @ExecutionStrategy(contextType = WorkspaceStatusAction.Context.class)
+ private class BazelWorkspaceStatusActionContext implements WorkspaceStatusAction.Context {
+ @Override
+ public ImmutableMap<String, Key> getStableKeys() {
+ return ImmutableMap.of();
+ }
+
+ @Override
+ public ImmutableMap<String, Key> getVolatileKeys() {
+ return ImmutableMap.of();
+ }
+ }
+
+
+ private class WorkspaceActionContextProvider implements ActionContextProvider {
+ @Override
+ public Iterable<ActionContext> getActionContexts() {
+ return ImmutableList.<ActionContext>of(new BazelWorkspaceStatusActionContext());
+ }
+
+ @Override
+ public void executorCreated(Iterable<ActionContext> usedContexts)
+ throws ExecutorInitException {
+ }
+
+ @Override
+ public void executionPhaseEnding() {
+ }
+
+ @Override
+ public void executionPhaseStarting(ActionInputFileCache actionInputFileCache,
+ ActionGraph actionGraph, Iterable<Artifact> topLevelArtifacts) throws ExecutorInitException,
+ InterruptedException {
+ }
+ }
+
+ private BlazeRuntime runtime;
+
+ @Override
+ public void beforeCommand(BlazeRuntime runtime, Command command) {
+ this.runtime = runtime;
+ }
+
+ @Override
+ public ActionContextProvider getActionContextProvider() {
+ return new WorkspaceActionContextProvider();
+ }
+
+ @Override
+ public WorkspaceStatusAction.Factory getWorkspaceStatusActionFactory() {
+ return new BazelStatusActionFactory();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java
new file mode 100644
index 0000000..1076f24
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java
@@ -0,0 +1,218 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import org.apache.commons.compress.archivers.ArchiveException;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.ArchiveStreamFactory;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.utils.IOUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+
+/**
+ * Creates decompressors to use on archive. Use {@link DecompressorFactory#create} to get the
+ * correct type of decompressor for the input archive, then call
+ * {@link Decompressor#decompress} to decompress it.
+ */
+public abstract class DecompressorFactory {
+
+
+ public static Decompressor create(Target target, Path archivePath)
+ throws DecompressorException {
+ String baseName = archivePath.getBaseName();
+
+ if (target.getTargetKind().startsWith(HttpJarRule.NAME + " ")) {
+ if (baseName.endsWith(".jar")) {
+ return new JarDecompressor(target, archivePath);
+ } else {
+ throw new DecompressorException(
+ "Expected " + HttpJarRule.NAME + " " + target.getName()
+ + " to create file with a .jar suffix (got " + archivePath + ")");
+ }
+ }
+
+ if (target.getTargetKind().startsWith(HttpArchiveRule.NAME + " ")) {
+ if (baseName.endsWith(".zip") || baseName.endsWith(".jar")) {
+ return new ZipDecompressor(archivePath);
+ } else {
+ throw new DecompressorException(
+ "Expected " + HttpArchiveRule.NAME + " " + target.getName()
+ + " to create file with a .zip or .jar suffix (got " + archivePath + ")");
+ }
+ }
+
+ throw new DecompressorException(
+ "No decompressor found for " + target.getTargetKind() + " rule " + target.getName()
+ + " (got " + archivePath + ")");
+ }
+
+ /**
+ * General decompressor for an archive. Should be overridden for each specific archive type.
+ */
+ public abstract static class Decompressor {
+ protected final Path archiveFile;
+
+ private Decompressor(Path archiveFile) {
+ this.archiveFile = archiveFile;
+ }
+
+ /**
+ * This is overridden by archive-specific decompression logic. Often this logic will create
+ * files and directories under the {@link Decompressor#archiveFile}'s parent directory.
+ *
+ * @return the path to the repository directory. That is, the returned path will be a directory
+ * containing a WORKSPACE file.
+ */
+ public abstract Path decompress() throws DecompressorException;
+ }
+
+ static class JarDecompressor extends Decompressor {
+ private final Target target;
+
+ public JarDecompressor(Target target, Path archiveFile) {
+ super(archiveFile);
+ this.target = target;
+ }
+
+ /**
+ * The .jar can be used compressed, so this just exposes it in a way Bazel can use.
+ *
+ * <p>It moves the jar from some-name/foo.jar to some-name/repository/jar/foo.jar and creates a
+ * BUILD file containing one entry: a .jar.
+ */
+ @Override
+ public Path decompress() throws DecompressorException {
+ Path destinationDirectory = archiveFile.getParentDirectory().getRelative("repository");
+ // Example: archiveFile is .external-repository/some-name/foo.jar.
+ String baseName = archiveFile.getBaseName();
+
+ try {
+ FileSystemUtils.createDirectoryAndParents(destinationDirectory);
+ // .external-repository/some-name/repository/WORKSPACE.
+ Path workspaceFile = destinationDirectory.getRelative("WORKSPACE");
+ FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"),
+ "# DO NOT EDIT: automatically generated WORKSPACE file for " + target.getTargetKind()
+ + " rule " + target.getName());
+ // .external-repository/some-name/repository/jar.
+ Path jarDirectory = destinationDirectory.getRelative("jar");
+ FileSystemUtils.createDirectoryAndParents(jarDirectory);
+ // .external-repository/some-name/repository/jar/foo.jar is a symbolic link to the jar in
+ // .external-repository/some-name.
+ Path jarSymlink = jarDirectory.getRelative(baseName);
+ if (!jarSymlink.exists()) {
+ jarSymlink.createSymbolicLink(archiveFile);
+ }
+ // .external-repository/some-name/repository/jar/BUILD defines the //jar target.
+ Path buildFile = jarDirectory.getRelative("BUILD");
+ FileSystemUtils.writeLinesAs(buildFile, Charset.forName("UTF-8"),
+ "# DO NOT EDIT: automatically generated BUILD file for " + target.getTargetKind()
+ + " rule " + target.getName(),
+ "java_import(",
+ " name = 'jar',",
+ " jars = ['" + baseName + "'],",
+ " visibility = ['//visibility:public']",
+ ")");
+ } catch (IOException e) {
+ throw new DecompressorException(e.getMessage());
+ }
+ return destinationDirectory;
+ }
+ }
+
+ private static class ZipDecompressor extends Decompressor {
+ public ZipDecompressor(Path archiveFile) {
+ super(archiveFile);
+ }
+
+ /**
+ * This unzips the zip file to a sibling directory of {@link Decompressor#archiveFile}. The
+ * zip file is expected to have the WORKSPACE file at the top level, e.g.:
+ *
+ * <pre>
+ * $ unzip -lf some-repo.zip
+ * Archive: ../repo.zip
+ * Length Date Time Name
+ * --------- ---------- ----- ----
+ * 0 2014-11-20 15:50 WORKSPACE
+ * 0 2014-11-20 16:10 foo/
+ * 236 2014-11-20 15:52 foo/BUILD
+ * ...
+ * </pre>
+ */
+ @Override
+ public Path decompress() throws DecompressorException {
+ Path destinationDirectory = archiveFile.getParentDirectory().getRelative("repository");
+ try (InputStream is = new FileInputStream(archiveFile.getPathString())) {
+ ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream(
+ ArchiveStreamFactory.ZIP, is);
+ ZipArchiveEntry entry = (ZipArchiveEntry) in.getNextEntry();
+ while (entry != null) {
+ extractZipEntry(in, entry, destinationDirectory);
+ entry = (ZipArchiveEntry) in.getNextEntry();
+ }
+ } catch (IOException | ArchiveException e) {
+ throw new DecompressorException(
+ "Error extracting " + archiveFile + " to " + destinationDirectory + ": "
+ + e.getMessage());
+ }
+ return destinationDirectory;
+ }
+
+ private void extractZipEntry(
+ ArchiveInputStream in, ZipArchiveEntry entry, Path destinationDirectory)
+ throws IOException, DecompressorException {
+ PathFragment relativePath = new PathFragment(entry.getName());
+ if (relativePath.isAbsolute()) {
+ throw new DecompressorException("Failed to extract " + relativePath
+ + ", zipped paths cannot be absolute");
+ }
+ Path outputPath = destinationDirectory.getRelative(relativePath);
+ FileSystemUtils.createDirectoryAndParents(outputPath.getParentDirectory());
+ if (entry.isDirectory()) {
+ FileSystemUtils.createDirectoryAndParents(outputPath);
+ } else {
+ try (OutputStream out = new FileOutputStream(new File(outputPath.getPathString()))) {
+ IOUtils.copy(in, out);
+ } catch (IOException e) {
+ throw new DecompressorException("Error writing " + outputPath + " from "
+ + archiveFile);
+ }
+ }
+ }
+ }
+
+ /**
+ * Exceptions thrown when something goes wrong decompressing an archive.
+ */
+ public static class DecompressorException extends Exception {
+ public DecompressorException(String message) {
+ super(message);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
new file mode 100644
index 0000000..1cd6db8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
@@ -0,0 +1,112 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.DecompressorException;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.FileValue;
+import com.google.devtools.build.lib.skyframe.RepositoryValue;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+/**
+ * Downloads a file over HTTP.
+ */
+public class HttpArchiveFunction extends RepositoryFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+ Rule rule = RepositoryFunction.getRule(repositoryName, HttpArchiveRule.NAME, env);
+ if (rule == null) {
+ return null;
+ }
+
+ return compute(env, rule);
+ }
+
+ protected FileValue createOutputDirectory(Environment env, String repositoryName)
+ throws RepositoryFunctionException {
+ // The output directory is always under .external-repository (to stay out of the way of
+ // artifacts from this repository) and uses the rule's name to avoid conflicts with other
+ // remote repository rules. For example, suppose you had the following WORKSPACE file:
+ //
+ // http_archive(name = "png", url = "http://example.com/downloads/png.tar.gz", sha256 = "...")
+ //
+ // This would download png.tar.gz to .external-repository/png/png.tar.gz.
+ Path outputDirectory = getExternalRepositoryDirectory().getRelative(repositoryName);
+ try {
+ FileSystemUtils.createDirectoryAndParents(outputDirectory);
+ } catch (IOException e) {
+ throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+ }
+ return getRepositoryDirectory(outputDirectory, env);
+ }
+
+ protected SkyValue compute(Environment env, Rule rule)
+ throws RepositoryFunctionException {
+ FileValue directoryValue = createOutputDirectory(env, rule.getName());
+ if (directoryValue == null) {
+ return null;
+ }
+ Path outputDirectory = directoryValue.realRootedPath().asPath();
+ AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
+ URL url = null;
+ try {
+ url = new URL(mapper.get("url", Type.STRING));
+ } catch (MalformedURLException e) {
+ throw new RepositoryFunctionException(
+ new EvalException(rule.getLocation(), "Error parsing URL: " + e.getMessage()),
+ Transience.PERSISTENT);
+ }
+ String sha256 = mapper.get("sha256", Type.STRING);
+ HttpDownloader downloader = new HttpDownloader(url, sha256, outputDirectory);
+ try {
+ Path archiveFile = downloader.download();
+ outputDirectory = DecompressorFactory.create(rule, archiveFile).decompress();
+ } catch (IOException e) {
+ // Assumes all IO errors transient.
+ throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+ } catch (DecompressorException e) {
+ throw new RepositoryFunctionException(new IOException(e.getMessage()), Transience.TRANSIENT);
+ }
+ return new RepositoryValue(outputDirectory, directoryValue);
+ }
+
+ @Override
+ public SkyFunctionName getSkyFunctionName() {
+ return SkyFunctionName.computed(HttpArchiveRule.NAME.toUpperCase());
+ }
+
+ @Override
+ public Class<? extends RuleDefinition> getRuleDefinition() {
+ return HttpArchiveRule.class;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
new file mode 100644
index 0000000..0f9ff44
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
@@ -0,0 +1,107 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+
+/**
+ * Helper class for downloading a file from a URL.
+ */
+public class HttpDownloader {
+ private static final int BUFFER_SIZE = 2048;
+
+ private final URL url;
+ private final String sha256;
+ private final Path outputDirectory;
+
+ HttpDownloader(URL url, String sha256, Path outputDirectory) {
+ this.url = url;
+ this.sha256 = sha256;
+ this.outputDirectory = outputDirectory;
+ }
+
+ /**
+ * Attempt to download a file from the repository's URL. Returns the path to the file downloaded.
+ */
+ public Path download() throws IOException {
+ String filename = new PathFragment(url.getPath()).getBaseName();
+ if (filename.isEmpty()) {
+ filename = "temp";
+ }
+ Path destination = outputDirectory.getRelative(filename);
+
+ try (OutputStream outputStream = destination.getOutputStream()) {
+ ReadableByteChannel rbc = getChannel(url);
+ ByteBuffer byteBuffer = ByteBuffer.allocate(BUFFER_SIZE);
+ while (rbc.read(byteBuffer) > 0) {
+ byteBuffer.flip();
+ while (byteBuffer.hasRemaining()) {
+ outputStream.write(byteBuffer.get());
+ }
+ }
+ } catch (IOException e) {
+ throw new IOException(
+ "Error downloading " + url + " to " + destination + ": " + e.getMessage());
+ }
+
+ try {
+ String downloadedSha256 = getSha256(destination);
+ if (!downloadedSha256.equals(sha256)) {
+ throw new IOException(
+ "Downloaded file at " + destination + " has SHA-256 of " + downloadedSha256
+ + ", does not match expected SHA-256 (" + sha256 + ")");
+ }
+ } catch (IOException e) {
+ throw new IOException(
+ "Could not hash file " + destination + ": " + e.getMessage() + ", expected SHA-256 of "
+ + sha256 + ")");
+ }
+ return destination;
+ }
+
+ @VisibleForTesting
+ protected ReadableByteChannel getChannel(URL url) throws IOException {
+ return Channels.newChannel(url.openStream());
+ }
+
+ private String getSha256(Path path) throws IOException {
+ Hasher hasher = Hashing.sha256().newHasher();
+
+ byte byteBuffer[] = new byte[BUFFER_SIZE];
+ try (InputStream stream = path.getInputStream()) {
+ int numBytesRead = stream.read(byteBuffer);
+ while (numBytesRead != -1) {
+ if (numBytesRead != 0) {
+ // If more than 0 bytes were read, add them to the hash.
+ hasher.putBytes(byteBuffer, 0, numBytesRead);
+ }
+ numBytesRead = stream.read(byteBuffer);
+ }
+ }
+ return hasher.hash().toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java
new file mode 100644
index 0000000..56e5e55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * Downloads a jar file from a URL.
+ */
+public class HttpJarFunction extends HttpArchiveFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+ Rule rule = RepositoryFunction.getRule(repositoryName, HttpJarRule.NAME, env);
+ if (rule == null) {
+ return null;
+ }
+ return compute(env, rule);
+ }
+
+ @Override
+ public SkyFunctionName getSkyFunctionName() {
+ return SkyFunctionName.computed(HttpJarRule.NAME.toUpperCase());
+ }
+
+ @Override
+ public Class<? extends RuleDefinition> getRuleDefinition() {
+ return HttpJarRule.class;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalRepositoryFunction.java
new file mode 100644
index 0000000..1a72dad
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalRepositoryFunction.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.FileValue;
+import com.google.devtools.build.lib.skyframe.RepositoryValue;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * Access a repository on the local filesystem.
+ */
+public class LocalRepositoryFunction extends RepositoryFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+ Rule rule = RepositoryFunction.getRule(repositoryName, LocalRepositoryRule.NAME, env);
+ if (rule == null) {
+ return null;
+ }
+
+ AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
+ String path = mapper.get("path", Type.STRING);
+ PathFragment pathFragment = new PathFragment(path);
+ if (!pathFragment.isAbsolute()) {
+ throw new RepositoryFunctionException(
+ new EvalException(
+ rule.getLocation(),
+ "In " + rule + " the 'path' attribute must specify an absolute path"),
+ Transience.PERSISTENT);
+ }
+ Path repositoryPath = getOutputBase().getFileSystem().getPath(pathFragment);
+ FileValue repositoryValue = getRepositoryDirectory(repositoryPath, env);
+ if (repositoryValue == null) {
+ return null;
+ }
+
+ if (!repositoryValue.isDirectory()) {
+ throw new RepositoryFunctionException(
+ new IOException(rule + " must specify an existing directory"), Transience.TRANSIENT);
+ }
+
+ return new RepositoryValue(repositoryPath, repositoryValue);
+ }
+
+ @Override
+ public SkyFunctionName getSkyFunctionName() {
+ return SkyFunctionName.computed(LocalRepositoryRule.NAME.toUpperCase());
+ }
+
+ @Override
+ public Class<? extends RuleDefinition> getRuleDefinition() {
+ return LocalRepositoryRule.class;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
new file mode 100644
index 0000000..4f83de6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
@@ -0,0 +1,189 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Ascii;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.DecompressorException;
+import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.JarDecompressor;
+import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.FileValue;
+import com.google.devtools.build.lib.skyframe.RepositoryValue;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.eclipse.aether.AbstractRepositoryListener;
+import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.RepositorySystemSession;
+import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
+import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.repository.LocalRepository;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.resolution.ArtifactRequest;
+import org.eclipse.aether.resolution.ArtifactResolutionException;
+import org.eclipse.aether.resolution.ArtifactResult;
+import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
+import org.eclipse.aether.spi.connector.transport.TransporterFactory;
+import org.eclipse.aether.transfer.AbstractTransferListener;
+import org.eclipse.aether.transport.file.FileTransporterFactory;
+import org.eclipse.aether.transport.http.HttpTransporterFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Implementation of maven_jar.
+ */
+public class MavenJarFunction extends HttpJarFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws RepositoryFunctionException {
+ RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+ Rule rule = RepositoryFunction.getRule(repositoryName, MavenJarRule.NAME, env);
+ if (rule == null) {
+ return null;
+ }
+
+ AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
+ FileValue outputDirectoryValue = createOutputDirectory(env, rule.getName());
+ if (outputDirectoryValue == null) {
+ return null;
+ }
+ Path outputDirectory = outputDirectoryValue.realRootedPath().asPath();
+ MavenDownloader downloader = new MavenDownloader(
+ mapper.get("group_id", Type.STRING),
+ mapper.get("artifact_id", Type.STRING),
+ mapper.get("version", Type.STRING),
+ outputDirectory);
+
+ List<String> repositories = mapper.get("repositories", Type.STRING_LIST);
+ if (repositories != null && !repositories.isEmpty()) {
+ downloader.setRepositories(repositories);
+ }
+
+ Path repositoryJar = null;
+ try {
+ repositoryJar = downloader.download();
+ } catch (IOException e) {
+ throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+ }
+
+ // Add a WORKSPACE file & BUILD file to the Maven jar.
+ JarDecompressor decompressor = new JarDecompressor(rule, repositoryJar);
+ Path repositoryDirectory = null;
+ try {
+ repositoryDirectory = decompressor.decompress();
+ } catch (DecompressorException e) {
+ throw new RepositoryFunctionException(new IOException(e.getMessage()), Transience.TRANSIENT);
+ }
+ FileValue repositoryFileValue = getRepositoryDirectory(repositoryDirectory, env);
+ if (repositoryFileValue == null) {
+ return null;
+ }
+ return new RepositoryValue(repositoryDirectory, repositoryFileValue);
+ }
+
+ @Override
+ public SkyFunctionName getSkyFunctionName() {
+ return SkyFunctionName.computed(Ascii.toUpperCase(MavenJarRule.NAME));
+ }
+
+ @Override
+ public Class<? extends RuleDefinition> getRuleDefinition() {
+ return MavenJarRule.class;
+ }
+
+ private static class MavenDownloader {
+ private static final String MAVEN_CENTRAL_URL = "http://central.maven.org/maven2/";
+
+ private final String groupId;
+ private final String artifactId;
+ private final String version;
+ private final Path outputDirectory;
+ private List<RemoteRepository> repositories;
+
+ MavenDownloader(String groupId, String artifactId, String version, Path outputDirectory) {
+ this.groupId = groupId;
+ this.artifactId = artifactId;
+ this.version = version;
+ this.outputDirectory = outputDirectory;
+
+ this.repositories = new ArrayList<>(Arrays.asList(
+ new RemoteRepository.Builder("central", "default", MAVEN_CENTRAL_URL)
+ .build()));
+ }
+
+ /**
+ * Customizes the set of Maven repositories to check. Takes a list of repository addresses.
+ */
+ public void setRepositories(List<String> repositoryUrls) {
+ repositories = Lists.newArrayList();
+ for (String repositoryUrl : repositoryUrls) {
+ repositories.add(new RemoteRepository.Builder(
+ "user-defined repository " + repositories.size(), "default", repositoryUrl).build());
+ }
+ }
+
+ public Path download() throws IOException {
+ RepositorySystem system = newRepositorySystem();
+ RepositorySystemSession session = newRepositorySystemSession(system);
+
+ ArtifactRequest artifactRequest = new ArtifactRequest();
+ Artifact artifact = new DefaultArtifact(groupId + ":" + artifactId + ":" + version);
+ artifactRequest.setArtifact(artifact);
+ artifactRequest.setRepositories(repositories);
+
+ try {
+ ArtifactResult artifactResult = system.resolveArtifact(session, artifactRequest);
+ artifact = artifactResult.getArtifact();
+ } catch (ArtifactResolutionException e) {
+ throw new IOException("Failed to fetch Maven dependency: " + e.getMessage());
+ }
+ return outputDirectory.getRelative(artifact.getFile().getAbsolutePath());
+ }
+
+ private RepositorySystemSession newRepositorySystemSession(RepositorySystem system) {
+ DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
+ LocalRepository localRepo = new LocalRepository(outputDirectory.getPathString());
+ session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
+ session.setTransferListener(new AbstractTransferListener() {});
+ session.setRepositoryListener(new AbstractRepositoryListener() {});
+ return session;
+ }
+
+ private RepositorySystem newRepositorySystem() {
+ DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
+ locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
+ locator.addService(TransporterFactory.class, FileTransporterFactory.class);
+ locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
+ return locator.getService(RepositorySystem.class);
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewLocalRepositoryFunction.java
new file mode 100644
index 0000000..b2d9f74
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewLocalRepositoryFunction.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.FileValue;
+import com.google.devtools.build.lib.skyframe.RepositoryValue;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+/**
+ * Create a repository from a directory on the local filesystem.
+ */
+public class NewLocalRepositoryFunction extends RepositoryFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+ Rule rule = RepositoryFunction.getRule(repositoryName, NewLocalRepositoryRule.NAME, env);
+ if (rule == null) {
+ return null;
+ }
+
+ // Given a rule that looks like this:
+ // new_local_repository(
+ // name = 'x',
+ // path = '/some/path/to/y',
+ // build_file = 'x.BUILD'
+ // )
+ // This creates the following directory structure:
+ // .external-repository/
+ // x/
+ // WORKSPACE
+ // x/
+ // BUILD -> <build_root>/x.BUILD
+ // y -> /some/path/to/y
+ //
+ // In the structure above, .external-repository/x is the repository directory and
+ // .external-repository/x/x is the package directory.
+ Path repositoryDirectory = getExternalRepositoryDirectory().getRelative(rule.getName());
+ Path outputDirectory = repositoryDirectory.getRelative(rule.getName());
+ try {
+ FileSystemUtils.createDirectoryAndParents(outputDirectory);
+ } catch (IOException e) {
+ throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+ }
+ FileValue directoryValue = getRepositoryDirectory(outputDirectory, env);
+ if (directoryValue == null) {
+ return null;
+ }
+
+ // Add x/WORKSPACE.
+ try {
+ Path workspaceFile = repositoryDirectory.getRelative("WORKSPACE");
+ FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"),
+ "# DO NOT EDIT: automatically generated WORKSPACE file for " + rule + "\n");
+ } catch (IOException e) {
+ throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+ }
+
+ AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
+ // Link x/x/y to /some/path/to/y.
+ String path = mapper.get("path", Type.STRING);
+ PathFragment pathFragment = new PathFragment(path);
+ if (!pathFragment.isAbsolute()) {
+ throw new RepositoryFunctionException(
+ new EvalException(
+ rule.getLocation(),
+ "In " + rule + " the 'path' attribute must specify an absolute path"),
+ Transience.PERSISTENT);
+ }
+ Path pathTarget = getOutputBase().getFileSystem().getPath(pathFragment);
+ Path symlinkPath = outputDirectory.getRelative(pathTarget.getBaseName());
+ if (createSymbolicLink(symlinkPath, pathTarget, env) == null) {
+ return null;
+ }
+
+ // Link x/x/BUILD to <build_root>/x.BUILD.
+ PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING));
+ Path buildFileTarget = getWorkspace().getRelative(buildFile);
+ if (buildFile.equals(PathFragment.EMPTY_FRAGMENT) || buildFile.isAbsolute()
+ || !buildFileTarget.exists()) {
+ throw new RepositoryFunctionException(
+ new EvalException(rule.getLocation(), "In " + rule
+ + " the 'build_file' attribute must specify a relative path to an existing file"),
+ Transience.PERSISTENT);
+ }
+ Path buildFilePath = outputDirectory.getRelative("BUILD");
+ if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) {
+ return null;
+ }
+
+ return new RepositoryValue(repositoryDirectory, directoryValue);
+ }
+
+ private FileValue createSymbolicLink(Path from, Path to, Environment env)
+ throws RepositoryFunctionException {
+ try {
+ if (!from.exists()) {
+ from.createSymbolicLink(to);
+ }
+ } catch (IOException e) {
+ throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+ }
+ FileValue fromValue = getRepositoryDirectory(from, env);
+ return fromValue;
+ }
+
+ @Override
+ public SkyFunctionName getSkyFunctionName() {
+ return SkyFunctionName.computed(NewLocalRepositoryRule.NAME.toUpperCase());
+ }
+
+ @Override
+ public Class<? extends RuleDefinition> getRuleDefinition() {
+ return NewLocalRepositoryRule.class;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java
new file mode 100644
index 0000000..f0af0c6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * Implements delegation to the correct repository fetcher.
+ */
+public class RepositoryDelegatorFunction implements SkyFunction {
+
+ // Mapping of rule class name to SkyFunction.
+ private final ImmutableMap<String, RepositoryFunction> handlers;
+
+ public RepositoryDelegatorFunction(
+ ImmutableMap<String, RepositoryFunction> handlers) {
+ this.handlers = handlers;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+ Rule rule = RepositoryFunction.getRule(repositoryName, null, env);
+ if (rule == null) {
+ return null;
+ }
+ RepositoryFunction handler = handlers.get(rule.getRuleClass());
+ if (handler == null) {
+ throw new IllegalStateException("Could not find handler for " + rule);
+ }
+ SkyKey key = new SkyKey(handler.getSkyFunctionName(), repositoryName);
+
+ try {
+ return env.getValueOrThrow(
+ key, NoSuchPackageException.class, IOException.class, EvalException.class);
+ } catch (NoSuchPackageException e) {
+ throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT);
+ } catch (IOException e) {
+ throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT);
+ } catch (EvalException e) {
+ throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java
new file mode 100644
index 0000000..906c38b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java
@@ -0,0 +1,181 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.ExternalPackage;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.skyframe.FileSymlinkCycleException;
+import com.google.devtools.build.lib.skyframe.FileValue;
+import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException;
+import com.google.devtools.build.lib.skyframe.PackageFunction;
+import com.google.devtools.build.lib.skyframe.PackageValue;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.io.IOException;
+
+import javax.annotation.Nullable;
+
+/**
+ * Parent class for repository-related Skyframe functions.
+ */
+public abstract class RepositoryFunction implements SkyFunction {
+ private static final String EXTERNAL_REPOSITORY_DIRECTORY = ".external-repository";
+ private BlazeDirectories directories;
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Gets Skyframe's name for this.
+ */
+ public abstract SkyFunctionName getSkyFunctionName();
+
+ /**
+ * Sets up output path information.
+ */
+ public void setDirectories(BlazeDirectories directories) {
+ this.directories = directories;
+ }
+
+ protected Path getExternalRepositoryDirectory() {
+ return RepositoryFunction.getExternalRepositoryDirectory(directories);
+ }
+
+ public static Path getExternalRepositoryDirectory(BlazeDirectories directories) {
+ return directories.getOutputBase().getRelative(EXTERNAL_REPOSITORY_DIRECTORY);
+ }
+
+ /**
+ * Gets the base directory repositories should be stored in locally.
+ */
+ protected Path getOutputBase() {
+ return directories.getOutputBase();
+ }
+
+ /**
+ * Gets the directory the WORKSPACE file for the build is in.
+ */
+ protected Path getWorkspace() {
+ return directories.getWorkspace();
+ }
+
+
+ /**
+ * Returns the RuleDefinition class for this type of repository.
+ */
+ public abstract Class<? extends RuleDefinition> getRuleDefinition();
+
+ /**
+ * Uses a remote repository name to fetch the corresponding Rule describing how to get it.
+ * This should be called from {@link SkyFunction#compute} functions, which should return null if
+ * this returns null. If {@code ruleClassName} is set, the rule found must have a matching rule
+ * class name.
+ */
+ @Nullable
+ public static Rule getRule(
+ RepositoryName repositoryName, @Nullable String ruleClassName, Environment env)
+ throws RepositoryFunctionException {
+ SkyKey packageKey = PackageValue.key(
+ PackageIdentifier.createInDefaultRepo(PackageFunction.EXTERNAL_PACKAGE_NAME));
+ PackageValue packageValue;
+ try {
+ packageValue = (PackageValue) env.getValueOrThrow(packageKey,
+ NoSuchPackageException.class);
+ } catch (NoSuchPackageException e) {
+ throw new RepositoryFunctionException(
+ new BuildFileNotFoundException(
+ PackageFunction.EXTERNAL_PACKAGE_NAME, "Could not load //external package"),
+ Transience.PERSISTENT);
+ }
+ if (packageValue == null) {
+ return null;
+ }
+ ExternalPackage externalPackage = (ExternalPackage) packageValue.getPackage();
+ Rule rule = externalPackage.getRepositoryInfo(repositoryName);
+ if (rule == null) {
+ throw new RepositoryFunctionException(
+ new BuildFileContainsErrorsException(
+ PackageFunction.EXTERNAL_PACKAGE_NAME,
+ "The repository named '" + repositoryName + "' could not be resolved"),
+ Transience.PERSISTENT);
+ }
+ Preconditions.checkState(ruleClassName == null || rule.getRuleClass().equals(ruleClassName),
+ "Got " + rule + ", was expecting a " + ruleClassName);
+ return rule;
+ }
+
+ /**
+ * Adds the repository's directory to the graph and, if it's a symlink, resolves it to an
+ * actual directory.
+ */
+ @Nullable
+ protected static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env)
+ throws RepositoryFunctionException {
+ SkyKey outputDirectoryKey = FileValue.key(RootedPath.toRootedPath(
+ repositoryDirectory, PathFragment.EMPTY_FRAGMENT));
+ try {
+ return (FileValue) env.getValueOrThrow(outputDirectoryKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class);
+ } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException e) {
+ throw new RepositoryFunctionException(
+ new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()),
+ Transience.PERSISTENT);
+ }
+ }
+
+ /**
+ * Exception thrown when something goes wrong accessing a remote repository.
+ *
+ * This exception should be used by child classes to limit the types of exceptions
+ * {@link RepositoryDelegatorFunction} has to know how to catch.
+ */
+ static final class RepositoryFunctionException extends SkyFunctionException {
+ public RepositoryFunctionException(NoSuchPackageException cause, Transience transience) {
+ super(cause, transience);
+ }
+
+ /**
+ * Error reading or writing to the filesystem.
+ */
+ public RepositoryFunctionException(IOException cause, Transience transience) {
+ super(cause, transience);
+ }
+
+ /**
+ * For errors in WORKSPACE file rules (e.g., malformed paths or URLs).
+ */
+ public RepositoryFunctionException(EvalException cause, Transience transience) {
+ super(cause, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelBaseRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelBaseRuleClasses.java
new file mode 100644
index 0000000..a1b27fe
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelBaseRuleClasses.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * The foundational rule templates to help in real rule construction. Only attributes truly common
+ * to all rules go in here. Attributes such as "out", "outs", "src" and "srcs" exhibit enough
+ * variation that we declare them explicitly for each rule. This leads to stricter error checking
+ * and prevents users from inadvertently using an attribute that doesn't actually do anything.
+ */
+public class BazelBaseRuleClasses {
+ public static final ImmutableSet<String> ALLOWED_RULE_CLASSES =
+ ImmutableSet.of("filegroup", "genrule", "Fileset");
+
+ /**
+ * A base rule for all binary rules.
+ */
+ @BlazeRule(name = "$binary_base_rule",
+ type = RuleClassType.ABSTRACT)
+ public static final class BinaryBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("args", STRING_LIST)
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("output_licenses", LICENSE))
+ .add(attr("$is_executable", BOOLEAN).value(true)
+ .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target"))
+ .build();
+ }
+ }
+
+ /**
+ * Rule class for rules in error.
+ */
+ @BlazeRule(name = "$error_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.BaseRule.class })
+ public static final class ErrorRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .publicByDefault()
+ .build();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
new file mode 100644
index 0000000..63aa4e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Bazel-specific configuration fragment.
+ */
+public class BazelConfiguration extends Fragment {
+ /**
+ * Loader for Google-specific settings.
+ */
+ public static class Loader implements ConfigurationFragmentFactory {
+ @Override
+ public Fragment create(ConfigurationEnvironment env, BuildOptions buildOptions)
+ throws InvalidConfigurationException {
+ return new BazelConfiguration();
+ }
+
+ @Override
+ public Class<? extends Fragment> creates() {
+ return BazelConfiguration.class;
+ }
+ }
+
+ public BazelConfiguration() {
+ }
+
+ @Override
+ public String getName() {
+ return "Bazel";
+ }
+
+ @Override
+ public String cacheKey() {
+ return "";
+ }
+
+ @Override
+ public void defineExecutables(ImmutableMap.Builder<String, PathFragment> builder) {
+ if (OS.getCurrent() == OS.WINDOWS) {
+ String path = System.getenv("BAZEL_SH");
+ if (path != null) {
+ builder.put("sh", new PathFragment(path));
+ return;
+ }
+ }
+ builder.put("sh", new PathFragment("/bin/bash"));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfigurationCollection.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfigurationCollection.java
new file mode 100644
index 0000000..1472b43
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfigurationCollection.java
@@ -0,0 +1,235 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Table;
+import com.google.devtools.build.lib.analysis.ConfigurationCollectionFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection.ConfigurationHolder;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection.Transitions;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppRuleClasses.CppTransition;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.Attribute.Transition;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Configuration collection used by the rules Bazel knows.
+ */
+public class BazelConfigurationCollection implements ConfigurationCollectionFactory {
+ @Override
+ @Nullable
+ public BuildConfiguration createConfigurations(
+ ConfigurationFactory configurationFactory,
+ PackageProviderForConfigurations loadedPackageProvider,
+ BuildOptions buildOptions,
+ Map<String, String> clientEnv,
+ EventHandler errorEventListener,
+ boolean performSanityCheck) throws InvalidConfigurationException {
+
+ // We cache all the related configurations for this target configuration in a cache that is
+ // dropped at the end of this method call. We instead rely on the cache for entire collections
+ // for caching the target and related configurations, and on a dedicated host configuration
+ // cache for the host configuration.
+ Cache<String, BuildConfiguration> cache =
+ CacheBuilder.newBuilder().<String, BuildConfiguration>build();
+
+ // Target configuration
+ BuildConfiguration targetConfiguration = configurationFactory.getConfiguration(
+ loadedPackageProvider, buildOptions, clientEnv, false, cache);
+ if (targetConfiguration == null) {
+ return null;
+ }
+
+ BuildConfiguration dataConfiguration = targetConfiguration;
+
+ // Host configuration
+ // Note that this passes in the dataConfiguration, not the target
+ // configuration. This is intentional.
+ BuildConfiguration hostConfiguration = getHostConfigurationFromRequest(configurationFactory,
+ loadedPackageProvider, clientEnv, dataConfiguration, buildOptions);
+ if (hostConfiguration == null) {
+ return null;
+ }
+
+ // Sanity check that the implicit labels are all in the transitive closure of explicit ones.
+ // This also registers all targets in the cache entry and validates them on subsequent requests.
+ Set<Label> reachableLabels = new HashSet<>();
+ if (performSanityCheck) {
+ // We allow the package provider to be null for testing.
+ for (Label label : buildOptions.getAllLabels().values()) {
+ try {
+ collectTransitiveClosure(loadedPackageProvider, reachableLabels, label);
+ } catch (NoSuchThingException e) {
+ // We've loaded the transitive closure of the labels-to-load above, and made sure that
+ // there are no errors loading it, so this can't happen.
+ throw new IllegalStateException(e);
+ }
+ }
+ sanityCheckImplicitLabels(reachableLabels, targetConfiguration);
+ sanityCheckImplicitLabels(reachableLabels, hostConfiguration);
+ }
+
+ BuildConfiguration result = setupTransitions(
+ targetConfiguration, dataConfiguration, hostConfiguration);
+ result.reportInvalidOptions(errorEventListener);
+ return result;
+ }
+
+ /**
+ * Gets the correct host configuration for this build. The behavior
+ * depends on the value of the --distinct_host_configuration flag.
+ *
+ * <p>With --distinct_host_configuration=false, we use identical configurations
+ * for the host and target, and you can ignore everything below. But please
+ * note: if you're cross-compiling for k8 on a piii machine, your build will
+ * fail. This is a stopgap measure.
+ *
+ * <p>Currently, every build is (in effect) a cross-compile, in the strict
+ * sense that host and target configurations are unequal, thus we do not
+ * issue a "cross-compiling" warning. (Perhaps we should?)
+ * *
+ * @param requestConfig the requested target (not host!) configuration for
+ * this build.
+ * @param buildOptions the configuration options used for the target configuration
+ */
+ @Nullable
+ private BuildConfiguration getHostConfigurationFromRequest(
+ ConfigurationFactory configurationFactory,
+ PackageProviderForConfigurations loadedPackageProvider, Map<String, String> clientEnv,
+ BuildConfiguration requestConfig, BuildOptions buildOptions)
+ throws InvalidConfigurationException {
+ BuildConfiguration.Options commonOptions = buildOptions.get(BuildConfiguration.Options.class);
+ if (!commonOptions.useDistinctHostConfiguration) {
+ return requestConfig;
+ } else {
+ BuildConfiguration hostConfig = configurationFactory.getHostConfiguration(
+ loadedPackageProvider, clientEnv, buildOptions, /*fallback=*/false);
+ if (hostConfig == null) {
+ return null;
+ }
+ return hostConfig;
+ }
+ }
+
+ static BuildConfiguration setupTransitions(BuildConfiguration targetConfiguration,
+ BuildConfiguration dataConfiguration, BuildConfiguration hostConfiguration) {
+ Set<BuildConfiguration> allConfigurations = ImmutableSet.of(targetConfiguration,
+ dataConfiguration, hostConfiguration);
+
+ Table<BuildConfiguration, Transition, ConfigurationHolder> transitionBuilder =
+ HashBasedTable.create();
+ for (BuildConfiguration from : allConfigurations) {
+ for (ConfigurationTransition transition : ConfigurationTransition.values()) {
+ BuildConfiguration to;
+ if (transition == ConfigurationTransition.HOST) {
+ to = hostConfiguration;
+ } else if (transition == ConfigurationTransition.DATA && from == targetConfiguration) {
+ to = dataConfiguration;
+ } else {
+ to = from;
+ }
+ transitionBuilder.put(from, transition, new ConfigurationHolder(to));
+ }
+ }
+
+ // TODO(bazel-team): This makes LIPO totally not work. Just a band-aid until we get around to
+ // implementing a way for the C++ rules to contribute this transition to the configuration
+ // collection.
+ for (BuildConfiguration config : allConfigurations) {
+ transitionBuilder.put(config, CppTransition.LIPO_COLLECTOR, new ConfigurationHolder(config));
+ transitionBuilder.put(config, CppTransition.TARGET_CONFIG_FOR_LIPO,
+ new ConfigurationHolder(config.isHostConfiguration() ? null : config));
+ }
+
+ for (BuildConfiguration config : allConfigurations) {
+ Transitions outgoingTransitions =
+ new BuildConfigurationCollection.Transitions(config, transitionBuilder.row(config));
+ // We allow host configurations to be shared between target configurations. In that case, the
+ // transitions may already be set.
+ // TODO(bazel-team): Check that the transitions are identical, or even better, change the
+ // code to set the host configuration transitions before we even create the target
+ // configuration.
+ if (config.isHostConfiguration() && config.getTransitions() != null) {
+ continue;
+ }
+ config.setConfigurationTransitions(outgoingTransitions);
+ }
+
+ return targetConfiguration;
+ }
+
+ /**
+ * Checks that the implicit labels are reachable from the loaded labels. The loaded labels are
+ * those returned from {@link BuildConfigurationKey#getLabelsToLoadUnconditionally()}, and the
+ * implicit ones are those that need to be available for late-bound attributes.
+ */
+ private void sanityCheckImplicitLabels(Collection<Label> reachableLabels,
+ BuildConfiguration config) throws InvalidConfigurationException {
+ for (Map.Entry<String, Label> entry : config.getImplicitLabels().entries()) {
+ if (!reachableLabels.contains(entry.getValue())) {
+ throw new InvalidConfigurationException("The required " + entry.getKey()
+ + " target is not transitively reachable from a command-line option: '"
+ + entry.getValue() + "'");
+ }
+ }
+ }
+
+ private void collectTransitiveClosure(PackageProviderForConfigurations loadedPackageProvider,
+ Set<Label> reachableLabels, Label from) throws NoSuchThingException {
+ if (!reachableLabels.add(from)) {
+ return;
+ }
+ Target fromTarget = loadedPackageProvider.getLoadedTarget(from);
+ if (fromTarget instanceof Rule) {
+ Rule rule = (Rule) fromTarget;
+ if (rule.getRuleClassObject().hasAttr("srcs", Type.LABEL_LIST)) {
+ // TODO(bazel-team): refine this. This visits "srcs" reachable under *any* configuration,
+ // not necessarily the configuration actually applied to the rule. We should correlate the
+ // two. However, doing so requires faithfully reflecting the configuration transitions that
+ // might happen as we traverse the dependency chain.
+ for (List<Label> labelsForConfiguration :
+ AggregatingAttributeMapper.of(rule).visitAttribute("srcs", Type.LABEL_LIST)) {
+ for (Label label : labelsForConfiguration) {
+ collectTransitiveClosure(loadedPackageProvider, reachableLabels, label);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
new file mode 100644
index 0000000..1280bdb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
@@ -0,0 +1,272 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Functions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.PrerequisiteValidator;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigRuleClasses;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.FragmentOptions;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.constraints.EnvironmentRule;
+import com.google.devtools.build.lib.bazel.rules.common.BazelActionListenerRule;
+import com.google.devtools.build.lib.bazel.rules.common.BazelExtraActionRule;
+import com.google.devtools.build.lib.bazel.rules.common.BazelFilegroupRule;
+import com.google.devtools.build.lib.bazel.rules.common.BazelTestSuiteRule;
+import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppRuleClasses;
+import com.google.devtools.build.lib.bazel.rules.genrule.BazelGenRuleRule;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaBinaryRule;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaBuildInfoFactory;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaImportRule;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaLibraryRule;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaPluginRule;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaTestRule;
+import com.google.devtools.build.lib.bazel.rules.objc.BazelIosTestRule;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShBinaryRule;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShLibraryRule;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShTestRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainRule;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppConfigurationLoader;
+import com.google.devtools.build.lib.rules.cpp.CppOptions;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration;
+import com.google.devtools.build.lib.rules.java.JavaConfigurationLoader;
+import com.google.devtools.build.lib.rules.java.JavaCpuSupplier;
+import com.google.devtools.build.lib.rules.java.JavaImportBaseRule;
+import com.google.devtools.build.lib.rules.java.JavaOptions;
+import com.google.devtools.build.lib.rules.java.JavaToolchainRule;
+import com.google.devtools.build.lib.rules.java.Jvm;
+import com.google.devtools.build.lib.rules.java.JvmConfigurationLoader;
+import com.google.devtools.build.lib.rules.objc.IosApplicationRule;
+import com.google.devtools.build.lib.rules.objc.IosDeviceRule;
+import com.google.devtools.build.lib.rules.objc.ObjcBinaryRule;
+import com.google.devtools.build.lib.rules.objc.ObjcBundleLibraryRule;
+import com.google.devtools.build.lib.rules.objc.ObjcBundleRule;
+import com.google.devtools.build.lib.rules.objc.ObjcCommandLineOptions;
+import com.google.devtools.build.lib.rules.objc.ObjcConfigurationLoader;
+import com.google.devtools.build.lib.rules.objc.ObjcFrameworkRule;
+import com.google.devtools.build.lib.rules.objc.ObjcImportRule;
+import com.google.devtools.build.lib.rules.objc.ObjcLibraryRule;
+import com.google.devtools.build.lib.rules.objc.ObjcOptionsRule;
+import com.google.devtools.build.lib.rules.objc.ObjcProtoLibraryRule;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses;
+import com.google.devtools.build.lib.rules.objc.ObjcXcodeprojRule;
+import com.google.devtools.build.lib.rules.workspace.BindRule;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkType;
+
+/**
+ * A rule class provider implementing the rules Bazel knows.
+ */
+public class BazelRuleClassProvider {
+
+ /**
+ * Used by the build encyclopedia generator.
+ */
+ public static ConfiguredRuleClassProvider create() {
+ ConfiguredRuleClassProvider.Builder builder =
+ new ConfiguredRuleClassProvider.Builder();
+ setup(builder);
+ return builder.build();
+ }
+
+ public static final JavaCpuSupplier JAVA_CPU_SUPPLIER = new JavaCpuSupplier() {
+ @Override
+ public String getJavaCpu(BuildOptions buildOptions, ConfigurationEnvironment env)
+ throws InvalidConfigurationException {
+ JavaOptions javaOptions = buildOptions.get(JavaOptions.class);
+ return javaOptions.javaCpu == null ? "default" : javaOptions.javaCpu;
+ }
+ };
+
+ private static class BazelPrerequisiteValidator implements PrerequisiteValidator {
+ @Override
+ public void validate(RuleContext.Builder context,
+ ConfiguredTarget prerequisite, Attribute attribute) {
+ validateDirectPrerequisiteVisibility(context, prerequisite, attribute.getName());
+ }
+
+ private void validateDirectPrerequisiteVisibility(
+ RuleContext.Builder context, ConfiguredTarget prerequisite, String attrName) {
+ Rule rule = context.getRule();
+ Target prerequisiteTarget = prerequisite.getTarget();
+ Label prerequisiteLabel = prerequisiteTarget.getLabel();
+ // We don't check the visibility of late-bound attributes, because it would break some
+ // features.
+ if (!context.getRule().getLabel().getPackageName().equals(
+ prerequisite.getTarget().getLabel().getPackageName())
+ && !context.isVisible(prerequisite)) {
+ if (!context.getConfiguration().checkVisibility()) {
+ context.ruleWarning(String.format("Target '%s' violates visibility of target "
+ + "'%s'. Continuing because --nocheck_visibility is active",
+ rule.getLabel(), prerequisiteLabel));
+ } else {
+ // Oddly enough, we use reportError rather than ruleError here.
+ context.reportError(rule.getLocation(),
+ String.format("Target '%s' is not visible from target '%s'. Check "
+ + "the visibility declaration of the former target if you think "
+ + "the dependency is legitimate",
+ prerequisiteLabel, rule.getLabel()));
+ }
+ }
+
+ if (prerequisiteTarget instanceof PackageGroup) {
+ if (!attrName.equals("visibility")) {
+ context.reportError(rule.getAttributeLocation(attrName),
+ "in " + attrName + " attribute of " + rule.getRuleClass()
+ + " rule " + rule.getLabel() + ": package group '"
+ + prerequisiteLabel + "' is misplaced here "
+ + "(they are only allowed in the visibility attribute)");
+ }
+ }
+ }
+ }
+
+ /**
+ * List of all build option classes in Blaze.
+ */
+ // TODO(bazel-team): make this private, remove from tests, then BuildOptions.of can be merged
+ // into RuleClassProvider.
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ public static final ImmutableList<Class<? extends FragmentOptions>> BUILD_OPTIONS =
+ ImmutableList.of(
+ BuildConfiguration.Options.class,
+ CppOptions.class,
+ JavaOptions.class,
+ ObjcCommandLineOptions.class
+ );
+
+ /**
+ * Java objects accessible from Skylark rule implementations using this module.
+ */
+ private static final ImmutableMap<String, SkylarkType> skylarkBuiltinJavaObects =
+ ImmutableMap.of(
+ "jvm", SkylarkType.of(Jvm.class),
+ "java_configuration", SkylarkType.of(JavaConfiguration.class),
+ "cpp", SkylarkType.of(CppConfiguration.class));
+
+ public static void setup(ConfiguredRuleClassProvider.Builder builder) {
+ builder
+ .addBuildInfoFactory(new BazelJavaBuildInfoFactory())
+ .setConfigurationCollectionFactory(new BazelConfigurationCollection())
+ .setPrerequisiteValidator(new BazelPrerequisiteValidator())
+ .setSkylarkAccessibleJavaClasses(skylarkBuiltinJavaObects);
+
+ for (Class<? extends FragmentOptions> fragmentOptions : BUILD_OPTIONS) {
+ builder.addConfigurationOptions(fragmentOptions);
+ }
+
+ builder.addRuleDefinition(BaseRuleClasses.BaseRule.class);
+ builder.addRuleDefinition(BaseRuleClasses.RuleBase.class);
+ builder.addRuleDefinition(BazelBaseRuleClasses.BinaryBaseRule.class);
+ builder.addRuleDefinition(BaseRuleClasses.TestBaseRule.class);
+ builder.addRuleDefinition(BazelBaseRuleClasses.ErrorRule.class);
+
+ builder.addRuleDefinition(EnvironmentRule.class);
+
+ builder.addRuleDefinition(ConfigRuleClasses.ConfigBaseRule.class);
+ builder.addRuleDefinition(ConfigRuleClasses.ConfigSettingRule.class);
+
+ builder.addRuleDefinition(BazelFilegroupRule.class);
+ builder.addRuleDefinition(BazelTestSuiteRule.class);
+ builder.addRuleDefinition(BazelGenRuleRule.class);
+
+ builder.addRuleDefinition(BazelShRuleClasses.ShRule.class);
+ builder.addRuleDefinition(BazelShLibraryRule.class);
+ builder.addRuleDefinition(BazelShBinaryRule.class);
+ builder.addRuleDefinition(BazelShTestRule.class);
+
+ builder.addRuleDefinition(CcToolchainRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcLinkingRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcDeclRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcBaseRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcBinaryBaseRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcBinaryRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcTestRule.class);
+
+ builder.addRuleDefinition(BazelCppRuleClasses.CcLibraryBaseRule.class);
+ builder.addRuleDefinition(BazelCppRuleClasses.CcLibraryRule.class);
+
+
+ builder.addRuleDefinition(BazelJavaRuleClasses.BaseJavaBinaryRule.class);
+ builder.addRuleDefinition(BazelJavaRuleClasses.IjarBaseRule.class);
+ builder.addRuleDefinition(BazelJavaRuleClasses.JavaBaseRule.class);
+ builder.addRuleDefinition(JavaImportBaseRule.class);
+ builder.addRuleDefinition(BazelJavaRuleClasses.JavaRule.class);
+ builder.addRuleDefinition(BazelJavaBinaryRule.class);
+ builder.addRuleDefinition(BazelJavaLibraryRule.class);
+ builder.addRuleDefinition(BazelJavaImportRule.class);
+ builder.addRuleDefinition(BazelJavaTestRule.class);
+ builder.addRuleDefinition(BazelJavaPluginRule.class);
+ builder.addRuleDefinition(JavaToolchainRule.class);
+
+ builder.addRuleDefinition(BazelIosTestRule.class);
+ builder.addRuleDefinition(IosDeviceRule.class);
+ builder.addRuleDefinition(ObjcBinaryRule.class);
+ builder.addRuleDefinition(ObjcBundleRule.class);
+ builder.addRuleDefinition(ObjcBundleLibraryRule.class);
+ builder.addRuleDefinition(ObjcFrameworkRule.class);
+ builder.addRuleDefinition(ObjcImportRule.class);
+ builder.addRuleDefinition(ObjcLibraryRule.class);
+ builder.addRuleDefinition(ObjcOptionsRule.class);
+ builder.addRuleDefinition(ObjcProtoLibraryRule.class);
+ builder.addRuleDefinition(ObjcXcodeprojRule.class);
+ builder.addRuleDefinition(ObjcRuleClasses.IosTestBaseRule.class);
+ builder.addRuleDefinition(ObjcRuleClasses.ObjcHasInfoplistRule.class);
+ builder.addRuleDefinition(ObjcRuleClasses.ObjcHasEntitlementsRule.class);
+ builder.addRuleDefinition(ObjcRuleClasses.ObjcCompilationRule.class);
+ builder.addRuleDefinition(ObjcRuleClasses.ObjcBaseResourcesRule.class);
+ builder.addRuleDefinition(IosApplicationRule.class);
+
+ builder.addRuleDefinition(BazelExtraActionRule.class);
+ builder.addRuleDefinition(BazelActionListenerRule.class);
+
+ builder.addRuleDefinition(BindRule.class);
+ builder.addRuleDefinition(HttpArchiveRule.class);
+ builder.addRuleDefinition(HttpJarRule.class);
+ builder.addRuleDefinition(LocalRepositoryRule.class);
+ builder.addRuleDefinition(MavenJarRule.class);
+ builder.addRuleDefinition(NewLocalRepositoryRule.class);
+
+ builder.addConfigurationFragment(new BazelConfiguration.Loader());
+ builder.addConfigurationFragment(new CppConfigurationLoader(
+ Functions.<String>identity()));
+ builder.addConfigurationFragment(new JvmConfigurationLoader(JAVA_CPU_SUPPLIER));
+ builder.addConfigurationFragment(new JavaConfigurationLoader(JAVA_CPU_SUPPLIER));
+ builder.addConfigurationFragment(new ObjcConfigurationLoader());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java
new file mode 100644
index 0000000..214b367
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java
@@ -0,0 +1,159 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.actions.ActionContextConsumer;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext;
+import com.google.devtools.build.lib.rules.cpp.CppLinkActionContext;
+import com.google.devtools.build.lib.rules.cpp.LocalGccStrategy;
+import com.google.devtools.build.lib.rules.cpp.LocalLinkStrategy;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.GotOptionsEvent;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Map;
+
+/**
+ * Module implementing the rule set of Bazel.
+ */
+public class BazelRulesModule extends BlazeModule {
+ /**
+ * Execution options affecting how we execute the build actions (but not their semantics).
+ */
+ public static class BazelExecutionOptions extends OptionsBase {
+ @Option(
+ name = "spawn_strategy",
+ defaultValue = "standalone",
+ category = "strategy",
+ help = "Specify how spawn actions are executed by default."
+ + "'standalone' means run all of them locally."
+ + "'sandboxed' means run them in namespaces based sandbox (available only on Linux)")
+ public String spawnStrategy;
+
+ @Option(
+ name = "genrule_strategy",
+ defaultValue = "standalone",
+ category = "strategy",
+ help = "Specify how to execute genrules."
+ + "'standalone' means run all of them locally."
+ + "'sandboxed' means run them in namespaces based sandbox (available only on Linux)")
+
+ public String genruleStrategy;
+ }
+
+ private static class BazelActionContextConsumer implements ActionContextConsumer {
+ BazelExecutionOptions options;
+
+ private BazelActionContextConsumer(BazelExecutionOptions options) {
+ this.options = options;
+
+ }
+ @Override
+ public Map<String, String> getSpawnActionContexts() {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+
+ builder.put("Genrule", options.genruleStrategy);
+
+ // TODO(bazel-team): put this in getActionContexts (key=SpawnActionContext.class) instead
+ builder.put("", options.spawnStrategy);
+
+ return builder.build();
+ }
+
+ @Override
+ public Map<Class<? extends ActionContext>, String> getActionContexts() {
+ ImmutableMap.Builder<Class<? extends ActionContext>, String> builder =
+ ImmutableMap.builder();
+ builder.put(CppCompileActionContext.class, "");
+ builder.put(CppLinkActionContext.class, "");
+ return builder.build();
+ }
+ }
+
+ private class BazelActionContextProvider implements ActionContextProvider {
+ @Override
+ public Iterable<ActionContext> getActionContexts() {
+ return ImmutableList.of(
+ new LocalGccStrategy(optionsProvider),
+ new LocalLinkStrategy());
+ }
+
+ @Override
+ public void executorCreated(Iterable<ActionContext> usedContexts)
+ throws ExecutorInitException {
+ }
+
+ @Override
+ public void executionPhaseStarting(ActionInputFileCache actionInputFileCache,
+ ActionGraph actionGraph, Iterable<Artifact> topLevelArtifacts)
+ throws ExecutorInitException, InterruptedException {
+ }
+
+ @Override
+ public void executionPhaseEnding() {
+ }
+ }
+
+ private BlazeRuntime runtime;
+ private OptionsProvider optionsProvider;
+
+ @Override
+ public void beforeCommand(BlazeRuntime blazeRuntime, Command command) {
+ this.runtime = blazeRuntime;
+ runtime.getEventBus().register(this);
+ }
+
+ @Override
+ public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+ return command.builds()
+ ? ImmutableList.<Class<? extends OptionsBase>>of(BazelExecutionOptions.class)
+ : ImmutableList.<Class<? extends OptionsBase>>of();
+ }
+
+ @Override
+ public ActionContextConsumer getActionContextConsumer() {
+ return new BazelActionContextConsumer(
+ optionsProvider.getOptions(BazelExecutionOptions.class));
+ }
+
+ @Override
+ public ActionContextProvider getActionContextProvider() {
+ return new BazelActionContextProvider();
+ }
+
+ @Subscribe
+ public void gotOptions(GotOptionsEvent event) {
+ optionsProvider = event.getOptions();
+ }
+
+ @Override
+ public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
+ BazelRuleClassProvider.setup(builder);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelActionListenerRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelActionListenerRule.java
new file mode 100644
index 0000000..eba1553
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelActionListenerRule.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.common;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.extra.ActionListener;
+
+/**
+ * Rule definition for action_listener rule.
+ */
+@BlazeRule(name = "action_listener",
+ ancestors = { BaseRuleClasses.RuleBase.class },
+ factoryClass = ActionListener.class)
+public final class BazelActionListenerRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ .add(attr("mnemonics", STRING_LIST).mandatory())
+ .add(attr("extra_actions", LABEL_LIST).mandatory()
+ .allowedRuleClasses("extra_action")
+ .allowedFileTypes())
+ .removeAttribute("deps")
+ .removeAttribute("data")
+ .removeAttribute(":action_listener")
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelExtraActionRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelExtraActionRule.java
new file mode 100644
index 0000000..fa73f93
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelExtraActionRule.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.common;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.extra.ExtraActionFactory;
+
+/**
+ * Rule definition for extra_action rule.
+ */
+@BlazeRule(name = "extra_action",
+ ancestors = { BaseRuleClasses.RuleBase.class },
+ factoryClass = ExtraActionFactory.class)
+public final class BazelExtraActionRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ .add(attr("tools", LABEL_LIST).cfg(HOST).allowedFileTypes().exec())
+ .add(attr("out_templates", STRING_LIST))
+ .add(attr("cmd", STRING).mandatory())
+ .add(attr("requires_action_output", BOOLEAN))
+ .removeAttribute("deps")
+ .removeAttribute(":action_listener")
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelFilegroupRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelFilegroupRule.java
new file mode 100644
index 0000000..0ff5cd0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelFilegroupRule.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.common;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.filegroup.Filegroup;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+/**
+ * Rule object implementing "filegroup".
+ */
+@BlazeRule(name = "filegroup",
+ ancestors = { BaseRuleClasses.BaseRule.class },
+ factoryClass = Filegroup.class)
+public final class BazelFilegroupRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ // filegroup ignores any filtering set with setSrcsAllowedFiles.
+ return builder
+ .add(attr("srcs", LABEL_LIST).allowedFileTypes(FileTypeSet.ANY_FILE))
+ .add(attr("data", LABEL_LIST).cfg(DATA).allowedFileTypes(FileTypeSet.ANY_FILE))
+ .add(attr("output_licenses", LICENSE))
+ .add(attr("path", STRING))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelTestSuiteRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelTestSuiteRule.java
new file mode 100644
index 0000000..54db469
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelTestSuiteRule.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.common;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.test.TestSuite;
+
+/**
+ * Rule object implementing "test_suite".
+ */
+@BlazeRule(name = "test_suite",
+ ancestors = { BaseRuleClasses.BaseRule.class },
+ factoryClass = TestSuite.class)
+public final class BazelTestSuiteRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .override(attr("testonly", BOOLEAN).value(true)
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("tests", LABEL_LIST).orderIndependent().allowedFileTypes()
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ .add(attr("suites", LABEL_LIST).orderIndependent().allowedFileTypes()
+ .nonconfigurable("policy decision: should be consistent across configurations"))
+ // This magic attribute contains all *test rules in the package, iff
+ // tests=[] and suites=[]:
+ .add(attr("$implicit_tests", LABEL_LIST)
+ .nonconfigurable("Accessed in TestTargetUtils without config context"))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcBinary.java
new file mode 100644
index 0000000..e3f62a1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcBinary.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.cpp;
+
+import com.google.devtools.build.lib.rules.cpp.CcBinary;
+
+/**
+ * Factory class for the {@code cc_binary} rule.
+ */
+public class BazelCcBinary extends CcBinary {
+ public BazelCcBinary() {
+ super(BazelCppSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibrary.java
new file mode 100644
index 0000000..ae38806
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibrary.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.cpp;
+
+import com.google.devtools.build.lib.rules.cpp.CcLibrary;
+
+/**
+ * Factory class for the {@code cc_library} rule.
+ */
+public class BazelCcLibrary extends CcLibrary {
+ public BazelCcLibrary() {
+ super(BazelCppSemantics.INSTANCE);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcTest.java
new file mode 100644
index 0000000..f42b1dc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcTest.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.cpp;
+
+import com.google.devtools.build.lib.rules.cpp.CcTest;
+
+/**
+ * Factory class for the {@code cc_test} rule.
+ */
+public class BazelCcTest extends CcTest {
+ public BazelCcTest() {
+ super(BazelCppSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java
new file mode 100644
index 0000000..4a1f3b6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java
@@ -0,0 +1,422 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.cpp;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromFunctions;
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_LIBRARY;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_PIC_LIBRARY;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ARCHIVE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_HEADER;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_SOURCE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.C_SOURCE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.OBJECT_FILE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_ARCHIVE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_OBJECT_FILE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.SHARED_LIBRARY;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.VERSIONED_SHARED_LIBRARY;
+
+import com.google.common.base.Predicates;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.bazel.rules.BazelBaseRuleClasses;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel;
+import com.google.devtools.build.lib.packages.Attribute.Transition;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.TriState;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CcLibrary;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppRuleClasses;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode;
+
+/**
+ * Rule class definitions for C++ rules.
+ */
+public class BazelCppRuleClasses {
+ static final SafeImplicitOutputsFunction CC_LIBRARY_DYNAMIC_LIB =
+ fromTemplates("%{dirname}lib%{basename}.so");
+
+ static final SafeImplicitOutputsFunction CC_BINARY_IMPLICIT_OUTPUTS =
+ fromFunctions(CppRuleClasses.CC_BINARY_STRIPPED, CppRuleClasses.CC_BINARY_DEBUG_PACKAGE);
+
+ static final FileTypeSet ALLOWED_SRC_FILES = FileTypeSet.of(
+ CPP_SOURCE,
+ C_SOURCE,
+ CPP_HEADER,
+ ASSEMBLER_WITH_C_PREPROCESSOR,
+ ARCHIVE,
+ PIC_ARCHIVE,
+ ALWAYS_LINK_LIBRARY,
+ ALWAYS_LINK_PIC_LIBRARY,
+ SHARED_LIBRARY,
+ VERSIONED_SHARED_LIBRARY,
+ OBJECT_FILE,
+ PIC_OBJECT_FILE);
+
+ static final String[] DEPS_ALLOWED_RULES = new String[] {
+ "cc_library",
+ };
+
+ /**
+ * Miscellaneous configuration transitions. It would be better not to have this - please don't add
+ * to it.
+ */
+ public static enum CppTransition implements Transition {
+ /**
+ * The configuration for LIPO information collection. Requesting this from a configuration that
+ * does not have lipo optimization enabled may result in an exception.
+ */
+ LIPO_COLLECTOR,
+
+ /**
+ * The corresponding (target) configuration.
+ */
+ TARGET_CONFIG_FOR_LIPO;
+
+ @Override
+ public boolean defaultsToSelf() {
+ return false;
+ }
+ }
+
+ private static final RuleClass.Configurator<BuildConfiguration, Rule> LIPO_ON_DEMAND =
+ new RuleClass.Configurator<BuildConfiguration, Rule>() {
+ @Override
+ public BuildConfiguration apply(Rule rule, BuildConfiguration configuration) {
+ BuildConfiguration toplevelConfig =
+ configuration.getConfiguration(CppTransition.TARGET_CONFIG_FOR_LIPO);
+ // If LIPO is enabled, override the default configuration.
+ if (toplevelConfig != null
+ && toplevelConfig.getFragment(CppConfiguration.class).isLipoOptimization()
+ && !configuration.isHostConfiguration()
+ && !configuration.getFragment(CppConfiguration.class).isLipoContextCollector()) {
+ // Switch back to data when the cc_binary is not the LIPO context.
+ return (rule.getLabel().equals(
+ toplevelConfig.getFragment(CppConfiguration.class).getLipoContextLabel()))
+ ? toplevelConfig
+ : configuration.getTransitions().getConfiguration(ConfigurationTransition.DATA);
+ }
+ return configuration;
+ }
+ };
+
+ /**
+ * Label of a pseudo-filegroup that contains all crosstool and libcfiles for
+ * all configurations, as specified on the command-line.
+ */
+ public static final String CROSSTOOL_LABEL = "//tools/defaults:crosstool";
+
+ public static final LateBoundLabel<BuildConfiguration> CC_TOOLCHAIN =
+ new LateBoundLabel<BuildConfiguration>(CROSSTOOL_LABEL) {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(CppConfiguration.class).getCcToolchainRuleLabel();
+ }
+ };
+
+ public static final LateBoundLabel<BuildConfiguration> DEFAULT_MALLOC =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(CppConfiguration.class).customMalloc();
+ }
+ };
+
+ public static final LateBoundLabel<BuildConfiguration> STL =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return getStl(rule, configuration);
+ }
+ };
+
+ /**
+ * Implementation for the :lipo_context_collector attribute.
+ */
+ public static final LateBoundLabel<BuildConfiguration> LIPO_CONTEXT_COLLECTOR =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ // This attribute connects a target to the LIPO context target configured with the
+ // lipo input collector configuration.
+ CppConfiguration cppConfiguration = configuration.getFragment(CppConfiguration.class);
+ return !cppConfiguration.isLipoContextCollector()
+ && (cppConfiguration.getLipoMode() == LipoMode.BINARY)
+ ? cppConfiguration.getLipoContextLabel()
+ : null;
+ }
+ };
+
+ /**
+ * Returns the STL prerequisite of the rule.
+ *
+ * <p>If rule has an implicit $stl attribute returns STL version set on the
+ * command line or if not set, the value of the $stl attribute. Returns
+ * {@code null} otherwise.
+ */
+ private static Label getStl(Rule rule, BuildConfiguration original) {
+ Label stl = null;
+ if (rule.getRuleClassObject().hasAttr("$stl", Type.LABEL)) {
+ Label stlConfigLabel = original.getFragment(CppConfiguration.class).getStl();
+ Label stlRuleLabel = RawAttributeMapper.of(rule).get("$stl", Type.LABEL);
+ if (stlConfigLabel == null) {
+ stl = stlRuleLabel;
+ } else if (!stlConfigLabel.equals(rule.getLabel()) && stlRuleLabel != null) {
+ // prevents self-reference and a cycle through standard STL in the dependency graph
+ stl = stlConfigLabel;
+ }
+ }
+ return stl;
+ }
+
+ /**
+ * Common attributes for all rules that create C++ links. This may
+ * include non-cc_* rules (e.g. py_binary).
+ */
+ @BlazeRule(name = "$cc_linking_rule",
+ type = RuleClassType.ABSTRACT)
+ public static final class CcLinkingRule implements RuleDefinition {
+ @Override
+ @SuppressWarnings("unchecked")
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr(":cc_toolchain", LABEL).value(CC_TOOLCHAIN))
+ .setPreferredDependencyPredicate(Predicates.<String>or(CPP_SOURCE, C_SOURCE, CPP_HEADER))
+ .build();
+ }
+ }
+
+ /**
+ * Common attributes for C++ rules.
+ */
+ @BlazeRule(name = "$cc_base_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { CcLinkingRule.class })
+ public static final class CcBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("copts", STRING_LIST))
+ .add(attr("$stl", LABEL).value(env.getLabel("//tools/cpp:stl")))
+ .add(attr(":stl", LABEL).value(STL))
+ .build();
+ }
+ }
+
+ /**
+ * Helper rule class.
+ */
+ @BlazeRule(name = "$cc_decl_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.RuleBase.class })
+ public static final class CcDeclRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("abi", STRING).value("$(ABI)"))
+ .add(attr("abi_deps", LABEL_LIST_DICT))
+ .add(attr("defines", STRING_LIST))
+ .add(attr("includes", STRING_LIST))
+ .add(attr(":lipo_context_collector", LABEL)
+ .cfg(CppTransition.LIPO_COLLECTOR)
+ .value(LIPO_CONTEXT_COLLECTOR))
+ .build();
+ }
+ }
+
+ /**
+ * Helper rule class.
+ */
+ @BlazeRule(name = "$cc_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { CcDeclRule.class, CcBaseRule.class })
+ public static final class CcRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("srcs", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedFileTypes(ALLOWED_SRC_FILES))
+ .override(attr("deps", LABEL_LIST)
+ .allowedRuleClasses(DEPS_ALLOWED_RULES)
+ .allowedFileTypes()
+ .skipAnalysisTimeFileTypeCheck())
+ .add(attr("linkopts", STRING_LIST))
+ .add(attr("nocopts", STRING))
+ .add(attr("hdrs_check", STRING).value("strict"))
+ .add(attr("linkstatic", BOOLEAN).value(true))
+ .override(attr("$stl", LABEL).value(new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ // Every cc_rule depends implicitly on STL to make
+ // sure that the correct headers are used for inclusion. The only exception is
+ // STL itself to avoid cycles in the dependency graph.
+ Label stl = env.getLabel("//tools/cpp:stl");
+ return rule.getLabel().equals(stl) ? null : stl;
+ }
+ }))
+ .build();
+ }
+ }
+
+ /**
+ * Helper rule class.
+ */
+ @BlazeRule(name = "$cc_binary_base",
+ type = RuleClassType.ABSTRACT,
+ ancestors = CcRule.class)
+ public static final class CcBinaryBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("malloc", LABEL)
+ .value(env.getLabel("//tools/cpp:malloc"))
+ .allowedFileTypes()
+ .allowedRuleClasses("cc_library"))
+ .add(attr(":default_malloc", LABEL).value(DEFAULT_MALLOC))
+ .add(attr("stamp", TRISTATE).value(TriState.AUTO))
+ .build();
+ }
+ }
+
+ /**
+ * Rule definition for cc_binary rules.
+ */
+ @BlazeRule(name = "cc_binary",
+ ancestors = { CcBinaryBaseRule.class,
+ BazelBaseRuleClasses.BinaryBaseRule.class },
+ factoryClass = BazelCcBinary.class)
+ public static final class CcBinaryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setImplicitOutputsFunction(CC_BINARY_IMPLICIT_OUTPUTS)
+ .add(attr("linkshared", BOOLEAN).value(false)
+ .nonconfigurable("used to *determine* the rule's configuration"))
+ .cfg(LIPO_ON_DEMAND)
+ .build();
+ }
+ }
+
+ /**
+ * Implementation for the :lipo_context attribute.
+ */
+ private static final LateBoundLabel<BuildConfiguration> LIPO_CONTEXT =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ Label result = configuration.getFragment(CppConfiguration.class).getLipoContextLabel();
+ return (rule == null || rule.getLabel().equals(result)) ? null : result;
+ }
+ };
+
+ /**
+ * Rule definition for cc_test rules.
+ */
+ @BlazeRule(name = "cc_test",
+ type = RuleClassType.TEST,
+ ancestors = { CcBinaryBaseRule.class, BaseRuleClasses.TestBaseRule.class },
+ factoryClass = BazelCcTest.class)
+ public static final class CcTestRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setImplicitOutputsFunction(CppRuleClasses.CC_BINARY_DEBUG_PACKAGE)
+ .override(attr("linkstatic", BOOLEAN).value(false))
+ .override(attr("stamp", TRISTATE).value(TriState.NO))
+ .add(attr(":lipo_context", LABEL).value(LIPO_CONTEXT))
+ .build();
+ }
+ }
+
+ /**
+ * Helper rule class.
+ */
+ @BlazeRule(name = "$cc_library",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { CcRule.class })
+ public static final class CcLibraryBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("hdrs", LABEL_LIST).orderIndependent().direct_compile_time_input()
+ .allowedFileTypes(CPP_HEADER))
+ .add(attr("linkstamp", LABEL).allowedFileTypes(CPP_SOURCE, C_SOURCE))
+ .build();
+ }
+ }
+
+ /**
+ * Rule definition for the cc_library rule.
+ */
+ @BlazeRule(name = "cc_library",
+ ancestors = { CcLibraryBaseRule.class},
+ factoryClass = BazelCcLibrary.class)
+ public static final class CcLibraryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ SafeImplicitOutputsFunction implicitOutputsFunction = new SafeImplicitOutputsFunction() {
+ @Override
+ public Iterable<String> getImplicitOutputs(AttributeMap rule) {
+ boolean alwaysLink = rule.get("alwayslink", Type.BOOLEAN);
+ boolean linkstatic = rule.get("linkstatic", Type.BOOLEAN);
+ SafeImplicitOutputsFunction staticLib = fromTemplates(
+ alwaysLink
+ ? "%{dirname}lib%{basename}.lo"
+ : "%{dirname}lib%{basename}.a");
+ SafeImplicitOutputsFunction allLibs =
+ linkstatic || CcLibrary.appearsToHaveNoObjectFiles(rule)
+ ? staticLib
+ : fromFunctions(staticLib, CC_LIBRARY_DYNAMIC_LIB);
+ return allLibs.getImplicitOutputs(rule);
+ }
+ };
+
+ return builder
+ .setImplicitOutputsFunction(implicitOutputsFunction)
+ .add(attr("alwayslink", BOOLEAN).
+ nonconfigurable("value is referenced in an ImplicitOutputsFunction"))
+ .add(attr("implements", LABEL_LIST)
+ .allowedFileTypes()
+ .allowedRuleClasses("cc_public_library$headers"))
+ .override(attr("linkstatic", BOOLEAN).value(false)
+ .nonconfigurable("value is referenced in an ImplicitOutputsFunction"))
+ .build();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppSemantics.java
new file mode 100644
index 0000000..3771e6c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppSemantics.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.rules.cpp.CppCompilationContext.Builder;
+import com.google.devtools.build.lib.rules.cpp.CppCompileActionBuilder;
+import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppHelper;
+import com.google.devtools.build.lib.rules.cpp.CppSemantics;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * C++ compilation semantics.
+ */
+public class BazelCppSemantics implements CppSemantics {
+ public static final CppSemantics INSTANCE = new BazelCppSemantics();
+
+ private BazelCppSemantics() {
+ }
+
+ @Override
+ public PathFragment getEffectiveSourcePath(Artifact source) {
+ return source.getRootRelativePath();
+ }
+
+ @Override
+ public void finalizeCompileActionBuilder(
+ RuleContext ruleContext, CppCompileActionBuilder actionBuilder) {
+ actionBuilder.setCppConfiguration(ruleContext.getFragment(CppConfiguration.class));
+ actionBuilder.setActionContext(CppCompileActionContext.class);
+ actionBuilder.addTransitiveMandatoryInputs(CppHelper.getToolchain(ruleContext).getCompile());
+ }
+
+ @Override
+ public void setupCompilationContext(RuleContext ruleContext, Builder contextBuilder) {
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/BazelGenRuleRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/BazelGenRuleRule.java
new file mode 100644
index 0000000..eabb4e9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/BazelGenRuleRule.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.genrule;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.Type;
+
+/**
+ * Rule definition for the genrule rule.
+ */
+@BlazeRule(name = "genrule",
+ ancestors = { BaseRuleClasses.RuleBase.class },
+ factoryClass = GenRule.class)
+public final class BazelGenRuleRule implements RuleDefinition {
+ public static final String GENRULE_SETUP_LABEL = "//tools/genrule:genrule-setup.sh";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setOutputToGenfiles()
+ .add(attr("srcs", LABEL_LIST)
+ .direct_compile_time_input()
+ .legacyAllowAnyFileType())
+ .add(attr("tools", LABEL_LIST).cfg(HOST).legacyAllowAnyFileType())
+ .add(attr("$genrule_setup", LABEL).cfg(HOST).value(env.getLabel(GENRULE_SETUP_LABEL)))
+ .add(attr("outs", OUTPUT_LIST).mandatory())
+ .add(attr("cmd", STRING).mandatory())
+ .add(attr("output_to_bindir", BOOLEAN).value(false)
+ .nonconfigurable("policy decision: no reason for this to depend on the configuration"))
+ .add(attr("local", BOOLEAN).value(false))
+ .add(attr("message", STRING))
+ .add(attr("output_licenses", LICENSE))
+ .add(attr("executable", BOOLEAN).value(false))
+ .add(attr("stamp", BOOLEAN).value(false))
+ .add(attr("heuristic_label_expansion", BOOLEAN).value(true))
+ .add(attr("$is_executable", BOOLEAN)
+ .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target")
+ .value(
+ new Attribute.ComputedDefault("outs", "executable") {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return (rule.get("outs", Type.OUTPUT_LIST).size() == 1)
+ && rule.get("executable", BOOLEAN);
+ }
+ }))
+ .removeAttribute("data")
+ .removeAttribute("deps")
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRule.java
new file mode 100644
index 0000000..d70f9a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRule.java
@@ -0,0 +1,219 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.genrule;
+
+import static com.google.devtools.build.lib.analysis.RunfilesProvider.withData;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.CommandHelper;
+import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An implementation of genrule.
+ */
+public class GenRule implements RuleConfiguredTargetFactory {
+
+ public static final String GENRULE_SETUP_CMD =
+ "source tools/genrule/genrule-setup.sh; ";
+
+ private Artifact getExecutable(RuleContext ruleContext, NestedSet<Artifact> filesToBuild) {
+ if (Iterables.size(filesToBuild) == 1) {
+ Artifact out = Iterables.getOnlyElement(filesToBuild);
+ if (ruleContext.attributes().get("executable", Type.BOOLEAN)) {
+ return out;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ final List<Artifact> resolvedSrcs = Lists.newArrayList();
+
+ final NestedSet<Artifact> filesToBuild =
+ NestedSetBuilder.wrap(Order.STABLE_ORDER, ruleContext.getOutputArtifacts());
+ if (filesToBuild.isEmpty()) {
+ ruleContext.attributeError("outs", "Genrules without outputs don't make sense");
+ }
+ if (ruleContext.attributes().get("executable", Type.BOOLEAN)
+ && Iterables.size(filesToBuild) > 1) {
+ ruleContext.attributeError("executable",
+ "if genrules produce executables, they are allowed only one output. "
+ + "If you need the executable=1 argument, then you should split this genrule into "
+ + "genrules producing single outputs");
+ }
+
+ ImmutableMap.Builder<Label, Iterable<Artifact>> labelMap = ImmutableMap.builder();
+ for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("srcs", Mode.TARGET)) {
+ Iterable<Artifact> files = dep.getProvider(FileProvider.class).getFilesToBuild();
+ Iterables.addAll(resolvedSrcs, files);
+ labelMap.put(dep.getLabel(), files);
+ }
+
+ CommandHelper commandHelper = new CommandHelper(ruleContext, ruleContext
+ .getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class), labelMap.build());
+
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ String baseCommand = commandHelper.resolveCommandAndExpandLabels(
+ ruleContext.attributes().get("heuristic_label_expansion", Type.BOOLEAN), false);
+
+ // Adds the genrule environment setup script before the actual shell command
+ String command = GENRULE_SETUP_CMD + baseCommand;
+
+ command = resolveCommand(ruleContext, command, resolvedSrcs, filesToBuild);
+
+ String message = ruleContext.attributes().get("message", Type.STRING);
+ if (message.isEmpty()) {
+ message = "Executing genrule";
+ }
+
+ ImmutableMap<String, String> env =
+ ruleContext.getConfiguration().getDefaultShellEnvironment();
+
+ Map<String, String> executionInfo = Maps.newLinkedHashMap();
+ executionInfo.putAll(TargetUtils.getExecutionInfo(ruleContext.getRule()));
+
+ if (ruleContext.attributes().get("local", Type.BOOLEAN)) {
+ executionInfo.put("local", "");
+ }
+
+ NestedSetBuilder<Artifact> inputs = NestedSetBuilder.stableOrder();
+ inputs.addAll(resolvedSrcs);
+ inputs.addAll(commandHelper.getResolvedTools());
+ FilesToRunProvider genruleSetup =
+ ruleContext.getPrerequisite("$genrule_setup", Mode.HOST, FilesToRunProvider.class);
+ inputs.addAll(genruleSetup.getFilesToRun());
+ List<String> argv = commandHelper.buildCommandLine(command, inputs, ".genrule_script.sh");
+
+ if (ruleContext.attributes().get("stamp", Type.BOOLEAN)) {
+ inputs.add(ruleContext.getAnalysisEnvironment().getStableWorkspaceStatusArtifact());
+ inputs.add(ruleContext.getAnalysisEnvironment().getVolatileWorkspaceStatusArtifact());
+ }
+
+ ruleContext.registerAction(new GenRuleAction(
+ ruleContext.getActionOwner(), inputs.build(), filesToBuild, argv, env,
+ ImmutableMap.copyOf(executionInfo), commandHelper.getRemoteRunfileManifestMap(),
+ message + ' ' + ruleContext.getLabel()));
+
+ RunfilesProvider runfilesProvider = withData(
+ // No runfiles provided if not a data dependency.
+ Runfiles.EMPTY,
+ // We only need to consider the outputs of a genrule
+ // No need to visit the dependencies of a genrule. They cross from the target into the host
+ // configuration, because the dependencies of a genrule are always built for the host
+ // configuration.
+ new Runfiles.Builder().addTransitiveArtifacts(filesToBuild).build());
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .setFilesToBuild(filesToBuild)
+ .setRunfilesSupport(null, getExecutable(ruleContext, filesToBuild))
+ .addProvider(RunfilesProvider.class, runfilesProvider)
+ .build();
+ }
+
+ private String resolveCommand(final RuleContext ruleContext, final String command,
+ final List<Artifact> resolvedSrcs, final NestedSet<Artifact> filesToBuild) {
+ return ruleContext.expandMakeVariables("cmd", command, new ConfigurationMakeVariableContext(
+ ruleContext.getRule().getPackage(), ruleContext.getConfiguration()) {
+ @Override
+ public String lookupMakeVariable(String name) throws ExpansionException {
+ if (name.equals("SRCS")) {
+ return Artifact.joinExecPaths(" ", resolvedSrcs);
+ } else if (name.equals("<")) {
+ return expandSingletonArtifact(resolvedSrcs, "$<", "input file");
+ } else if (name.equals("OUTS")) {
+ return Artifact.joinExecPaths(" ", filesToBuild);
+ } else if (name.equals("@")) {
+ return expandSingletonArtifact(filesToBuild, "$@", "output file");
+ } else if (name.equals("@D")) {
+ // The output directory. If there is only one filename in outs,
+ // this expands to the directory containing that file. If there are
+ // multiple filenames, this variable instead expands to the
+ // package's root directory in the genfiles tree, even if all the
+ // generated files belong to the same subdirectory!
+ if (Iterables.size(filesToBuild) == 1) {
+ Artifact outputFile = Iterables.getOnlyElement(filesToBuild);
+ PathFragment relativeOutputFile = outputFile.getExecPath();
+ if (relativeOutputFile.segmentCount() <= 1) {
+ // This should never happen, since the path should contain at
+ // least a package name and a file name.
+ throw new IllegalStateException("$(@D) for genrule " + ruleContext.getLabel()
+ + " has less than one segment");
+ }
+ return relativeOutputFile.getParentDirectory().getPathString();
+ } else {
+ PathFragment dir;
+ if (ruleContext.getRule().hasBinaryOutput()) {
+ dir = ruleContext.getConfiguration().getBinFragment();
+ } else {
+ dir = ruleContext.getConfiguration().getGenfilesFragment();
+ }
+ PathFragment relPath = ruleContext.getRule().getLabel().getPackageFragment();
+ return dir.getRelative(relPath).getPathString();
+ }
+ } else {
+ return super.lookupMakeVariable(name);
+ }
+ }
+ }
+ );
+ }
+
+ // Returns the path of the sole element "artifacts", generating an exception
+ // with an informative error message iff the set is not a singleton.
+ //
+ // Used to expand "$<", "$@"
+ private String expandSingletonArtifact(Iterable<Artifact> artifacts,
+ String variable,
+ String artifactName)
+ throws ExpansionException {
+ if (Iterables.isEmpty(artifacts)) {
+ throw new ExpansionException("variable '" + variable
+ + "' : no " + artifactName);
+ } else if (Iterables.size(artifacts) > 1) {
+ throw new ExpansionException("variable '" + variable
+ + "' : more than one " + artifactName);
+ }
+ return Iterables.getOnlyElement(artifacts).getExecPathString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRuleAction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRuleAction.java
new file mode 100644
index 0000000..0a9b3e7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRuleAction.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.genrule;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * A spawn action for genrules. Genrules are handled specially in that inputs and outputs are
+ * checked for directories.
+ */
+public final class GenRuleAction extends SpawnAction {
+
+ private static final ResourceSet GENRULE_RESOURCES =
+ // Not chosen scientifically/carefully. 300MB memory, 100% CPU, 20% of total I/O.
+ new ResourceSet(300, 1.0, 0.0);
+
+ public GenRuleAction(ActionOwner owner,
+ Iterable<Artifact> inputs,
+ Iterable<Artifact> outputs,
+ List<String> argv,
+ ImmutableMap<String, String> environment,
+ ImmutableMap<String, String> executionInfo,
+ ImmutableMap<PathFragment, Artifact> runfilesManifests,
+ String progressMessage) {
+ super(owner, inputs, outputs, GENRULE_RESOURCES,
+ CommandLine.of(argv, false), environment, executionInfo, progressMessage,
+ runfilesManifests,
+ "Genrule", null);
+ }
+
+ @Override
+ protected void internalExecute(
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
+ EventHandler reporter = actionExecutionContext.getExecutor().getEventHandler();
+ checkInputsForDirectories(reporter, actionExecutionContext.getMetadataHandler());
+ super.internalExecute(actionExecutionContext);
+ checkOutputsForDirectories(reporter);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java
new file mode 100644
index 0000000..04713c2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.devtools.build.lib.rules.java.JavaBinary;
+
+/**
+ * Implementation of {@code java_binary} with Bazel semantics.
+ */
+public class BazelJavaBinary extends JavaBinary {
+ public BazelJavaBinary() {
+ super(BazelJavaSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java
new file mode 100644
index 0000000..279cbdb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.BazelBaseRuleClasses;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.BaseJavaBinaryRule;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for the java_binary rule.
+ */
+@BlazeRule(name = "java_binary",
+ ancestors = { BaseJavaBinaryRule.class,
+ BazelBaseRuleClasses.BinaryBaseRule.class },
+ factoryClass = BazelJavaBinary.class)
+public final class BazelJavaBinaryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_BINARY_IMPLICIT_OUTPUTS)
+ .override(attr("$is_executable", BOOLEAN).nonconfigurable("automatic").value(
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.get("create_executable", BOOLEAN);
+ }
+ }))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBuildInfoFactory.java
new file mode 100644
index 0000000..db33897
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBuildInfoFactory.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.rules.java.BuildInfoPropertiesTranslator;
+import com.google.devtools.build.lib.rules.java.GenericBuildInfoPropertiesTranslator;
+import com.google.devtools.build.lib.rules.java.JavaBuildInfoFactory;
+
+import java.util.Map;
+
+/**
+ * BuildInfoFactory for Java.
+ */
+public class BazelJavaBuildInfoFactory extends JavaBuildInfoFactory {
+ private static final Map<String, String> VOLATILE_KEYS = ImmutableMap
+ .<String, String>builder()
+ .put("build.time", "%BUILD_TIME%")
+ .put("build.timestamp.as.int", "%BUILD_TIMESTAMP%")
+ .put("build.timestamp", "%BUILD_TIMESTAMP%")
+ .build();
+
+ private static final Map<String, String> NONVOLATILE_KEYS = ImmutableMap
+ .<String, String>builder()
+ .build();
+
+ private static final Map<String, String> REDACTED_KEYS = ImmutableMap
+ .<String, String>builder()
+ .put("build.time", "Thu Jan 01 00:00:00 1970 (0)")
+ .put("build.timestamp.as.int", "0")
+ .put("build.timestamp", "Thu Jan 01 00:00:00 1970 (0)")
+ .build();
+
+ @Override
+ protected BuildInfoPropertiesTranslator createVolatileTranslator() {
+ return new GenericBuildInfoPropertiesTranslator(VOLATILE_KEYS);
+ }
+
+ @Override
+ protected BuildInfoPropertiesTranslator createNonVolatileTranslator() {
+ return new GenericBuildInfoPropertiesTranslator(NONVOLATILE_KEYS);
+ }
+
+ @Override
+ protected BuildInfoPropertiesTranslator createRedactedTranslator() {
+ return new GenericBuildInfoPropertiesTranslator(REDACTED_KEYS);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImport.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImport.java
new file mode 100644
index 0000000..6c7dcd4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImport.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.devtools.build.lib.rules.java.JavaImport;
+
+/**
+ * Implementation of {@code java_import} with Bazel semantics.
+ */
+public class BazelJavaImport extends JavaImport {
+ public BazelJavaImport() {
+ super(BazelJavaSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImportRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImportRule.java
new file mode 100644
index 0000000..132df23
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImportRule.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.ANY_EDGE;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.IjarBaseRule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.java.JavaImportBaseRule;
+
+/**
+ * Rule definition for the java_import rule.
+ */
+@BlazeRule(name = "java_import",
+ ancestors = { JavaImportBaseRule.class, IjarBaseRule.class },
+ factoryClass = BazelJavaImport.class)
+public final class BazelJavaImportRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(exports) -->
+ Targets to make available to users of this rule.
+ ${SYNOPSIS}
+ See <a href="#java_library.exports">java_library.exports</a>.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("exports", LABEL_LIST)
+ .allowedRuleClasses(ImmutableSet.of(
+ "java_library", "java_import", "cc_library", "cc_binary"))
+ .allowedFileTypes() // none allowed
+ .validityPredicate(ANY_EDGE))
+ .build();
+
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibrary.java
new file mode 100644
index 0000000..6af9450
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibrary.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.devtools.build.lib.rules.java.JavaLibrary;
+
+/**
+ * Implementation of {@code java_library} with Bazel semantics.
+ */
+public class BazelJavaLibrary extends JavaLibrary {
+ public BazelJavaLibrary() {
+ super(BazelJavaSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibraryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibraryRule.java
new file mode 100644
index 0000000..04c8a0f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibraryRule.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.JavaRule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Common attributes for Java rules.
+ */
+@BlazeRule(name = "java_library",
+ ancestors = { JavaRule.class },
+ factoryClass = BazelJavaLibrary.class)
+public final class BazelJavaLibraryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+
+ return builder
+ .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_LIBRARY_IMPLICIT_OUTPUTS)
+ .add(attr("exports", LABEL_LIST)
+ .allowedRuleClasses(BazelJavaRuleClasses.ALLOWED_RULES_IN_DEPS)
+ .allowedFileTypes(/*May not have files in exports!*/))
+ .add(attr("neverlink", BOOLEAN).value(false))
+ .override(attr("javacopts", STRING_LIST))
+ .add(attr("exported_plugins", LABEL_LIST).cfg(HOST).allowedRuleClasses("java_plugin")
+ .legacyAllowAnyFileType())
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPlugin.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPlugin.java
new file mode 100644
index 0000000..e6d3478
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPlugin.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.devtools.build.lib.rules.java.JavaPlugin;
+
+/**
+ * Implementation of the {@code java_plugin} rule for bazel.
+ */
+public class BazelJavaPlugin extends JavaPlugin {
+
+ public BazelJavaPlugin() {
+ super(BazelJavaSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPluginRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPluginRule.java
new file mode 100644
index 0000000..cbb9411
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPluginRule.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for the java_plugin rule.
+ */
+@BlazeRule(name = "java_plugin",
+ ancestors = { BazelJavaLibraryRule.class },
+ factoryClass = BazelJavaPlugin.class)
+public final class BazelJavaPluginRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_LIBRARY_IMPLICIT_OUTPUTS)
+ .override(builder.copy("deps").validityPredicate(Attribute.ANY_EDGE))
+ .override(builder.copy("srcs").validityPredicate(Attribute.ANY_EDGE))
+ .add(attr("processor_class", STRING))
+ .removeAttribute("runtime_deps")
+ .removeAttribute("exports")
+ .removeAttribute("exported_plugins")
+ .build();
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaRuleClasses.java
new file mode 100644
index 0000000..663b82a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaRuleClasses.java
@@ -0,0 +1,173 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromFunctions;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppRuleClasses;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.PredicateWithMessage;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.RuleClass.PackageNameConstraint;
+import com.google.devtools.build.lib.packages.TriState;
+import com.google.devtools.build.lib.rules.java.JavaSemantics;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.Set;
+
+/**
+ * Rule class definitions for Java rules.
+ */
+public class BazelJavaRuleClasses {
+
+ public static final PredicateWithMessage<Rule> JAVA_PACKAGE_NAMES = new PackageNameConstraint(
+ PackageNameConstraint.ANY_SEGMENT, "java", "javatests");
+
+ public static final ImplicitOutputsFunction JAVA_BINARY_IMPLICIT_OUTPUTS =
+ fromFunctions(JavaSemantics.JAVA_BINARY_CLASS_JAR, JavaSemantics.JAVA_BINARY_SOURCE_JAR,
+ JavaSemantics.JAVA_BINARY_DEPLOY_JAR, JavaSemantics.JAVA_BINARY_DEPLOY_SOURCE_JAR);
+
+ static final ImplicitOutputsFunction JAVA_LIBRARY_IMPLICIT_OUTPUTS =
+ fromFunctions(JavaSemantics.JAVA_LIBRARY_CLASS_JAR, JavaSemantics.JAVA_LIBRARY_SOURCE_JAR);
+
+ /**
+ * Common attributes for rules that depend on ijar.
+ */
+ @BlazeRule(name = "$ijar_base_rule",
+ type = RuleClassType.ABSTRACT)
+ public static final class IjarBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("$ijar", LABEL).cfg(HOST).exec().value(env.getLabel("//tools/defaults:ijar")))
+ .setPreferredDependencyPredicate(JavaSemantics.JAVA_SOURCE)
+ .build();
+ }
+ }
+
+
+ /**
+ * Common attributes for Java rules.
+ */
+ @BlazeRule(name = "$java_base_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { IjarBaseRule.class })
+ public static final class JavaBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr(":jvm", LABEL).cfg(HOST).value(JavaSemantics.JVM))
+ .add(attr(":host_jdk", LABEL).cfg(HOST).value(JavaSemantics.HOST_JDK))
+ .add(attr(":java_toolchain", LABEL).value(JavaSemantics.JAVA_TOOLCHAIN))
+ .add(attr("$java_langtools", LABEL).cfg(HOST)
+ .value(env.getLabel("//tools/defaults:java_langtools")))
+ .add(attr("$javac_bootclasspath", LABEL).cfg(HOST)
+ .value(env.getLabel(JavaSemantics.JAVAC_BOOTCLASSPATH_LABEL)))
+ .add(attr("$javabuilder", LABEL).cfg(HOST)
+ .value(env.getLabel(JavaSemantics.JAVABUILDER_LABEL)))
+ .add(attr("$singlejar", LABEL).cfg(HOST)
+ .value(env.getLabel(JavaSemantics.SINGLEJAR_LABEL)))
+ .build();
+ }
+ }
+
+ static final Set<String> ALLOWED_RULES_IN_DEPS = ImmutableSet.of(
+ "cc_binary", // NB: linkshared=1
+ "cc_library",
+ "genrule",
+ "genproto", // TODO(bazel-team): we should filter using providers instead (skylark rule).
+ "java_import",
+ "java_library",
+ "sh_binary",
+ "sh_library");
+
+ /**
+ * Common attributes for Java rules.
+ */
+ @BlazeRule(name = "$java_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.RuleBase.class, JavaBaseRule.class })
+ public static final class JavaRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .override(builder.copy("deps")
+ .allowedFileTypes(JavaSemantics.JAR)
+ .allowedRuleClasses(ALLOWED_RULES_IN_DEPS)
+ .skipAnalysisTimeFileTypeCheck())
+ .add(attr("runtime_deps", LABEL_LIST)
+ .allowedFileTypes(JavaSemantics.JAR)
+ .allowedRuleClasses(ALLOWED_RULES_IN_DEPS)
+ .skipAnalysisTimeFileTypeCheck())
+ .add(attr("srcs", LABEL_LIST)
+ .orderIndependent()
+ .direct_compile_time_input()
+ .allowedFileTypes(JavaSemantics.JAVA_SOURCE, JavaSemantics.JAR,
+ JavaSemantics.SOURCE_JAR, JavaSemantics.PROPERTIES))
+ .add(attr("resources", LABEL_LIST).orderIndependent()
+ .allowedFileTypes(FileTypeSet.ANY_FILE))
+ .add(attr("plugins", LABEL_LIST).cfg(HOST).allowedRuleClasses("java_plugin")
+ .legacyAllowAnyFileType())
+ .add(attr(":java_plugins", LABEL_LIST)
+ .cfg(HOST)
+ .allowedRuleClasses("java_plugin")
+ .silentRuleClassFilter()
+ .value(JavaSemantics.JAVA_PLUGINS))
+ .add(attr("javacopts", STRING_LIST))
+ .build();
+ }
+ }
+
+ /**
+ * Base class for rule definitions producing Java binaries.
+ */
+ @BlazeRule(name = "$base_java_binary",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { JavaRule.class,
+ // java_binary and java_test require the crosstool C++ runtime
+ // libraries (libstdc++.so, libgcc_s.so).
+ // TODO(bazel-team): Add tests for Java+dynamic runtime.
+ BazelCppRuleClasses.CcLinkingRule.class })
+ public static final class BaseJavaBinaryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr("classpath_resources", LABEL_LIST).legacyAllowAnyFileType())
+ .add(attr("jvm_flags", STRING_LIST))
+ .add(attr("main_class", STRING))
+ .add(attr("create_executable", BOOLEAN).nonconfigurable("internal").value(true))
+ .add(attr("deploy_manifest_lines", STRING_LIST))
+ .add(attr("stamp", TRISTATE).value(TriState.AUTO))
+ .add(attr(":java_launcher", LABEL).value(JavaSemantics.JAVA_LAUNCHER)) // blaze flag
+ .build();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java
new file mode 100644
index 0000000..b301161
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java
@@ -0,0 +1,341 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction;
+import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.ComputedSubstitution;
+import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution;
+import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Template;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder;
+import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression;
+import com.google.devtools.build.lib.rules.java.DirectDependencyProvider;
+import com.google.devtools.build.lib.rules.java.DirectDependencyProvider.Dependency;
+import com.google.devtools.build.lib.rules.java.JavaCommon;
+import com.google.devtools.build.lib.rules.java.JavaCompilationArtifacts;
+import com.google.devtools.build.lib.rules.java.JavaCompilationHelper;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration;
+import com.google.devtools.build.lib.rules.java.JavaHelper;
+import com.google.devtools.build.lib.rules.java.JavaPrimaryClassProvider;
+import com.google.devtools.build.lib.rules.java.JavaSemantics;
+import com.google.devtools.build.lib.rules.java.JavaTargetAttributes;
+import com.google.devtools.build.lib.rules.java.JavaUtil;
+import com.google.devtools.build.lib.rules.java.Jvm;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Semantics for Bazel Java rules
+ */
+public class BazelJavaSemantics implements JavaSemantics {
+
+ public static final BazelJavaSemantics INSTANCE = new BazelJavaSemantics();
+
+ private static final Template STUB_SCRIPT =
+ Template.forResource(BazelJavaSemantics.class, "java_stub_template.txt");
+
+ public static final InstrumentationSpec GREEDY_COLLECTION_SPEC = new InstrumentationSpec(
+ FileTypeSet.of(FileType.of(".sh"), JavaSemantics.JAVA_SOURCE),
+ "srcs", "deps", "data");
+
+ private BazelJavaSemantics() {
+ }
+
+ private boolean isJavaBinaryOrJavaTest(RuleContext ruleContext) {
+ String ruleClass = ruleContext.getRule().getRuleClass();
+ return ruleClass.equals("java_binary") || ruleClass.equals("java_test");
+ }
+
+ @Override
+ public void checkRule(RuleContext ruleContext, JavaCommon javaCommon) {
+ if (isJavaBinaryOrJavaTest(ruleContext)) {
+ checkMainClass(ruleContext, javaCommon);
+ }
+ }
+
+ private String getMainClassInternal(RuleContext ruleContext) {
+ return ruleContext.getRule().isAttrDefined("main_class", Type.STRING)
+ ? ruleContext.attributes().get("main_class", Type.STRING) : "";
+ }
+
+ private void checkMainClass(RuleContext ruleContext, JavaCommon javaCommon) {
+ boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN);
+ String mainClass = getMainClassInternal(ruleContext);
+
+ if (!createExecutable && !mainClass.isEmpty()) {
+ ruleContext.ruleError("main class must not be specified when executable is not created");
+ }
+
+ if (createExecutable && mainClass.isEmpty()) {
+ if (javaCommon.getSrcsArtifacts().isEmpty()) {
+ ruleContext.ruleError(
+ "need at least one of 'main_class', 'use_testrunner' or Java source files");
+ }
+ mainClass = javaCommon.determinePrimaryClass(javaCommon.getSrcsArtifacts());
+ if (mainClass == null) {
+ ruleContext.ruleError("cannot determine main class for launching "
+ + "(found neither a source file '" + ruleContext.getTarget().getName()
+ + ".java', nor a main_class attribute, and package name "
+ + "doesn't include 'java' or 'javatests')");
+ }
+ }
+ }
+
+ @Override
+ public String getMainClass(RuleContext ruleContext, JavaCommon javaCommon) {
+ checkMainClass(ruleContext, javaCommon);
+ return getMainClassInternal(ruleContext);
+ }
+
+ @Override
+ public ImmutableList<Artifact> collectResources(RuleContext ruleContext) {
+ if (!ruleContext.getRule().isAttrDefined("resources", Type.LABEL_LIST)) {
+ return ImmutableList.of();
+ }
+
+ return ruleContext.getPrerequisiteArtifacts("resources", Mode.TARGET).list();
+ }
+
+ @Override
+ public Artifact createInstrumentationMetadataArtifact(
+ AnalysisEnvironment analysisEnvironment, Artifact outputJar) {
+ return null;
+ }
+
+ @Override
+ public void buildJavaCommandLine(Collection<Artifact> outputs, BuildConfiguration configuration,
+ CustomCommandLine.Builder result) {
+ }
+
+ @Override
+ public void createStubAction(RuleContext ruleContext, final JavaCommon javaCommon,
+ List<String> jvmFlags, Artifact executable, String javaStartClass,
+ String javaExecutable) {
+
+ Preconditions.checkNotNull(jvmFlags);
+ Preconditions.checkNotNull(executable);
+ Preconditions.checkNotNull(javaStartClass);
+ Preconditions.checkNotNull(javaExecutable);
+ BuildConfiguration config = ruleContext.getConfiguration();
+
+ List<Substitution> arguments = new ArrayList<>();
+ arguments.add(Substitution.of("%javabin%", javaExecutable));
+ arguments.add(Substitution.of("%needs_runfiles%",
+ config.getFragment(Jvm.class).getJavaExecutable().isAbsolute() ? "0" : "1"));
+ arguments.add(new ComputedSubstitution("%classpath%") {
+ @Override
+ public String getValue() {
+ StringBuilder buffer = new StringBuilder();
+ Iterable<Artifact> jars = javaCommon.getRuntimeClasspath();
+ appendRunfilesRelativeEntries(buffer, jars, ':');
+ return buffer.toString();
+ }
+ });
+
+ arguments.add(Substitution.of("%java_start_class%",
+ ShellEscaper.escapeString(javaStartClass)));
+ arguments.add(Substitution.ofSpaceSeparatedList("%jvm_flags%", jvmFlags));
+
+ ruleContext.registerAction(new TemplateExpansionAction(
+ ruleContext.getActionOwner(), executable, STUB_SCRIPT, arguments, true));
+ }
+
+ /**
+ * Builds a class path by concatenating the root relative paths of the artifacts separated by the
+ * delimiter. Each relative path entry is prepended with "${RUNPATH}" which will be expanded by
+ * the stub script at runtime, to either "${JAVA_RUNFILES}/" or if we are lucky, the empty
+ * string.
+ *
+ * @param buffer the buffer to use for concatenating the entries
+ * @param artifacts the entries to concatenate in the buffer
+ * @param delimiter the delimiter character to separate the entries
+ */
+ private static void appendRunfilesRelativeEntries(StringBuilder buffer,
+ Iterable<Artifact> artifacts, char delimiter) {
+ for (Artifact artifact : artifacts) {
+ if (buffer.length() > 0) {
+ buffer.append(delimiter);
+ }
+ buffer.append("${RUNPATH}");
+ buffer.append(artifact.getRootRelativePath().getPathString());
+ }
+ }
+
+ @Override
+ public void addRunfilesForBinary(RuleContext ruleContext, Artifact launcher,
+ Runfiles.Builder runfilesBuilder) {
+ }
+
+ @Override
+ public void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder) {
+ }
+
+ @Override
+ public void collectTargetsTreatedAsDeps(
+ RuleContext ruleContext, ImmutableList.Builder<TransitiveInfoCollection> builder) {
+ }
+
+ @Override
+ public InstrumentationSpec getCoverageInstrumentationSpec() {
+ return GREEDY_COLLECTION_SPEC.withAttributes("srcs", "deps", "data", "exports", "runtime_deps");
+ }
+
+ @Override
+ public Iterable<String> getExtraJavacOpts(RuleContext ruleContext) {
+ return ImmutableList.<String>of();
+ }
+
+ @Override
+ public void addProviders(RuleContext ruleContext,
+ JavaCommon javaCommon,
+ List<String> jvmFlags,
+ Artifact classJar,
+ Artifact srcJar,
+ Artifact gensrcJar,
+ ImmutableMap<Artifact, Artifact> compilationToRuntimeJarMap,
+ JavaCompilationHelper helper,
+ NestedSetBuilder<Artifact> filesBuilder,
+ RuleConfiguredTargetBuilder ruleBuilder) {
+ if (!isJavaBinaryOrJavaTest(ruleContext)) {
+ Artifact outputDepsProto = helper.getOutputDepsProtoArtifact();
+ if (outputDepsProto != null && helper.getStrictJavaDeps() != StrictDepsMode.OFF) {
+ ImmutableList<Dependency> strictDependencies =
+ javaCommon.computeStrictDepsFromJavaAttributes(helper.getAttributes());
+ ruleBuilder.add(DirectDependencyProvider.class,
+ new DirectDependencyProvider(strictDependencies));
+ }
+ } else {
+ boolean createExec = ruleContext.attributes().get("create_executable", Type.BOOLEAN);
+ ruleBuilder.add(JavaPrimaryClassProvider.class,
+ new JavaPrimaryClassProvider(createExec ? getMainClassInternal(ruleContext) : null));
+ }
+ }
+
+
+ @Override
+ public Iterable<String> getJvmFlags(RuleContext ruleContext, JavaCommon javaCommon,
+ Artifact launcher, List<String> userJvmFlags) {
+ return userJvmFlags;
+ }
+
+ @Override
+ public String addCoverageSupport(JavaCompilationHelper helper,
+ JavaTargetAttributes.Builder attributes,
+ Artifact executable, Artifact instrumentationMetadata,
+ JavaCompilationArtifacts.Builder javaArtifactsBuilder, String mainClass) {
+ return mainClass;
+ }
+
+ @Override
+ public boolean useStrictJavaDeps(BuildConfiguration configuration) {
+ return true;
+ }
+
+ @Override
+ public CustomCommandLine buildSingleJarCommandLine(BuildConfiguration configuration,
+ Artifact output, String mainClass, ImmutableList<String> manifestLines,
+ Iterable<Artifact> buildInfoFiles, ImmutableList<Artifact> resources,
+ Iterable<Artifact> classpath, boolean includeBuildData,
+ Compression compression, Artifact launcher) {
+ return DeployArchiveBuilder.defaultSingleJarCommandLine(output, mainClass, manifestLines,
+ buildInfoFiles, resources, classpath, includeBuildData, compression, launcher).build();
+ }
+
+ @Override
+ public Collection<Artifact> translate(RuleContext ruleContext, JavaConfiguration javaConfig,
+ List<Artifact> messages) {
+ return ImmutableList.<Artifact>of();
+ }
+
+ @Override
+ public Artifact getLauncher(RuleContext ruleContext, JavaCommon common,
+ DeployArchiveBuilder deployArchiveBuilder, Runfiles.Builder runfilesBuilder,
+ List<String> jvmFlags, JavaTargetAttributes.Builder attributesBuilder) {
+ return JavaHelper.launcherArtifactForTarget(this, ruleContext);
+ }
+
+ @Override
+ public void addDependenciesForRunfiles(RuleContext ruleContext, Runfiles.Builder builder) {
+ }
+
+ @Override
+ public boolean forceUseJavaLauncherTarget(RuleContext ruleContext) {
+ return false;
+ }
+
+ @Override
+ public void addArtifactToJavaTargetAttribute(JavaTargetAttributes.Builder builder,
+ Artifact srcArtifact) {
+ }
+
+ @Override
+ public void commonDependencyProcessing(RuleContext ruleContext,
+ JavaTargetAttributes.Builder attributes,
+ Collection<? extends TransitiveInfoCollection> deps) {
+ }
+
+ @Override
+ public Collection<ActionInput> getExtraJavaCompileOutputs(PathFragment classDirectory) {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public PathFragment getJavaResourcePath(PathFragment path) {
+ PathFragment javaPath = JavaUtil.getJavaPath(path);
+ return javaPath == null ? path : javaPath;
+ }
+
+ @Override
+ public List<String> getExtraArguments(RuleContext ruleContext, JavaCommon javaCommon) {
+ if (ruleContext.getRule().getRuleClass().equals("java_test")) {
+ if (ruleContext.getConfiguration().getTestArguments().isEmpty()
+ && !ruleContext.attributes().isAttributeValueExplicitlySpecified("args")) {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (Artifact artifact : javaCommon.getSrcsArtifacts()) {
+ PathFragment path = artifact.getRootRelativePath();
+ String className = JavaUtil.getJavaFullClassname(FileSystemUtils.removeExtension(path));
+ if (className != null) {
+ builder.add(className);
+ }
+ }
+ return builder.build();
+ }
+ }
+ return ImmutableList.<String>of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java
new file mode 100644
index 0000000..ca94814
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.java.JavaBinary;
+
+/**
+ * An implementation of {@code java_test} rules.
+ */
+public class BazelJavaTest extends JavaBinary implements RuleConfiguredTargetFactory {
+ public BazelJavaTest() {
+ super(BazelJavaSemantics.INSTANCE);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java
new file mode 100644
index 0000000..fe881b7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.BaseJavaBinaryRule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.TriState;
+import com.google.devtools.build.lib.rules.java.JavaSemantics;
+
+/**
+ * Rule definition for the java_test rule.
+ */
+@BlazeRule(name = "java_test",
+ type = RuleClassType.TEST,
+ ancestors = { BaseJavaBinaryRule.class,
+ BaseRuleClasses.TestBaseRule.class },
+ factoryClass = BazelJavaTest.class)
+public final class BazelJavaTestRule implements RuleDefinition {
+
+ private static final String JUNIT4_RUNNER = "org.junit.runner.JUnitCore";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_BINARY_IMPLICIT_OUTPUTS)
+ .override(attr("main_class", STRING).value(JUNIT4_RUNNER))
+ .override(attr("stamp", TRISTATE).value(TriState.NO))
+ .override(attr(":java_launcher", LABEL).value(JavaSemantics.JAVA_LAUNCHER))
+ .build();
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt
new file mode 100644
index 0000000..a17246f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt
@@ -0,0 +1,195 @@
+#!/bin/bash --posix
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script was generated from java_stub_template.txt. Please
+# don't edit it directly.
+#
+# If present, these flags should either be at the beginning of the command
+# line, or they should be wrapped in a --wrapper_script_flag=FLAG argument.
+#
+# --debug Launch the JVM in remote debugging mode listening
+# --debug=<port> to the specified port or the port set in the
+# DEFAULT_JVM_DEBUG_PORT environment variable (e.g.
+# 'export DEFAULT_JVM_DEBUG_PORT=8000') or else the
+# default port of 5005. The JVM starts suspended
+# unless the DEFAULT_JVM_DEBUG_SUSPEND environment
+# variable is set to 'n'.
+# --main_advice=<class> Run an alternate main class with the usual main
+# program and arguments appended as arguments.
+# --main_advice_classpath=<classpath>
+# Prepend additional class path entries.
+# --jvm_flag=<flag> Pass <flag> to the "java" command itself.
+# <flag> may contain spaces. Can be used multiple times.
+# --jvm_flags=<flags> Pass space-separated flags to the "java" command
+# itself. Can be used multiple times.
+# --singlejar Start the program from the packed-up deployment
+# jar rather than from the classpath.
+# --print_javabin Print the location of java executable binary and exit.
+#
+# The remainder of the command line is passed to the program.
+
+# Make it easy to insert 'set -x' or similar commands when debugging problems with this script.
+eval "$JAVA_STUB_DEBUG"
+
+# Prevent problems where the caller has exported CLASSPATH, causing our
+# computed value to be copied into the environment and double-counted
+# against the argv limit.
+unset CLASSPATH
+
+JVM_FLAGS_CMDLINE=()
+
+# Processes an argument for the wrapper. Returns 0 if the given argument
+# was recognized as an argument for this wrapper, and 1 if it was not.
+function process_wrapper_argument() {
+ case "$1" in
+ --debug) JVM_DEBUG_PORT="${DEFAULT_JVM_DEBUG_PORT:-5005}" ;;
+ --debug=*) JVM_DEBUG_PORT="${1#--debug=}" ;;
+ --main_advice=*) MAIN_ADVICE="${1#--main_advice=}" ;;
+ --main_advice_classpath=*) MAIN_ADVICE_CLASSPATH="${1#--main_advice_classpath=}" ;;
+ --jvm_flag=*) JVM_FLAGS_CMDLINE+=( "${1#--jvm_flag=}" ) ;;
+ --jvm_flags=*) JVM_FLAGS_CMDLINE+=( ${1#--jvm_flags=} ) ;;
+ --singlejar) SINGLEJAR=1 ;;
+ --print_javabin) PRINT_JAVABIN=1 ;;
+ *)
+ return 1 ;;
+ esac
+ return 0
+}
+
+die() {
+ printf "%s: $1\n" "$0" "${@:2}" >&2
+ exit 1
+}
+
+# Parse arguments sequentially until the first unrecognized arg is encountered.
+# Scan the remaining args for --wrapper_script_flag=X options and process them.
+ARGS=()
+for ARG in "$@"; do
+ if [[ "$ARG" == --wrapper_script_flag=* ]]; then
+ process_wrapper_argument "${ARG#--wrapper_script_flag=}" \
+ || die "invalid wrapper argument '%s'" "$ARG"
+ elif [[ "${#ARGS}" > 0 ]] || ! process_wrapper_argument "$ARG"; then
+ ARGS+=( "$ARG" )
+ fi
+done
+
+# Find our runfiles tree. We need this to construct the classpath
+# (unless --singlejar was passed).
+#
+# Call this program X. X was generated by a java_binary or java_test rule.
+# X may be invoked in many ways:
+# 1a) directly by a user, with $0 in the output tree
+# 1b) via 'bazel run' (similar to case 1a)
+# 2) directly by a user, with $0 in X's runfiles tree
+# 3) by another program Y which has a data dependency on X, with $0 in Y's runfiles tree
+# 4) via 'bazel test'
+# 5) by a genrule cmd, with $0 in the output tree
+# 6) case 3 in the context of a genrule
+#
+# For case 1, $0 will be a regular file, and the runfiles tree will be
+# at $0.runfiles.
+# For case 2 or 3, $0 will be a symlink to the file seen in case 1.
+# For case 4, $JAVA_RUNFILES and $TEST_SRCDIR should already be set.
+# Case 5 is handled like case 1.
+# Case 6 is handled like case 3.
+
+case "$0" in
+ /*) self="$0" ;;
+ *) self="$PWD/$0" ;;
+esac
+
+if [[ "$SINGLEJAR" != 1 || "%needs_runfiles%" == 1 ]]; then
+ if [[ -z "$JAVA_RUNFILES" ]]; then
+ while true; do
+ if [[ -e "$self.runfiles" ]]; then
+ JAVA_RUNFILES="$self.runfiles"
+ break
+ fi
+ if [[ $self == *.runfiles/* ]]; then
+ JAVA_RUNFILES="${self%.runfiles/*}.runfiles"
+ # don't break; this value is only a last resort for case 6b
+ fi
+ if [[ ! -L "$self" ]]; then
+ break
+ fi
+ readlink="$(readlink "$self")"
+ if [[ "$readlink" = /* ]]; then
+ self="$readlink"
+ else
+ # resolve relative symlink
+ self="${self%/*}/$readlink"
+ fi
+ done
+ if [[ -n "$JAVA_RUNFILES" ]]; then
+ export TEST_SRCDIR=${TEST_SRCDIR:-$JAVA_RUNFILES}
+ elif [[ -f "${self}_deploy.jar" && "%needs_runfiles%" == 0 ]]; then
+ SINGLEJAR=1;
+ else
+ die 'Cannot locate runfiles directory. (Set $JAVA_RUNFILES to inhibit searching.)'
+ fi
+ fi
+fi
+
+# Set JAVABIN to the path to the JVM launcher.
+%javabin%
+
+if [[ "$PRINT_JAVABIN" == 1 || "%java_start_class%" == "--print_javabin" ]]; then
+ echo -n "$JAVABIN"
+ exit 0
+fi
+
+if [[ "$SINGLEJAR" == 1 ]]; then
+ CLASSPATH="${self}_deploy.jar"
+ # Check for the deploy jar now. If it doesn't exist, we can print a
+ # more helpful error message than the JVM.
+ [[ -r "$CLASSPATH" ]] \
+ || die "Option --singlejar was passed, but %s does not exist.\n (You may need to build it explicitly.)" "$CLASSPATH"
+else
+ # Create the shortest classpath we can, by making it relative if possible.
+ RUNPATH="${JAVA_RUNFILES}/"
+ RUNPATH="${RUNPATH#$PWD/}"
+ CLASSPATH=%classpath%
+fi
+
+if [[ -n "$JVM_DEBUG_PORT" ]]; then
+ JVM_DEBUG_SUSPEND=${DEFAULT_JVM_DEBUG_SUSPEND:-"y"}
+ JVM_DEBUG_FLAGS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=${JVM_DEBUG_SUSPEND},address=${JVM_DEBUG_PORT}"
+fi
+
+if [[ -n "$MAIN_ADVICE_CLASSPATH" ]]; then
+ CLASSPATH="${MAIN_ADVICE_CLASSPATH}:${CLASSPATH}"
+fi
+
+# Check if TEST_TMPDIR is available to use for scratch.
+if [[ -n "$TEST_TMPDIR" && -d "$TEST_TMPDIR" ]]; then
+ JVM_FLAGS+=" -Djava.io.tmpdir=$TEST_TMPDIR"
+fi
+
+ARGS=(
+ ${JVM_DEBUG_FLAGS}
+ ${JVM_FLAGS}
+ %jvm_flags%
+ "${JVM_FLAGS_CMDLINE[@]}"
+ ${MAIN_ADVICE}
+ %java_start_class%
+ "${ARGS[@]}")
+
+# Linux per-arg limit MAX_ARG_STRLEN == 128k!
+if (("${#CLASSPATH}" > 120000)); then
+ set +o posix # Enable process substitution.
+ exec $JAVABIN -classpath @<(echo $CLASSPATH) "${ARGS[@]}"
+else
+ exec $JAVABIN -classpath $CLASSPATH "${ARGS[@]}"
+fi
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTest.java
new file mode 100644
index 0000000..f45ca08
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTest.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.objc;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.objc.IosTest;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon;
+import com.google.devtools.build.lib.rules.objc.XcodeProvider;
+
+/**
+ * Implementation for ios_test rule in Bazel.
+ */
+public final class BazelIosTest extends IosTest {
+ static final String IOS_TEST_ON_BAZEL_ATTR = "$ios_test_on_bazel";
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext, ObjcCommon common,
+ XcodeProvider xcodeProvider, NestedSet<Artifact> filesToBuild) throws InterruptedException {
+ Artifact testRunner = ruleContext.getPrerequisiteArtifact(IOS_TEST_ON_BAZEL_ATTR, Mode.TARGET);
+ Runfiles runfiles = new Runfiles.Builder()
+ .addArtifact(testRunner)
+ .build();
+ RunfilesSupport runfilesSupport =
+ RunfilesSupport.withExecutable(ruleContext, runfiles, testRunner);
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .setFilesToBuild(NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(filesToBuild)
+ .add(testRunner)
+ .build())
+ .add(XcodeProvider.class, xcodeProvider)
+ .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles))
+ .setRunfilesSupport(runfilesSupport, testRunner)
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTestRule.java
new file mode 100644
index 0000000..114d454
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTestRule.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.rules.objc.ApplicationSupport;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses;
+import com.google.devtools.build.lib.rules.objc.XcodeSupport;
+
+/**
+ * Rule definition for the ios_test rule.
+ */
+@BlazeRule(name = "ios_test",
+ type = RuleClassType.TEST,
+ ancestors = { ObjcRuleClasses.IosTestBaseRule.class,
+ BaseRuleClasses.TestBaseRule.class },
+ factoryClass = BazelIosTest.class)
+public final class BazelIosTestRule implements RuleDefinition {
+ @Override
+ public RuleClass build(RuleClass.Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ /*<!-- #BLAZE_RULE(ios_test).IMPLICIT_OUTPUTS -->
+ <ul>
+ <li><code><var>name</var>.ipa</code>: the test bundle as an
+ <code>.ipa</code> file
+ <li><code><var>name</var>.xcodeproj/project.pbxproj: An Xcode project file which can be
+ used to develop or build on a Mac.</li>
+ </ul>
+ <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/
+ .setImplicitOutputsFunction(
+ ImplicitOutputsFunction.fromFunctions(ApplicationSupport.IPA, XcodeSupport.PBXPROJ))
+ .add(attr(BazelIosTest.IOS_TEST_ON_BAZEL_ATTR, LABEL)
+ .value(env.getLabel("//tools/objc:ios_test_on_bazel")).exec())
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = ios_test, TYPE = TEST, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule provides a way to build iOS unit tests written in KIF, GTM and XCTest test frameworks
+on both iOS simulator and real devices.
+</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShBinaryRule.java
new file mode 100644
index 0000000..39d4a0c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShBinaryRule.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.BazelBaseRuleClasses;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses.ShRule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for the sh_binary rule.
+ */
+@BlazeRule(name = "sh_binary",
+ ancestors = { ShRule.class, BazelBaseRuleClasses.BinaryBaseRule.class },
+ factoryClass = ShBinary.class)
+public final class BazelShBinaryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder.add(
+ attr("bash_version", STRING)
+ .value(BazelShRuleClasses.DEFAULT_BASH_VERSION)
+ .allowedValues(BazelShRuleClasses.BASH_VERSION_ALLOWED_VALUES)).build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShLibraryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShLibraryRule.java
new file mode 100644
index 0000000..9d9640b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShLibraryRule.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses.ShRule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for the sh_library rule.
+ */
+@BlazeRule(name = "sh_library",
+ ancestors = { ShRule.class },
+ factoryClass = ShLibrary.class)
+public final class BazelShLibraryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(sh_library).ATTRIBUTE(deps) -->
+ The list of other targets to be aggregated in to this "library" target.
+ <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/>
+ See general comments about <code>deps</code>
+ at <a href="#common-attributes">Attributes common to all build rules</a>.
+ You should use this attribute to list other
+ <code>sh_library</code> or <code>proto_library</code> rules that provide
+ interpreted program source code depended on by the code in
+ <code>srcs</code>. If you depend on a <code>proto_library</code> target,
+ the proto sources in that target will be included in this library, but
+ no generated files will be built.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+
+ /* <!-- #BLAZE_RULE(sh_library).ATTRIBUTE(srcs) -->
+ The list of input files.
+ <i>(List of <a href="build-ref.html#labels">labels</a>,
+ optional)</i><br/>
+ You should use this attribute to list interpreted program
+ source files that belong to this package, such as additional
+ files containing Bourne shell subroutines, loaded via the shell's
+ <code>source</code> or <code>.</code> command.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .override(attr("srcs", LABEL_LIST).legacyAllowAnyFileType())
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = sh_library, TYPE = LIBRARY, FAMILY = Shell) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>
+ The main use for this rule is to aggregate together a logical
+ "library" consisting of related scripts—programs in an
+ interpreted language that does not require compilation or linking,
+ such as the Bourne shell—and any data those programs need at
+ run-time. Such "libraries" can then be used from
+ the <code>data</code> attribute of one or
+ more <code>sh_binary</code> rules.
+</p>
+
+<p>
+ Historically, a second use was to aggregate a collection of data files
+ together, to ensure that they are available at runtime in
+ the <code>.runfiles</code> area of one or more <code>*_binary</code>
+ rules (not necessarily <code>sh_binary</code>).
+ However, the <a href="#filegroup"><code>filegroup()</code></a> rule
+ should be used now; it is intended to replace this use of
+ <code>sh_library</code>.
+</p>
+
+<p>
+ In interpreted programming languages, there's not always a clear
+ distinction between "code" and "data": after all, the program is
+ just "data" from the interpreter's point of view. For this reason
+ (and historical accident) this rule has three attributes which are
+ all essentially equivalent: <code>srcs</code>, <code>deps</code>
+ and <code>data</code>.
+ The recommended usage of each attribute is mentioned below. The
+ current implementation does not distinguish the elements of these lists.
+ All three attributes accept rules, source files and derived files.
+</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="sh_library_examples">Examples</h4>
+
+<pre class="code">
+sh_library(
+ name = "foo",
+ data = [
+ ":foo_service_script", # a sh_binary with srcs
+ ":deploy_foo", # another sh_binary with srcs
+ ],
+)
+</pre>
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShRuleClasses.java
new file mode 100644
index 0000000..6f66465
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShRuleClasses.java
@@ -0,0 +1,101 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.Attribute.AllowedValueSet;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.PredicateWithMessage;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+import java.util.Collection;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Rule definitions for rule classes implementing shell support.
+ */
+public final class BazelShRuleClasses {
+
+ static final Collection<String> ALLOWED_RULES_IN_DEPS_WITH_WARNING = ImmutableSet.of(
+ "filegroup", "Fileset", "genrule", "sh_binary", "sh_test", "test_suite");
+
+ /**
+ * Common attributes for shell rules.
+ */
+ @BlazeRule(name = "$sh_target",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.RuleBase.class })
+ public static final class ShRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ .add(attr("srcs", LABEL_LIST).mandatory().legacyAllowAnyFileType())
+ .override(builder.copy("deps")
+ .allowedRuleClasses("sh_library", "proto_library")
+ .allowedRuleClassesWithWarning(ALLOWED_RULES_IN_DEPS_WITH_WARNING)
+ .allowedFileTypes())
+ .build();
+ }
+ }
+
+ /**
+ * Defines the file name of an sh_binary's implicit .sar (script package) output.
+ */
+ static final ImplicitOutputsFunction SAR_PACKAGE_FILENAME =
+ fromTemplates("%{name}.sar");
+
+ /**
+ * Convenience structure for the bash dependency combinations defined
+ * by BASH_BINARY_BINDINGS.
+ */
+ static class BashBinaryBinding {
+ public final String execPath;
+ public BashBinaryBinding(@Nullable String execPath) {
+ this.execPath = execPath;
+ }
+ }
+
+ /**
+ * Attribute value specifying the local system's bash version.
+ */
+ static final String SYSTEM_BASH_VERSION = "system";
+
+ static final Map<String, BashBinaryBinding> BASH_BINARY_BINDINGS =
+ ImmutableMap.of(
+ // "system": don't package any bash with the target, but rather use whatever is
+ // available on the system the script is run on.
+ SYSTEM_BASH_VERSION, new BashBinaryBinding("/bin/bash")
+ );
+
+ static final String DEFAULT_BASH_VERSION = SYSTEM_BASH_VERSION;
+
+ // TODO(bazel-team): refactor sh_binary and sh_base to have a common root
+ // with srcs and bash_version attributes
+ static final PredicateWithMessage<Object> BASH_VERSION_ALLOWED_VALUES =
+ new AllowedValueSet(BASH_BINARY_BINDINGS.keySet());
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShTestRule.java
new file mode 100644
index 0000000..4a1e51f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShTestRule.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses.ShRule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Rule definition for the sh_test rule.
+ */
+@BlazeRule(name = "sh_test",
+ type = RuleClassType.TEST,
+ ancestors = { ShRule.class, BaseRuleClasses.TestBaseRule.class },
+ factoryClass = ShTest.class)
+public final class BazelShTestRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ .add(attr("bash_version", STRING)
+ .value(BazelShRuleClasses.DEFAULT_BASH_VERSION)
+ .allowedValues(BazelShRuleClasses.BASH_VERSION_ALLOWED_VALUES))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java
new file mode 100644
index 0000000..4e6ba81
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.actions.ExecutableSymlinkAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for the sh_binary rule.
+ */
+public class ShBinary implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ ImmutableList<Artifact> srcs = ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list();
+ if (srcs.size() != 1) {
+ ruleContext.attributeError("srcs", "you must specify exactly one file in 'srcs'");
+ return null;
+ }
+
+ Artifact symlink = ruleContext.createOutputArtifact();
+ Artifact src = srcs.get(0);
+ Artifact executableScript = getExecutableScript(ruleContext, src);
+ // The interpretation of this deceptively simple yet incredibly generic rule is complicated
+ // by the distinction between targets and (not properly encapsulated) artifacts. It depends
+ // on the notion of other rule's "files-to-build" sets, which are undocumented, making it
+ // impossible to give a precise definition of what this rule does in all cases (e.g. what
+ // happens when srcs = ['x', 'y'] but 'x' is an empty filegroup?). This is a pervasive
+ // problem in Blaze.
+ ruleContext.registerAction(
+ new ExecutableSymlinkAction(ruleContext.getActionOwner(), executableScript, symlink));
+
+ NestedSet<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder()
+ .add(src)
+ .add(executableScript) // May be the same as src, in which case set semantics apply.
+ .add(symlink)
+ .build();
+ Runfiles runfiles = new Runfiles.Builder()
+ .addTransitiveArtifacts(filesToBuild)
+ .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES)
+ .build();
+ RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable(
+ ruleContext, runfiles, symlink);
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .setFilesToBuild(filesToBuild)
+ .setRunfilesSupport(runfilesSupport, symlink)
+ .addProvider(RunfilesProvider.class, RunfilesProvider.simple(runfiles))
+ .build();
+ }
+
+ /**
+ * Hook for sh_test to provide the executable.
+ *
+ * @param ruleContext
+ * @param src
+ */
+ protected Artifact getExecutableScript(RuleContext ruleContext, Artifact src) {
+ return src;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShLibrary.java
new file mode 100644
index 0000000..e2744ea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShLibrary.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for the sh_library rule.
+ */
+public class ShLibrary implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ NestedSet<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder()
+ .addAll(ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list())
+ .addAll(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET).list())
+ .addAll(ruleContext.getPrerequisiteArtifacts("data", Mode.DATA).list())
+ .build();
+ Runfiles runfiles = new Runfiles.Builder()
+ .addTransitiveArtifacts(filesToBuild)
+ .build();
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .setFilesToBuild(filesToBuild)
+ .addProvider(RunfilesProvider.class, RunfilesProvider.simple(runfiles))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShTest.java
new file mode 100644
index 0000000..cc965aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShTest.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.rules.sh;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Implementation for sh_test rules.
+ */
+public class ShTest extends ShBinary implements RuleConfiguredTargetFactory {
+
+ @Override
+ protected Artifact getExecutableScript(RuleContext ruleContext, Artifact src) {
+ if (ruleContext.attributes().get("bash_version", Type.STRING)
+ .equals(BazelShRuleClasses.SYSTEM_BASH_VERSION)) {
+ return src;
+ }
+
+ // What *will* this script run with the wrapper?
+ PathFragment newOutput = src.getRootRelativePath().getParentDirectory().getRelative(
+ ruleContext.getLabel().getName() + "_runner.sh");
+ Artifact testRunner = ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ newOutput, ruleContext.getConfiguration().getBinDirectory());
+
+ String bashPath = BazelShRuleClasses.BASH_BINARY_BINDINGS
+ .get(BazelShRuleClasses.SYSTEM_BASH_VERSION).execPath;
+
+ // Generate the runner contents.
+ String runnerContents =
+ "#!/bin/bash\n"
+ + bashPath + " \"" + src.getRootRelativePath().getPathString() + "\" \"$@\"\n";
+
+ ruleContext.registerAction(
+ new FileWriteAction(ruleContext.getActionOwner(), testRunner, runnerContents, true));
+ return testRunner;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java
new file mode 100644
index 0000000..c7f3677
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Rule definition for the http_archive rule.
+ */
+@BlazeRule(name = HttpArchiveRule.NAME,
+ type = RuleClassType.WORKSPACE,
+ ancestors = { WorkspaceBaseRule.class },
+ factoryClass = WorkspaceConfiguredTargetFactory.class)
+public class HttpArchiveRule implements RuleDefinition {
+
+ public static final String NAME = "http_archive";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(url) -->
+ A URL to an archive file containing a Bazel repository
+
+ <p>This must be an http URL that ends with .zip. There is no support for authentication or
+ redirection.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("url", STRING).mandatory())
+ /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(sha256) -->
+ The expected SHA-256 hash of the file downloaded
+
+ <p>This must match the SHA-256 hash of the file downloaded.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("sha256", STRING).mandatory())
+ .setWorkspaceOnly()
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = http_archive, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>Downloads a Bazel repository as a compressed archive file, decompresses it, and makes its
+ targets available for binding.</p>
+
+<p>Only Zip-formatted archives with the .zip extension are supported.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="http_archive_examples">Examples</h4>
+
+<p>Suppose the current repository contains the source code for a chat program, rooted at the
+ directory <i>~/chat-app</i>. It needs to depend on an SSL library which is available from
+ <i>http://example.com/openssl.zip</i>. This .zip file contains the following directory
+ structure:</p>
+
+<pre class="code">
+WORKSPACE
+src/
+ BUILD
+ openssl.cc
+ openssl.h
+</pre>
+
+<p><i>src/BUILD</i> contains the following target definition:</p>
+
+<pre class="code">
+cc_library(
+ name = "openssl-lib",
+ srcs = ["openssl.cc"],
+ hdrs = ["openssl.h"],
+)
+</pre>
+
+<p>Targets in the <i>~/chat-app</i> repository can depend on this target if the following lines are
+ added to <i>~/chat-app/WORKSPACE</i>:</p>
+
+<pre class="code">
+http_archive(
+ name = "my-ssl",
+ url = "http://example.com/openssl.zip",
+ sha256 = "03a58ac630e59778f328af4bcc4acb4f80208ed4",
+)
+
+bind(
+ name = "openssl",
+ actual = "@my-ssl//src:openssl-lib",
+)
+</pre>
+
+<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p>
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpJarRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpJarRule.java
new file mode 100644
index 0000000..862c6d7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpJarRule.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Rule definition for the http_jar rule.
+ */
+@BlazeRule(name = HttpJarRule.NAME,
+ type = RuleClassType.WORKSPACE,
+ ancestors = { WorkspaceBaseRule.class },
+ factoryClass = WorkspaceConfiguredTargetFactory.class)
+public class HttpJarRule implements RuleDefinition {
+
+ public static final String NAME = "http_jar";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(http_jar).ATTRIBUTE(url) -->
+ A URL to an archive file containing a Bazel repository
+
+ <p>This must be an http or https URL that ends with .jar. Redirects are not followed.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("url", STRING).mandatory())
+ /* <!-- #BLAZE_RULE(http_jar).ATTRIBUTE(sha256) -->
+ The expected SHA-256 of the file downloaded
+
+ <p>This must match the SHA-256 of the file downloaded.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("sha256", STRING).mandatory())
+ .setWorkspaceOnly()
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = http_jar, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>Downloads a jar from a URL and makes it available to be used as a Java dependency.</p>
+
+<p>Downloaded files must have a .jar extension.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="http_jar_examples">Examples</h4>
+
+<p>Suppose the current repository contains the source code for a chat program, rooted at the
+ directory <i>~/chat-app</i>. It needs to depend on an SSL library which is available from
+ <i>http://example.com/openssl-0.2.jar</i>.</p>
+
+<p>Targets in the <i>~/chat-app</i> repository can depend on this target if the following lines are
+ added to <i>~/chat-app/WORKSPACE</i>:</p>
+
+<pre class="code">
+http_jar(
+ name = "my-ssl",
+ url = "http://example.com/openssl-0.2.jar",
+ sha256 = "03a58ac630e59778f328af4bcc4acb4f80208ed4",
+)
+
+bind(
+ name = "openssl",
+ actual = "@my-ssl//jar:openssl-0.2.jar",
+)
+</pre>
+
+<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p>
+
+<!-- #END_BLAZE_RULE -->*/
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/LocalRepositoryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/LocalRepositoryRule.java
new file mode 100644
index 0000000..b904bc5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/LocalRepositoryRule.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Rule definition for the local_repository rule.
+ */
+@BlazeRule(name = LocalRepositoryRule.NAME,
+ type = RuleClassType.WORKSPACE,
+ ancestors = { WorkspaceBaseRule.class },
+ factoryClass = WorkspaceConfiguredTargetFactory.class)
+public class LocalRepositoryRule implements RuleDefinition {
+
+ public static final String NAME = "local_repository";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(local_repository).ATTRIBUTE(path) -->
+ The path to the local repository's directory.
+
+ <p>This must be an absolute path to the directory containing the repository's
+ <i>WORKSPACE</i> file.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("path", STRING).mandatory())
+ .setWorkspaceOnly()
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = local_repository, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>Allows targets from a local directory to be bound. This means that the current repository can
+ use targets defined in this other directory. See the <a href="#bind_examples">bind section</a>
+ for more details.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="local_repository_examples">Examples</h4>
+
+<p>Suppose the current repository is a chat client, rooted at the directory <i>~/chat-app</i>. It
+ would like to use an SSL library which is defined in a different repository: <i>~/ssl</i>. The
+ SSL library has a target <code>//src:openssl-lib</code>.</p>
+
+<p>The user can add a dependency on this target by adding the following lines to
+ <i>~/chat-app/WORKSPACE</i>:</p>
+
+<pre class="code">
+local_repository(
+ name = "my-ssl",
+ path = "/home/user/ssl",
+)
+
+bind(
+ name = "openssl",
+ actual = "@my-ssl//src:openssl-lib",
+)
+</pre>
+
+<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p>
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/MavenJarRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/MavenJarRule.java
new file mode 100644
index 0000000..f4496dd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/MavenJarRule.java
@@ -0,0 +1,127 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.Type;
+
+/**
+ * Rule definition for the maven_jar rule.
+ */
+@BlazeRule(name = MavenJarRule.NAME,
+ type = RuleClassType.WORKSPACE,
+ ancestors = { WorkspaceBaseRule.class },
+ factoryClass = WorkspaceConfiguredTargetFactory.class)
+public class MavenJarRule implements RuleDefinition {
+
+ public static final String NAME = "maven_jar";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(artifact_id) -->
+ The artifactId of the Maven dependency.
+
+ <p>Required.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("artifact_id", Type.STRING).mandatory())
+ /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(group_id) -->
+ The groupId of the Maven dependency.
+
+ <p>Required.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("group_id", Type.STRING).mandatory())
+ /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(version) -->
+ The version of the Maven dependency.
+
+ <p>Required.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("version", Type.STRING).mandatory())
+ /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(repositories) -->
+ A list of repositories to use to attempt to fetch the jar.
+
+ <p>Defaults to Maven Central ("repo1.maven.org"). If repositories are specified, they will
+ be checked in the order listed here (Maven Central will not be checked in this case,
+ unless it is on the list).</p>
+
+ <p><b>To be implemented: add a maven_repositories rule that allows a list of repositories
+ to be labeled.</b></p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("repositories", Type.STRING_LIST))
+ /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(exclusions) -->
+ Transitive dependencies of this dependency that should not be downloaded.
+
+ <p>Defaults to None: Bazel will download all of the dependencies requested by the Maven
+ dependency. If exclusions are specified, they will not be downloaded.</p>
+
+ <p>Exclusions are specified in the format "<group_id>:<artifact_id>", for example,
+ "com.google.guava:guava".</p>
+
+ <p><b>Not yet implemented.</b></p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("exclusions", Type.STRING_LIST))
+ .setWorkspaceOnly()
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = maven_jar, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>Downloads a jar from Maven and makes it available to be used as a Java dependency.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="http_jar_examples">Examples</h4>
+
+Suppose that the current repostory contains a java_library target that needs to depend on Guava.
+Using Maven, this dependency would be defined in the pom.xml file as:
+
+<pre>
+<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
+</dependency>
+</pre>
+
+In Bazel, the following lines can be added to the WORKSPACE file:
+
+<pre>
+maven_jar(
+ name = "guava",
+ group_id = "com.google.guava",
+ artifact_id = "guava",
+ version = "18.0",
+)
+
+bind(
+ name = "guava-jar",
+ actual = "@guava//jar"
+)
+</pre>
+
+Then the java_library can depend on <code>//external:guava-jar</code>.
+
+<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p>
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewLocalRepositoryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewLocalRepositoryRule.java
new file mode 100644
index 0000000..0061d3d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewLocalRepositoryRule.java
@@ -0,0 +1,135 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Rule definition for the new_repository rule.
+ */
+@BlazeRule(name = NewLocalRepositoryRule.NAME,
+ type = RuleClassType.WORKSPACE,
+ ancestors = { WorkspaceBaseRule.class },
+ factoryClass = WorkspaceConfiguredTargetFactory.class)
+public class NewLocalRepositoryRule implements RuleDefinition {
+ public static final String NAME = "new_local_repository";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(new_local_repository).ATTRIBUTE(path) -->
+ A path on the local filesystem.
+
+ <p>This must be an absolute path to an existing file or a directory.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("path", STRING).mandatory())
+ /* <!-- #BLAZE_RULE(new_local_repository).ATTRIBUTE(build_file) -->
+ A file to use as a BUILD file for this directory.
+
+ <p>This path must be relative to the build's workspace. The file does not need to be named
+ BUILD, but can be (something like BUILD.new-repo-name may work well for distinguishing it
+ from the repository's actual BUILD files.</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("build_file", STRING).mandatory())
+ .setWorkspaceOnly()
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = new_local_repository, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>Allows a local directory to be turned into a Bazel repository. This means that the current
+ repository can define and use targets from anywhere on the filesystem.</p>
+
+<p>This rule creates a Bazel repository by creating a WORKSPACE file and subdirectory containing
+symlinks to the BUILD file and path given. The build file should create targets relative to the
+path, which can then be bound and used by the current build.
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="new_local_repository_examples">Examples</h4>
+
+<p>Suppose the current repository is a chat client, rooted at the directory <i>~/chat-app</i>. It
+ would like to use an SSL library which is defined in a different directory: <i>~/ssl</i>.</p>
+
+<p>The user can add a dependency by creating a BUILD file for the SSL library
+(~/chat-app/BUILD.my-ssl) containing:
+
+<pre class="code">
+java_library(
+ name = "openssl",
+ srcs = glob(['ssl/*.java'])
+)
+</pre>
+
+<p>Then they can add the following lines to <i>~/chat-app/WORKSPACE</i>:</p>
+
+<pre class="code">
+new_local_repository(
+ name = "my-ssl",
+ path = "/home/user/ssl",
+ build_file = "BUILD.my-ssl",
+)
+
+bind(
+ name = "openssl",
+ actual = "@my-ssl//my-ssl:openssl",
+)
+</pre>
+
+<p>This will create a @my-ssl repository containing a my-ssl package that contains a symlink to
+/home/user/ssl named ssl (so the BUILD file must refer to paths within /home/user/ssl relative to
+ssl).</p>
+
+<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p>
+
+<p>You can also use <code>new_local_repository</code> to include single files, not just
+directories. For example, suppose you had a jar file at /home/username/Downloads/piano.jar. You
+could add just that file to your build by adding the following to your WORKSPACE file:
+
+<pre class="code">
+new_local_repository(
+ name = "piano",
+ path = "/home/username/Downloads/piano.jar",
+ build_file = "BUILD.piano",
+)
+
+bind(
+ name = "music",
+ actual = "@piano//piano:play-music",
+)
+</pre>
+
+<p>And creating the following BUILD file:</p>
+
+<pre class="code">
+java_import(
+ name = "play-music",
+ jars = ["piano.jar"],
+)
+</pre>
+
+Then targets can depend on //external:music to use piano.jar.
+
+<!-- #END_BLAZE_RULE -->*/
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceBaseRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceBaseRule.java
new file mode 100644
index 0000000..2719cb8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceBaseRule.java
@@ -0,0 +1,34 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Base rule for rules in the WORKSPACE file.
+ */
+@BlazeRule(name = "$workspace_base_rule",
+ type = RuleClassType.ABSTRACT)
+public class WorkspaceBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceConfiguredTargetFactory.java
new file mode 100644
index 0000000..6162228
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceConfiguredTargetFactory.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation of workspace rules. Generally, these don't have any providers, since they
+ * "forward" to the SkyFunctions which actually create the repositories and then are accessed via
+ * "normal" rules.
+ */
+public class WorkspaceConfiguredTargetFactory implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .build();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
new file mode 100644
index 0000000..95fbde5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
@@ -0,0 +1,532 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Converters.RangeConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * A BuildRequest represents a single invocation of the build tool by a user.
+ * A request specifies a list of targets to be built for a single
+ * configuration, a pair of output/error streams, and additional options such
+ * as --keep_going, --jobs, etc.
+ */
+public class BuildRequest implements OptionsClassProvider {
+ private static final String DEFAULT_SYMLINK_PREFIX_MARKER = "...---:::@@@DEFAULT@@@:::--...";
+
+ /**
+ * A converter for symlink prefixes that defaults to {@code Constants.PRODUCT_NAME} and a
+ * minus sign if the option is not given.
+ *
+ * <p>Required because you cannot specify a non-constant value in annotation attributes.
+ */
+ public static class SymlinkPrefixConverter implements Converter<String> {
+ @Override
+ public String convert(String input) throws OptionsParsingException {
+ return input.equals(DEFAULT_SYMLINK_PREFIX_MARKER)
+ ? Constants.PRODUCT_NAME + "-"
+ : input;
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a string";
+ }
+ }
+
+ /**
+ * Options interface--can be used to parse command-line arguments.
+ *
+ * See also ExecutionOptions; from the user's point of view, there's no
+ * qualitative difference between these two sets of options.
+ */
+ public static class BuildRequestOptions extends OptionsBase {
+
+ /* "Execution": options related to the execution of a build: */
+
+ @Option(name = "jobs",
+ abbrev = 'j',
+ defaultValue = "200",
+ category = "strategy",
+ help = "The number of concurrent jobs to run. "
+ + "0 means build sequentially. Values above " + MAX_JOBS
+ + " are not allowed.")
+ public int jobs;
+
+ @Option(name = "progress_report_interval",
+ defaultValue = "0",
+ category = "verbosity",
+ converter = ProgressReportIntervalConverter.class,
+ help = "The number of seconds to wait between two reports on"
+ + " still running jobs. The default value 0 means to use"
+ + " the default 10:30:60 incremental algorithm.")
+ public int progressReportInterval;
+
+ @Option(name = "show_builder_stats",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "If set, parallel builder will report worker-related statistics.")
+ public boolean useBuilderStatistics;
+
+ @Option(name = "explain",
+ defaultValue = "null",
+ category = "verbosity",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "Causes Blaze to explain each executed step of the build. "
+ + "The explanation is written to the specified log file.")
+ public PathFragment explanationPath;
+
+ @Option(name = "verbose_explanations",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Increases the verbosity of the explanations issued if --explain is enabled. "
+ + "Has no effect if --explain is not enabled.")
+ public boolean verboseExplanations;
+
+ @Deprecated
+ @Option(name = "dump_makefile",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "this flag has no effect.")
+ public boolean dumpMakefile;
+
+ @Deprecated
+ @Option(name = "dump_action_graph",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "this flag has no effect.")
+
+ public boolean dumpActionGraph;
+
+ @Deprecated
+ @Option(name = "dump_action_graph_for_package",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "undocumented",
+ help = "this flag has no effect.")
+ public List<String> dumpActionGraphForPackage = new ArrayList<>();
+
+ @Deprecated
+ @Option(name = "dump_action_graph_with_middlemen",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "this flag has no effect.")
+ public boolean dumpActionGraphWithMiddlemen;
+
+ @Deprecated
+ @Option(name = "dump_providers",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "This is a no-op.")
+ public boolean dumpProviders;
+
+ @Option(name = "incremental_builder",
+ deprecationWarning = "incremental_builder is now a no-op and will be removed in an"
+ + " upcoming Blaze release",
+ defaultValue = "true",
+ category = "strategy",
+ help = "Enables an incremental builder aimed at faster "
+ + "incremental builds. Currently it has the greatest effect on null"
+ + "builds.")
+ public boolean useIncrementalDependencyChecker;
+
+ @Deprecated
+ @Option(name = "dump_targets",
+ defaultValue = "null",
+ category = "undocumented",
+ help = "this flag has no effect.")
+ public String dumpTargets;
+
+ @Deprecated
+ @Option(name = "dump_host_deps",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Deprecated")
+ public boolean dumpHostDeps;
+
+ @Deprecated
+ @Option(name = "dump_to_stdout",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Deprecated")
+ public boolean dumpToStdout;
+
+ @Option(name = "analyze",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Execute the analysis phase; this is the usual behaviour. "
+ + "Specifying --noanalyze causes the build to stop before starting the "
+ + "analysis phase, returning zero iff the package loading completed "
+ + "successfully; this mode is useful for testing.")
+ public boolean performAnalysisPhase;
+
+ @Option(name = "build",
+ defaultValue = "true",
+ category = "what",
+ help = "Execute the build; this is the usual behaviour. "
+ + "Specifying --nobuild causes the build to stop before executing the "
+ + "build actions, returning zero iff the package loading and analysis "
+ + "phases completed successfully; this mode is useful for testing "
+ + "those phases.")
+ public boolean performExecutionPhase;
+
+ @Option(name = "compile_only",
+ defaultValue = "false",
+ category = "what",
+ help = "If specified, Blaze will only build files that are generated by lightweight "
+ + "compilation actions, skipping more expensive build steps (such as linking).")
+ public boolean compileOnly;
+
+ @Option(name = "compilation_prerequisites_only",
+ defaultValue = "false",
+ category = "what",
+ help = "If specified, Blaze will only build files that are prerequisites to compilation "
+ + "of the given target (for example, generated source files and headers) without "
+ + "building the target itself. This flag is ignored if --compile_only is enabled.")
+ public boolean compilationPrerequisitesOnly;
+
+ @Option(name = "output_groups",
+ converter = Converters.CommaSeparatedOptionListConverter.class,
+ allowMultiple = true,
+ defaultValue = "",
+ category = "undocumented",
+ help = "Specifies, which output groups of the top-level target to build.")
+ public List<String> outputGroups;
+
+ @Option(name = "show_result",
+ defaultValue = "1",
+ category = "verbosity",
+ help = "Show the results of the build. For each "
+ + "target, state whether or not it was brought up-to-date, and if "
+ + "so, a list of output files that were built. The printed files "
+ + "are convenient strings for copy+pasting to the shell, to "
+ + "execute them.\n"
+ + "This option requires an integer argument, which "
+ + "is the threshold number of targets above which result "
+ + "information is not printed. "
+ + "Thus zero causes suppression of the message and MAX_INT "
+ + "causes printing of the result to occur always. The default is one.")
+ public int maxResultTargets;
+
+ @Option(name = "announce",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Deprecated. No-op.",
+ deprecationWarning = "This option is now deprecated and is a no-op")
+ public boolean announce;
+
+ @Option(name = "symlink_prefix",
+ defaultValue = DEFAULT_SYMLINK_PREFIX_MARKER,
+ converter = SymlinkPrefixConverter.class,
+ category = "misc",
+ help = "The prefix that is prepended to any of the convenience symlinks that are created "
+ + "after a build. If '/' is passed, then no symlinks are created and no warning is "
+ + "emitted."
+ )
+ public String symlinkPrefix;
+
+ @Option(name = "experimental_multi_cpu",
+ converter = Converters.CommaSeparatedOptionListConverter.class,
+ allowMultiple = true,
+ defaultValue = "",
+ category = "semantics",
+ help = "This flag allows specifying multiple target CPUs. If this is specified, "
+ + "the --cpu option is ignored.")
+ public List<String> multiCpus;
+
+ @Option(name = "experimental_check_output_files",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Check for modifications made to the output files of a build. Consider setting "
+ + "this flag to false to see the effect on incremental build times.")
+ public boolean checkOutputFiles;
+ }
+
+ /**
+ * Converter for progress_report_interval: [0, 3600].
+ */
+ public static class ProgressReportIntervalConverter extends RangeConverter {
+ public ProgressReportIntervalConverter() {
+ super(0, 3600);
+ }
+ }
+
+ private static final int MAX_JOBS = 2000;
+ private static final int JOBS_TOO_HIGH_WARNING = 1000;
+
+ private final UUID id;
+ private final LoadingCache<Class<? extends OptionsBase>, Optional<OptionsBase>> optionsCache;
+
+ /** A human-readable description of all the non-default option settings. */
+ private final String optionsDescription;
+
+ /**
+ * The name of the Blaze command that the user invoked.
+ * Used for --announce.
+ */
+ private final String commandName;
+
+ private final OutErr outErr;
+ private final List<String> targets;
+
+ private long startTimeMillis = 0; // milliseconds since UNIX epoch.
+
+ private boolean runningInEmacs = false;
+ private boolean runTests = false;
+
+ private static final List<Class<? extends OptionsBase>> MANDATORY_OPTIONS = ImmutableList.of(
+ BuildRequestOptions.class,
+ PackageCacheOptions.class,
+ LoadingPhaseRunner.Options.class,
+ BuildView.Options.class,
+ ExecutionOptions.class);
+
+ private BuildRequest(String commandName,
+ final OptionsProvider options,
+ final OptionsProvider startupOptions,
+ List<String> targets,
+ OutErr outErr,
+ UUID id,
+ long startTimeMillis) {
+ this.commandName = commandName;
+ this.optionsDescription = OptionsUtils.asShellEscapedString(options);
+ this.outErr = outErr;
+ this.targets = targets;
+ this.id = id;
+ this.startTimeMillis = startTimeMillis;
+ this.optionsCache = CacheBuilder.newBuilder()
+ .build(new CacheLoader<Class<? extends OptionsBase>, Optional<OptionsBase>>() {
+ @Override
+ public Optional<OptionsBase> load(Class<? extends OptionsBase> key) throws Exception {
+ OptionsBase result = options.getOptions(key);
+ if (result == null && startupOptions != null) {
+ result = startupOptions.getOptions(key);
+ }
+
+ return Optional.fromNullable(result);
+ }
+ });
+
+ for (Class<? extends OptionsBase> optionsClass : MANDATORY_OPTIONS) {
+ Preconditions.checkNotNull(getOptions(optionsClass));
+ }
+ }
+
+ /**
+ * Returns a unique identifier that universally identifies this build.
+ */
+ public UUID getId() {
+ return id;
+ }
+
+ /**
+ * Returns the name of the Blaze command that the user invoked.
+ */
+ public String getCommandName() {
+ return commandName;
+ }
+
+ /**
+ * Set to true if this build request was initiated by Emacs.
+ * (Certain output formatting may be necessary.)
+ */
+ public void setRunningInEmacs() {
+ runningInEmacs = true;
+ }
+
+ boolean isRunningInEmacs() {
+ return runningInEmacs;
+ }
+
+ /**
+ * Enables test execution for this build request.
+ */
+ public void setRunTests() {
+ runTests = true;
+ }
+
+ /**
+ * Returns true if tests should be run by the build tool.
+ */
+ public boolean shouldRunTests() {
+ return runTests;
+ }
+
+ /**
+ * Returns the (immutable) list of targets to build in commandline
+ * form.
+ */
+ public List<String> getTargets() {
+ return targets;
+ }
+
+ /**
+ * Returns the output/error streams to which errors and progress messages
+ * should be sent during the fulfillment of this request.
+ */
+ public OutErr getOutErr() {
+ return outErr;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T extends OptionsBase> T getOptions(Class<T> clazz) {
+ try {
+ return (T) optionsCache.get(clazz).orNull();
+ } catch (ExecutionException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Returns the set of command-line options specified for this request.
+ */
+ public BuildRequestOptions getBuildOptions() {
+ return getOptions(BuildRequestOptions.class);
+ }
+
+ /**
+ * Returns the set of options related to the loading phase.
+ */
+ public PackageCacheOptions getPackageCacheOptions() {
+ return getOptions(PackageCacheOptions.class);
+ }
+
+ /**
+ * Returns the set of options related to the loading phase.
+ */
+ public LoadingPhaseRunner.Options getLoadingOptions() {
+ return getOptions(LoadingPhaseRunner.Options.class);
+ }
+
+ /**
+ * Returns the set of command-line options related to the view specified for
+ * this request.
+ */
+ public BuildView.Options getViewOptions() {
+ return getOptions(BuildView.Options.class);
+ }
+
+ /**
+ * Returns the human-readable description of the non-default options
+ * for this build request.
+ */
+ public String getOptionsDescription() {
+ return optionsDescription;
+ }
+
+ /**
+ * Return the time (according to System.currentTimeMillis()) at which the
+ * service of this request was started.
+ */
+ public long getStartTime() {
+ return startTimeMillis;
+ }
+
+ /**
+ * Validates the options for this BuildRequest.
+ *
+ * <p>Issues warnings or throws {@code InvalidConfigurationException} for option settings that
+ * conflict.
+ *
+ * @return list of warnings
+ */
+ public List<String> validateOptions() throws InvalidConfigurationException {
+ List<String> warnings = new ArrayList<>();
+ // Validate "jobs".
+ int jobs = getBuildOptions().jobs;
+ if (jobs < 0 || jobs > MAX_JOBS) {
+ throw new InvalidConfigurationException(String.format(
+ "Invalid parameter for --jobs: %d. Only values 0 <= jobs <= %d are allowed.", jobs,
+ MAX_JOBS));
+ }
+ if (jobs > JOBS_TOO_HIGH_WARNING) {
+ warnings.add(
+ String.format("High value for --jobs: %d. You may run into memory issues", jobs));
+ }
+
+ // Validate other BuildRequest options.
+ if (getBuildOptions().verboseExplanations && getBuildOptions().explanationPath == null) {
+ warnings.add("--verbose_explanations has no effect when --explain=<file> is not enabled");
+ }
+ if (getBuildOptions().compileOnly && getBuildOptions().compilationPrerequisitesOnly) {
+ throw new InvalidConfigurationException(
+ "--compile_only and --compilation_prerequisites_only are not compatible");
+ }
+
+ return warnings;
+ }
+
+ /** Creates a new TopLevelArtifactContext from this build request. */
+ public TopLevelArtifactContext getTopLevelArtifactContext() {
+ return new TopLevelArtifactContext(getCommandName(),
+ getBuildOptions().compileOnly, getBuildOptions().compilationPrerequisitesOnly,
+ getOptions(ExecutionOptions.class).testStrategy.equals("exclusive"),
+ ImmutableSet.<String>copyOf(getBuildOptions().outputGroups), shouldRunTests());
+ }
+
+ public String getSymlinkPrefix() {
+ return getBuildOptions().symlinkPrefix;
+ }
+
+ public ImmutableSortedSet<String> getMultiCpus() {
+ return ImmutableSortedSet.copyOf(getBuildOptions().multiCpus);
+ }
+
+ public static BuildRequest create(String commandName, OptionsProvider options,
+ OptionsProvider startupOptions,
+ List<String> targets, OutErr outErr, UUID commandId, long commandStartTime) {
+
+ BuildRequest request = new BuildRequest(commandName, options, startupOptions, targets, outErr,
+ commandId, commandStartTime);
+
+ // All this, just to pass a global boolean from the client to the server. :(
+ if (options.getOptions(BlazeCommandEventHandler.Options.class).runningInEmacs) {
+ request.setRunningInEmacs();
+ }
+
+ return request;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java
new file mode 100644
index 0000000..22c36f8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java
@@ -0,0 +1,196 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.util.ExitCode;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import javax.annotation.Nullable;
+
+/**
+ * Contains information about the result of a build. While BuildRequest is immutable, this class is
+ * mutable.
+ */
+public final class BuildResult {
+ private long startTimeMillis = 0; // milliseconds since UNIX epoch.
+ private long stopTimeMillis = 0;
+
+ private Throwable crash = null;
+ private boolean catastrophe = false;
+ private ExitCode exitCondition = ExitCode.BLAZE_INTERNAL_ERROR;
+ private Collection<ConfiguredTarget> actualTargets;
+ private Collection<ConfiguredTarget> testTargets;
+ private Collection<ConfiguredTarget> successfulTargets;
+
+ public BuildResult(long startTimeMillis) {
+ this.startTimeMillis = startTimeMillis;
+ }
+
+ /**
+ * Record the time (according to System.currentTimeMillis()) at which the
+ * service of this request was completed.
+ */
+ public void setStopTime(long stopTimeMillis) {
+ this.stopTimeMillis = stopTimeMillis;
+ }
+
+ /**
+ * Return the time (according to System.currentTimeMillis()) at which the
+ * service of this request was completed.
+ */
+ public long getStopTime() {
+ return stopTimeMillis;
+ }
+
+ /**
+ * Returns the elapsed time in seconds for the service of this request. Not
+ * defined for requests that have not been serviced.
+ */
+ public double getElapsedSeconds() {
+ if (startTimeMillis == 0 || stopTimeMillis == 0) {
+ throw new IllegalStateException("BuildRequest has not been serviced");
+ }
+ return (stopTimeMillis - startTimeMillis) / 1000.0;
+ }
+
+ public void setExitCondition(ExitCode exitCondition) {
+ this.exitCondition = exitCondition;
+ }
+
+ /**
+ * True iff the build request has been successfully completed.
+ */
+ public boolean getSuccess() {
+ return exitCondition == ExitCode.SUCCESS;
+ }
+
+ /**
+ * Gets the Blaze exit condition.
+ */
+ public ExitCode getExitCondition() {
+ return exitCondition;
+ }
+
+ /**
+ * Sets the RuntimeException / Error that induced a Blaze crash.
+ */
+ public void setUnhandledThrowable(Throwable crash) {
+ Preconditions.checkState(crash == null ||
+ ((crash instanceof RuntimeException) || (crash instanceof Error)));
+ this.crash = crash;
+ }
+
+ /**
+ * Sets a "catastrophe": A build failure severe enough to halt a keep_going build.
+ */
+ public void setCatastrophe() {
+ this.catastrophe = true;
+ }
+
+ /**
+ * Was the build a "catastrophe": A build failure severe enough to halt a keep_going build.
+ */
+ public boolean wasCatastrophe() {
+ return catastrophe;
+ }
+
+ /**
+ * Gets the Blaze crash Throwable. Null if Blaze did not crash.
+ */
+ public Throwable getUnhandledThrowable() {
+ return crash;
+ }
+
+ /**
+ * @see #getActualTargets
+ */
+ public void setActualTargets(Collection<ConfiguredTarget> actualTargets) {
+ this.actualTargets = actualTargets;
+ }
+
+ /**
+ * Returns the actual set of targets which we attempted to build. This value
+ * is set during the build, after the target patterns have been parsed and
+ * resolved. If --keep_going is specified, this set may exclude targets that
+ * could not be found or successfully analyzed. It may be examined after the
+ * build. May be null even after the build, if there were errors in the
+ * loading or analysis phases.
+ */
+ public Collection<ConfiguredTarget> getActualTargets() {
+ return actualTargets;
+ }
+
+ /**
+ * @see #getTestTargets
+ */
+ public void setTestTargets(@Nullable Collection<ConfiguredTarget> testTargets) {
+ this.testTargets = testTargets == null ? null : Collections.unmodifiableCollection(testTargets);
+ }
+
+ /**
+ * Returns the actual unmodifiable collection of targets which we attempted to
+ * test. This value is set at the end of the build analysis phase, after the
+ * test target patterns have been parsed and resolved. If --keep_going is
+ * specified, this collection may exclude targets that could not be found or
+ * successfully analyzed. It may be examined after the build. May be null even
+ * after the build, if there were errors in the loading or analysis phases or
+ * if testing was not requested.
+ */
+ public Collection<ConfiguredTarget> getTestTargets() {
+ return testTargets;
+ }
+
+ /**
+ * @see #getSuccessfulTargets
+ */
+ void setSuccessfulTargets(Collection<ConfiguredTarget> successfulTargets) {
+ this.successfulTargets = successfulTargets;
+ }
+
+ /**
+ * Returns the set of targets which successfully built. This value
+ * is set at the end of the build, after the target patterns have been parsed
+ * and resolved and after attempting to build the targets. If --keep_going
+ * is specified, this set may exclude targets that could not be found or
+ * successfully analyzed, or could not be built. It may be examined after
+ * the build. May be null if the execution phase was not attempted, as
+ * may happen if there are errors in the loading phase, for example.
+ */
+ public Collection<ConfiguredTarget> getSuccessfulTargets() {
+ return successfulTargets;
+ }
+
+ /** For debugging. */
+ @Override
+ @SuppressWarnings("deprecation")
+ public String toString() {
+ // We need to be compatible with Guava, so we use this, even though it is deprecated.
+ return Objects.toStringHelper(this)
+ .add("startTimeMillis", startTimeMillis)
+ .add("stopTimeMillis", stopTimeMillis)
+ .add("crash", crash)
+ .add("catastrophe", catastrophe)
+ .add("exitCondition", exitCondition)
+ .add("actualTargets", actualTargets)
+ .add("testTargets", testTargets)
+ .add("successfulTargets", successfulTargets)
+ .toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
new file mode 100644
index 0000000..a27cc50
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
@@ -0,0 +1,540 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.analysis.AnalysisPhaseCompleteEvent;
+import com.google.devtools.build.lib.analysis.BuildInfoEvent;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult;
+import com.google.devtools.build.lib.analysis.ConfigurationsCreatedEvent;
+import com.google.devtools.build.lib.analysis.ConfiguredAttributeMapper;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LicensesProvider;
+import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense;
+import com.google.devtools.build.lib.analysis.MakeEnvironmentEvent;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.Callback;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * Provides the bulk of the implementation of the 'blaze build' command.
+ *
+ * <p>The various concrete build command classes handle the command options and request
+ * setup, then delegate the handling of the request (the building of targets) to this class.
+ *
+ * <p>The main entry point is {@link #buildTargets}.
+ *
+ * <p>This class is always instantiated and managed as a singleton, being constructed and held by
+ * {@link BlazeRuntime}. This is so multiple kinds of build commands can share this single
+ * instance.
+ *
+ * <p>Most of analysis is handled in {@link BuildView}, and execution in {@link ExecutionTool}.
+ */
+public class BuildTool {
+
+ private static final Logger LOG = Logger.getLogger(BuildTool.class.getName());
+
+ protected final BlazeRuntime runtime;
+
+ /**
+ * Constructs a BuildTool.
+ *
+ * @param runtime a reference to the blaze runtime.
+ */
+ public BuildTool(BlazeRuntime runtime) {
+ this.runtime = runtime;
+ }
+
+ /**
+ * The crux of the build system. Builds the targets specified in the request using the specified
+ * Executor.
+ *
+ * <p>Performs loading, analysis and execution for the specified set of targets, honoring the
+ * configuration options in the BuildRequest. Returns normally iff successful, throws an exception
+ * otherwise.
+ *
+ * <p>Callers must ensure that {@link #stopRequest} is called after this method, even if it
+ * throws.
+ *
+ * <p>The caller is responsible for setting up and syncing the package cache.
+ *
+ * <p>During this function's execution, the actualTargets and successfulTargets
+ * fields of the request object are set.
+ *
+ * @param request the build request that this build tool is servicing, which specifies various
+ * options; during this method's execution, the actualTargets and successfulTargets fields
+ * of the request object are populated
+ * @param result the build result that is the mutable result of this build
+ * @param validator target validator
+ */
+ public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator)
+ throws BuildFailedException, LocalEnvironmentException,
+ InterruptedException, ViewCreationFailedException,
+ TargetParsingException, LoadingFailedException, ExecutorInitException,
+ AbruptExitException, InvalidConfigurationException, TestExecException {
+ validateOptions(request);
+ BuildOptions buildOptions = runtime.createBuildOptions(request);
+ // Sync the package manager before sending the BuildStartingEvent in runLoadingPhase()
+ runtime.setupPackageCache(request.getPackageCacheOptions(),
+ DefaultsPackage.getDefaultsPackageContent(buildOptions));
+
+ ExecutionTool executionTool = null;
+ LoadingResult loadingResult = null;
+ BuildConfigurationCollection configurations = null;
+ try {
+ getEventBus().post(new BuildStartingEvent(runtime.getOutputFileSystem(), request));
+ LOG.info("Build identifier: " + request.getId());
+ executionTool = new ExecutionTool(runtime, request);
+ if (needsExecutionPhase(request.getBuildOptions())) {
+ // Initialize the execution tool early if we need it. This hides the latency of setting up
+ // the execution backends.
+ executionTool.init();
+ }
+
+ // Loading phase.
+ loadingResult = runLoadingPhase(request, validator);
+
+ // Create the build configurations.
+ if (!request.getMultiCpus().isEmpty()) {
+ getReporter().handle(Event.warn(
+ "The --experimental_multi_cpu option is _very_ experimental and only intended for "
+ + "internal testing at this time. If you do not work on the build tool, then you "
+ + "should stop now!"));
+ if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) {
+ throw new InvalidConfigurationException(
+ "The experimental setting to select multiple CPUs is only supported for 'build' and "
+ + "'test' right now!");
+ }
+ }
+ configurations = getConfigurations(
+ runtime.getBuildConfigurationKey(buildOptions, request.getMultiCpus()),
+ request.getViewOptions().keepGoing);
+
+ getEventBus().post(new ConfigurationsCreatedEvent(configurations));
+ runtime.throwPendingException();
+ if (configurations.getTargetConfigurations().size() == 1) {
+ // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case
+ // where there's only a single configuration, but we don't send an event in the multi-config
+ // case. Can we do better? [multi-config]
+ getEventBus().post(new MakeEnvironmentEvent(
+ configurations.getTargetConfigurations().get(0).getMakeEnvironment()));
+ }
+ LOG.info("Configurations created");
+
+ // Analysis phase.
+ AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations);
+ result.setActualTargets(analysisResult.getTargetsToBuild());
+ result.setTestTargets(analysisResult.getTargetsToTest());
+
+ reportTargets(analysisResult);
+
+ // Execution phase.
+ if (needsExecutionPhase(request.getBuildOptions())) {
+ executionTool.executeBuild(analysisResult, result, runtime.getSkyframeExecutor(),
+ configurations, mergePackageRoots(loadingResult.getPackageRoots(),
+ runtime.getSkyframeExecutor().getPackageRoots()));
+ }
+
+ String delayedErrorMsg = analysisResult.getError();
+ if (delayedErrorMsg != null) {
+ throw new BuildFailedException(delayedErrorMsg);
+ }
+ } catch (RuntimeException e) {
+ // Print an error message for unchecked runtime exceptions. This does not concern Error
+ // subclasses such as OutOfMemoryError.
+ request.getOutErr().printErrLn("Unhandled exception thrown during build; message: " +
+ e.getMessage());
+ throw e;
+ } finally {
+ // Delete dirty nodes to ensure that they do not accumulate indefinitely.
+ long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc;
+ if (versionWindow != -1) {
+ runtime.getSkyframeExecutor().deleteOldNodes(versionWindow);
+ }
+
+ if (executionTool != null) {
+ executionTool.shutdown();
+ }
+ // The workspace status actions will not run with certain flags, or if an error
+ // occurs early in the build. Tell a lie so that the event is not missing.
+ // If multiple build_info events are sent, only the first is kept, so this does not harm
+ // successful runs (which use the workspace status action).
+ getEventBus().post(new BuildInfoEvent(
+ runtime.getworkspaceStatusActionFactory().createDummyWorkspaceStatus()));
+ }
+
+ if (loadingResult != null && loadingResult.hasTargetPatternError()) {
+ throw new BuildFailedException("execution phase successful, but there were errors " +
+ "parsing the target pattern");
+ }
+ }
+
+ private ImmutableMap<PathFragment, Path> mergePackageRoots(
+ ImmutableMap<PackageIdentifier, Path> first,
+ ImmutableMap<PackageIdentifier, Path> second) {
+ Map<PathFragment, Path> builder = Maps.newHashMap();
+ for (Map.Entry<PackageIdentifier, Path> entry : first.entrySet()) {
+ builder.put(entry.getKey().getPackageFragment(), entry.getValue());
+ }
+ for (Map.Entry<PackageIdentifier, Path> entry : second.entrySet()) {
+ if (first.containsKey(entry.getKey())) {
+ Preconditions.checkState(first.get(entry.getKey()).equals(entry.getValue()));
+ } else {
+ // This could overwrite entries from first in other repositories.
+ builder.put(entry.getKey().getPackageFragment(), entry.getValue());
+ }
+ }
+ return ImmutableMap.copyOf(builder);
+ }
+
+ private void reportExceptionError(Exception e) {
+ if (e.getMessage() != null) {
+ getReporter().handle(Event.error(e.getMessage()));
+ }
+ }
+ /**
+ * The crux of the build system. Builds the targets specified in the request using the specified
+ * Executor.
+ *
+ * <p>Performs loading, analysis and execution for the specified set of targets, honoring the
+ * configuration options in the BuildRequest. Returns normally iff successful, throws an exception
+ * otherwise.
+ *
+ * <p>The caller is responsible for setting up and syncing the package cache.
+ *
+ * <p>During this function's execution, the actualTargets and successfulTargets
+ * fields of the request object are set.
+ *
+ * @param request the build request that this build tool is servicing, which specifies various
+ * options; during this method's execution, the actualTargets and successfulTargets fields
+ * of the request object are populated
+ * @param validator target validator
+ * @return the result as a {@link BuildResult} object
+ */
+ public BuildResult processRequest(BuildRequest request, TargetValidator validator) {
+ BuildResult result = new BuildResult(request.getStartTime());
+ runtime.getEventBus().register(result);
+ Throwable catastrophe = null;
+ ExitCode exitCode = ExitCode.BLAZE_INTERNAL_ERROR;
+ try {
+ buildTargets(request, result, validator);
+ exitCode = ExitCode.SUCCESS;
+ } catch (BuildFailedException e) {
+ if (e.isErrorAlreadyShown()) {
+ // The actual error has already been reported by the Builder.
+ } else {
+ reportExceptionError(e);
+ }
+ if (e.isCatastrophic()) {
+ result.setCatastrophe();
+ }
+ exitCode = ExitCode.BUILD_FAILURE;
+ } catch (InterruptedException e) {
+ exitCode = ExitCode.INTERRUPTED;
+ getReporter().handle(Event.error("build interrupted"));
+ getEventBus().post(new BuildInterruptedEvent());
+ } catch (TargetParsingException | LoadingFailedException | ViewCreationFailedException e) {
+ exitCode = ExitCode.PARSING_FAILURE;
+ reportExceptionError(e);
+ } catch (TestExecException e) {
+ // ExitCode.SUCCESS means that build was successful. Real return code of program
+ // is going to be calculated in TestCommand.doTest().
+ exitCode = ExitCode.SUCCESS;
+ reportExceptionError(e);
+ } catch (InvalidConfigurationException e) {
+ exitCode = ExitCode.COMMAND_LINE_ERROR;
+ reportExceptionError(e);
+ } catch (AbruptExitException e) {
+ exitCode = e.getExitCode();
+ reportExceptionError(e);
+ result.setCatastrophe();
+ } catch (Throwable throwable) {
+ catastrophe = throwable;
+ Throwables.propagate(throwable);
+ } finally {
+ stopRequest(request, result, catastrophe, exitCode);
+ }
+
+ return result;
+ }
+
+ protected final BuildConfigurationCollection getConfigurations(BuildConfigurationKey key,
+ boolean keepGoing)
+ throws InvalidConfigurationException, InterruptedException {
+ SkyframeExecutor executor = runtime.getSkyframeExecutor();
+ // TODO(bazel-team): consider a possibility of moving ConfigurationFactory construction into
+ // skyframe.
+ return executor.createConfigurations(keepGoing, runtime.getConfigurationFactory(), key);
+ }
+
+ @VisibleForTesting
+ protected final LoadingResult runLoadingPhase(final BuildRequest request,
+ final TargetValidator validator)
+ throws LoadingFailedException, TargetParsingException, InterruptedException,
+ AbruptExitException {
+ Profiler.instance().markPhase(ProfilePhase.LOAD);
+ runtime.throwPendingException();
+
+ final boolean keepGoing = request.getViewOptions().keepGoing;
+
+ Callback callback = new Callback() {
+ @Override
+ public void notifyTargets(Collection<Target> targets) throws LoadingFailedException {
+ if (validator != null) {
+ validator.validateTargets(targets, keepGoing);
+ }
+ }
+
+ @Override
+ public void notifyVisitedPackages(Set<PackageIdentifier> visitedPackages) {
+ runtime.getSkyframeExecutor().updateLoadedPackageSet(visitedPackages);
+ }
+ };
+
+ LoadingResult result = runtime.getLoadingPhaseRunner().execute(getReporter(),
+ getEventBus(), request.getTargets(), request.getLoadingOptions(),
+ runtime.createBuildOptions(request).getAllLabels(), keepGoing,
+ request.shouldRunTests(), callback);
+ runtime.throwPendingException();
+ return result;
+ }
+
+ /**
+ * Performs the initial phases 0-2 of the build: Setup, Loading and Analysis.
+ * <p>
+ * Postcondition: On success, populates the BuildRequest's set of targets to
+ * build.
+ *
+ * @return null if loading / analysis phases were successful; a useful error
+ * message if loading or analysis phase errors were encountered and
+ * request.keepGoing.
+ * @throws InterruptedException if the current thread was interrupted.
+ * @throws ViewCreationFailedException if analysis failed for any reason.
+ */
+ private AnalysisResult runAnalysisPhase(BuildRequest request, LoadingResult loadingResult,
+ BuildConfigurationCollection configurations)
+ throws InterruptedException, ViewCreationFailedException {
+ Stopwatch timer = Stopwatch.createStarted();
+ if (!request.getBuildOptions().performAnalysisPhase) {
+ getReporter().handle(Event.progress("Loading complete."));
+ LOG.info("No analysis requested, so finished");
+ return AnalysisResult.EMPTY;
+ }
+
+ getReporter().handle(Event.progress("Loading complete. Analyzing..."));
+ Profiler.instance().markPhase(ProfilePhase.ANALYZE);
+
+ AnalysisResult analysisResult = getView().update(loadingResult, configurations,
+ request.getViewOptions(), request.getTopLevelArtifactContext(), getReporter(),
+ getEventBus());
+
+ // TODO(bazel-team): Merge these into one event.
+ getEventBus().post(new AnalysisPhaseCompleteEvent(analysisResult.getTargetsToBuild(),
+ getView().getTargetsVisited(), timer.stop().elapsed(TimeUnit.MILLISECONDS)));
+ getEventBus().post(new TestFilteringCompleteEvent(analysisResult.getTargetsToBuild(),
+ analysisResult.getTargetsToTest()));
+
+ // Check licenses.
+ // We check licenses if the first target configuration has license checking enabled. Right now,
+ // it is not possible to have multiple target configurations with different settings for this
+ // flag, which allows us to take this short cut.
+ boolean checkLicenses = configurations.getTargetConfigurations().get(0).checkLicenses();
+ if (checkLicenses) {
+ Profiler.instance().markPhase(ProfilePhase.LICENSE);
+ validateLicensingForTargets(analysisResult.getTargetsToBuild(),
+ request.getViewOptions().keepGoing);
+ }
+
+ return analysisResult;
+ }
+
+ private static boolean needsExecutionPhase(BuildRequestOptions options) {
+ return options.performAnalysisPhase && options.performExecutionPhase;
+ }
+
+ /**
+ * Stops processing the specified request.
+ *
+ * <p>This logs the build result, cleans up and stops the clock.
+ *
+ * @param request the build request that this build tool is servicing
+ * @param crash Any unexpected RuntimeException or Error. May be null
+ * @param exitCondition A suggested exit condition from either the build logic or
+ * a thrown exception somewhere along the way.
+ */
+ public void stopRequest(BuildRequest request, BuildResult result, Throwable crash,
+ ExitCode exitCondition) {
+ Preconditions.checkState((crash == null) || (exitCondition != ExitCode.SUCCESS));
+ result.setUnhandledThrowable(crash);
+ result.setExitCondition(exitCondition);
+ // The stop time has to be captured before we send the BuildCompleteEvent.
+ result.setStopTime(runtime.getClock().currentTimeMillis());
+ getEventBus().post(new BuildCompleteEvent(request, result));
+ }
+
+ private void reportTargets(AnalysisResult analysisResult) {
+ Collection<ConfiguredTarget> targetsToBuild = analysisResult.getTargetsToBuild();
+ Collection<ConfiguredTarget> targetsToTest = analysisResult.getTargetsToTest();
+ if (targetsToTest != null) {
+ int testCount = targetsToTest.size();
+ int targetCount = targetsToBuild.size() - testCount;
+ if (targetCount == 0) {
+ getReporter().handle(Event.info("Found "
+ + testCount + (testCount == 1 ? " test target..." : " test targets...")));
+ } else {
+ getReporter().handle(Event.info("Found "
+ + targetCount + (targetCount == 1 ? " target and " : " targets and ")
+ + testCount + (testCount == 1 ? " test target..." : " test targets...")));
+ }
+ } else {
+ int targetCount = targetsToBuild.size();
+ getReporter().handle(Event.info("Found "
+ + targetCount + (targetCount == 1 ? " target..." : " targets...")));
+ }
+ }
+
+ /**
+ * Validates the options for this BuildRequest.
+ *
+ * <p>Issues warnings for the use of deprecated options, and warnings or errors for any option
+ * settings that conflict.
+ */
+ @VisibleForTesting
+ public void validateOptions(BuildRequest request) throws InvalidConfigurationException {
+ for (String issue : request.validateOptions()) {
+ getReporter().handle(Event.warn(issue));
+ }
+ }
+
+ /**
+ * Takes a set of configured targets, and checks if the distribution methods
+ * declared for the targets are compatible with the constraints imposed by
+ * their prerequisites' licenses.
+ *
+ * @param configuredTargets the targets to check
+ * @param keepGoing if false, and a licensing error is encountered, both
+ * generates an error message on the reporter, <em>and</em> throws an
+ * exception. If true, then just generates a message on the reporter.
+ * @throws ViewCreationFailedException if the license checking failed (and not
+ * --keep_going)
+ */
+ private void validateLicensingForTargets(Iterable<ConfiguredTarget> configuredTargets,
+ boolean keepGoing) throws ViewCreationFailedException {
+ for (ConfiguredTarget configuredTarget : configuredTargets) {
+ final Target target = configuredTarget.getTarget();
+
+ if (TargetUtils.isTestRule(target)) {
+ continue; // Tests are exempt from license checking
+ }
+
+ final Set<DistributionType> distribs = target.getDistributions();
+ BuildConfiguration config = configuredTarget.getConfiguration();
+ boolean staticallyLinked = (config != null) && config.performsStaticLink();
+ staticallyLinked |= (config != null) && (target instanceof Rule)
+ && ((Rule) target).getRuleClassObject().hasAttr("linkopts", Type.STRING_LIST)
+ && ConfiguredAttributeMapper.of((RuleConfiguredTarget) configuredTarget)
+ .get("linkopts", Type.STRING_LIST).contains("-static");
+
+ LicensesProvider provider = configuredTarget.getProvider(LicensesProvider.class);
+ if (provider != null) {
+ NestedSet<TargetLicense> licenses = provider.getTransitiveLicenses();
+ for (TargetLicense targetLicense : licenses) {
+ if (!targetLicense.getLicense().checkCompatibility(
+ distribs, target, targetLicense.getLabel(), getReporter(), staticallyLinked)) {
+ if (!keepGoing) {
+ throw new ViewCreationFailedException("Build aborted due to licensing error");
+ }
+ }
+ }
+ } else if (configuredTarget.getTarget() instanceof InputFile) {
+ // Input file targets do not provide licenses because they do not
+ // depend on the rule where their license is taken from. This is usually
+ // not a problem, because the transitive collection of licenses always
+ // hits the rule they come from, except when the input file is a
+ // top-level target. Thus, we need to handle that case specially here.
+ //
+ // See FileTarget#getLicense for more information about the handling of
+ // license issues with File targets.
+ License license = configuredTarget.getTarget().getLicense();
+ if (!license.checkCompatibility(distribs, target, configuredTarget.getLabel(),
+ getReporter(), staticallyLinked)) {
+ if (!keepGoing) {
+ throw new ViewCreationFailedException("Build aborted due to licensing error");
+ }
+ }
+ }
+ }
+ }
+
+ public BuildView getView() {
+ return runtime.getView();
+ }
+
+ private Reporter getReporter() {
+ return runtime.getReporter();
+ }
+
+ private EventBus getEventBus() {
+ return runtime.getEventBus();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java
new file mode 100644
index 0000000..5b3229c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+/**
+ * Event that is raised when the action and artifact metadata caches are saved at the end of the
+ * build. Contains statistics.
+ */
+public class CachesSavedEvent {
+ /** Cache serialization statistics. */
+ private final long actionCacheSaveTimeInMillis;
+ private final long actionCacheSizeInBytes;
+
+ public CachesSavedEvent(
+ long actionCacheSaveTimeInMillis,
+ long actionCacheSizeInBytes) {
+ this.actionCacheSaveTimeInMillis = actionCacheSaveTimeInMillis;
+ this.actionCacheSizeInBytes = actionCacheSizeInBytes;
+ }
+
+ public long getActionCacheSaveTimeInMillis() {
+ return actionCacheSaveTimeInMillis;
+ }
+
+ public long getActionCacheSizeInBytes() {
+ return actionCacheSizeInBytes;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
new file mode 100644
index 0000000..74143cc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Event signaling the end of the execution phase. Contains statistics about the action cache,
+ * the metadata cache and about last file save times.
+ */
+public class ExecutionFinishedEvent {
+ /** The mtime of the most recently saved source file when the build starts. */
+ private long lastFileSaveTimeInMillis;
+
+ /**
+ * The (filename, mtime) pairs of all files saved between the last build's
+ * start time and the current build's start time. Only applies to builds
+ * running with existing Blaze servers. Currently disabled.
+ */
+ private Map<String, Long> changedFileSaveTimes = new HashMap<>();
+
+ public ExecutionFinishedEvent(Map<String, Long> changedFileSaveTimes,
+ long lastFileSaveTimeInMillis) {
+ this.changedFileSaveTimes = ImmutableMap.copyOf(changedFileSaveTimes);
+ this.lastFileSaveTimeInMillis = lastFileSaveTimeInMillis;
+ }
+
+ public long getLastFileSaveTimeInMillis() {
+ return lastFileSaveTimeInMillis;
+ }
+
+ public Map<String, Long> getChangedFileSaveTimes() {
+ return changedFileSaveTimes;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
new file mode 100644
index 0000000..771cfe6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -0,0 +1,875 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Table;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionContextConsumer;
+import com.google.devtools.build.lib.actions.ActionContextMarker;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BlazeExecutor;
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.LocalHostCapacity;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult;
+import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToCompileProvider;
+import com.google.devtools.build.lib.analysis.InputFileConfiguredTarget;
+import com.google.devtools.build.lib.analysis.OutputFileConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TempsProvider;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.exec.CheckUpToDateFilter;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.exec.OutputService;
+import com.google.devtools.build.lib.exec.SingleBuildFileCache;
+import com.google.devtools.build.lib.exec.SourceManifestActionContextImpl;
+import com.google.devtools.build.lib.exec.SymlinkTreeStrategy;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.rules.fileset.FilesetActionContext;
+import com.google.devtools.build.lib.rules.fileset.FilesetActionContextImpl;
+import com.google.devtools.build.lib.rules.test.TestActionContext;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.skyframe.Builder;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * This class manages the execution phase. The entry point is {@link #executeBuild}.
+ *
+ * <p>This is only intended for use by {@link BuildTool}.
+ *
+ * <p>This class contains an ActionCache, and refers to the BlazeRuntime's BuildView and
+ * PackageCache.
+ *
+ * @see BuildTool
+ * @see BuildView
+ */
+public class ExecutionTool {
+ private static class StrategyConverter {
+ private Table<Class<? extends ActionContext>, String, ActionContext> classMap =
+ HashBasedTable.create();
+ private Map<Class<? extends ActionContext>, ActionContext> defaultClassMap =
+ new HashMap<>();
+
+ /**
+ * Aggregates all {@link ActionContext}s that are in {@code contextProviders}.
+ */
+ @SuppressWarnings("unchecked")
+ private StrategyConverter(Iterable<ActionContextProvider> contextProviders) {
+ for (ActionContextProvider provider : contextProviders) {
+ for (ActionContext strategy : provider.getActionContexts()) {
+ ExecutionStrategy annotation =
+ strategy.getClass().getAnnotation(ExecutionStrategy.class);
+ if (annotation != null) {
+ defaultClassMap.put(annotation.contextType(), strategy);
+
+ for (String name : annotation.name()) {
+ classMap.put(annotation.contextType(), name, strategy);
+ }
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T extends ActionContext> T getStrategy(Class<T> clazz, String name) {
+ return (T) (name.isEmpty() ? defaultClassMap.get(clazz) : classMap.get(clazz, name));
+ }
+
+ private String getValidValues(Class<? extends ActionContext> context) {
+ return Joiner.on(", ").join(Ordering.natural().sortedCopy(classMap.row(context).keySet()));
+ }
+
+ private String getUserFriendlyName(Class<? extends ActionContext> context) {
+ ActionContextMarker marker = context.getAnnotation(ActionContextMarker.class);
+ return marker != null
+ ? marker.name()
+ : context.getSimpleName();
+ }
+ }
+
+ static final Logger LOG = Logger.getLogger(ExecutionTool.class.getName());
+
+ private final BlazeRuntime runtime;
+ private final BuildRequest request;
+ private BlazeExecutor executor;
+ private ActionInputFileCache fileCache;
+ private List<ActionContextProvider> actionContextProviders;
+
+ private Map<String, ActionContext> spawnStrategyMap = new HashMap<>();
+ private List<ActionContext> strategies = new ArrayList<>();
+
+ ExecutionTool(BlazeRuntime runtime, BuildRequest request) throws ExecutorInitException {
+ this.runtime = runtime;
+ this.request = request;
+
+ List<ActionContextConsumer> actionContextConsumers = new ArrayList<>();
+ actionContextProviders = new ArrayList<>();
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ ActionContextProvider provider = module.getActionContextProvider();
+ if (provider != null) {
+ actionContextProviders.add(provider);
+ }
+
+ ActionContextConsumer consumer = module.getActionContextConsumer();
+ if (consumer != null) {
+ actionContextConsumers.add(consumer);
+ }
+ }
+
+ actionContextProviders.add(new FilesetActionContextImpl.Provider(
+ runtime.getReporter(), runtime.getWorkspaceName()));
+
+ strategies.add(new SourceManifestActionContextImpl(runtime.getRunfilesPrefix()));
+ strategies.add(new SymlinkTreeStrategy(runtime.getOutputService(), runtime.getBinTools()));
+
+ StrategyConverter strategyConverter = new StrategyConverter(actionContextProviders);
+ strategies.add(strategyConverter.getStrategy(FilesetActionContext.class, ""));
+ strategies.add(strategyConverter.getStrategy(WorkspaceStatusAction.Context.class, ""));
+
+ for (ActionContextConsumer consumer : actionContextConsumers) {
+ // There are many different SpawnActions, and we want to control the action context they use
+ // independently from each other, for example, to run genrules locally and Java compile action
+ // in prod. Thus, for SpawnActions, we decide the action context to use not only based on the
+ // context class, but also the mnemonic of the action.
+ for (Map.Entry<String, String> entry : consumer.getSpawnActionContexts().entrySet()) {
+ SpawnActionContext context =
+ strategyConverter.getStrategy(SpawnActionContext.class, entry.getValue());
+ if (context == null) {
+ throw makeExceptionForInvalidStrategyValue(entry.getValue(), "spawn",
+ strategyConverter.getValidValues(SpawnActionContext.class));
+ }
+
+ spawnStrategyMap.put(entry.getKey(), context);
+ }
+
+ for (Map.Entry<Class<? extends ActionContext>, String> entry :
+ consumer.getActionContexts().entrySet()) {
+ ActionContext context = strategyConverter.getStrategy(entry.getKey(), entry.getValue());
+ if (context != null) {
+ strategies.add(context);
+ } else if (!entry.getValue().isEmpty()) {
+ // If the action context consumer requested the default value (by passing in the empty
+ // string), we do not throw the exception, because we assume that whoever put together
+ // the modules in this Blaze binary knew what they were doing.
+ throw makeExceptionForInvalidStrategyValue(entry.getValue(),
+ strategyConverter.getUserFriendlyName(entry.getKey()),
+ strategyConverter.getValidValues(entry.getKey()));
+ }
+ }
+ }
+
+ // If tests are to be run during build, too, we have to explicitly load the test action context.
+ if (request.shouldRunTests()) {
+ String testStrategyValue = request.getOptions(ExecutionOptions.class).testStrategy;
+ ActionContext context = strategyConverter.getStrategy(TestActionContext.class,
+ testStrategyValue);
+ if (context == null) {
+ throw makeExceptionForInvalidStrategyValue(testStrategyValue, "test",
+ strategyConverter.getValidValues(TestActionContext.class));
+ }
+ strategies.add(context);
+ }
+ }
+
+ private static ExecutorInitException makeExceptionForInvalidStrategyValue(String value,
+ String strategy, String validValues) {
+ return new ExecutorInitException(String.format(
+ "'%s' is an invalid value for %s strategy. Valid values are: %s", value, strategy,
+ validValues), ExitCode.COMMAND_LINE_ERROR);
+ }
+
+ Executor getExecutor() throws ExecutorInitException {
+ if (executor == null) {
+ executor = createExecutor();
+ }
+ return executor;
+ }
+
+ /**
+ * Creates an executor for the current set of blaze runtime, execution options, and request.
+ */
+ private BlazeExecutor createExecutor()
+ throws ExecutorInitException {
+ return new BlazeExecutor(
+ runtime.getDirectories().getExecRoot(),
+ runtime.getDirectories().getOutputPath(),
+ getReporter(),
+ getEventBus(),
+ runtime.getClock(),
+ request,
+ request.getOptions(ExecutionOptions.class).verboseFailures,
+ request.getOptions(ExecutionOptions.class).showSubcommands,
+ strategies,
+ spawnStrategyMap,
+ actionContextProviders);
+ }
+
+ void init() throws ExecutorInitException {
+ createToolsSymlinks();
+ getExecutor();
+ }
+
+ void shutdown() {
+ for (ActionContextProvider actionContextProvider : actionContextProviders) {
+ actionContextProvider.executionPhaseEnding();
+ }
+ }
+
+ /**
+ * Performs the execution phase (phase 3) of the build, in which the Builder
+ * is applied to the action graph to bring the targets up to date. (This
+ * function will return prior to execution-proper if --nobuild was specified.)
+ *
+ * @param analysisResult the analysis phase output
+ * @param buildResult the mutable build result
+ * @param skyframeExecutor the skyframe executor (if any)
+ * @param packageRoots package roots collected from loading phase and BuildConfigutaionCollection
+ * creation
+ */
+ void executeBuild(AnalysisResult analysisResult,
+ BuildResult buildResult, @Nullable SkyframeExecutor skyframeExecutor,
+ BuildConfigurationCollection configurations,
+ ImmutableMap<PathFragment, Path> packageRoots)
+ throws BuildFailedException, InterruptedException, AbruptExitException, TestExecException,
+ ViewCreationFailedException {
+ Stopwatch timer = Stopwatch.createStarted();
+ prepare(packageRoots, configurations);
+
+ ActionGraph actionGraph = analysisResult.getActionGraph();
+
+ // Get top-level artifacts.
+ ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild();
+
+ // If --nobuild is specified, this request completes successfully without
+ // execution.
+ if (!request.getBuildOptions().performExecutionPhase) {
+ return;
+ }
+
+ // Create symlinks only after we've verified that we're actually
+ // supposed to build something.
+ if (getWorkspace().getFileSystem().supportsSymbolicLinks()) {
+ List<BuildConfiguration> targetConfigurations =
+ getView().getConfigurationCollection().getTargetConfigurations();
+ // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case where
+ // there's only a single configuration, but we don't create any symlinks in the multi-config
+ // case. Can we do better? [multi-config]
+ if (targetConfigurations.size() == 1) {
+ OutputDirectoryLinksUtils.createOutputDirectoryLinks(
+ runtime.getWorkspaceName(), getWorkspace(), getExecRoot(),
+ runtime.getOutputPath(), getReporter(), targetConfigurations.get(0),
+ request.getSymlinkPrefix());
+ }
+ }
+
+ OutputService outputService = runtime.getOutputService();
+ if (outputService != null) {
+ outputService.startBuild();
+ } else {
+ startLocalOutputBuild(); // TODO(bazel-team): this could be just another OutputService
+ }
+
+ ActionCache actionCache = getActionCache();
+ Builder builder = createBuilder(request, executor, actionCache, skyframeExecutor);
+
+ //
+ // Execution proper. All statements below are logically nested in
+ // begin/end pairs. No early returns or exceptions please!
+ //
+
+ Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets();
+ getEventBus().post(new ExecutionStartingEvent(configuredTargets));
+
+ getReporter().handle(Event.progress("Building..."));
+
+ // Conditionally record dependency-checker log:
+ ExplanationHandler explanationHandler =
+ installExplanationHandler(request.getBuildOptions().explanationPath,
+ request.getOptionsDescription());
+
+ Set<ConfiguredTarget> builtTargets = new HashSet<>();
+ boolean interrupted = false;
+ try {
+ Iterable<Artifact> allArtifactsForProviders = Iterables.concat(additionalArtifacts,
+ TopLevelArtifactHelper.getAllArtifactsToBuild(
+ analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext()),
+ TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest()));
+ if (request.isRunningInEmacs()) {
+ // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs
+ request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'");
+ }
+ for (ActionContextProvider actionContextProvider : actionContextProviders) {
+ actionContextProvider.executionPhaseStarting(
+ fileCache,
+ actionGraph,
+ allArtifactsForProviders);
+ }
+ executor.executionPhaseStarting();
+ skyframeExecutor.drainChangedFiles();
+
+ if (request.getViewOptions().discardAnalysisCache) {
+ // Free memory by removing cache entries that aren't going to be needed. Note that in
+ // skyframe full, this destroys the action graph as well, so we can only do it after the
+ // action graph is no longer needed.
+ getView().clearAnalysisCache(analysisResult.getTargetsToBuild());
+ actionGraph = null;
+ }
+
+ configureResourceManager(request);
+
+ Profiler.instance().markPhase(ProfilePhase.EXECUTE);
+
+ builder.buildArtifacts(additionalArtifacts,
+ analysisResult.getParallelTests(),
+ analysisResult.getExclusiveTests(),
+ analysisResult.getTargetsToBuild(),
+ executor, builtTargets,
+ request.getBuildOptions().explanationPath != null);
+
+ } catch (InterruptedException e) {
+ interrupted = true;
+ throw e;
+ } finally {
+ if (request.isRunningInEmacs()) {
+ request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
+ }
+ if (!interrupted) {
+ getReporter().handle(Event.progress("Building complete."));
+ }
+
+ // Transfer over source file "last save time" stats so the remote logger can find them.
+ runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0));
+
+ // Disable system load polling (noop if it was not enabled).
+ ResourceManager.instance().setAutoSensing(false);
+ executor.executionPhaseEnding();
+ for (ActionContextProvider actionContextProvider : actionContextProviders) {
+ actionContextProvider.executionPhaseEnding();
+ }
+
+ Profiler.instance().markPhase(ProfilePhase.FINISH);
+
+ if (!interrupted) {
+ saveCaches(actionCache);
+ }
+
+ long startTime = Profiler.nanoTimeMaybe();
+ determineSuccessfulTargets(buildResult, configuredTargets, builtTargets, timer);
+ showBuildResult(request, buildResult, configuredTargets);
+ Preconditions.checkNotNull(buildResult.getSuccessfulTargets());
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Show results");
+ if (explanationHandler != null) {
+ uninstallExplanationHandler(explanationHandler);
+ }
+ // Finalize output service last, so that if we do throw an exception, we know all the other
+ // code has already run.
+ if (runtime.getOutputService() != null) {
+ boolean isBuildSuccessful =
+ buildResult.getSuccessfulTargets().size() == configuredTargets.size();
+ runtime.getOutputService().finalizeBuild(isBuildSuccessful);
+ }
+ }
+ }
+
+ private void prepare(ImmutableMap<PathFragment, Path> packageRoots,
+ BuildConfigurationCollection configurations)
+ throws ViewCreationFailedException {
+ // Prepare for build.
+ Profiler.instance().markPhase(ProfilePhase.PREPARE);
+
+ // Create some tools symlinks / cleanup per-build state
+ createActionLogDirectory();
+
+ // Plant the symlink forest.
+ plantSymlinkForest(packageRoots, configurations);
+ }
+
+ private void createToolsSymlinks() throws ExecutorInitException {
+ try {
+ runtime.getBinTools().setupBuildTools();
+ } catch (ExecException e) {
+ throw new ExecutorInitException("Tools symlink creation failed: "
+ + e.getMessage() + "; build aborted", e);
+ }
+ }
+
+ private void plantSymlinkForest(ImmutableMap<PathFragment, Path> packageRoots,
+ BuildConfigurationCollection configurations) throws ViewCreationFailedException {
+ try {
+ FileSystemUtils.deleteTreesBelowNotPrefixed(getExecRoot(),
+ new String[] { ".", "_", Constants.PRODUCT_NAME + "-"});
+ // Delete the build configuration's temporary directories
+ for (BuildConfiguration configuration : configurations.getTargetConfigurations()) {
+ configuration.prepareForExecutionPhase();
+ }
+ FileSystemUtils.plantLinkForest(packageRoots, getExecRoot());
+ } catch (IOException e) {
+ throw new ViewCreationFailedException("Source forest creation failed: " + e.getMessage()
+ + "; build aborted", e);
+ }
+ }
+
+ private void createActionLogDirectory() throws ViewCreationFailedException {
+ Path directory = runtime.getDirectories().getActionConsoleOutputDirectory();
+ try {
+ if (directory.exists()) {
+ FileSystemUtils.deleteTree(directory);
+ }
+ directory.createDirectory();
+ } catch (IOException ex) {
+ throw new ViewCreationFailedException("couldn't delete action output directory: " +
+ ex.getMessage());
+ }
+ }
+
+ /**
+ * Prepare for a local output build.
+ */
+ private void startLocalOutputBuild() throws BuildFailedException {
+ long startTime = Profiler.nanoTimeMaybe();
+
+ try {
+ Path outputPath = runtime.getOutputPath();
+ Path localOutputPath = runtime.getDirectories().getLocalOutputPath();
+
+ if (outputPath.isSymbolicLink()) {
+ // Remove the existing symlink first.
+ outputPath.delete();
+ if (localOutputPath.exists()) {
+ // Pre-existing local output directory. Move to outputPath.
+ localOutputPath.renameTo(outputPath);
+ }
+ }
+ } catch (IOException e) {
+ throw new BuildFailedException(e.getMessage());
+ } finally {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO,
+ "Starting local output build");
+ }
+ }
+
+ /**
+ * If a path is supplied, creates and installs an ExplanationHandler. Returns
+ * an instance on success. Reports an error and returns null otherwise.
+ */
+ private ExplanationHandler installExplanationHandler(PathFragment explanationPath,
+ String allOptions) {
+ if (explanationPath == null) {
+ return null;
+ }
+ ExplanationHandler handler;
+ try {
+ handler = new ExplanationHandler(
+ getWorkspace().getRelative(explanationPath).getOutputStream(),
+ allOptions);
+ } catch (IOException e) {
+ getReporter().handle(Event.warn(String.format(
+ "Cannot write explanation of rebuilds to file '%s': %s",
+ explanationPath, e.getMessage())));
+ return null;
+ }
+ getReporter().handle(
+ Event.info("Writing explanation of rebuilds to '" + explanationPath + "'"));
+ getReporter().addHandler(handler);
+ return handler;
+ }
+
+ /**
+ * Uninstalls the specified ExplanationHandler (if any) and closes the log
+ * file.
+ */
+ private void uninstallExplanationHandler(ExplanationHandler handler) {
+ if (handler != null) {
+ getReporter().removeHandler(handler);
+ handler.log.close();
+ }
+ }
+
+ /**
+ * An ErrorEventListener implementation that records DEPCHECKER events into a log
+ * file, iff the --explain flag is specified during a build.
+ */
+ private static class ExplanationHandler implements EventHandler {
+
+ private final PrintWriter log;
+
+ private ExplanationHandler(OutputStream log, String optionsDescription) {
+ this.log = new PrintWriter(log);
+ this.log.println("Build options: " + optionsDescription);
+ }
+
+
+ @Override
+ public void handle(Event event) {
+ if (event.getKind() == EventKind.DEPCHECKER) {
+ log.println(event.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Computes the result of the build. Sets the list of successful (up-to-date)
+ * targets in the request object.
+ *
+ * @param configuredTargets The configured targets whose artifacts are to be
+ * built.
+ * @param timer A timer that was started when the execution phase started.
+ */
+ private void determineSuccessfulTargets(BuildResult result,
+ Collection<ConfiguredTarget> configuredTargets, Set<ConfiguredTarget> builtTargets,
+ Stopwatch timer) {
+ // Maintain the ordering by copying builtTargets into a LinkedHashSet in the same iteration
+ // order as configuredTargets.
+ Collection<ConfiguredTarget> successfulTargets = new LinkedHashSet<>();
+ for (ConfiguredTarget target : configuredTargets) {
+ if (builtTargets.contains(target)) {
+ successfulTargets.add(target);
+ }
+ }
+ getEventBus().post(
+ new ExecutionPhaseCompleteEvent(timer.stop().elapsed(TimeUnit.MILLISECONDS)));
+ result.setSuccessfulTargets(successfulTargets);
+ }
+
+ /**
+ * Shows the result of the build. Information includes the list of up-to-date
+ * and failed targets and list of output artifacts for successful targets
+ *
+ * @param request The build request, which specifies various options.
+ * @param configuredTargets The configured targets whose artifacts are to be
+ * built.
+ *
+ * TODO(bazel-team): (2010) refactor into using Reporter and info/progress events
+ */
+ private void showBuildResult(BuildRequest request, BuildResult result,
+ Collection<ConfiguredTarget> configuredTargets) {
+ // NOTE: be careful what you print! We don't want to create a consistency
+ // problem where the summary message and the exit code disagree. The logic
+ // here is already complex.
+
+ // Filter the targets we care about into two buckets:
+ Collection<ConfiguredTarget> succeeded = new ArrayList<>();
+ Collection<ConfiguredTarget> failed = new ArrayList<>();
+ for (ConfiguredTarget target : configuredTargets) {
+ // TODO(bazel-team): this is quite ugly. Add a marker provider for this check.
+ if (target instanceof InputFileConfiguredTarget) {
+ // Suppress display of source files (because we do no work to build them).
+ continue;
+ }
+ if (target.getTarget() instanceof Rule) {
+ Rule rule = (Rule) target.getTarget();
+ if (rule.getRuleClass().contains("$")) {
+ // Suppress display of hidden rules
+ continue;
+ }
+ }
+ if (target instanceof OutputFileConfiguredTarget) {
+ // Suppress display of generated files (because they appear underneath
+ // their generating rule), EXCEPT those ones which are not part of the
+ // filesToBuild of their generating rule (e.g. .par, _deploy.jar
+ // files), OR when a user explicitly requests an output file but not
+ // its rule.
+ TransitiveInfoCollection generatingRule =
+ getView().getGeneratingRule((OutputFileConfiguredTarget) target);
+ if (CollectionUtils.containsAll(
+ generatingRule.getProvider(FileProvider.class).getFilesToBuild(),
+ target.getProvider(FileProvider.class).getFilesToBuild()) &&
+ configuredTargets.contains(generatingRule)) {
+ continue;
+ }
+ }
+
+ Collection<ConfiguredTarget> successfulTargets = result.getSuccessfulTargets();
+ (successfulTargets.contains(target) ? succeeded : failed).add(target);
+ }
+
+ // Suppress summary if --show_result value is exceeded:
+ if (succeeded.size() + failed.size() > request.getBuildOptions().maxResultTargets) {
+ return;
+ }
+
+ OutErr outErr = request.getOutErr();
+
+ for (ConfiguredTarget target : succeeded) {
+ Label label = target.getLabel();
+ // For up-to-date targets report generated artifacts, but only
+ // if they have associated action and not middleman artifacts.
+ boolean headerFlag = true;
+ for (Artifact artifact : getFilesToBuild(target, request)) {
+ if (!artifact.isSourceArtifact()) {
+ if (headerFlag) {
+ outErr.printErr("Target " + label + " up-to-date:\n");
+ headerFlag = false;
+ }
+ outErr.printErrLn(" " +
+ OutputDirectoryLinksUtils.getPrettyPath(artifact.getPath(),
+ runtime.getWorkspaceName(), getWorkspace(), request.getSymlinkPrefix()));
+ }
+ }
+ if (headerFlag) {
+ outErr.printErr(
+ "Target " + label + " up-to-date (nothing to build)\n");
+ }
+ }
+
+ for (ConfiguredTarget target : failed) {
+ outErr.printErr("Target " + target.getLabel() + " failed to build\n");
+
+ // For failed compilation, it is still useful to examine temp artifacts,
+ // (ie, preprocessed and assembler files).
+ TempsProvider tempsProvider = target.getProvider(TempsProvider.class);
+ if (tempsProvider != null) {
+ for (Artifact temp : tempsProvider.getTemps()) {
+ if (temp.getPath().exists()) {
+ outErr.printErrLn(" See temp at " +
+ OutputDirectoryLinksUtils.getPrettyPath(temp.getPath(),
+ runtime.getWorkspaceName(), getWorkspace(), request.getSymlinkPrefix()));
+ }
+ }
+ }
+ }
+ if (!failed.isEmpty() && !request.getOptions(ExecutionOptions.class).verboseFailures) {
+ outErr.printErr("Use --verbose_failures to see the command lines of failed build steps.\n");
+ }
+ }
+
+ /**
+ * Gets all the files to build for a given target and build request.
+ * There may be artifacts that should be built which are not represented in the
+ * configured target graph. Currently, this only occurs when "--save_temps" is on.
+ *
+ * @param target configured target
+ * @param request the build request
+ * @return artifacts to build
+ */
+ private static Collection<Artifact> getFilesToBuild(ConfiguredTarget target,
+ BuildRequest request) {
+ ImmutableSet.Builder<Artifact> result = ImmutableSet.builder();
+ if (request.getBuildOptions().compileOnly) {
+ FilesToCompileProvider provider = target.getProvider(FilesToCompileProvider.class);
+ if (provider != null) {
+ result.addAll(provider.getFilesToCompile());
+ }
+ } else if (request.getBuildOptions().compilationPrerequisitesOnly) {
+ CompilationPrerequisitesProvider provider =
+ target.getProvider(CompilationPrerequisitesProvider.class);
+ if (provider != null) {
+ result.addAll(provider.getCompilationPrerequisites());
+ }
+ } else {
+ FileProvider provider = target.getProvider(FileProvider.class);
+ if (provider != null) {
+ result.addAll(provider.getFilesToBuild());
+ }
+ }
+ TempsProvider tempsProvider = target.getProvider(TempsProvider.class);
+ if (tempsProvider != null) {
+ result.addAll(tempsProvider.getTemps());
+ }
+
+ return result.build();
+ }
+
+ private ActionCache getActionCache() throws LocalEnvironmentException {
+ try {
+ return runtime.getPersistentActionCache();
+ } catch (IOException e) {
+ // TODO(bazel-team): (2010) Ideally we should just remove all cache data and reinitialize
+ // caches.
+ LoggingUtil.logToRemote(Level.WARNING, "Failed to initialize action cache: "
+ + e.getMessage(), e);
+ throw new LocalEnvironmentException("couldn't create action cache: " + e.getMessage()
+ + ". If error persists, use 'blaze clean'");
+ }
+ }
+
+ private Builder createBuilder(BuildRequest request,
+ Executor executor,
+ ActionCache actionCache,
+ SkyframeExecutor skyframeExecutor) {
+ BuildRequest.BuildRequestOptions options = request.getBuildOptions();
+ boolean verboseExplanations = options.verboseExplanations;
+ boolean keepGoing = request.getViewOptions().keepGoing;
+
+ Path actionOutputRoot = runtime.getDirectories().getActionConsoleOutputDirectory();
+ Predicate<Action> executionFilter = CheckUpToDateFilter.fromOptions(
+ request.getOptions(ExecutionOptions.class));
+
+ // jobs should have been verified in BuildRequest#validateOptions().
+ Preconditions.checkState(options.jobs >= -1);
+ int actualJobs = options.jobs == 0 ? 1 : options.jobs; // Treat 0 jobs as a single task.
+
+ // Unfortunately, the exec root cache is not shared with caches in the remote execution
+ // client.
+ fileCache = createBuildSingleFileCache(executor.getExecRoot());
+ skyframeExecutor.setActionOutputRoot(actionOutputRoot);
+ return new SkyframeBuilder(skyframeExecutor,
+ new ActionCacheChecker(actionCache, getView().getArtifactFactory(), executionFilter,
+ verboseExplanations),
+ keepGoing, actualJobs, options.checkOutputFiles, fileCache,
+ request.getBuildOptions().progressReportInterval);
+ }
+
+ private void configureResourceManager(BuildRequest request) {
+ ResourceManager resourceMgr = ResourceManager.instance();
+ ExecutionOptions options = request.getOptions(ExecutionOptions.class);
+ if (options.availableResources != null) {
+ resourceMgr.setAvailableResources(options.availableResources);
+ resourceMgr.setRamUtilizationPercentage(100);
+ } else {
+ resourceMgr.setAvailableResources(LocalHostCapacity.getLocalHostCapacity());
+ resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage);
+ if (options.useResourceAutoSense) {
+ getReporter().handle(
+ Event.warn("Not using resource autosense due to known responsiveness issues"));
+ }
+ ResourceManager.instance().setAutoSensing(/*autosense=*/false);
+ }
+ }
+
+ /**
+ * Writes the cache files to disk, reporting any errors that occurred during
+ * writing.
+ */
+ private void saveCaches(ActionCache actionCache) {
+ long actionCacheSizeInBytes = 0;
+ long actionCacheSaveTime;
+
+ long startTime = BlazeClock.nanoTime();
+ try {
+ LOG.info("saving action cache...");
+ actionCacheSizeInBytes = actionCache.save();
+ LOG.info("action cache saved");
+ } catch (IOException e) {
+ getReporter().handle(Event.error("I/O error while writing action log: " + e.getMessage()));
+ } finally {
+ long stopTime = BlazeClock.nanoTime();
+ actionCacheSaveTime =
+ TimeUnit.MILLISECONDS.convert(stopTime - startTime, TimeUnit.NANOSECONDS);
+ Profiler.instance().logSimpleTask(startTime, stopTime,
+ ProfilerTask.INFO, "Saving action cache");
+ }
+
+ runtime.getEventBus().post(new CachesSavedEvent(
+ actionCacheSaveTime, actionCacheSizeInBytes));
+ }
+
+ private ActionInputFileCache createBuildSingleFileCache(Path execRoot) {
+ String cwd = execRoot.getPathString();
+ FileSystem fs = runtime.getDirectories().getFileSystem();
+
+ ActionInputFileCache cache = null;
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ ActionInputFileCache pluggable = module.createActionInputCache(cwd, fs);
+ if (pluggable != null) {
+ Preconditions.checkState(cache == null);
+ cache = pluggable;
+ }
+ }
+
+ if (cache == null) {
+ cache = new SingleBuildFileCache(cwd, fs);
+ }
+ return cache;
+ }
+
+ private Reporter getReporter() {
+ return runtime.getReporter();
+ }
+
+ private EventBus getEventBus() {
+ return runtime.getEventBus();
+ }
+
+ private BuildView getView() {
+ return runtime.getView();
+ }
+
+ private Path getWorkspace() {
+ return runtime.getWorkspace();
+ }
+
+ private Path getExecRoot() {
+ return runtime.getExecRoot();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java b/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java
new file mode 100644
index 0000000..7890b22
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+
+/**
+ * An exception that signals that something is wrong with the user's environment
+ * that he can fix. Used to report the problem of having no free space left in
+ * the blaze output directory.
+ *
+ * <p>Note that this is a much higher level exception then the similarly named
+ * EnvironmentExecException, which is thrown from the base Client and Strategy
+ * layers of Blaze.
+ *
+ * <p>This exception is only thrown when we've decided that the build has, in
+ * fact, failed and we should exit.
+ */
+public class LocalEnvironmentException extends AbruptExitException {
+
+ public LocalEnvironmentException(String message) {
+ super(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
+ }
+
+ public LocalEnvironmentException(Throwable cause) {
+ super(ExitCode.LOCAL_ENVIRONMENTAL_ERROR, cause);
+ }
+
+ public LocalEnvironmentException(String message, Throwable cause) {
+ super(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
new file mode 100644
index 0000000..094b7bc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
@@ -0,0 +1,184 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.base.Joiner;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Static utilities for managing output directory symlinks.
+ */
+public class OutputDirectoryLinksUtils {
+ public static final String OUTPUT_SYMLINK_NAME = Constants.PRODUCT_NAME + "-out";
+
+ // Used in getPrettyPath() method below.
+ private static final String[] LINKS = { "bin", "genfiles", "includes" };
+
+ private static final String NO_CREATE_SYMLINKS_PREFIX = "/";
+
+ private static String execRootSymlink(String workspaceName) {
+ return Constants.PRODUCT_NAME + "-" + workspaceName;
+ }
+ /**
+ * Attempts to create convenience symlinks in the workspaceDirectory and in
+ * execRoot to the output area and to the configuration-specific output
+ * directories. Issues a warning if it fails, e.g. because workspaceDirectory
+ * is readonly.
+ */
+ public static void createOutputDirectoryLinks(String workspaceName,
+ Path workspace, Path execRoot, Path outputPath,
+ EventHandler eventHandler, BuildConfiguration targetConfig, String symlinkPrefix) {
+ if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) {
+ return;
+ }
+ List<String> failures = new ArrayList<>();
+
+ // Make the two non-specific links from the workspace to the output area,
+ // and the configuration-specific links in both the workspace and the execution root dirs.
+ // NB! Keep in sync with removeOutputDirectoryLinks below.
+ createLink(workspace, OUTPUT_SYMLINK_NAME, outputPath, failures);
+
+ // Points to execroot
+ createLink(workspace, execRootSymlink(workspaceName), execRoot, failures);
+ createLink(workspace, symlinkPrefix + "bin", targetConfig.getBinDirectory().getPath(),
+ failures);
+ createLink(workspace, symlinkPrefix + "testlogs", targetConfig.getTestLogsDirectory().getPath(),
+ failures);
+ createLink(workspace, symlinkPrefix + "genfiles", targetConfig.getGenfilesDirectory().getPath(),
+ failures);
+ if (!failures.isEmpty()) {
+ eventHandler.handle(Event.warn(String.format(
+ "failed to create one or more convenience symlinks for prefix '%s':\n %s",
+ symlinkPrefix, Joiner.on("\n ").join(failures))));
+ }
+ }
+
+ /**
+ * Returns a convenient path to the specified file, relativizing it and using output-dir symlinks
+ * if possible. Otherwise, return a path relative to the workspace directory if possible.
+ * Otherwise, return the absolute path.
+ *
+ * <p>This method must be called after the symlinks are created at the end of a build. If called
+ * before, the pretty path may be incorrect if the symlinks end up pointing somewhere new.
+ */
+ public static PathFragment getPrettyPath(Path file, String workspaceName,
+ Path workspaceDirectory, String symlinkPrefix) {
+ for (String link : LINKS) {
+ PathFragment result = relativize(file, workspaceDirectory, symlinkPrefix + link);
+ if (result != null) {
+ return result;
+ }
+ }
+
+ PathFragment result = relativize(file, workspaceDirectory, execRootSymlink(workspaceName));
+ if (result != null) {
+ return result;
+ }
+
+ result = relativize(file, workspaceDirectory, OUTPUT_SYMLINK_NAME);
+ if (result != null) {
+ return result;
+ }
+
+ return file.asFragment();
+ }
+
+ // Helper to getPrettyPath. Returns file, relativized w.r.t. the referent of
+ // "linkname", or null if it was a not a child.
+ private static PathFragment relativize(Path file, Path workspaceDirectory, String linkname) {
+ PathFragment link = new PathFragment(linkname);
+ try {
+ Path dir = workspaceDirectory.getRelative(link);
+ PathFragment levelOneLinkTarget = dir.readSymbolicLink();
+ if (levelOneLinkTarget.isAbsolute() &&
+ file.startsWith(dir = file.getRelative(levelOneLinkTarget))) {
+ return link.getRelative(file.relativeTo(dir));
+ }
+ } catch (IOException e) {
+ /* ignore */
+ }
+ return null;
+ }
+
+ /**
+ * Attempts to remove the convenience symlinks in the workspace directory.
+ *
+ * <p>Issues a warning if it fails, e.g. because workspaceDirectory is readonly.
+ * Also cleans up any child directories created by a custom prefix.
+ *
+ * @param workspace the runtime's workspace
+ * @param eventHandler the error eventHandler
+ * @param symlinkPrefix the symlink prefix which should be removed
+ */
+ public static void removeOutputDirectoryLinks(String workspaceName, Path workspace,
+ EventHandler eventHandler, String symlinkPrefix) {
+ if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) {
+ return;
+ }
+ List<String> failures = new ArrayList<>();
+
+ removeLink(workspace, OUTPUT_SYMLINK_NAME, failures);
+ removeLink(workspace, execRootSymlink(workspaceName), failures);
+ removeLink(workspace, symlinkPrefix + "bin", failures);
+ removeLink(workspace, symlinkPrefix + "testlogs", failures);
+ removeLink(workspace, symlinkPrefix + "genfiles", failures);
+ FileSystemUtils.removeDirectoryAndParents(workspace, new PathFragment(symlinkPrefix));
+ if (!failures.isEmpty()) {
+ eventHandler.handle(Event.warn(String.format(
+ "failed to remove one or more convenience symlinks for prefix '%s':\n %s", symlinkPrefix,
+ Joiner.on("\n ").join(failures))));
+ }
+ }
+
+ /**
+ * Helper to createOutputDirectoryLinks that creates a symlink from base + name to target.
+ */
+ private static boolean createLink(Path base, String name, Path target, List<String> failures) {
+ try {
+ FileSystemUtils.ensureSymbolicLink(base.getRelative(name), target);
+ return true;
+ } catch (IOException e) {
+ failures.add(String.format("%s -> %s: %s", name, target.getPathString(), e.getMessage()));
+ return false;
+ }
+ }
+
+ /**
+ * Helper to removeOutputDirectoryLinks that removes one of the Blaze convenience symbolic links.
+ */
+ private static boolean removeLink(Path base, String name, List<String> failures) {
+ Path link = base.getRelative(name);
+ try {
+ if (link.exists(Symlinks.NOFOLLOW)) {
+ ExecutionTool.LOG.finest("Removing " + link);
+ link.delete();
+ }
+ return true;
+ } catch (IOException e) {
+ failures.add(String.format("%s: %s", name, e.getMessage()));
+ return false;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
new file mode 100644
index 0000000..779515a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
@@ -0,0 +1,355 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.BuilderUtils;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog;
+import com.google.devtools.build.lib.skyframe.ActionExecutionValue;
+import com.google.devtools.build.lib.skyframe.Builder;
+import com.google.devtools.build.lib.skyframe.SkyFunctions;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.TargetCompletionValue;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.text.NumberFormat;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A {@link Builder} implementation driven by Skyframe.
+ */
+@VisibleForTesting
+public class SkyframeBuilder implements Builder {
+
+ private final SkyframeExecutor skyframeExecutor;
+ private final boolean keepGoing;
+ private final int numJobs;
+ private final boolean checkOutputFiles;
+ private final ActionInputFileCache fileCache;
+ private final ActionCacheChecker actionCacheChecker;
+ private final int progressReportInterval;
+
+ @VisibleForTesting
+ public SkyframeBuilder(SkyframeExecutor skyframeExecutor, ActionCacheChecker actionCacheChecker,
+ boolean keepGoing, int numJobs, boolean checkOutputFiles,
+ ActionInputFileCache fileCache, int progressReportInterval) {
+ this.skyframeExecutor = skyframeExecutor;
+ this.actionCacheChecker = actionCacheChecker;
+ this.keepGoing = keepGoing;
+ this.numJobs = numJobs;
+ this.checkOutputFiles = checkOutputFiles;
+ this.fileCache = fileCache;
+ this.progressReportInterval = progressReportInterval;
+ }
+
+ @Override
+ public void buildArtifacts(Set<Artifact> artifacts,
+ Set<ConfiguredTarget> parallelTests,
+ Set<ConfiguredTarget> exclusiveTests,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Executor executor,
+ Set<ConfiguredTarget> builtTargets,
+ boolean explain)
+ throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException {
+ skyframeExecutor.prepareExecution(checkOutputFiles);
+ skyframeExecutor.setFileCache(fileCache);
+ // Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a
+ // synchronized collection), so unsynchronized access to this variable is unsafe while it runs.
+ ExecutionProgressReceiver executionProgressReceiver =
+ new ExecutionProgressReceiver(Preconditions.checkNotNull(builtTargets),
+ countTestActions(exclusiveTests), skyframeExecutor.getEventBus());
+ ResourceManager.instance().setEventBus(skyframeExecutor.getEventBus());
+
+ boolean success = false;
+ EvaluationResult<?> result;
+
+ ActionExecutionStatusReporter statusReporter = ActionExecutionStatusReporter.create(
+ skyframeExecutor.getReporter(), executor, skyframeExecutor.getEventBus());
+
+ AtomicBoolean isBuildingExclusiveArtifacts = new AtomicBoolean(false);
+ ActionExecutionInactivityWatchdog watchdog = new ActionExecutionInactivityWatchdog(
+ executionProgressReceiver.createInactivityMonitor(statusReporter),
+ executionProgressReceiver.createInactivityReporter(statusReporter,
+ isBuildingExclusiveArtifacts), progressReportInterval);
+
+ skyframeExecutor.setActionExecutionProgressReportingObjects(executionProgressReceiver,
+ executionProgressReceiver, statusReporter);
+ watchdog.start();
+
+ try {
+ result = skyframeExecutor.buildArtifacts(executor, artifacts, targetsToBuild, parallelTests,
+ /*exclusiveTesting=*/false, keepGoing, explain, numJobs, actionCacheChecker,
+ executionProgressReceiver);
+ // progressReceiver is finished, so unsynchronized access to builtTargets is now safe.
+ success = processResult(result, keepGoing, skyframeExecutor);
+
+ Preconditions.checkState(
+ !success || result.keyNames().size()
+ == (artifacts.size() + targetsToBuild.size() + parallelTests.size()),
+ "Build reported as successful but not all artifacts and targets built: %s, %s",
+ result, artifacts);
+
+ // Run exclusive tests: either tagged as "exclusive" or is run in an invocation with
+ // --test_output=streamed.
+ isBuildingExclusiveArtifacts.set(true);
+ for (ConfiguredTarget exclusiveTest : exclusiveTests) {
+ // Since only one artifact is being built at a time, we don't worry about an artifact being
+ // built and then the build being interrupted.
+ result = skyframeExecutor.buildArtifacts(executor, ImmutableSet.<Artifact>of(),
+ targetsToBuild, ImmutableSet.of(exclusiveTest), /*exclusiveTesting=*/true, keepGoing,
+ explain, numJobs, actionCacheChecker, null);
+ boolean exclusiveSuccess = processResult(result, keepGoing, skyframeExecutor);
+ Preconditions.checkState(!exclusiveSuccess || !result.keyNames().isEmpty(),
+ "Build reported as successful but test %s not executed: %s",
+ exclusiveTest, result);
+ success &= exclusiveSuccess;
+ }
+ } finally {
+ watchdog.stop();
+ ResourceManager.instance().unsetEventBus();
+ skyframeExecutor.setActionExecutionProgressReportingObjects(null, null, null);
+ statusReporter.unregisterFromEventBus();
+ }
+
+ if (!success) {
+ throw new BuildFailedException();
+ }
+ }
+
+ private static boolean resultHasCatastrophicError(EvaluationResult<?> result) {
+ for (ErrorInfo errorInfo : result.errorMap().values()) {
+ if (errorInfo.isCatastrophic()) {
+ return true;
+ }
+ }
+ // An unreported catastrophe manifests with hasError() being true but no errors visible.
+ return result.hasError() && result.errorMap().isEmpty();
+ }
+
+ /**
+ * Process the Skyframe update, taking into account the keepGoing setting.
+ *
+ * Returns false if the update() failed, but we should continue. Returns true on success.
+ * Throws on fail-fast failures.
+ */
+ private static boolean processResult(EvaluationResult<?> result, boolean keepGoing,
+ SkyframeExecutor skyframeExecutor) throws BuildFailedException, TestExecException {
+ if (result.hasError()) {
+ boolean hasCycles = false;
+ for (Map.Entry<SkyKey, ErrorInfo> entry : result.errorMap().entrySet()) {
+ Iterable<CycleInfo> cycles = entry.getValue().getCycleInfo();
+ skyframeExecutor.reportCycles(cycles, entry.getKey());
+ hasCycles |= !Iterables.isEmpty(cycles);
+ }
+ if (keepGoing && !resultHasCatastrophicError(result)) {
+ return false;
+ }
+ if (hasCycles || result.errorMap().isEmpty()) {
+ // error map may be empty in the case of a catastrophe.
+ throw new BuildFailedException();
+ } else {
+ // Need to wrap exception for rethrowCause.
+ BuilderUtils.rethrowCause(
+ new Exception(Preconditions.checkNotNull(result.getError().getException())));
+ }
+ }
+ return true;
+ }
+
+ private static int countTestActions(Iterable<ConfiguredTarget> testTargets) {
+ int count = 0;
+ for (ConfiguredTarget testTarget : testTargets) {
+ count += TestProvider.getTestStatusArtifacts(testTarget).size();
+ }
+ return count;
+ }
+
+ /**
+ * Listener for executed actions and built artifacts. We use a listener so that we have an
+ * accurate set of successfully run actions and built artifacts, even if the build is interrupted.
+ */
+ private static final class ExecutionProgressReceiver implements EvaluationProgressReceiver,
+ SkyframeActionExecutor.ProgressSupplier, SkyframeActionExecutor.ActionCompletedReceiver {
+ private static final NumberFormat PROGRESS_MESSAGE_NUMBER_FORMATTER;
+
+ // Must be thread-safe!
+ private final Set<ConfiguredTarget> builtTargets;
+ private final Set<SkyKey> enqueuedActions = Sets.newConcurrentHashSet();
+ private final Set<Action> completedActions = Sets.newConcurrentHashSet();
+ private final Object activityIndicator = new Object();
+ /** Number of exclusive tests. To be accounted for in progress messages. */
+ private final int exclusiveTestsCount;
+ private final EventBus eventBus;
+
+ static {
+ PROGRESS_MESSAGE_NUMBER_FORMATTER = NumberFormat.getIntegerInstance(Locale.ENGLISH);
+ PROGRESS_MESSAGE_NUMBER_FORMATTER.setGroupingUsed(true);
+ }
+
+ /**
+ * {@code builtTargets} is accessed through a synchronized set, and so no other access to it
+ * is permitted while this receiver is active.
+ */
+ ExecutionProgressReceiver(Set<ConfiguredTarget> builtTargets, int exclusiveTestsCount,
+ EventBus eventBus) {
+ this.builtTargets = Collections.synchronizedSet(builtTargets);
+ this.exclusiveTestsCount = exclusiveTestsCount;
+ this.eventBus = eventBus;
+ }
+
+ @Override
+ public void invalidated(SkyValue node, InvalidationState state) {}
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ if (ActionExecutionValue.isReportWorthyAction(skyKey)) {
+ // Remember all enqueued actions for the benefit of progress reporting.
+ // We discover most actions early in the build, well before we start executing them.
+ // Some of these will be cache hits and won't be executed, so we'll need to account for them
+ // in the evaluated method too.
+ enqueuedActions.add(skyKey);
+ }
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue node, EvaluationState state) {
+ SkyFunctionName type = skyKey.functionName();
+ if (type == SkyFunctions.TARGET_COMPLETION) {
+ TargetCompletionValue val = (TargetCompletionValue) node;
+ ConfiguredTarget target = val.getConfiguredTarget();
+ builtTargets.add(target);
+ eventBus.post(TargetCompleteEvent.createSuccessful(target));
+ } else if (type == SkyFunctions.ACTION_EXECUTION) {
+ // Remember all completed actions, regardless of having been cached or really executed.
+ actionCompleted((Action) skyKey.argument());
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>This method adds the action to {@link #completedActions} and notifies the
+ * {@link #activityIndicator}.
+ *
+ * <p>We could do this only in the {@link #evaluated} method too, but as it happens the action
+ * executor tells the reporter about the completed action before the node is inserted into the
+ * graph, so the reporter would find out about the completed action sooner than we could
+ * have updated {@link #completedActions}, which would result in incorrect numbers on the
+ * progress messages. However we have to store completed actions in {@link #evaluated} too,
+ * because that's the only place we get notified about completed cached actions.
+ */
+ @Override
+ public void actionCompleted(Action a) {
+ if (ActionExecutionValue.isReportWorthyAction(a)) {
+ completedActions.add(a);
+ synchronized (activityIndicator) {
+ activityIndicator.notifyAll();
+ }
+ }
+ }
+
+ @Override
+ public String getProgressString() {
+ return String.format("[%s / %s]",
+ PROGRESS_MESSAGE_NUMBER_FORMATTER.format(completedActions.size()),
+ PROGRESS_MESSAGE_NUMBER_FORMATTER.format(exclusiveTestsCount + enqueuedActions.size()));
+ }
+
+ ActionExecutionInactivityWatchdog.InactivityMonitor createInactivityMonitor(
+ final ActionExecutionStatusReporter statusReporter) {
+ return new ActionExecutionInactivityWatchdog.InactivityMonitor() {
+
+ @Override
+ public boolean hasStarted() {
+ return !enqueuedActions.isEmpty();
+ }
+
+ @Override
+ public int getPending() {
+ return statusReporter.getCount();
+ }
+
+ @Override
+ public int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException {
+ synchronized (activityIndicator) {
+ int before = completedActions.size();
+ long startTime = BlazeClock.instance().currentTimeMillis();
+ while (true) {
+ activityIndicator.wait(timeoutMilliseconds);
+
+ int completed = completedActions.size() - before;
+ long now = 0;
+ if (completed > 0 || (startTime + timeoutMilliseconds) <= (now = BlazeClock.instance()
+ .currentTimeMillis())) {
+ // Some actions completed, or timeout fully elapsed.
+ return completed;
+ } else {
+ // Spurious Wakeup -- no actions completed and there's still time to wait.
+ timeoutMilliseconds -= now - startTime; // account for elapsed wait time
+ startTime = now;
+ }
+ }
+ }
+ }
+ };
+ }
+
+ ActionExecutionInactivityWatchdog.InactivityReporter createInactivityReporter(
+ final ActionExecutionStatusReporter statusReporter,
+ final AtomicBoolean isBuildingExclusiveArtifacts) {
+ return new ActionExecutionInactivityWatchdog.InactivityReporter() {
+ @Override
+ public void maybeReportInactivity() {
+ // Do not report inactivity if we are currently running an exclusive test or a streaming
+ // action (in practice only tests can stream and it implicitly makes them exclusive).
+ if (!isBuildingExclusiveArtifacts.get()) {
+ statusReporter.showCurrentlyExecutingActions(
+ ExecutionProgressReceiver.this.getProgressString() + " ");
+ }
+ }
+ };
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java b/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java
new file mode 100644
index 0000000..e6eed80
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
+
+import java.util.Collection;
+
+/**
+ * Validator for targets.
+ *
+ * <p>Used in "blaze run" to make sure that we are building exactly one binary target.
+ */
+public interface TargetValidator {
+
+ /**
+ * Hook for subclasses to validate a build request before building begins.
+ * Implementors should print warnings for invalid targets iff keepGoing.
+ *
+ * @param targets The targets to build.
+ * @throws LoadingFailedException if the request is not valid for some reason.
+ */
+ void validateTargets(Collection<Target> targets, boolean keepGoing)
+ throws LoadingFailedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java
new file mode 100644
index 0000000..e9278e6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildResult;
+
+/**
+ * This event is fired from BuildTool#stopRequest().
+ */
+public final class BuildCompleteEvent {
+ private final BuildResult result;
+
+ /**
+ * Construct the BuildStartingEvent.
+ * @param request the build request.
+ */
+ public BuildCompleteEvent(BuildRequest request, BuildResult result) {
+ this.result = result;
+ }
+
+ /**
+ * @return the build summary
+ */
+ public BuildResult getResult() {
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java
new file mode 100644
index 0000000..02a5d8b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java
@@ -0,0 +1,22 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+/**
+ * This event is fired from {@code AbstractBuildCommand#doBuild} to indicate
+ * that the user interrupted the build with control-C.
+ */
+public class BuildInterruptedEvent {
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java
new file mode 100644
index 0000000..714534d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+
+/**
+ * This event is fired from BuildTool#startRequest().
+ * At this point, the set of target patters are known, but have
+ * yet to be parsed.
+ */
+public class BuildStartingEvent {
+ private final String outputFileSystem;
+ private final BuildRequest request;
+
+ /**
+ * Construct the BuildStartingEvent.
+ * @param request the build request.
+ */
+ public BuildStartingEvent(String outputFileSystem, BuildRequest request) {
+ this.outputFileSystem = outputFileSystem;
+ this.request = request;
+ }
+
+ /**
+ * @return the output file system.
+ */
+ public String getOutputFileSystem() {
+ return outputFileSystem;
+ }
+
+ /**
+ * @return the active BuildRequest.
+ */
+ public BuildRequest getRequest() {
+ return request;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java
new file mode 100644
index 0000000..cf57960
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+/**
+ * This event is fired after the execution phase is complete.
+ */
+public class ExecutionPhaseCompleteEvent {
+ private final long timeInMs;
+
+ /**
+ * Construct the event.
+ *
+ * @param timeInMs time for execution phase in milliseconds.
+ */
+ public ExecutionPhaseCompleteEvent(long timeInMs) {
+ this.timeInMs = timeInMs;
+ }
+
+ public long getTimeInMs() {
+ return timeInMs;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java
new file mode 100644
index 0000000..c2b4f77
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.buildtool.ExecutionTool;
+
+import java.util.Collection;
+
+/**
+ * This event is fired from {@link ExecutionTool#executeBuild} to indicate that the execution phase
+ * of the build is starting.
+ */
+public class ExecutionStartingEvent {
+ private final Collection<TransitiveInfoCollection> targets;
+
+ /**
+ * Construct the event with a set of targets.
+ * @param targets Remaining active targets.
+ */
+ public ExecutionStartingEvent(Collection<? extends TransitiveInfoCollection> targets) {
+ this.targets = ImmutableList.copyOf(targets);
+ }
+
+ /**
+ * @return The set of active targets remaining, which is a subset
+ * of the targets in the user request.
+ */
+ public Collection<TransitiveInfoCollection> getTargets() {
+ return targets;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java
new file mode 100644
index 0000000..c380456
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+
+import java.util.Collection;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * This event is fired after test filtering.
+ *
+ * The test filtering phase always expands test_suite rules, so
+ * the set of active targets should never contain test_suites.
+ */
+@Immutable
+public class TestFilteringCompleteEvent {
+ private final Collection<ConfiguredTarget> targets;
+ private final Collection<ConfiguredTarget> testTargets;
+
+ /**
+ * Construct the event.
+ * @param targets The set of active targets that remain.
+ * @param testTargets The collection of tests to be run. May be null.
+ */
+ public TestFilteringCompleteEvent(
+ Collection<? extends ConfiguredTarget> targets,
+ Collection<? extends ConfiguredTarget> testTargets) {
+ this.targets = ImmutableList.copyOf(targets);
+ this.testTargets = testTargets == null ? null : ImmutableList.copyOf(testTargets);
+ if (testTargets == null) {
+ return;
+ }
+
+ for (ConfiguredTarget testTarget : testTargets) {
+ Preconditions.checkState(testTarget.getProvider(TestProvider.class) != null);
+ }
+ }
+
+ /**
+ * @return The set of active targets remaining. This is a subset of
+ * the targets that passed analysis, after test_suite expansion.
+ */
+ public Collection<ConfiguredTarget> getTargets() {
+ return targets;
+ }
+
+ /**
+ * @return The set of test targets to be run. May be null.
+ */
+ public Collection<ConfiguredTarget> getTestTargets() {
+ return testTargets;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/LabelValidator.java b/src/main/java/com/google/devtools/build/lib/cmdline/LabelValidator.java
new file mode 100644
index 0000000..50b3379
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/cmdline/LabelValidator.java
@@ -0,0 +1,289 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.cmdline;
+
+import com.google.common.base.CharMatcher;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * The canonical place to parse and validate Blaze labels.
+ */
+public final class LabelValidator {
+
+ /**
+ * Matches punctuation in target names which requires quoting in a blaze query.
+ */
+ private static final CharMatcher PUNCTUATION_REQUIRING_QUOTING = CharMatcher.anyOf("+,=~");
+
+ /**
+ * Matches punctuation in target names which doesn't require quoting in a blaze query.
+ *
+ * Note that . is also allowed in target names, and doesn't require quoting, but has restrictions
+ * on its surrounding characters; see {@link #validateTargetName(String)}.
+ */
+ private static final CharMatcher PUNCTUATION_NOT_REQUIRING_QUOTING = CharMatcher.anyOf("_-@");
+
+ /**
+ * Matches characters allowed in target names regardless of context.
+ *
+ * Note that the only other characters allowed in target names are / and . but they have
+ * restrictions around surrounding characters; see {@link #validateTargetName(String)}.
+ */
+ private static final CharMatcher ALWAYS_ALLOWED_TARGET_CHARACTERS =
+ CharMatcher.JAVA_LETTER_OR_DIGIT
+ .or(PUNCTUATION_REQUIRING_QUOTING)
+ .or(PUNCTUATION_NOT_REQUIRING_QUOTING);
+
+ private static final String PACKAGE_NAME_ERROR =
+ "package names may contain only A-Z, a-z, 0-9, '/', '-' and '_'";
+
+ /**
+ * Performs validity checking of the specified package name. Returns null on success or an error
+ * message otherwise.
+ *
+ * @param packageName the name of the package
+ * @return null if {@code name} is valid or an error string if any part
+ * of the package name is invalid
+ */
+ @Nullable
+ public static String validatePackageName(String packageName) {
+ int len = packageName.length();
+ if (len == 0) {
+ return "empty package name";
+ }
+ char first = packageName.charAt(0);
+ if (first < 'a' || first > 'z') {
+ return "package names must start with a lowercase ASCII letter";
+ }
+
+ // Fast path for packages with '.' in their name
+ if (packageName.lastIndexOf('.') != -1) {
+ return PACKAGE_NAME_ERROR;
+ }
+
+ // Check for any character outside of [/0-9A-Z_a-z-]. Try to evaluate the
+ // conditional quickly (by looking in decreasing order of character class
+ // likelihood).
+ for (int i = len - 1; i >= 0; --i) {
+ char c = packageName.charAt(i);
+ if ((c < 'a' || c > 'z') && c != '/' && c != '_' && c != '-' &&
+ (c < '0' || c > '9') && (c < 'A' || c > 'Z')) {
+ return PACKAGE_NAME_ERROR;
+ }
+ }
+
+ if (packageName.contains("//")) {
+ return "package names may not contain '//' path separators";
+ }
+ if (packageName.endsWith("/")) {
+ return "package names may not end with '/'";
+ }
+ return null; // ok
+ }
+
+ /**
+ * Performs validity checking of the specified target name. Returns null on success or an error
+ * message otherwise.
+ */
+ @Nullable
+ public static String validateTargetName(String targetName) {
+ // TODO(bazel-team): (2011) allow labels equaling '.' or ending in '/.' for now. If we ever
+ // actually configure the target we will report an error, but they will be permitted for
+ // data directories.
+
+ // TODO(bazel-team): (2011) Get rid of this code once we have reached critical mass and can
+ // pressure developers to clean up their BUILD files.
+
+ // Code optimized for the common case: success.
+ int len = targetName.length();
+ if (len == 0) {
+ return "empty target name";
+ }
+ // Forbidden start chars:
+ char c = targetName.charAt(0);
+ if (c == '/') {
+ return "target names may not start with '/'";
+ } else if (c == '.') {
+ if (targetName.startsWith("../") || targetName.equals("..")) {
+ return "target names may not contain up-level references '..'";
+ } else if (targetName.equals(".")) {
+ return null; // See comment above; ideally should be an error.
+ } else if (targetName.startsWith("./")) {
+ return "target names may not contain '.' as a path segment";
+ }
+ }
+
+ // Give a friendly error message on CRs in target names
+ if (targetName.endsWith("\r")) {
+ return "target names may not end with carriage returns " +
+ "(perhaps the input source is CRLF-terminated)";
+ }
+
+ for (int ii = 0; ii < len; ++ii) {
+ c = targetName.charAt(ii);
+ if (ALWAYS_ALLOWED_TARGET_CHARACTERS.matches(c)) {
+ continue;
+ }
+ if (c == '.') {
+ continue;
+ }
+ if (c == '/') {
+ if (targetName.substring(ii).startsWith("/../")) {
+ return "target names may not contain up-level references '..'";
+ } else if (targetName.substring(ii).startsWith("/./")) {
+ return "target names may not contain '.' as a path segment";
+ } else if (targetName.substring(ii).startsWith("//")) {
+ return "target names may not contain '//' path separators";
+ }
+ continue;
+ }
+ if (CharMatcher.JAVA_ISO_CONTROL.matches(c)) {
+ return "target names may not contain non-printable characters: '" +
+ String.format("\\x%02X", (int) c) + "'";
+ }
+ return "target names may not contain '" + c + "'";
+ }
+ // Forbidden end chars:
+ if (c == '.' && targetName.endsWith("/..")) {
+ return "target names may not contain up-level references '..'";
+ } else if (c == '.' && targetName.endsWith("/.")) {
+ return null; // See comment above; ideally should be an error.
+ }
+ if (c == '/') {
+ return "target names may not end with '/'";
+ }
+ return null; // ok
+ }
+
+ /**
+ * Validate the label and parse it into a pair of package name and target name. If the label is
+ * not valid, it throws an {@link BadLabelException}.
+ *
+ * <p>It accepts these forms of labels:
+ * <pre>
+ * //foo/bar
+ * //foo/bar:quux
+ * //foo/bar: (undocumented, but accepted)
+ * </pre>
+ */
+ public static PackageAndTarget validateAbsoluteLabel(String absName) throws BadLabelException {
+ PackageAndTarget result = parseAbsoluteLabel(absName);
+ String packageName = result.getPackageName();
+ String targetName = result.getTargetName();
+ String error = validatePackageName(packageName);
+ if (error != null) {
+ error = "invalid package name '" + packageName + "': " + error;
+ // This check is just for a more helpful error message,
+ // i.e. valid target name, invalid package name, colon-free label form
+ // used => probably they meant "//foo:bar.c" not "//foo/bar.c".
+ if (packageName.endsWith("/" + targetName)) {
+ error += " (perhaps you meant \":" + targetName + "\"?)";
+ }
+ throw new BadLabelException(error);
+ }
+ error = validateTargetName(targetName);
+ if (error != null) {
+ error = "invalid target name '" + targetName + "': " + error;
+ throw new BadLabelException(error);
+ }
+ return result;
+ }
+
+ /**
+ * Parses the given absolute label by verifying that it starts with "//". If it contains a ':',
+ * then the part after that is the target name within the package, and the part before that (but
+ * without the leading "//") is the package name. However, it performs no validation on these two
+ * pieces.
+ *
+ * <p>Use of this method is generally not recommended.
+ *
+ * @throws NullPointerException if {@code absName} is {@code null}
+ * @throws BadLabelException if {@code absName} starts with "//"
+ */
+ public static PackageAndTarget parseAbsoluteLabel(String absName) throws BadLabelException {
+ if (!absName.startsWith("//")) {
+ throw new BadLabelException("invalid label: " + absName);
+ }
+ // Find the package/suffix separation:
+ int colonIndex = absName.indexOf(':');
+ int splitAt = colonIndex >= 0 ? colonIndex : absName.length();
+ String packageName = absName.substring("//".length(), splitAt);
+ String suffix = absName.substring(splitAt);
+ // ('suffix' is empty, or starts with a colon.)
+
+ // "If packagename and version are elided, the colon is not necessary."
+ String targetName = suffix.isEmpty()
+ // Target name is last package segment: (works in slash-free case too.)
+ ? packageName.substring(packageName.lastIndexOf('/') + 1)
+ // Target name is what's after colon:
+ : suffix.substring(1);
+
+ return new PackageAndTarget(packageName, targetName);
+ }
+
+ /**
+ * A pair of package and target names. Note that having an instance of this does not imply that
+ * the package or target names are actually valid.
+ */
+ public static class PackageAndTarget {
+ private final String packageName;
+ private final String targetName;
+
+ public PackageAndTarget(String packageName, String targetName) {
+ this.packageName = packageName;
+ this.targetName = targetName;
+ }
+
+ public String getPackageName() {
+ return packageName;
+ }
+
+ public String getTargetName() {
+ return targetName;
+ }
+
+ @Override
+ public String toString() {
+ return "//" + packageName + ":" + targetName;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(packageName, targetName);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || o.getClass() != getClass()) {
+ return false;
+ }
+ PackageAndTarget otherTarget = (PackageAndTarget) o;
+ return Objects.equals(otherTarget.targetName, targetName)
+ && Objects.equals(otherTarget.packageName, packageName);
+ }
+ }
+
+ /**
+ * An exception to notify the caller that a label could not be parsed.
+ */
+ public static class BadLabelException extends Exception {
+ public BadLabelException(String msg) {
+ super(msg);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/ResolvedTargets.java b/src/main/java/com/google/devtools/build/lib/cmdline/ResolvedTargets.java
new file mode 100644
index 0000000..806cd61
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/cmdline/ResolvedTargets.java
@@ -0,0 +1,170 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.cmdline;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+import java.util.Collection;
+import java.util.Set;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Contains the result of the target pattern evaluation. This is a specialized container class for
+ * the result of target pattern resolution. There is no restriction on the element type, but it will
+ * usually be {@code Target}.
+ */
+@Immutable
+public final class ResolvedTargets<T> {
+ private static final ResolvedTargets<?> FAILED_RESULT =
+ new ResolvedTargets<>(ImmutableSet.of(), ImmutableSet.of(), true);
+
+ private static final ResolvedTargets<?> EMPTY_RESULT =
+ new ResolvedTargets<>(ImmutableSet.of(), ImmutableSet.of(), false);
+
+ @SuppressWarnings("unchecked")
+ public static <T> ResolvedTargets<T> failed() {
+ return (ResolvedTargets<T>) FAILED_RESULT;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> ResolvedTargets<T> empty() {
+ return (ResolvedTargets<T>) EMPTY_RESULT;
+ }
+
+ public static <T> ResolvedTargets<T> of(T target) {
+ return new ResolvedTargets<>(ImmutableSet.<T>of(target), false);
+ }
+
+ private final boolean hasError;
+ private final ImmutableSet<T> targets;
+ private final ImmutableSet<T> filteredTargets;
+
+ public ResolvedTargets(Set<T> targets, Set<T> filteredTargets, boolean hasError) {
+ this.targets = ImmutableSet.copyOf(targets);
+ this.filteredTargets = ImmutableSet.copyOf(filteredTargets);
+ this.hasError = hasError;
+ }
+
+ public ResolvedTargets(Set<T> targets, boolean hasError) {
+ this.targets = ImmutableSet.copyOf(targets);
+ this.filteredTargets = ImmutableSet.of();
+ this.hasError = hasError;
+ }
+
+ public boolean hasError() {
+ return hasError;
+ }
+
+ public ImmutableSet<T> getTargets() {
+ return targets;
+ }
+
+ public ImmutableSet<T> getFilteredTargets() {
+ return filteredTargets;
+ }
+
+ /**
+ * Returns a builder using concurrent sets, as long as you don't call filter.
+ */
+ public static <T> ResolvedTargets.Builder<T> concurrentBuilder() {
+ return new ResolvedTargets.Builder<>(
+ Sets.<T>newConcurrentHashSet(),
+ Sets.<T>newConcurrentHashSet());
+ }
+
+ public static <T> ResolvedTargets.Builder<T> builder() {
+ return new ResolvedTargets.Builder<>();
+ }
+
+ public static final class Builder<T> {
+ private Set<T> targets;
+ private Set<T> filteredTargets;
+ private volatile boolean hasError = false;
+
+ private Builder() {
+ this(Sets.<T>newLinkedHashSet(), Sets.<T>newLinkedHashSet());
+ }
+
+ private Builder(Set<T> targets, Set<T> filteredTargets) {
+ this.targets = targets;
+ this.filteredTargets = filteredTargets;
+ }
+
+ public ResolvedTargets<T> build() {
+ return new ResolvedTargets<>(targets, filteredTargets, hasError);
+ }
+
+ public Builder<T> merge(ResolvedTargets<T> other) {
+ removeAll(other.filteredTargets);
+ addAll(other.targets);
+ if (other.hasError) {
+ hasError = true;
+ }
+ return this;
+ }
+
+ public Builder<T> add(T target) {
+ targets.add(target);
+ filteredTargets.remove(target);
+ return this;
+ }
+
+ public Builder<T> addAll(Collection<T> targets) {
+ this.targets.addAll(targets);
+ this.filteredTargets.removeAll(targets);
+ return this;
+ }
+
+ public void remove(T target) {
+ targets.remove(target);
+ filteredTargets.add(target);
+ }
+
+ public Builder<T> removeAll(Collection<T> targets) {
+ this.filteredTargets.addAll(targets);
+ this.targets.removeAll(targets);
+ return this;
+ }
+
+ public Builder<T> filter(Predicate<T> predicate) {
+ Set<T> oldTargets = targets;
+ targets = Sets.newLinkedHashSet();
+ for (T target : oldTargets) {
+ if (predicate.apply(target)) {
+ add(target);
+ } else {
+ remove(target);
+ }
+ }
+ return this;
+ }
+
+ public Builder<T> setError() {
+ this.hasError = true;
+ return this;
+ }
+
+ public Builder<T> mergeError(boolean hasError) {
+ this.hasError |= hasError;
+ return this;
+ }
+
+ public boolean isEmpty() {
+ return targets.isEmpty();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/TargetParsingException.java b/src/main/java/com/google/devtools/build/lib/cmdline/TargetParsingException.java
new file mode 100644
index 0000000..044bcca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/cmdline/TargetParsingException.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.cmdline;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Indicates that a target label cannot be parsed.
+ */
+public class TargetParsingException extends Exception {
+ public TargetParsingException(String message) {
+ super(Preconditions.checkNotNull(message));
+ }
+
+ public TargetParsingException(String message, Throwable cause) {
+ super(Preconditions.checkNotNull(message), cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java
new file mode 100644
index 0000000..6677970
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java
@@ -0,0 +1,464 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.cmdline;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.LabelValidator.BadLabelException;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Represents a target pattern. Target patterns are a generalization of labels to include
+ * wildcards for finding all packages recursively beneath some root, and for finding all targets
+ * within a package.
+ *
+ * <p>Note that this class does not handle negative patterns ("-//foo/bar"); these must be handled
+ * one level up. In particular, the query language comes with built-in support for negative
+ * patterns.
+ *
+ * <p>In order to resolve target patterns, you need an implementation of {@link
+ * TargetPatternResolver}. This class is thread-safe if the corresponding instance is thread-safe.
+ *
+ * <p>See lib/blaze/commands/target-syntax.txt for details.
+ */
+public abstract class TargetPattern {
+
+ private static final Splitter SLASH_SPLITTER = Splitter.on('/');
+ private static final Joiner SLASH_JOINER = Joiner.on('/');
+
+ private static final Parser DEFAULT_PARSER = new Parser("");
+
+ private final Type type;
+
+ /**
+ * Returns a parser with no offset. Note that the Parser class is immutable, so this method may
+ * return the same instance on subsequent calls.
+ */
+ public static Parser defaultParser() {
+ return DEFAULT_PARSER;
+ }
+
+ private static String removeSuffix(String s, String suffix) {
+ if (s.endsWith(suffix)) {
+ return s.substring(0, s.length() - suffix.length());
+ } else {
+ throw new IllegalArgumentException(s + ", " + suffix);
+ }
+ }
+
+ /**
+ * Normalizes the given relative path by resolving {@code //}, {@code /./} and {@code x/../}
+ * pieces. Note that leading {@code ".."} segments are not removed, so the returned string can
+ * have leading {@code ".."} segments.
+ *
+ * @throws IllegalArgumentException if the path is absolute, i.e. starts with a @{code '/'}
+ */
+ @VisibleForTesting
+ static String normalize(String path) {
+ Preconditions.checkArgument(!path.startsWith("/"));
+ Iterator<String> it = SLASH_SPLITTER.split(path).iterator();
+ List<String> pieces = new ArrayList<>();
+ while (it.hasNext()) {
+ String piece = it.next();
+ if (".".equals(piece) || piece.isEmpty()) {
+ continue;
+ }
+ if ("..".equals(piece)) {
+ if (pieces.isEmpty()) {
+ pieces.add(piece);
+ continue;
+ }
+ String predecessor = pieces.remove(pieces.size() - 1);
+ if ("..".equals(predecessor)) {
+ pieces.add(piece);
+ pieces.add(piece);
+ }
+ continue;
+ }
+ pieces.add(piece);
+ }
+ return SLASH_JOINER.join(pieces);
+ }
+
+ private TargetPattern(Type type) {
+ // Don't allow inheritance outside this class.
+ this.type = type;
+ }
+
+ /**
+ * Return the type of the pattern. Examples include "below package" like "foo/..." and "single
+ * target" like "//x:y".
+ */
+ public Type getType() {
+ return type;
+ }
+
+ /**
+ * Evaluates the current target pattern and returns the result.
+ */
+ public abstract <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver)
+ throws TargetParsingException, InterruptedException,
+ TargetPatternResolver.MissingDepException;
+
+ private static final class SingleTarget extends TargetPattern {
+
+ private final String targetName;
+
+ private SingleTarget(String targetName) {
+ super(Type.SINGLE_TARGET);
+ this.targetName = targetName;
+ }
+
+ @Override
+ public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver)
+ throws TargetParsingException, InterruptedException,
+ TargetPatternResolver.MissingDepException {
+ return resolver.getExplicitTarget(targetName);
+ }
+ }
+
+ private static final class InterpretPathAsTarget extends TargetPattern {
+
+ private final String path;
+
+ private InterpretPathAsTarget(String path) {
+ super(Type.PATH_AS_TARGET);
+ this.path = normalize(path);
+ }
+
+ @Override
+ public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver)
+ throws TargetParsingException, InterruptedException,
+ TargetPatternResolver.MissingDepException {
+ if (resolver.isPackage(path)) {
+ // User has specified a package name. Issue a helpful error message.
+ throw new TargetParsingException("ambiguous target pattern: '" + path + "' is "
+ + "the name of a package; use '" + path + ":all' to mean \"all "
+ + "rules in this package\", '" + path + "/...' to mean \"all rules recursively "
+ + "beneath this package\", or '//" + path + "' to mean \"the default rule in this "
+ + "package\"");
+ }
+
+ List<String> pieces = SLASH_SPLITTER.splitToList(path);
+
+ // Interprets the label as a file target. This loop stops as soon as the
+ // first BUILD file is found (i.e. longest prefix match).
+ for (int i = pieces.size() - 1; i > 0; i--) {
+ String packageName = SLASH_JOINER.join(pieces.subList(0, i));
+ if (resolver.isPackage(packageName)) {
+ String targetName = SLASH_JOINER.join(pieces.subList(i, pieces.size()));
+ return resolver.getExplicitTarget("//" + packageName + ":" + targetName);
+ }
+ }
+
+ throw new TargetParsingException(
+ "couldn't determine target from filename '" + path + "'");
+ }
+ }
+
+ private static final class TargetsInPackage extends TargetPattern {
+
+ private final String originalPattern;
+ private final String pattern;
+ private final String suffix;
+ private final boolean isAbsolute;
+ private final boolean rulesOnly;
+ private final boolean checkWildcardConflict;
+
+ private TargetsInPackage(String originalPattern, String pattern, String suffix,
+ boolean isAbsolute, boolean rulesOnly, boolean checkWildcardConflict) {
+ super(Type.TARGETS_IN_PACKAGE);
+ this.originalPattern = originalPattern;
+ this.pattern = pattern;
+ this.suffix = suffix;
+ this.isAbsolute = isAbsolute;
+ this.rulesOnly = rulesOnly;
+ this.checkWildcardConflict = checkWildcardConflict;
+ }
+
+ @Override
+ public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver)
+ throws TargetParsingException, InterruptedException,
+ TargetPatternResolver.MissingDepException {
+ if (checkWildcardConflict) {
+ ResolvedTargets<T> targets = getWildcardConflict(resolver);
+ if (targets != null) {
+ return targets;
+ }
+ }
+ return resolver.getTargetsInPackage(originalPattern, removeSuffix(pattern, suffix),
+ rulesOnly);
+ }
+
+ /**
+ * There's a potential ambiguity if '//foo/bar:all' refers to an actual target. In this case, we
+ * use the the target but print a warning.
+ *
+ * @return the Target corresponding to the given pattern, if the pattern is absolute and there
+ * is such a target. Otherwise, return null.
+ */
+ private <T> ResolvedTargets<T> getWildcardConflict(TargetPatternResolver<T> resolver)
+ throws InterruptedException, TargetPatternResolver.MissingDepException {
+ if (!isAbsolute) {
+ return null;
+ }
+
+ T target = resolver.getTargetOrNull("//" + pattern);
+ if (target != null) {
+ String name = pattern.lastIndexOf(':') != -1
+ ? pattern.substring(pattern.lastIndexOf(':') + 1)
+ : pattern.substring(pattern.lastIndexOf('/') + 1);
+ resolver.warn(String.format("The Blaze target pattern '%s' is ambiguous: '%s' is " +
+ "both a wildcard, and the name of an existing %s; " +
+ "using the latter interpretation",
+ "//" + pattern, ":" + name,
+ resolver.getTargetKind(target)));
+ try {
+ return resolver.getExplicitTarget("//" + pattern);
+ } catch (TargetParsingException e) {
+ throw new IllegalStateException(
+ "getTargetOrNull() returned non-null, so target should exist", e);
+ }
+ }
+ return null;
+ }
+ }
+
+ private static final class TargetsBelowPackage extends TargetPattern {
+
+ private final String originalPattern;
+ private final String pathPrefix;
+ private final boolean rulesOnly;
+
+ private TargetsBelowPackage(String originalPattern, String pathPrefix, boolean rulesOnly) {
+ super(Type.TARGETS_BELOW_PACKAGE);
+ this.originalPattern = originalPattern;
+ this.pathPrefix = pathPrefix;
+ this.rulesOnly = rulesOnly;
+ }
+
+ @Override
+ public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver)
+ throws TargetParsingException, InterruptedException,
+ TargetPatternResolver.MissingDepException {
+ return resolver.findTargetsBeneathDirectory(originalPattern, pathPrefix, rulesOnly);
+ }
+ }
+
+ @Immutable
+ public static final class Parser {
+ // TODO(bazel-team): Merge the Label functionality that requires similar constants into this
+ // class.
+ /**
+ * The set of target-pattern suffixes which indicate wildcards over all <em>rules</em> in a
+ * single package.
+ */
+ private static final List<String> ALL_RULES_IN_SUFFIXES = ImmutableList.of(
+ "all");
+
+ /**
+ * The set of target-pattern suffixes which indicate wildcards over all <em>targets</em> in a
+ * single package.
+ */
+ private static final List<String> ALL_TARGETS_IN_SUFFIXES = ImmutableList.of(
+ "*",
+ "all-targets");
+
+ private static final List<String> SUFFIXES;
+
+ static {
+ SUFFIXES = ImmutableList.<String>builder()
+ .addAll(ALL_RULES_IN_SUFFIXES)
+ .addAll(ALL_TARGETS_IN_SUFFIXES)
+ .add("/...")
+ .build();
+ }
+
+ /**
+ * Returns whether the given pattern is simple, i.e., not starting with '-' and using none of
+ * the target matching suffixes.
+ */
+ public static boolean isSimpleTargetPattern(String pattern) {
+ if (pattern.startsWith("-")) {
+ return false;
+ }
+
+ for (String suffix : SUFFIXES) {
+ if (pattern.endsWith(":" + suffix)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Directory prefix to use when resolving relative labels (rather than absolute ones). For
+ * example, if the working directory is "<workspace root>/foo", then this should be "foo",
+ * which will make patterns such as "bar:bar" be resolved as "//foo/bar:bar". This makes the
+ * command line a bit more convenient to use.
+ */
+ private final String relativeDirectory;
+
+ /**
+ * Creates a new parser with the given offset for relative patterns.
+ */
+ public Parser(String relativeDirectory) {
+ this.relativeDirectory = relativeDirectory;
+ }
+
+ /**
+ * Parses the given pattern, and throws an exception if the pattern is invalid.
+ *
+ * @return a target pattern corresponding to the pattern parsed
+ * @throws TargetParsingException if the pattern is invalid
+ */
+ public TargetPattern parse(String pattern) throws TargetParsingException {
+ // The structure of this method is by cases, according to the usage string
+ // constant (see lib/blaze/commands/target-syntax.txt).
+
+ String originalPattern = pattern;
+ final boolean isAbsolute = pattern.startsWith("//");
+
+ // We now absolutize non-absolute target patterns.
+ pattern = isAbsolute ? pattern.substring(2) : absolutize(pattern);
+ // Check for common errors.
+ if (pattern.startsWith("/")) {
+ throw new TargetParsingException("not a relative path or label: '" + pattern + "'");
+ }
+ if (pattern.isEmpty()) {
+ throw new TargetParsingException("the empty string is not a valid target");
+ }
+
+ // Transform "/BUILD" suffix into ":BUILD" to accept //foo/bar/BUILD
+ // syntax as a synonym to //foo/bar:BUILD.
+ if (pattern.endsWith("/BUILD")) {
+ pattern = pattern.substring(0, pattern.length() - 6) + ":BUILD";
+ }
+
+ int colonIndex = pattern.lastIndexOf(':');
+ String packagePart = colonIndex < 0 ? pattern : pattern.substring(0, colonIndex);
+ String targetPart = colonIndex < 0 ? "" : pattern.substring(colonIndex + 1);
+
+ if (packagePart.equals("...")) {
+ packagePart = "/..."; // special case this for easier parsing
+ }
+
+ if (packagePart.endsWith("/")) {
+ throw new TargetParsingException("The package part of '" + originalPattern
+ + "' should not end in a slash");
+ }
+
+ if (packagePart.endsWith("/...")) {
+ String realPackagePart = removeSuffix(packagePart, "/...");
+ if (targetPart.isEmpty() || ALL_RULES_IN_SUFFIXES.contains(targetPart)) {
+ return new TargetsBelowPackage(originalPattern, realPackagePart, true);
+ } else if (ALL_TARGETS_IN_SUFFIXES.contains(targetPart)) {
+ return new TargetsBelowPackage(originalPattern, realPackagePart, false);
+ }
+ }
+
+ if (ALL_RULES_IN_SUFFIXES.contains(targetPart)) {
+ return new TargetsInPackage(
+ originalPattern, pattern, ":" + targetPart, isAbsolute, true, true);
+ }
+
+ if (ALL_TARGETS_IN_SUFFIXES.contains(targetPart)) {
+ return new TargetsInPackage(
+ originalPattern, pattern, ":" + targetPart, isAbsolute, false, true);
+ }
+
+
+ if (isAbsolute || pattern.contains(":")) {
+ String fullLabel = "//" + pattern;
+ try {
+ LabelValidator.validateAbsoluteLabel(fullLabel);
+ } catch (BadLabelException e) {
+ String error = "invalid target format '" + originalPattern + "': " + e.getMessage();
+ throw new TargetParsingException(error);
+ }
+ return new SingleTarget(fullLabel);
+ }
+
+ // This is a stripped-down version of interpretPathAsTarget that does no I/O. We have a basic
+ // relative path. e.g. "foo/bar/Wiz.java". The strictest correct check we can do here (without
+ // I/O) is just to ensure that there is *some* prefix that is a valid package-name. It's
+ // sufficient to test the first segment. This is really a rather weak check; perhaps we should
+ // just eliminate it.
+ int slashIndex = pattern.indexOf('/');
+ if (slashIndex < 0) {
+ throw new TargetParsingException("ambiguous target pattern: '" + pattern + "' could "
+ + "potentially be the name of a package; use '" + pattern + ":all' to mean \"all "
+ + "rules in this package\", '" + pattern + "/...' to mean \"all rules recursively "
+ + "beneath this package\", or '//" + pattern + "' to mean \"the default rule in this "
+ + "package\"");
+ }
+ String errorMessage = LabelValidator.validatePackageName(pattern.substring(0, slashIndex));
+ if (errorMessage != null) {
+ throw new TargetParsingException("Bad target pattern '" + originalPattern + "': " +
+ errorMessage);
+ }
+ return new InterpretPathAsTarget(pattern);
+ }
+
+ /**
+ * Absolutizes the target pattern to the offset.
+ * Patterns starting with "/" are absolute and not modified.
+ *
+ * If the offset is "foo":
+ * absolutize(":bar") --> "foo:bar"
+ * absolutize("bar") --> "foo/bar"
+ * absolutize("/biz/bar") --> "biz/bar" (absolute)
+ * absolutize("biz:bar") --> "foo/biz:bar"
+ *
+ * @param pattern The target pattern to parse.
+ * @return the pattern, absolutized to the offset if approprate.
+ */
+ private String absolutize(String pattern) {
+ if (relativeDirectory.isEmpty() || pattern.startsWith("/")) {
+ return pattern;
+ }
+
+ // It seems natural to use {@link PathFragment#getRelative()} here,
+ // but it doesn't work when the pattern starts with ":".
+ // "foo".getRelative(":all") would return "foo/:all", where we
+ // really want "foo:all".
+ return pattern.startsWith(":")
+ ? relativeDirectory + pattern
+ : relativeDirectory + "/" + pattern;
+ }
+ }
+
+ /**
+ * The target pattern type (targets below package, in package, explicit target, etc.)
+ */
+ public enum Type {
+ /** A path interpreted as a target, eg "foo/bar/baz" */
+ PATH_AS_TARGET,
+ /** An explicit target, eg "//foo:bar." */
+ SINGLE_TARGET,
+ /** Targets below a package, eg "foo/...". */
+ TARGETS_BELOW_PACKAGE,
+ /** Target in a package, eg "foo:all". */
+ TARGETS_IN_PACKAGE;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/TargetPatternResolver.java b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPatternResolver.java
new file mode 100644
index 0000000..109179f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPatternResolver.java
@@ -0,0 +1,94 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.cmdline;
+
+/**
+ * A callback interface that is used during the process of converting target patterns (such as
+ * <code>//foo:all</code>) into one or more lists of targets (such as <code>//foo:foo,
+ * //foo:bar</code>). During a call to {@link TargetPattern#eval}, the {@link TargetPattern} makes
+ * calls to this interface to implement the target pattern semantics. The generic type {@code T} is
+ * only for compile-time type safety; there are no requirements to the actual type.
+ */
+public interface TargetPatternResolver<T> {
+
+ /**
+ * Reports the given warning.
+ */
+ void warn(String msg);
+
+ /**
+ * Returns a single target corresponding to the given name, or null. This method may only throw an
+ * exception if the current thread was interrupted.
+ */
+ T getTargetOrNull(String targetName) throws InterruptedException, MissingDepException;
+
+ /**
+ * Returns a single target corresponding to the given name, or an empty or failed result.
+ */
+ ResolvedTargets<T> getExplicitTarget(String targetName)
+ throws TargetParsingException, InterruptedException, MissingDepException;
+
+ /**
+ * Returns the set containing the targets found in the given package. The specified directory is
+ * not necessarily a valid package name. If {@code rulesOnly} is true, then this method should
+ * only return rules in the given package.
+ *
+ * @param originalPattern the original target pattern for error reporting purposes
+ * @param packageName the name of the package
+ * @param rulesOnly whether to return rules only
+ */
+ ResolvedTargets<T> getTargetsInPackage(String originalPattern, String packageName,
+ boolean rulesOnly) throws TargetParsingException, InterruptedException, MissingDepException;
+
+ /**
+ * Returns the set containing the targets found below the given {@code pathPrefix}. Conceptually,
+ * this method should look for all packages that start with the {@code pathPrefix} (as a proper
+ * prefix directory, i.e., "foo/ba" is not a proper prefix of "foo/bar/"), and then collect all
+ * targets in each such package (subject to {@code rulesOnly}) as if calling {@link
+ * #getTargetsInPackage}. The specified directory is not necessarily a valid package name.
+ *
+ * <p>Note that the {@code pathPrefix} can be empty, which corresponds to the "//..." pattern.
+ * Implementations may choose not to support this case and throw an exception instead, or may
+ * restrict the set of directories that are considered by default.
+ *
+ * <p>If the {@code pathPrefix} points to a package, then that package should also be part of the
+ * result.
+ *
+ * @param originalPattern the original target pattern for error reporting purposes
+ * @param pathPrefix the directory in which to look for packages
+ * @param rulesOnly whether to return rules only
+ */
+ ResolvedTargets<T> findTargetsBeneathDirectory(String originalPattern, String pathPrefix,
+ boolean rulesOnly) throws TargetParsingException, InterruptedException, MissingDepException;
+
+ /**
+ * Returns true, if and only if the given name corresponds to a package, i.e., a file with the
+ * name {@code packageName/BUILD} exists.
+ */
+ boolean isPackage(String packageName) throws MissingDepException;
+
+ /**
+ * Returns the target kind of the given target, for example {@code cc_library rule}.
+ */
+ String getTargetKind(T target);
+
+ /**
+ * A missing dependency is needed before target parsing can proceed. Currently used only in
+ * skyframe to notify the framework of missing dependencies.
+ */
+ // TODO(bazel-team): Avoid this use of exception for expected control flow management.
+ public class MissingDepException extends Exception {
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/CollectionUtils.java b/src/main/java/com/google/devtools/build/lib/collect/CollectionUtils.java
new file mode 100644
index 0000000..d7b04bb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/CollectionUtils.java
@@ -0,0 +1,225 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Utilities for collection classes.
+ */
+public final class CollectionUtils {
+
+ private CollectionUtils() {}
+
+ /**
+ * Given a collection of elements and an equivalence relation, returns a new
+ * unordered collection of the disjoint subsets of those elements which are
+ * equivalent under the specified relation.
+ *
+ * <p>Note: the Comparator needs only to implement the less-strict contract
+ * of EquivalenceRelation (q.v.). (Hopefully this will one day be a
+ * superinterface of Comparator.)
+ *
+ * @param elements the collection of elements to be partitioned. May
+ * contain duplicates.
+ * @param equivalenceRelation an equivalence relation over the elements.
+ * @return a collection of sets of elements that are equivalent under the
+ * specified relation.
+ */
+ public static <T> Collection<Set<T>> partition(Collection<T> elements,
+ Comparator<T> equivalenceRelation) {
+ // TODO(bazel-team): (2009) O(n*m) where n=|elements| and m=|eqClasses|; i.e.,
+ // quadratic. Use Tarjan's algorithm instead.
+ List<Set<T>> eqClasses = new ArrayList<>();
+ for (T element : elements) {
+ boolean found = false;
+ for (Set<T> eqClass : eqClasses) {
+ if (equivalenceRelation.compare(eqClass.iterator().next(),
+ element) == 0) {
+ eqClass.add(element);
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ Set<T> eqClass = new HashSet<>();
+ eqClass.add(element);
+ eqClasses.add(eqClass);
+ }
+ }
+ return eqClasses;
+ }
+
+ /**
+ * See partition(Collection, Comparator).
+ */
+ public static <T> Collection<Set<T>> partition(Collection<T> elements,
+ final EquivalenceRelation<T> equivalenceRelation) {
+ return partition(elements, new Comparator<T>() {
+ @Override
+ public int compare(T o1, T o2) {
+ return equivalenceRelation.compare(o1, o2);
+ }
+ });
+ }
+
+ /**
+ * Returns the set of all elements in the given collection that appear more than once.
+ * @param input some collection.
+ * @return the set of repeated elements. May return an empty set, but never null.
+ */
+ public static <T> Set<T> duplicatedElementsOf(Collection<T> input) {
+ Set<T> duplicates = new HashSet<>();
+ Set<T> elementSet = new HashSet<>();
+ for (T el : input) {
+ if (!elementSet.add(el)) {
+ duplicates.add(el);
+ }
+ }
+ return duplicates;
+ }
+
+ /**
+ * Returns an immutable list of all non-null parameters in the order in which
+ * they are specified.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> ImmutableList<T> asListWithoutNulls(T... elements) {
+ ImmutableList.Builder<T> builder = ImmutableList.builder();
+ for (T element : elements) {
+ if (element != null) {
+ builder.add(element);
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns true if the given iterable can be verified to be immutable.
+ *
+ * <p>Note that if this method returns false, that does not mean that the iterable is mutable.
+ */
+ public static <T> boolean isImmutable(Iterable<T> iterable) {
+ return iterable instanceof ImmutableList<?>
+ || iterable instanceof ImmutableSet<?>
+ || iterable instanceof IterablesChain<?>
+ || iterable instanceof NestedSet<?>
+ || iterable instanceof ImmutableIterable<?>;
+ }
+
+ /**
+ * Throws a runtime exception if the given iterable can not be verified to be immutable.
+ */
+ public static <T> void checkImmutable(Iterable<T> iterable) {
+ Preconditions.checkState(isImmutable(iterable), iterable.getClass());
+ }
+
+ /**
+ * Given an iterable, returns an immutable iterable with the same contents.
+ */
+ public static <T> Iterable<T> makeImmutable(Iterable<T> iterable) {
+ if (isImmutable(iterable)) {
+ return iterable;
+ } else {
+ return ImmutableList.copyOf(iterable);
+ }
+ }
+
+ /**
+ * Converts a set of enum values to a bit field. Requires that the enum contains at most 32
+ * elements.
+ */
+ public static <T extends Enum<T>> int toBits(Set<T> values) {
+ int result = 0;
+ for (T value : values) {
+ // <p>Note that when the 32. bit is set, the integer becomes negative (because that is the
+ // sign bit). This does not affect the function of the bitwise operators, so it is fine.
+ Preconditions.checkArgument(value.ordinal() < 32);
+ result |= (1 << value.ordinal());
+ }
+
+ return result;
+ }
+
+ /**
+ * Converts a set of enum values to a bit field. Requires that the enum contains at most 32
+ * elements.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T extends Enum<T>> int toBits(T... values) {
+ return toBits(ImmutableSet.copyOf(values));
+ }
+
+ /**
+ * Converts a bit field to a set of enum values. Requires that the enum contains at most 32
+ * elements.
+ */
+ public static <T extends Enum<T>> EnumSet<T> fromBits(int value, Class<T> clazz) {
+ T[] elements = clazz.getEnumConstants();
+ Preconditions.checkArgument(elements.length <= 32);
+ ArrayList<T> result = new ArrayList<>();
+ for (T element : elements) {
+ if ((value & (1 << element.ordinal())) != 0) {
+ result.add(element);
+ }
+ }
+
+ return result.isEmpty() ? EnumSet.noneOf(clazz) : EnumSet.copyOf(result);
+ }
+
+ /**
+ * Returns whether an {@link Iterable} is a superset of another one.
+ */
+ public static <T> boolean containsAll(Iterable<T> superset, Iterable<T> subset) {
+ return ImmutableSet.copyOf(superset).containsAll(ImmutableList.copyOf(subset));
+ }
+
+ /**
+ * Returns an ImmutableMap of ImmutableMaps created from the Map of Maps parameter.
+ */
+ public static <KEY_1, KEY_2, VALUE> ImmutableMap<KEY_1, ImmutableMap<KEY_2, VALUE>> toImmutable(
+ Map<KEY_1, Map<KEY_2, VALUE>> map) {
+ ImmutableMap.Builder<KEY_1, ImmutableMap<KEY_2, VALUE>> builder = ImmutableMap.builder();
+ for (Map.Entry<KEY_1, Map<KEY_2, VALUE>> entry : map.entrySet()) {
+ builder.put(entry.getKey(), ImmutableMap.copyOf(entry.getValue()));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns a copy of the Map of Maps parameter.
+ */
+ public static <KEY_1, KEY_2, VALUE> Map<KEY_1, Map<KEY_2, VALUE>> copyOf(
+ Map<KEY_1, ? extends Map<KEY_2, VALUE>> map) {
+ Map<KEY_1, Map<KEY_2, VALUE>> result = new HashMap<>();
+ for (Map.Entry<KEY_1, ? extends Map<KEY_2, VALUE>> entry : map.entrySet()) {
+ result.put(entry.getKey(), new HashMap<>(entry.getValue()));
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/CompactHashSet.java b/src/main/java/com/google/devtools/build/lib/collect/CompactHashSet.java
new file mode 100644
index 0000000..5ee2417
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/CompactHashSet.java
@@ -0,0 +1,604 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+/*
+ * Copyright (C) 2012 The Guava Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.devtools.build.lib.collect;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.lang.reflect.Array;
+import java.util.AbstractSet;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * CompactHashSet is an implementation of a Set. All optional operations (adding and
+ * removing) are supported. The elements can be any objects.
+ *
+ * <p>{@code contains(x)}, {@code add(x)} and {@code remove(x)}, are all (expected and amortized)
+ * constant time operations. Expected in the hashtable sense (depends on the hash function
+ * doing a good job of distributing the elements to the buckets to a distribution not far from
+ * uniform), and amortized since some operations can trigger a hash table resize.
+ *
+ * <p>Unlike {@code java.util.HashSet}, iteration is only proportional to the actual
+ * {@code size()}, which is optimal, and <i>not</i> the size of the internal hashtable,
+ * which could be much larger than {@code size()}. Furthermore, this structure only depends
+ * on a fixed number of arrays; {@code add(x)} operations <i>do not</i> create objects
+ * for the garbage collector to deal with, and for every element added, the garbage collector
+ * will have to traverse {@code 1.5} references on average, in the marking phase, not {@code 5.0}
+ * as in {@code java.util.HashSet}.
+ *
+ * <p>If there are no removals, then {@link #iterator iteration} order is the same as insertion
+ * order. Any removal invalidates any ordering guarantees.
+ */
+// TODO(bazel-team): This was branched of an internal version of guava. If the class is released, we
+// should remove this again.
+public class CompactHashSet<E> extends AbstractSet<E> implements Serializable {
+ // TODO(bazel-team): cache all field accesses in local vars
+
+ // A partial copy of com.google.common.collect.Hashing.
+ private static final int C1 = 0xcc9e2d51;
+ private static final int C2 = 0x1b873593;
+
+ /*
+ * This method was rewritten in Java from an intermediate step of the Murmur hash function in
+ * http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp, which contained the
+ * following header:
+ *
+ * MurmurHash3 was written by Austin Appleby, and is placed in the public domain. The author
+ * hereby disclaims copyright to this source code.
+ */
+ private static int smear(int hashCode) {
+ return C2 * Integer.rotateLeft(hashCode * C1, 15);
+ }
+
+ private static int smearedHash(@Nullable Object o) {
+ return smear((o == null) ? 0 : o.hashCode());
+ }
+
+ private static final int MAX_TABLE_SIZE = Ints.MAX_POWER_OF_TWO;
+
+ private static int closedTableSize(int expectedEntries, double loadFactor) {
+ // Get the recommended table size.
+ // Round down to the nearest power of 2.
+ expectedEntries = Math.max(expectedEntries, 2);
+ int tableSize = Integer.highestOneBit(expectedEntries);
+ // Check to make sure that we will not exceed the maximum load factor.
+ if (expectedEntries > (int) (loadFactor * tableSize)) {
+ tableSize <<= 1;
+ return (tableSize > 0) ? tableSize : MAX_TABLE_SIZE;
+ }
+ return tableSize;
+ }
+
+ /**
+ * Creates an empty {@code CompactHashSet} instance.
+ */
+ public static <E> CompactHashSet<E> create() {
+ return new CompactHashSet<E>();
+ }
+
+ /**
+ * Creates a <i>mutable</i> {@code CompactHashSet} instance containing the elements
+ * of the given collection in unspecified order.
+ *
+ * @param collection the elements that the set should contain
+ * @return a new {@code CompactHashSet} containing those elements (minus duplicates)
+ */
+ public static <E> CompactHashSet<E> create(Collection<? extends E> collection) {
+ CompactHashSet<E> set = createWithExpectedSize(collection.size());
+ set.addAll(collection);
+ return set;
+ }
+
+ /**
+ * Creates a <i>mutable</i> {@code CompactHashSet} instance containing the given
+ * elements in unspecified order.
+ *
+ * @param elements the elements that the set should contain
+ * @return a new {@code CompactHashSet} containing those elements (minus duplicates)
+ */
+ @SafeVarargs
+ public static <E> CompactHashSet<E> create(E... elements) {
+ CompactHashSet<E> set = createWithExpectedSize(elements.length);
+ Collections.addAll(set, elements);
+ return set;
+ }
+
+ /**
+ * Creates a {@code CompactHashSet} instance, with a high enough "initial capacity"
+ * that it <i>should</i> hold {@code expectedSize} elements without growth.
+ *
+ * @param expectedSize the number of elements you expect to add to the returned set
+ * @return a new, empty {@code CompactHashSet} with enough capacity to hold {@code
+ * expectedSize} elements without resizing
+ * @throws IllegalArgumentException if {@code expectedSize} is negative
+ */
+ public static <E> CompactHashSet<E> createWithExpectedSize(int expectedSize) {
+ return new CompactHashSet<E>(expectedSize);
+ }
+
+ private static final int MAXIMUM_CAPACITY = 1 << 30;
+
+ // TODO(bazel-team): decide, and inline, load factor. 0.75?
+ private static final float DEFAULT_LOAD_FACTOR = 1.0f;
+
+ /**
+ * Bitmask that selects the low 32 bits.
+ */
+ private static final long NEXT_MASK = (1L << 32) - 1;
+
+ /**
+ * Bitmask that selects the high 32 bits.
+ */
+ private static final long HASH_MASK = ~NEXT_MASK;
+
+ // TODO(bazel-team): decide default size
+ private static final int DEFAULT_SIZE = 3;
+
+ static final int UNSET = -1;
+
+ /**
+ * The hashtable. Its values are indexes to both the elements and entries arrays.
+ *
+ * Currently, the UNSET value means "null pointer", and any non negative value x is
+ * the actual index.
+ *
+ * Its size must be a power of two.
+ */
+ private transient int[] table;
+
+ /**
+ * Contains the logical entries, in the range of [0, size()). The high 32 bits of each
+ * long is the smeared hash of the element, whereas the low 32 bits is the "next" pointer
+ * (pointing to the next entry in the bucket chain). The pointers in [size(), entries.length)
+ * are all "null" (UNSET).
+ */
+ private transient long[] entries;
+
+ /**
+ * The elements contained in the set, in the range of [0, size()).
+ */
+ transient Object[] elements;
+
+ /**
+ * The load factor.
+ */
+ transient float loadFactor;
+
+ /**
+ * Keeps track of modifications of this set, to make it possible to throw
+ * ConcurrentModificationException in the iterator. Note that we choose not to
+ * make this volatile, so we do less of a "best effort" to track such errors,
+ * for better performance.
+ */
+ transient int modCount;
+
+ /**
+ * When we have this many elements, resize the hashtable.
+ */
+ private transient int threshold;
+
+ /**
+ * The number of elements contained in the set.
+ */
+ private transient int size;
+
+ /**
+ * Constructs a new empty instance of {@code CompactHashSet}.
+ */
+ CompactHashSet() {
+ init(DEFAULT_SIZE, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Constructs a new instance of {@code CompactHashSet} with the specified capacity.
+ *
+ * @param expectedSize the initial capacity of this {@code CompactHashSet}.
+ */
+ CompactHashSet(int expectedSize) {
+ init(expectedSize, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * Pseudoconstructor for serialization support.
+ */
+ void init(int expectedSize, float loadFactor) {
+ Preconditions.checkArgument(expectedSize >= 0, "Initial capacity must be non-negative");
+ Preconditions.checkArgument(loadFactor > 0, "Illegal load factor");
+ int buckets = closedTableSize(expectedSize, loadFactor);
+ this.table = newTable(buckets);
+ this.loadFactor = loadFactor;
+ this.elements = new Object[expectedSize];
+ this.entries = newEntries(expectedSize);
+ this.threshold = Math.max(1, (int) (buckets * loadFactor));
+ }
+
+ private static int[] newTable(int size) {
+ int[] array = new int[size];
+ Arrays.fill(array, UNSET);
+ return array;
+ }
+
+ private static long[] newEntries(int size) {
+ long[] array = new long[size];
+ Arrays.fill(array, UNSET);
+ return array;
+ }
+
+ private static int getHash(long entry) {
+ return (int) (entry >>> 32);
+ }
+
+ /**
+ * Returns the index, or UNSET if the pointer is "null"
+ */
+ private static int getNext(long entry) {
+ return (int) entry;
+ }
+
+ /**
+ * Returns a new entry value by changing the "next" index of an existing entry
+ */
+ private static long swapNext(long entry, int newNext) {
+ return (HASH_MASK & entry) | (NEXT_MASK & newNext);
+ }
+
+ private int hashTableMask() {
+ return table.length - 1;
+ }
+
+ @Override
+ public boolean add(@Nullable E object) {
+ long[] entries = this.entries;
+ Object[] elements = this.elements;
+ int hash = smearedHash(object);
+ int tableIndex = hash & hashTableMask();
+ int newEntryIndex = this.size; // current size, and pointer to the entry to be appended
+ int next = table[tableIndex];
+ if (next == UNSET) { // uninitialized bucket
+ table[tableIndex] = newEntryIndex;
+ } else {
+ int last;
+ long entry;
+ do {
+ last = next;
+ entry = entries[next];
+ if (getHash(entry) == hash && Objects.equals(object, elements[next])) {
+ return false;
+ }
+ next = getNext(entry);
+ } while (next != UNSET);
+ entries[last] = swapNext(entry, newEntryIndex);
+ }
+ if (newEntryIndex == Integer.MAX_VALUE) {
+ throw new IllegalStateException("Cannot contain more than Integer.MAX_VALUE elements!");
+ }
+ int newSize = newEntryIndex + 1;
+ resizeMeMaybe(newSize);
+ insertEntry(newEntryIndex, object, hash);
+ this.size = newSize;
+ if (newEntryIndex >= threshold) {
+ resizeTable(2 * table.length);
+ }
+ modCount++;
+ return true;
+ }
+
+ /**
+ * Creates a fresh entry with the specified object at the specified position in the entry
+ * arrays.
+ */
+ void insertEntry(int entryIndex, E object, int hash) {
+ this.entries[entryIndex] = ((long) hash << 32) | (NEXT_MASK & UNSET);
+ this.elements[entryIndex] = object;
+ }
+
+ /**
+ * Returns currentSize + 1, after resizing the entries storage if necessary.
+ */
+ private void resizeMeMaybe(int newSize) {
+ int entriesSize = entries.length;
+ if (newSize > entriesSize) {
+ int newCapacity = entriesSize + Math.max(1, entriesSize >>> 1);
+ if (newCapacity < 0) {
+ newCapacity = Integer.MAX_VALUE;
+ }
+ if (newCapacity != entriesSize) {
+ resizeEntries(newCapacity);
+ }
+ }
+ }
+
+ /**
+ * Resizes the internal entries array to the specified capacity, which may be greater or less
+ * than the current capacity.
+ */
+ void resizeEntries(int newCapacity) {
+ this.elements = Arrays.copyOf(elements, newCapacity);
+ long[] entries = this.entries;
+ int oldSize = entries.length;
+ entries = Arrays.copyOf(entries, newCapacity);
+ if (newCapacity > oldSize) {
+ Arrays.fill(entries, oldSize, newCapacity, UNSET);
+ }
+ this.entries = entries;
+ }
+
+ private void resizeTable(int newCapacity) { // newCapacity always a power of two
+ int[] oldTable = table;
+ int oldCapacity = oldTable.length;
+ if (oldCapacity >= MAXIMUM_CAPACITY) {
+ threshold = Integer.MAX_VALUE;
+ return;
+ }
+ int newThreshold = 1 + (int) (newCapacity * loadFactor);
+ int[] newTable = newTable(newCapacity);
+ long[] entries = this.entries;
+
+ int mask = newTable.length - 1;
+ for (int i = 0; i < size; i++) {
+ long oldEntry = entries[i];
+ int hash = getHash(oldEntry);
+ int tableIndex = hash & mask;
+ int next = newTable[tableIndex];
+ newTable[tableIndex] = i;
+ entries[i] = ((long) hash << 32) | (NEXT_MASK & next);
+ }
+
+ this.threshold = newThreshold;
+ this.table = newTable;
+ }
+
+ @Override
+ public boolean contains(@Nullable Object object) {
+ int hash = smearedHash(object);
+ int next = table[hash & hashTableMask()];
+ while (next != UNSET) {
+ long entry = entries[next];
+ if (getHash(entry) == hash && Objects.equals(object, elements[next])) {
+ return true;
+ }
+ next = getNext(entry);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean remove(@Nullable Object object) {
+ return remove(object, smearedHash(object));
+ }
+
+ private boolean remove(Object object, int hash) {
+ int tableIndex = hash & hashTableMask();
+ int next = table[tableIndex];
+ if (next == UNSET) {
+ return false;
+ }
+ int last = UNSET;
+ do {
+ if (getHash(entries[next]) == hash && Objects.equals(object, elements[next])) {
+ if (last == UNSET) {
+ // we need to update the root link from table[]
+ table[tableIndex] = getNext(entries[next]);
+ } else {
+ // we need to update the link from the chain
+ entries[last] = swapNext(entries[last], getNext(entries[next]));
+ }
+
+ moveEntry(next);
+ size--;
+ modCount++;
+ return true;
+ }
+ last = next;
+ next = getNext(entries[next]);
+ } while (next != UNSET);
+ return false;
+ }
+
+ /**
+ * Moves the last entry in the entry array into {@code dstIndex}, and nulls out its old position.
+ */
+ void moveEntry(int dstIndex) {
+ int srcIndex = size() - 1;
+ if (dstIndex < srcIndex) {
+ // move last entry to deleted spot
+ elements[dstIndex] = elements[srcIndex];
+ elements[srcIndex] = null;
+
+ // move the last entry to the removed spot, just like we moved the element
+ long lastEntry = entries[srcIndex];
+ entries[dstIndex] = lastEntry;
+ entries[srcIndex] = UNSET;
+
+ // also need to update whoever's "next" pointer was pointing to the last entry place
+ // reusing "tableIndex" and "next"; these variables were no longer needed
+ int tableIndex = getHash(lastEntry) & hashTableMask();
+ int lastNext = table[tableIndex];
+ if (lastNext == srcIndex) {
+ // we need to update the root pointer
+ table[tableIndex] = dstIndex;
+ } else {
+ // we need to update a pointer in an entry
+ int previous;
+ long entry;
+ do {
+ previous = lastNext;
+ lastNext = getNext(entry = entries[lastNext]);
+ } while (lastNext != srcIndex);
+ // here, entries[previous] points to the old entry location; update it
+ entries[previous] = swapNext(entry, dstIndex);
+ }
+ } else {
+ elements[dstIndex] = null;
+ entries[dstIndex] = UNSET;
+ }
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return new Iterator<E>() {
+ int expectedModCount = modCount;
+ boolean nextCalled = false;
+ int index = 0;
+
+ @Override
+ public boolean hasNext() {
+ return index < size;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public E next() {
+ checkForConcurrentModification();
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ nextCalled = true;
+ return (E) elements[index++];
+ }
+
+ @Override
+ public void remove() {
+ checkForConcurrentModification();
+ Preconditions.checkState(nextCalled, "no calls to next() since the last call to remove()");
+ expectedModCount++;
+ index--;
+ CompactHashSet.this.remove(elements[index], getHash(entries[index]));
+ nextCalled = false;
+ }
+
+ private void checkForConcurrentModification() {
+ if (modCount != expectedModCount) {
+ throw new ConcurrentModificationException();
+ }
+ }
+ };
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return size == 0;
+ }
+
+ @Override
+ public Object[] toArray() {
+ return Arrays.copyOf(elements, size);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T[] toArray(T[] a) {
+ if (a.length < size) {
+ a = (T[]) Array.newInstance(a.getClass().getComponentType(), size);
+ }
+ System.arraycopy(elements, 0, a, 0, size);
+ return a;
+ }
+
+ /**
+ * Ensures that this {@code CompactHashSet} has the smallest representation in memory,
+ * given its current size.
+ */
+ public void trimToSize() {
+ int size = this.size;
+ if (size < entries.length) {
+ resizeEntries(size);
+ }
+ // size / loadFactor gives the table size of the appropriate load factor,
+ // but that may not be a power of two. We floor it to a power of two by
+ // keeping its highest bit. But the smaller table may have a load factor
+ // larger than what we want; then we want to go to the next power of 2 if we can
+ int minimumTableSize = Math.max(1, Integer.highestOneBit((int) (size / loadFactor)));
+ if (minimumTableSize < MAXIMUM_CAPACITY) {
+ double load = (double) size / minimumTableSize;
+ if (load > loadFactor) {
+ minimumTableSize <<= 1; // increase to next power if possible
+ }
+ }
+
+ if (minimumTableSize < table.length) {
+ resizeTable(minimumTableSize);
+ }
+ }
+
+ @Override
+ public void clear() {
+ modCount++;
+ Arrays.fill(elements, 0, size, null);
+ Arrays.fill(table, UNSET);
+ Arrays.fill(entries, UNSET);
+ this.size = 0;
+ }
+
+ private void writeObject(ObjectOutputStream stream) throws IOException {
+ stream.defaultWriteObject();
+ stream.writeInt(table.length);
+ stream.writeFloat(loadFactor);
+ stream.writeInt(size);
+ for (E e : this) {
+ stream.writeObject(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException {
+ stream.defaultReadObject();
+ int length = stream.readInt();
+ float loadFactor = stream.readFloat();
+ int elementCount = stream.readInt();
+ try {
+ init(length, loadFactor);
+ } catch (IllegalArgumentException e) {
+ throw new InvalidObjectException(e.getMessage());
+ }
+ for (int i = elementCount; --i >= 0;) {
+ E element = (E) stream.readObject();
+ add(element);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/EquivalenceRelation.java b/src/main/java/com/google/devtools/build/lib/collect/EquivalenceRelation.java
new file mode 100644
index 0000000..1596523
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/EquivalenceRelation.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.collect;
+
+/**
+ * A comparison function, which imposes an equivalence relation on some
+ * collection of objects.
+ *
+ * <p>The ordering imposed by an EquivalenceRelation <tt>e</tt> on a set of
+ * elements <tt>S</tt> is said to be <i>consistent with equals</i> if and only
+ * if <tt>(compare((Object)e1, (Object)e2)==0)</tt> has the same boolean value
+ * as <tt>e1.equals((Object)e2)</tt> for every <tt>e1</tt> and <tt>e2</tt> in
+ * <tt>S</tt>.<p>
+ *
+ * <p>Unlike {@link java.util.Comparator}, whose implementations are often
+ * consistent with equals, the applications for which EquivalenceRelation
+ * instances are used means that its implementations rarely are. They may are
+ * usually more or less discriminative than the default equivalence relation
+ * for the type.
+ *
+ * <p>For example, consider possible equivalence relations for {@link
+ * java.lang.Integer}: the default equivalence defined by Integer.equals() is
+ * based on the integer value is represents, but two alternative equivalences
+ * would be {@link EquivalenceRelation#IDENTITY} (object identity—a more
+ * discriminative relation) or <i>parity</i> (under which all even numbers, odd
+ * numbers are considered equivalent to each other—a less discriminative
+ * relation).
+ */
+public interface EquivalenceRelation<T> {
+ // This should be a superinterface of Comparator.
+
+ /**
+ * Compares its two arguments for equivalence. Returns zero if they are
+ * considered equivalent, or non-zero otherwise.<p>
+ *
+ * The implementor must ensure that the relation is
+ *
+ * reflexive (<tt>compare(x,x)==0</tt> for all x),
+ *
+ * symmetric (<tt>compare(x,y)==compare(y,x)<tt> for all x, y),
+ *
+ * and transitive <tt>(compare(x, y)==0 && compare(y,
+ * z)==0</tt> implies <tt>compare(x, z)==0</tt>.<p>
+ *
+ * @param o1 the first object to be compared.
+ * @param o2 the second object to be compared.
+ * @return zero if the two objects are equivalent; some other integer value
+ * otherwise.
+ * @throws ClassCastException if the arguments' types prevent them from
+ * being compared by this EquivalenceRelation.
+ */
+ int compare(T o1, T o2);
+
+ /**
+ * The object-identity equivalence relation. This is the strictest possible
+ * equivalence relation for objects, and considers two values equal iff they
+ * are references to the same object instance.
+ */
+ public static final EquivalenceRelation<?> IDENTITY =
+ new EquivalenceRelation<Object>() {
+ @Override
+ public int compare(Object o1, Object o2) {
+ return o1 == o2 ? 0 : -1;
+ }
+ };
+
+ /**
+ * The default equivalence relation for type T, using T.equals(). This
+ * relation considers two values equivalent if either they are both null, or
+ * o1.equals(o2).
+ */
+ public static final EquivalenceRelation<?> DEFAULT =
+ new EquivalenceRelation<Object>() {
+ @Override
+ public int compare(Object o1, Object o2) {
+ return (o1 == null ? o2 == null : o1.equals(o2))
+ ? 0
+ : -1;
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/ImmutableIterable.java b/src/main/java/com/google/devtools/build/lib/collect/ImmutableIterable.java
new file mode 100644
index 0000000..74fab83
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/ImmutableIterable.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.collect;
+
+import java.util.Iterator;
+
+/**
+ * A wrapper that signals the immutability of a certain iterable.
+ *
+ * <p>Intended for use in scenarios when you have an iterable that is de facto immutable,
+ * but is not recognized as such by {@link CollectionUtils#checkImmutable(Iterable)}.
+ *
+ * <p>Only use this when you know that the contents of the underlying iterable will never change,
+ * or you will be setting yourself up for aliasing bugs.
+ */
+public final class ImmutableIterable<T> implements Iterable<T> {
+
+ private final Iterable<T> iterable;
+
+ private ImmutableIterable(Iterable<T> iterable) {
+ this.iterable = iterable;
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return iterable.iterator();
+ }
+
+ /**
+ * Creates an {@link ImmutableIterable} instance.
+ */
+ // Use a factory method in order to avoid having to specify generic arguments.
+ public static <T> ImmutableIterable<T> from(Iterable<T> iterable) {
+ return new ImmutableIterable<>(iterable);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimap.java b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimap.java
new file mode 100644
index 0000000..2163378
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimap.java
@@ -0,0 +1,394 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultiset;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multiset;
+
+import java.util.AbstractCollection;
+import java.util.AbstractMap;
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A immutable multimap implementation for multimaps with comparable keys. It uses a sorted array
+ * and binary search to return the correct values. It's only purpose is to save memory - it consumes
+ * only about half the memory of the equivalent ImmutableListMultimap. Only a few methods are
+ * efficiently implemented: {@link #isEmpty} is O(1), {@link #get} and {@link #containsKey} are
+ * O(log(n)), and {@link #asMap} and {@link #values} refer to the parent instance. All other methods
+ * can take O(n) or even make a copy of the contents.
+ *
+ * <p>This implementation supports neither {@code null} keys nor {@code null} values.
+ */
+public final class ImmutableSortedKeyListMultimap<K extends Comparable<K>, V>
+ implements ListMultimap<K, V> {
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static final ImmutableSortedKeyListMultimap EMPTY_MULTIMAP =
+ new ImmutableSortedKeyListMultimap(new Comparable<?>[0], new List<?>[0]);
+
+ /** Returns the empty multimap. */
+ @SuppressWarnings("unchecked")
+ public static <K extends Comparable<K>, V> ImmutableSortedKeyListMultimap<K, V> of() {
+ // Safe because the multimap will never hold any elements.
+ return EMPTY_MULTIMAP;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <K extends Comparable<K>, V> ImmutableSortedKeyListMultimap<K, V> copyOf(
+ Multimap<K, V> data) {
+ if (data.isEmpty()) {
+ return EMPTY_MULTIMAP;
+ }
+ if (data instanceof ImmutableSortedKeyListMultimap) {
+ return (ImmutableSortedKeyListMultimap<K, V>) data;
+ }
+ Set<K> keySet = data.keySet();
+ int size = keySet.size();
+ K[] sortedKeys = (K[]) new Comparable<?>[size];
+ int index = 0;
+ for (K key : keySet) {
+ sortedKeys[index++] = Preconditions.checkNotNull(key);
+ }
+ Arrays.sort(sortedKeys);
+ List<V>[] values = (List<V>[]) new List<?>[size];
+ for (int i = 0; i < size; i++) {
+ values[i] = ImmutableList.copyOf(data.get(sortedKeys[i]));
+ }
+ return new ImmutableSortedKeyListMultimap<>(sortedKeys, values);
+ }
+
+ public static <K extends Comparable<K>, V> Builder<K, V> builder() {
+ return new Builder<>();
+ }
+
+ /**
+ * A builder class for ImmutableSortedKeyListMultimap<K, V> instances.
+ */
+ public static final class Builder<K extends Comparable<K>, V> {
+ private final Multimap<K, V> builderMultimap = ArrayListMultimap.create();
+
+ Builder() {
+ // Not public so you must call builder() instead.
+ }
+
+ public ImmutableSortedKeyListMultimap<K, V> build() {
+ return ImmutableSortedKeyListMultimap.copyOf(builderMultimap);
+ }
+
+ public Builder<K, V> put(K key, V value) {
+ builderMultimap.put(Preconditions.checkNotNull(key), Preconditions.checkNotNull(value));
+ return this;
+ }
+
+ public Builder<K, V> putAll(K key, Collection<? extends V> values) {
+ Collection<V> valueList = builderMultimap.get(Preconditions.checkNotNull(key));
+ for (V value : values) {
+ valueList.add(Preconditions.checkNotNull(value));
+ }
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Builder<K, V> putAll(K key, V... values) {
+ return putAll(Preconditions.checkNotNull(key), Arrays.asList(values));
+ }
+
+ public Builder<K, V> putAll(Multimap<? extends K, ? extends V> multimap) {
+ for (Map.Entry<? extends K, ? extends Collection<? extends V>> entry
+ : multimap.asMap().entrySet()) {
+ putAll(entry.getKey(), entry.getValue());
+ }
+ return this;
+ }
+ }
+
+ /**
+ * An implementation for the Multimap.asMap method. Note that AbstractMap already provides
+ * implementations for all methods except {@link #entrySet}, but we override a few here because we
+ * can do it much faster than the existing entrySet-based implementations. Also note that it
+ * inherits the type parameters K and V from the parent class.
+ */
+ private class AsMap extends AbstractMap<K, Collection<V>> {
+
+ AsMap() {
+ }
+
+ @Override
+ public int size() {
+ return sortedKeys.length;
+ }
+
+ @Override
+ public boolean containsKey(Object key) {
+ return ImmutableSortedKeyListMultimap.this.containsKey(key);
+ }
+
+ @Override
+ public Collection<V> get(Object key) {
+ int index = Arrays.binarySearch(sortedKeys, key);
+ // Note the different semantic between Map and Multimap.
+ return index >= 0 ? values[index] : null;
+ }
+
+ @Override
+ public Collection<V> remove(Object key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Set<Entry<K, Collection<V>>> entrySet() {
+ ImmutableSet.Builder<Entry<K, Collection<V>>> builder = ImmutableSet.builder();
+ for (int i = 0; i < sortedKeys.length; i++) {
+ builder.add(new SimpleImmutableEntry<K, Collection<V>>(sortedKeys[i], values[i]));
+ }
+ return builder.build();
+ }
+ }
+
+ private class ValuesCollection extends AbstractCollection<V> {
+
+ ValuesCollection() {
+ }
+
+ @Override
+ public int size() {
+ return ImmutableSortedKeyListMultimap.this.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return sortedKeys.length == 0;
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ return ImmutableSortedKeyListMultimap.this.containsValue(o);
+ }
+
+ @Override
+ public Iterator<V> iterator() {
+ if (isEmpty()) {
+ return Collections.emptyIterator();
+ }
+ return new AbstractIterator<V>() {
+ private int currentList = 0;
+ private int currentIndex = 0;
+
+ @Override
+ protected V computeNext() {
+ if (currentList >= values.length) {
+ return endOfData();
+ }
+ V result = values[currentList].get(currentIndex);
+ // Find the next list/index pair.
+ currentIndex++;
+ if (currentIndex >= values[currentList].size()) {
+ currentIndex = 0;
+ currentList++;
+ }
+ return result;
+ }
+ };
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private final K[] sortedKeys;
+ private final List<V>[] values;
+
+ private ImmutableSortedKeyListMultimap(K[] sortedKeys, List<V>[] values) {
+ this.sortedKeys = sortedKeys;
+ this.values = values;
+ }
+
+ @Override
+ public int size() {
+ int result = 0;
+ for (List<V> list : values) {
+ result += list.size();
+ }
+ return result;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return sortedKeys.length == 0;
+ }
+
+ @Override
+ public boolean containsKey(Object key) {
+ int index = Arrays.binarySearch(sortedKeys, key);
+ return index >= 0;
+ }
+
+ @Override
+ public boolean containsValue(Object value) {
+ for (List<V> list : values) {
+ if (list.contains(value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean containsEntry(Object key, Object value) {
+ int index = Arrays.binarySearch(sortedKeys, key);
+ if (index >= 0) {
+ return values[index].contains(value);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean put(K key, V value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean remove(Object key, Object value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean putAll(K key, Iterable<? extends V> values) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean putAll(Multimap<? extends K, ? extends V> multimap) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<V> replaceValues(K key, Iterable<? extends V> values) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<V> removeAll(Object key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<V> get(K key) {
+ int index = Arrays.binarySearch(sortedKeys, key);
+ return index >= 0 ? values[index] : ImmutableList.<V>of();
+ }
+
+ @Override
+ public Set<K> keySet() {
+ return ImmutableSet.copyOf(sortedKeys);
+ }
+
+ @Override
+ public Multiset<K> keys() {
+ return ImmutableMultiset.copyOf(sortedKeys);
+ }
+
+ @Override
+ public Collection<V> values() {
+ return new ValuesCollection();
+ }
+
+ @Override
+ public Collection<Entry<K, V>> entries() {
+ ImmutableList.Builder<Entry<K, V>> builder = ImmutableList.builder();
+ for (int i = 0; i < sortedKeys.length; i++) {
+ for (V value : values[i]) {
+ builder.add(new SimpleImmutableEntry<K, V>(sortedKeys[i], value));
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Note that only {@code get} and {@code containsKey} are implemented efficiently on the
+ * returned map.
+ */
+ @Override
+ public Map<K, Collection<V>> asMap() {
+ return new AsMap();
+ }
+
+ @Override
+ public String toString() {
+ return asMap().toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return asMap().hashCode();
+ }
+
+ @Override
+ public boolean equals(@Nullable Object object) {
+ if (this == object) {
+ return true;
+ }
+ if (object instanceof Multimap) {
+ Multimap<?, ?> that = (Multimap<?, ?>) object;
+ return asMap().equals(that.asMap());
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMap.java b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMap.java
new file mode 100644
index 0000000..e8f4621
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMap.java
@@ -0,0 +1,310 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.AbstractCollection;
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A immutable map implementation for maps with comparable keys. It uses a sorted array
+ * and binary search to return the correct values. Its only purpose is to save memory - for n
+ * entries, it consumes 8n + 64 bytes, much less than a normal HashMap (43n + 128) or an
+ * ImmutableMap (35n + 81).
+ *
+ * <p>Only a few methods are efficiently implemented: {@link #isEmpty} is O(1), {@link #get} and
+ * {@link #containsKey} are O(log(n)), using binary search; {@link #keySet} and {@link #values}
+ * refer to the parent instance. All other methods can take O(n) or even make a copy of the
+ * contents.
+ *
+ * <p>This implementation supports neither {@code null} keys nor {@code null} values.
+ *
+ * @param <K> the type of keys maintained by this map; keys must be comparable
+ * @param <V> the type of mapped values
+ */
+public final class ImmutableSortedKeyMap<K extends Comparable<K>, V> implements Map<K, V> {
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static final ImmutableSortedKeyMap EMPTY_MAP =
+ new ImmutableSortedKeyMap(new Comparable<?>[0], new Object[0]);
+
+ /** Returns the empty multimap. */
+ @SuppressWarnings("unchecked")
+ public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> of() {
+ // Safe because the multimap will never hold any elements.
+ return EMPTY_MAP;
+ }
+
+ public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> of(K key0, V value0) {
+ return ImmutableSortedKeyMap.<K, V>builder()
+ .put(key0, value0)
+ .build();
+ }
+
+ public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> of(
+ K key0, V value0, K key1, V value1) {
+ return ImmutableSortedKeyMap.<K, V>builder()
+ .put(key0, value0)
+ .put(key1, value1)
+ .build();
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> copyOf(Map<K, V> data) {
+ if (data.isEmpty()) {
+ return EMPTY_MAP;
+ }
+ if (data instanceof ImmutableSortedKeyMap) {
+ return (ImmutableSortedKeyMap<K, V>) data;
+ }
+ Set<K> keySet = data.keySet();
+ int size = keySet.size();
+ K[] sortedKeys = (K[]) new Comparable<?>[size];
+ int index = 0;
+ for (K key : keySet) {
+ sortedKeys[index] = Preconditions.checkNotNull(key);
+ index++;
+ }
+ Arrays.sort(sortedKeys);
+ V[] values = (V[]) new Object[size];
+ for (int i = 0; i < size; i++) {
+ values[i] = data.get(sortedKeys[i]);
+ }
+ return new ImmutableSortedKeyMap<>(sortedKeys, values);
+ }
+
+ public static <K extends Comparable<K>, V> Builder<K, V> builder() {
+ return new Builder<>();
+ }
+
+ /**
+ * A builder class for ImmutableSortedKeyListMultimap<K, V> instances.
+ */
+ public static final class Builder<K extends Comparable<K>, V> {
+ private final Map<K, V> builderMap = new HashMap<>();
+
+ Builder() {
+ // Not public so you must call builder() instead.
+ }
+
+ public ImmutableSortedKeyMap<K, V> build() {
+ return ImmutableSortedKeyMap.copyOf(builderMap);
+ }
+
+ public Builder<K, V> put(K key, V value) {
+ builderMap.put(Preconditions.checkNotNull(key), Preconditions.checkNotNull(value));
+ return this;
+ }
+
+ public Builder<K, V> putAll(Map<? extends K, ? extends V> map) {
+ for (Map.Entry<? extends K, ? extends V> entry : map.entrySet()) {
+ put(entry.getKey(), entry.getValue());
+ }
+ return this;
+ }
+ }
+
+ private class ValuesCollection extends AbstractCollection<V> {
+
+ ValuesCollection() {
+ }
+
+ @Override
+ public int size() {
+ return ImmutableSortedKeyMap.this.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return sortedKeys.length == 0;
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ return ImmutableSortedKeyMap.this.containsValue(o);
+ }
+
+ @Override
+ public Iterator<V> iterator() {
+ if (isEmpty()) {
+ return Collections.emptyIterator();
+ }
+ return new AbstractIterator<V>() {
+ private int currentIndex = 0;
+
+ @Override
+ protected V computeNext() {
+ if (currentIndex >= values.length) {
+ return endOfData();
+ }
+ return values[currentIndex++];
+ }
+ };
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private final K[] sortedKeys;
+ private final V[] values;
+
+ private ImmutableSortedKeyMap(K[] sortedKeys, V[] values) {
+ this.sortedKeys = sortedKeys;
+ this.values = values;
+ }
+
+ @Override
+ public int size() {
+ return sortedKeys.length;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return sortedKeys.length == 0;
+ }
+
+ @Override
+ public boolean containsKey(@Nullable Object key) {
+ if (key == null) {
+ return false;
+ }
+ int index = Arrays.binarySearch(sortedKeys, key);
+ return index >= 0;
+ }
+
+ @Override
+ public boolean containsValue(@Nullable Object value) {
+ if (value == null) {
+ return false;
+ }
+ for (V v : values) {
+ if (v.equals(value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public V put(K key, V value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public V remove(Object key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void putAll(Map<? extends K, ? extends V> map) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public V get(@Nullable Object key) {
+ if (key == null) {
+ return null;
+ }
+ int index = Arrays.binarySearch(sortedKeys, key);
+ return index >= 0 ? values[index] : null;
+ }
+
+ @Override
+ public Set<K> keySet() {
+ return ImmutableSet.copyOf(sortedKeys);
+ }
+
+ @Override
+ public Collection<V> values() {
+ return new ValuesCollection();
+ }
+
+ @Override
+ public Set<Entry<K, V>> entrySet() {
+ ImmutableSet.Builder<Entry<K, V>> builder = ImmutableSet.builder();
+ for (int i = 0; i < sortedKeys.length; i++) {
+ builder.add(new SimpleImmutableEntry<K, V>(sortedKeys[i], values[i]));
+ }
+ return builder.build();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ result.append('{');
+ for (int i = 0; i < sortedKeys.length; i++) {
+ if (i != 0) {
+ result.append(", ");
+ }
+ result.append(sortedKeys[i]).append('=').append(values[i]);
+ }
+ result.append('}');
+ return result.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ int h = 0;
+ for (Entry<K, V> entry : entrySet()) {
+ h += entry.hashCode();
+ }
+ return h;
+ }
+
+ @Override
+ public boolean equals(@Nullable Object object) {
+ if (this == object) {
+ return true;
+ }
+ if (object instanceof Map) {
+ throw new UnsupportedOperationException();
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/IterablesChain.java b/src/main/java/com/google/devtools/build/lib/collect/IterablesChain.java
new file mode 100644
index 0000000..15182b6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/IterablesChain.java
@@ -0,0 +1,159 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.collect;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * An immutable chain of immutable Iterables.
+ *
+ * <p>This class is defined for the sole purpose of being able to check for immutability
+ * (using instanceof). Otherwise, we could use a plain Iterable (as returned by
+ * {@code Iterables.concat()}).
+ *
+ * @see CollectionUtils#checkImmutable(Iterable)
+ */
+public final class IterablesChain<T> implements Iterable<T> {
+
+ private final Iterable<T> chain;
+
+ private IterablesChain(Iterable<T> chain) {
+ this.chain = chain;
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return chain.iterator();
+ }
+
+ public static <T> Builder<T> builder() {
+ return new Builder<>();
+ }
+
+ @Override
+ public String toString() {
+ return "[" + Joiner.on(", ").join(this) + "]";
+ }
+
+ /**
+ * Builder for IterablesChain.
+ *
+ *
+ */
+ public static class Builder<T> {
+ private List<Iterable<T>> iterables = new ArrayList<>();
+ private boolean deduplicate;
+
+ private Builder() {
+ }
+
+ /**
+ * Adds an immutable iterable to the end of the chain.
+ *
+ * <p>If the iterable can not be confirmed to be immutable, a runtime error is thrown.
+ */
+ public Builder<T> add(Iterable<T> iterable) {
+ CollectionUtils.checkImmutable(iterable);
+ if (!Iterables.isEmpty(iterable)) {
+ iterables.add(iterable);
+ }
+ return this;
+ }
+
+ /**
+ * Adds a single element to the chain.
+ */
+ public Builder<T> addElement(T element) {
+ iterables.add(ImmutableList.of(element));
+ return this;
+ }
+
+ /**
+ * Returns true if the chain is empty.
+ */
+ public boolean isEmpty() {
+ return iterables.isEmpty();
+ }
+
+ /**
+ * If this is called, the the resulting {@link IterablesChain} object uses a hash set to remove
+ * duplicate elements.
+ */
+ public Builder<T> deduplicate() {
+ this.deduplicate = true;
+ return this;
+ }
+
+ /**
+ * Builds an iterable that iterates through all elements in this chain.
+ */
+ public IterablesChain<T> build() {
+ if (isEmpty()) {
+ return new IterablesChain<>(ImmutableList.<T>of());
+ }
+ Iterable<T> concat = Iterables.concat(ImmutableList.copyOf(iterables));
+ return new IterablesChain<>(deduplicate ? new Deduper<>(concat) : concat);
+ }
+ }
+
+ /**
+ * An iterable implementation that removes duplicate elements (as determined by equals), using a
+ * hash set.
+ */
+ private static final class Deduper<T> implements Iterable<T> {
+ private final Iterable<T> iterable;
+
+ public Deduper(Iterable<T> iterable) {
+ this.iterable = iterable;
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return new DedupingIterator<T>(iterable.iterator());
+ }
+ }
+
+ /**
+ * An iterator implementation that removes duplicate elements (as determined by equals), using a
+ * hash set.
+ */
+ private static final class DedupingIterator<T> extends AbstractIterator<T> {
+ private final HashSet<T> set = new HashSet<>();
+ private final Iterator<T> it;
+
+ public DedupingIterator(Iterator<T> it) {
+ this.it = it;
+ }
+
+ @Override
+ protected T computeNext() {
+ while (it.hasNext()) {
+ T next = it.next();
+ if (set.add(next)) {
+ return next;
+ }
+ }
+ return endOfData();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpander.java
new file mode 100644
index 0000000..5ac8610
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpander.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableCollection;
+
+/**
+ * A nested set expander that implements left-to-right postordering.
+ *
+ * <p>For example, for the nested set {B, D, {A, C}}, the iteration order is "A C B D"
+ * (child-first).
+ *
+ * <p>This type of set would typically be used for artifacts where elements of nested sets go before
+ * the direct members of a set, for example in the case of constructing Java classpaths.
+ */
+final class CompileOrderExpander<E> implements NestedSetExpander<E> {
+
+ // We suppress unchecked warning so that we can access the internal raw structure of the
+ // NestedSet.
+ @SuppressWarnings("unchecked")
+ @Override
+ public void expandInto(NestedSet<E> set, Uniqueifier uniqueifier,
+ ImmutableCollection.Builder<E> builder) {
+ for (NestedSet<E> subset : set.transitiveSets()) {
+ if (!subset.isEmpty() && uniqueifier.isUnique(subset)) {
+ expandInto(subset, uniqueifier, builder);
+ }
+ }
+
+ // This switch is here to compress the memo used by the uniqueifier
+ for (Object e : set.directMembers()) {
+ if (uniqueifier.isUnique(e)) {
+ builder.add((E) e);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderNestedSetFactory.java
new file mode 100644
index 0000000..178f9a5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderNestedSetFactory.java
@@ -0,0 +1,152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Compile order {@code NestedSet} factory.
+ */
+final class CompileOrderNestedSetFactory implements NestedSetFactory {
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(Object[] directs) {
+ return new CompileOnlyDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) {
+ return new CompileOrderImmutableListDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) {
+ return new CompileOneDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct,
+ NestedSet<E> transitive) {
+ return new CompileManyDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) {
+ return new CompileOnlyOneTransitiveNestedSet<>(transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) {
+ return new CompileOnlyTransitivesNestedSet<>(transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) {
+ return new CompileOneDirectManyTransitive<>(direct, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ return new CompileManyDirectManyTransitive<>(directs, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirect(E element) {
+ return new CompileSingleDirectNestedSet<>(element);
+ }
+
+ private static class CompileOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> {
+
+ CompileOnlyDirectsNestedSet(Object[] directs) { super(directs); }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileOneDirectOneTransitiveNestedSet<E> extends
+ OneDirectOneTransitiveNestedSet<E> {
+
+ private CompileOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> {
+
+ private CompileOneDirectManyTransitive(Object direct, NestedSet[] transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> {
+
+ private CompileManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ super(directs, transitives);
+ }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileOnlyOneTransitiveNestedSet<E> extends OnlyOneTransitiveNestedSet<E> {
+
+ private CompileOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileManyDirectOneTransitiveNestedSet<E> extends
+ ManyDirectOneTransitiveNestedSet<E> {
+
+ private CompileManyDirectOneTransitiveNestedSet(Object[] direct,
+ NestedSet<E> transitive) { super(direct, transitive); }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> {
+
+ private CompileOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+
+ private static class CompileOrderImmutableListDirectsNestedSet<E> extends
+ ImmutableListDirectsNestedSet<E> {
+
+ private CompileOrderImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); }
+
+ @Override
+ public Order getOrder() {
+ return Order.COMPILE_ORDER;
+ }
+ }
+
+ private static class CompileSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> {
+
+ private CompileSingleDirectNestedSet(E element) { super(element); }
+
+ @Override
+ public Order getOrder() { return Order.COMPILE_ORDER; }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/EmptyNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/EmptyNestedSet.java
new file mode 100644
index 0000000..5889ba8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/EmptyNestedSet.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * An empty nested set.
+ */
+final class EmptyNestedSet<E> extends NestedSet<E> {
+ private static final NestedSet[] EMPTY_NESTED_SET = new NestedSet[0];
+ private static final Object[] EMPTY_ELEMENTS = new Object[0];
+ private final Order order;
+
+ EmptyNestedSet(Order type) {
+ this.order = type;
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return ImmutableList.<E>of().iterator();
+ }
+
+ @Override
+ Object[] directMembers() {
+ return EMPTY_ELEMENTS;
+ }
+
+ @Override
+ NestedSet[] transitiveSets() {
+ return EMPTY_NESTED_SET;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return true;
+ }
+
+ @Override
+ public List<E> toList() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Set<E> toSet() {
+ return ImmutableSet.of();
+ }
+
+ @Override
+ public String toString() {
+ return "{}";
+ }
+
+ @Override
+ public Order getOrder() {
+ return order;
+ }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ return other != null && getOrder() == other.getOrder() && other.isEmpty();
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Objects.hash(getOrder());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/ImmutableListDirectsNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ImmutableListDirectsNestedSet.java
new file mode 100644
index 0000000..12bf222
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ImmutableListDirectsNestedSet.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-optimized NestedSet implementation for NestedSets without transitive dependencies that
+ * allows us to share an ImmutableList.
+ */
+abstract class ImmutableListDirectsNestedSet<E> extends NestedSet<E> {
+
+ private static final NestedSet[] EMPTY = new NestedSet[0];
+ private final ImmutableList<E> directDeps;
+
+ public ImmutableListDirectsNestedSet(ImmutableList<E> directDeps) {
+ this.directDeps = directDeps;
+ }
+
+ @Override
+ public abstract Order getOrder();
+
+ @Override
+ Object[] directMembers() {
+ return directDeps.toArray();
+ }
+
+ @Override
+ NestedSet[] transitiveSets() {
+ return EMPTY;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return directDeps.isEmpty();
+ }
+
+ /**
+ * Currently all the Order implementations return the direct elements in the same order if they do
+ * not have transitive elements. So we skip calling order.getExpander().
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<E> toList() {
+ return directDeps;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Set<E> toSet() {
+ return ImmutableSet.copyOf(directDeps);
+ }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null) {
+ return false;
+ }
+ return getOrder().equals(other.getOrder())
+ && other instanceof ImmutableListDirectsNestedSet
+ && directDeps.equals(((ImmutableListDirectsNestedSet) other).directDeps);
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return directDeps.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpander.java
new file mode 100644
index 0000000..603ac15
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpander.java
@@ -0,0 +1,105 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * A nested set expander that implements a variation of left-to-right preordering.
+ *
+ * <p>For example, for the nested set {A, C, {B, D}}, the iteration order is "A C B D"
+ * (parent-first).
+ *
+ * <p>This type of set would typically be used for artifacts where elements of
+ * nested sets go after the direct members of a set, for example when providing
+ * a list of libraries to the C++ compiler.
+ *
+ * <p>The custom ordering has the property that elements of nested sets always come
+ * before elements of descendant nested sets. Left-to-right order is preserved if
+ * possible, both for items and for references to nested sets.
+ *
+ * <p>The left-to-right pre-order-like ordering is implemented by running a
+ * right-to-left postorder traversal and then reversing the result.
+ *
+ * <p>The reason naive left-to left-to-right preordering is not used here is that
+ * it does not handle diamond-like structures properly. For example, take the
+ * following structure (nesting downwards):
+ *
+ * <pre>
+ * A
+ * / \
+ * B C
+ * \ /
+ * D
+ * </pre>
+ *
+ * <p>Naive preordering would produce "A B D C", which does not preserve the
+ * "parent before child" property: C is a parent of D, so C should come before
+ * D. Either "A B C D" or "A C B D" would be acceptable. This implementation
+ * returns the first option of the two so that left-to-right order is preserved.
+ *
+ * <p>In case the nested sets form a tree, the ordering algorithm is equivalent to
+ * standard left-to-right preorder.
+ *
+ * <p>Sometimes it may not be possible to preserve left-to-right order:
+ *
+ * <pre>
+ * A
+ * / \
+ * B C
+ * / \ / \
+ * \ E /
+ * \ /
+ * \ /
+ * D
+ * </pre>
+ *
+ * <p>The left branch (B) would indicate "D E" ordering and the right branch (C)
+ * dictates "E D". In such cases ordering is decided by the rightmost branch
+ * because of the list reversing behind the scenes, so the ordering in the final
+ * enumeration will be "E D".
+ */
+
+final class LinkOrderExpander<E> implements NestedSetExpander<E> {
+ @Override
+ public void expandInto(NestedSet<E> nestedSet, Uniqueifier uniqueifier,
+ ImmutableCollection.Builder<E> builder) {
+ ImmutableList.Builder<E> result = ImmutableList.builder();
+ internalEnumerate(nestedSet, uniqueifier, result);
+ builder.addAll(result.build().reverse());
+ }
+
+ // We suppress unchecked warning so that we can access the internal raw structure of the
+ // NestedSet.
+ @SuppressWarnings("unchecked")
+ private void internalEnumerate(NestedSet<E> set, Uniqueifier uniqueifier,
+ ImmutableCollection.Builder<E> builder) {
+ NestedSet[] transitiveSets = set.transitiveSets();
+ for (int i = transitiveSets.length - 1; i >= 0; i--) {
+ NestedSet<E> subset = transitiveSets[i];
+ if (!subset.isEmpty() && uniqueifier.isUnique(subset)) {
+ internalEnumerate(subset, uniqueifier, builder);
+ }
+ }
+
+ Object[] directMembers = set.directMembers();
+ for (int i = directMembers.length - 1; i >= 0; i--) {
+ Object e = directMembers[i];
+ if (uniqueifier.isUnique(e)) {
+ builder.add((E) e);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderNestedSetFactory.java
new file mode 100644
index 0000000..9e23793
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderNestedSetFactory.java
@@ -0,0 +1,152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Link order {@code NestedSet} factory.
+ */
+final class LinkOrderNestedSetFactory implements NestedSetFactory {
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(Object[] directs) {
+ return new LinkOnlyDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) {
+ return new LinkImmutableListDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) {
+ return new LinkOneDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct,
+ NestedSet<E> transitive) {
+ return new LinkManyDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) {
+ return new LinkOnlyOneTransitiveNestedSet<>(transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) {
+ return new LinkOnlyTransitivesNestedSet<>(transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) {
+ return new LinkOneDirectManyTransitive<>(direct, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ return new LinkManyDirectManyTransitive<>(directs, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirect(E element) {
+ return new LinkSingleDirectNestedSet<>(element);
+ }
+
+ private static class LinkOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> {
+
+ LinkOnlyDirectsNestedSet(Object[] directs) { super(directs); }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkOneDirectOneTransitiveNestedSet<E> extends
+ OneDirectOneTransitiveNestedSet<E> {
+
+ private LinkOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> {
+
+ private LinkOneDirectManyTransitive(Object direct, NestedSet[] transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> {
+
+ private LinkManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ super(directs, transitives);
+ }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkOnlyOneTransitiveNestedSet<E> extends OnlyOneTransitiveNestedSet<E> {
+
+ private LinkOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkManyDirectOneTransitiveNestedSet<E> extends
+ ManyDirectOneTransitiveNestedSet<E> {
+
+ private LinkManyDirectOneTransitiveNestedSet(Object[] direct,
+ NestedSet<E> transitive) { super(direct, transitive); }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> {
+
+ private LinkOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+
+ private static class LinkImmutableListDirectsNestedSet<E> extends
+ ImmutableListDirectsNestedSet<E> {
+
+ private LinkImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); }
+
+ @Override
+ public Order getOrder() {
+ return Order.LINK_ORDER;
+ }
+ }
+
+ private static class LinkSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> {
+
+ private LinkSingleDirectNestedSet(E element) { super(element); }
+
+ @Override
+ public Order getOrder() { return Order.LINK_ORDER; }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectManyTransitive.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectManyTransitive.java
new file mode 100644
index 0000000..05ba2e8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectManyTransitive.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * NestedSet implementation that can have many direct elements and many transitive
+ * {@code NestedSet}s.
+ */
+abstract class ManyDirectManyTransitive<E> extends MemoizedUniquefierNestedSet<E> {
+
+ private final Object[] directs;
+ private final NestedSet[] transitives;
+ private Object memo;
+
+ ManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ this.directs = directs;
+ this.transitives = transitives;
+ }
+
+ @Override
+ Object getMemo() { return memo; }
+
+ @Override
+ void setMemo(Object memo) { this.memo = memo; }
+
+ @Override
+ Object[] directMembers() { return directs; }
+
+ @Override
+ NestedSet[] transitiveSets() { return transitives; }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other != null
+ && getOrder().equals(other.getOrder())
+ && other instanceof ManyDirectManyTransitive
+ && Arrays.equals(directs, ((ManyDirectManyTransitive) other).directs)
+ && Arrays.equals(transitives, ((ManyDirectManyTransitive) other).transitives);
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Objects.hash(getOrder(), Arrays.hashCode(directs), Arrays.hashCode(transitives)); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectOneTransitiveNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectOneTransitiveNestedSet.java
new file mode 100644
index 0000000..cdb4f04
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectOneTransitiveNestedSet.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-efficient implementation for the case where we have many direct elements and one
+ * transitive NestedSet.
+ */
+abstract class ManyDirectOneTransitiveNestedSet<E> extends MemoizedUniquefierNestedSet<E> {
+
+ private final Object[] directs;
+ private final NestedSet<E> transitive;
+ private Object memo;
+
+ public ManyDirectOneTransitiveNestedSet(Object[] directs, NestedSet<E> transitive) {
+ this.directs = directs;
+ this.transitive = transitive;
+ }
+
+ @Override
+ Object getMemo() { return memo; }
+
+ @Override
+ void setMemo(Object memo) { this.memo = memo; }
+
+ @Override
+ Object[] directMembers() { return directs; }
+
+ @Override
+ NestedSet[] transitiveSets() { return new NestedSet[]{transitive}; }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other != null
+ && getOrder().equals(other.getOrder())
+ && other instanceof ManyDirectOneTransitiveNestedSet
+ && Arrays.equals(directs, ((ManyDirectOneTransitiveNestedSet) other).directs)
+ && transitive == ((ManyDirectOneTransitiveNestedSet) other).transitive;
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Objects.hash(getOrder(), Arrays.hashCode(directs), transitive); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/MemoizedUniquefierNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/MemoizedUniquefierNestedSet.java
new file mode 100644
index 0000000..2a7f1b6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/MemoizedUniquefierNestedSet.java
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A NestedSet that keeps a memoized uniquifier so that it is faster to fill a set.
+ *
+ * <p>This class does not keep the memoized object itself so that we can take advantage of the
+ * memory field alignment (Memory alignment does not put in the same structure the fields of a
+ * class and its extensions).
+ */
+public abstract class MemoizedUniquefierNestedSet<E> extends NestedSet<E> {
+
+ @Override
+ public List<E> toList() {
+ ImmutableList.Builder<E> builder = new ImmutableList.Builder<>();
+ memoizedFill(builder);
+ return builder.build();
+ }
+
+ @Override
+ public Set<E> toSet() {
+ ImmutableSet.Builder<E> builder = new ImmutableSet.Builder<>();
+ memoizedFill(builder);
+ return builder.build();
+ }
+
+ /**
+ * It does not make sense to have a {@code MemoizedUniquefierNestedSet} if it is empty.
+ */
+ @Override
+ public boolean isEmpty() { return false; }
+
+ abstract Object getMemo();
+
+ abstract void setMemo(Object object);
+
+ /**
+ * Fill a collection builder by using a memoized {@code Uniqueifier} for faster uniqueness check.
+ */
+ final void memoizedFill(ImmutableCollection.Builder<E> builder) {
+ Uniqueifier memoed;
+ synchronized (this) {
+ Object memo = getMemo();
+ if (memo == null) {
+ RecordingUniqueifier uniqueifier = new RecordingUniqueifier();
+ getOrder().<E>expander().expandInto(this, uniqueifier, builder);
+ setMemo(uniqueifier.getMemo());
+ return;
+ } else {
+ memoed = RecordingUniqueifier.createReplayUniqueifier(memo);
+ }
+ }
+ getOrder().<E>expander().expandInto(this, memoed, builder);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpander.java
new file mode 100644
index 0000000..6c49103
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpander.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableCollection;
+
+/**
+ * A nested set expander that implements naive left-to-right preordering.
+ *
+ * <p>For example, for the nested set {B, D, {A, C}}, the iteration order is "B D A C".
+ *
+ * <p>This implementation is intended for backwards-compatible nested set replacements of code that
+ * uses naive preordering.
+ *
+ * <p>The implementation is called naive because it does no special treatment of dependency graphs
+ * that are not trees. For such graphs the property of parent-before-dependencies in the iteration
+ * order will not be upheld. For example, the diamond-shape graph A->{B, C}, B->{D}, C->{D} will be
+ * enumerated as "A B D C" rather than "A B C D" or "A C B D".
+ *
+ * <p>The difference from {@link LinkOrderNestedSet} is that this implementation gives priority to
+ * left-to-right order over dependencies-after-parent ordering. Note that the latter is usually more
+ * important, so please use {@link LinkOrderNestedSet} whenever possible.
+ */
+final class NaiveLinkOrderExpander<E> implements NestedSetExpander<E> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void expandInto(NestedSet<E> set, Uniqueifier uniqueifier,
+ ImmutableCollection.Builder<E> builder) {
+
+ for (Object e : set.directMembers()) {
+ if (uniqueifier.isUnique(e)) {
+ builder.add((E) e);
+ }
+ }
+
+ for (NestedSet<E> subset : set.transitiveSets()) {
+ if (!subset.isEmpty() && uniqueifier.isUnique(subset)) {
+ expandInto(subset, uniqueifier, builder);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderNestedSetFactory.java
new file mode 100644
index 0000000..4677938
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderNestedSetFactory.java
@@ -0,0 +1,153 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * NaiveLink order {@code NestedSet} factory.
+ */
+final class NaiveLinkOrderNestedSetFactory implements NestedSetFactory {
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(Object[] directs) {
+ return new NaiveLinkOnlyDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) {
+ return new NaiveLinkImmutableListDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) {
+ return new NaiveLinkOneDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct,
+ NestedSet<E> transitive) {
+ return new NaiveLinkManyDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) {
+ return new NaiveLinkOnlyOneTransitiveNestedSet<>(transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) {
+ return new NaiveLinkOnlyTransitivesNestedSet<>(transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) {
+ return new NaiveLinkOneDirectManyTransitive<>(direct, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ return new NaiveLinkManyDirectManyTransitive<>(directs, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirect(final E element) {
+ return new NaiveLinkSingleDirectNestedSet<>(element);
+ }
+
+ private static class NaiveLinkOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> {
+
+ NaiveLinkOnlyDirectsNestedSet(Object[] directs) { super(directs); }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkOneDirectOneTransitiveNestedSet<E> extends
+ OneDirectOneTransitiveNestedSet<E> {
+
+ private NaiveLinkOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> {
+
+ private NaiveLinkOneDirectManyTransitive(Object direct, NestedSet[] transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> {
+
+ private NaiveLinkManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ super(directs, transitives);
+ }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkOnlyOneTransitiveNestedSet<E>
+ extends OnlyOneTransitiveNestedSet<E> {
+
+ private NaiveLinkOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkManyDirectOneTransitiveNestedSet<E> extends
+ ManyDirectOneTransitiveNestedSet<E> {
+
+ private NaiveLinkManyDirectOneTransitiveNestedSet(Object[] direct,
+ NestedSet<E> transitive) { super(direct, transitive); }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> {
+
+ private NaiveLinkOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+
+ private static class NaiveLinkImmutableListDirectsNestedSet<E> extends
+ ImmutableListDirectsNestedSet<E> {
+
+ private NaiveLinkImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); }
+
+ @Override
+ public Order getOrder() {
+ return Order.NAIVE_LINK_ORDER;
+ }
+ }
+
+ private static class NaiveLinkSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> {
+
+ private NaiveLinkSingleDirectNestedSet(E element) { super(element); }
+
+ @Override
+ public Order getOrder() { return Order.NAIVE_LINK_ORDER; }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSet.java
new file mode 100644
index 0000000..da074e0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSet.java
@@ -0,0 +1,127 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.base.Joiner;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A list-like iterable that supports efficient nesting.
+ *
+ * @see NestedSetBuilder
+ */
+public abstract class NestedSet<E> implements Iterable<E>, Serializable {
+
+ NestedSet() {}
+
+ /**
+ * Returns the ordering of this nested set.
+ */
+ public abstract Order getOrder();
+
+ /**
+ * Returns a collection of elements added to this specific set in an implementation-specified
+ * order.
+ *
+ * <p>Elements from subsets are not taken into account.
+ *
+ * <p>The reason for using Object[] instead of E[] is that when we build the NestedSet we
+ * would need to have access to the specific class that E represents in order to create an E
+ * array. Since this method is only designed to be used internally it is fine to keep it as
+ * Object[].
+ *
+ * <p>Callers of this method should only consume the objects and not modify the array.
+ */
+ abstract Object[] directMembers();
+
+ /**
+ * Returns the collection of sets included as subsets in this set.
+ *
+ * <p>Callers of this method should only consume the objects and not modify the array.
+ */
+ abstract NestedSet[] transitiveSets();
+
+ /**
+ * Returns true if the set is empty.
+ */
+ public abstract boolean isEmpty();
+
+ /**
+ * Returns a collection of all unique elements of this set (including subsets)
+ * in an implementation-specified order as a {@code Collection}.
+ *
+ * <p>If you do not need a Collection and an Iterable is enough, use the
+ * nested set itself as an Iterable.
+ */
+ public Collection<E> toCollection() {
+ return toList();
+ }
+
+ /**
+ * Returns a collection of all unique elements of this set (including subsets)
+ * in an implementation-specified order as a {code List}.
+ *
+ * <p>Use {@link #toCollection} when possible for better efficiency.
+ */
+ public abstract List<E> toList();
+
+ /**
+ * Returns a collection of all unique elements of this set (including subsets)
+ * in an implementation-specified order as a {@code Set}.
+ *
+ * <p>Use {@link #toCollection} when possible for better efficiency.
+ */
+ public abstract Set<E> toSet();
+
+ /**
+ * Returns true if this set is equal to {@code other} based on the top-level
+ * elements and object identity (==) of direct subsets. As such, this function
+ * can fail to equate {@code this} with another {@code NestedSet} that holds
+ * the same elements. It will never fail to detect that two {@code NestedSet}s
+ * are different, however.
+ *
+ * @param other the {@code NestedSet} to compare against.
+ */
+ public abstract boolean shallowEquals(@Nullable NestedSet<? extends E> other);
+
+ /**
+ * Returns a hash code that produces a notion of identity that is consistent with
+ * {@link #shallowEquals}. In other words, if two {@code NestedSet}s are equal according
+ * to {@code #shallowEquals}, then they return the same {@code shallowHashCode}.
+ *
+ * <p>The main reason for having these separate functions instead of reusing
+ * the standard equals/hashCode is to minimize accidental use, since they are
+ * different from both standard Java objects and collection-like objects.
+ */
+ public abstract int shallowHashCode();
+
+ @Override
+ public String toString() {
+ String members = Joiner.on(", ").join(directMembers());
+ String nestedSets = Joiner.on(", ").join(transitiveSets());
+ String separator = members.length() > 0 && nestedSets.length() > 0 ? ", " : "";
+ return "{" + members + separator + nestedSets + "}";
+ }
+
+ @Override
+ public Iterator<E> iterator() { return new NestedSetLazyIterator<>(this); }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetBuilder.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetBuilder.java
new file mode 100644
index 0000000..327fcdb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetBuilder.java
@@ -0,0 +1,253 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import static com.google.common.collect.Iterables.getOnlyElement;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+import java.util.LinkedHashSet;
+
+/**
+ * A builder for nested sets.
+ *
+ * <p>The builder supports the standard builder interface (that is, {@code #add}, {@code #addAll}
+ * and {@code #addTransitive} followed by {@code build}), in addition to shortcut methods
+ * {@code #wrap} and {@code #of}.
+ */
+public final class NestedSetBuilder<E> {
+
+ private final Order order;
+ private final LinkedHashSet<E> items = new LinkedHashSet<>();
+ private final LinkedHashSet<NestedSet<? extends E>> transitiveSets = new LinkedHashSet<>();
+
+ public NestedSetBuilder(Order order) {
+ this.order = order;
+ }
+
+ /** Returns whether the set to be built is empty. */
+ public boolean isEmpty() {
+ return items.isEmpty() && transitiveSets.isEmpty();
+ }
+
+ /**
+ * Add an element.
+ *
+ * <p>Preserves ordering of added elements. Discards duplicate values.
+ * Throws an exception if a null value is passed in.
+ *
+ * <p>The collections of the direct members of the set and the nested sets are
+ * kept separate, so the order between multiple add/addAll calls matters,
+ * and the order between multiple addTransitive calls matters, but the order
+ * between add/addAll and addTransitive does not.
+ *
+ * @return the builder.
+ */
+ @SuppressWarnings("unchecked") // B is the type of the concrete subclass
+ public NestedSetBuilder<E> add(E element) {
+ Preconditions.checkNotNull(element);
+ items.add(element);
+ return this;
+ }
+
+ /**
+ * Adds a collection of elements to the set.
+ *
+ * <p>This is equivalent to invoking {@code add} for every item of the collection in iteration
+ * order.
+ *
+ * <p>The collections of the direct members of the set and the nested sets are kept separate, so
+ * the order between multiple add/addAll calls matters, and the order between multiple
+ * addTransitive calls matters, but the order between add/addAll and addTransitive does not.
+ *
+ * @return the builder.
+ */
+ @SuppressWarnings("unchecked") // B is the type of the concrete subclass
+ public NestedSetBuilder<E> addAll(Iterable<? extends E> elements) {
+ Preconditions.checkNotNull(elements);
+ Iterables.addAll(items, elements);
+ return this;
+ }
+
+ /**
+ * @deprecated Use {@link #addTransitive} to avoid excessive memory use.
+ */
+ @Deprecated
+ public NestedSetBuilder<E> addAll(NestedSet<E> elements) {
+ // Do not delete this method, or else addAll(Iterable) calls with a NestedSet argument
+ // will not be flagged.
+ Iterable<E> it = elements;
+ addAll(it);
+ return this;
+ }
+
+ /**
+ * Adds another nested set to this set.
+ *
+ * <p>Preserves ordering of added nested sets. Discards duplicate values. Throws an exception if
+ * a null value is passed in.
+ *
+ * <p>The collections of the direct members of the set and the nested sets are kept separate, so
+ * the order between multiple add/addAll calls matters, and the order between multiple
+ * addTransitive calls matters, but the order between add/addAll and addTransitive does not.
+ *
+ * <p>An error will be thrown if the ordering of {@code subset} is incompatible with the ordering
+ * of this set. Either they must match or this set must be a {@code STABLE_ORDER} set.
+ *
+ * @return the builder.
+ */
+ public NestedSetBuilder<E> addTransitive(NestedSet<? extends E> subset) {
+ Preconditions.checkNotNull(subset);
+ if (subset.getOrder() != order && order != Order.STABLE_ORDER
+ && subset.getOrder() != Order.STABLE_ORDER) {
+ // Note that this check is not strictly necessary, although keeping the nested set types
+ // consistent helps readability and protects against bugs. The polymorphism regarding
+ // STABLE_ORDER is allowed in order to be able to, e.g., include an arbitrary nested set in
+ // the inputs of an action, or include a nested set that is indifferent to its order in
+ // multiple nested sets.
+ throw new IllegalStateException(subset.getOrder() + " != " + order);
+ }
+ if (!subset.isEmpty()) {
+ transitiveSets.add(subset);
+ }
+ return this;
+ }
+
+ /**
+ * Builds the actual nested set.
+ *
+ * <p>This method may be called multiple times with interleaved {@link #add}, {@link #addAll} and
+ * {@link #addTransitive} calls.
+ */
+ // Casting from LinkedHashSet<NestedSet<? extends E>> to LinkedHashSet<NestedSet<E>> by way of
+ // LinkedHashSet<?>.
+ @SuppressWarnings("unchecked")
+ public NestedSet<E> build() {
+ if (isEmpty()) {
+ return order.emptySet();
+ }
+
+ // This cast is safe because NestedSets are immutable -- we will never try to add an element to
+ // these nested sets, only to retrieve elements from them. Thus, treating them as NestedSet<E>
+ // is safe.
+ LinkedHashSet<NestedSet<E>> transitiveSetsCast =
+ (LinkedHashSet<NestedSet<E>>) (LinkedHashSet<?>) transitiveSets;
+ if (items.isEmpty() && (transitiveSetsCast.size() == 1)) {
+ NestedSet<E> candidate = getOnlyElement(transitiveSetsCast);
+ if (candidate.getOrder().equals(order)) {
+ return candidate;
+ }
+ }
+ int transitiveSize = transitiveSets.size();
+ int directSize = items.size();
+
+ switch (transitiveSize) {
+ case 0:
+ switch (directSize) {
+ case 0:
+ return order.emptySet();
+ case 1:
+ return order.factory.oneDirect(getOnlyElement(items));
+ default:
+ return order.factory.onlyDirects(items.toArray());
+ }
+ case 1:
+ switch (directSize) {
+ case 0:
+ return order.factory.onlyOneTransitive(getOnlyElement(transitiveSetsCast));
+ case 1:
+ return order.factory.oneDirectOneTransitive(getOnlyElement(items),
+ getOnlyElement(transitiveSetsCast));
+ default:
+ return order.factory.manyDirectsOneTransitive(items.toArray(),
+ getOnlyElement(transitiveSetsCast));
+ }
+ default:
+ switch (directSize) {
+ case 0:
+ return order.factory.onlyManyTransitives(
+ transitiveSetsCast.toArray(new NestedSet[transitiveSize]));
+ case 1:
+ return order.factory.oneDirectManyTransitive(getOnlyElement(items), transitiveSetsCast
+ .toArray(new NestedSet[transitiveSize]));
+ default:
+ return order.factory.manyDirectManyTransitive(items.toArray(),
+ transitiveSetsCast.toArray(new NestedSet[transitiveSize]));
+ }
+ }
+ }
+
+ /**
+ * Creates a nested set from a given list of items.
+ *
+ * <p>If the list of items is an {@link ImmutableList}, reuses the list as the backing store for
+ * the nested set.
+ */
+ public static <E> NestedSet<E> wrap(Order order, Iterable<E> wrappedItems) {
+ ImmutableList<E> wrappedList = ImmutableList.copyOf(wrappedItems);
+ if (wrappedList.isEmpty()) {
+ return order.emptySet();
+ } else if (wrappedList.size() == 1) {
+ return order.factory.oneDirect(getOnlyElement(wrappedItems));
+ } else {
+ return order.factory.onlyDirects(wrappedList);
+ }
+ }
+
+
+ /**
+ * Creates a nested set with the given list of items as its elements.
+ */
+ @SuppressWarnings("unchecked")
+ public static <E> NestedSet<E> create(Order order, E... elems) {
+ return wrap(order, ImmutableList.copyOf(elems));
+ }
+
+ /**
+ * Creates an empty nested set.
+ */
+ public static <E> NestedSet<E> emptySet(Order order) {
+ return order.emptySet();
+ }
+
+ /**
+ * Creates a builder for stable order nested sets.
+ */
+ public static <E> NestedSetBuilder<E> stableOrder() {
+ return new NestedSetBuilder<>(Order.STABLE_ORDER);
+ }
+
+ /**
+ * Creates a builder for compile order nested sets.
+ */
+ public static <E> NestedSetBuilder<E> compileOrder() {
+ return new NestedSetBuilder<>(Order.COMPILE_ORDER);
+ }
+
+ /**
+ * Creates a builder for link order nested sets.
+ */
+ public static <E> NestedSetBuilder<E> linkOrder() {
+ return new NestedSetBuilder<>(Order.LINK_ORDER);
+ }
+
+ /**
+ * Creates a builder for naive link order nested sets.
+ */
+ public static <E> NestedSetBuilder<E> naiveLinkOrder() {
+ return new NestedSetBuilder<>(Order.NAIVE_LINK_ORDER);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetExpander.java
new file mode 100644
index 0000000..c04c39d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetExpander.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableCollection;
+
+/**
+ * An expander that converts a nested set into a flattened collection.
+ *
+ * <p>Expanders are initialized statically (there is one for each order), so they should
+ * contain no state and all methods must be threadsafe.
+ */
+interface NestedSetExpander<E> {
+ /**
+ * Flattens the NestedSet into the builder.
+ */
+ void expandInto(NestedSet<E> nestedSet, Uniqueifier uniqueifier,
+ ImmutableCollection.Builder<E> builder);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetFactory.java
new file mode 100644
index 0000000..99fb8bb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetFactory.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Factory methods for creating {@link NestedSet}s of specific shapes. This allows the
+ * implementation to be memory efficient (e.g. a specialized implementation for the case where
+ * there are only direct elements, etc).
+ *
+ * <p>It's intended for each {@link Order} to have its own factory implementation. That way we can
+ * be even more efficient since the {@link NestedSet}s instances don't need to store their
+ * {@link Order}.
+ */
+interface NestedSetFactory {
+
+ /** Create a NestedSet with just one direct element and not transitive elements. */
+ <E> NestedSet<E> oneDirect(E element);
+
+ /** Create a NestedSet with only direct elements. */
+ <E> NestedSet<E> onlyDirects(Object[] directs);
+
+ /** Create a NestedSet with only direct elements potentially sharing the ImmutableList. */
+ <E> NestedSet<E> onlyDirects(ImmutableList<E> directs);
+
+ /** Create a NestedSet with one direct element and one transitive {@code NestedSet}. */
+ <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive);
+
+ /** Create a NestedSet with many direct elements and one transitive {@code NestedSet}. */
+ <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct, NestedSet<E> transitive);
+
+ /** Create a NestedSet with no direct elements and one transitive {@code NestedSet.} */
+ <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive);
+
+ /** Create a NestedSet with no direct elements and many transitive {@code NestedSet}s. */
+ <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives);
+
+ /** Create a NestedSet with one direct elements and many transitive {@code NestedSet}s. */
+ <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitive);
+
+ /** Create a NestedSet with many direct elements and many transitive {@code NestedSet}s. */
+ <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitive);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetLazyIterator.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetLazyIterator.java
new file mode 100644
index 0000000..c873d56
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetLazyIterator.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Iterator;
+
+/**
+ * A NestedSet iterator that only expands the NestedSet when the first element is requested. This
+ * allows code that calls unconditionally to {@code hasNext} to check if the iterator is empty
+ * to not expand the nested set.
+ */
+final class NestedSetLazyIterator<E> implements Iterator<E> {
+
+ private NestedSet<E> nestedSet;
+ private Iterator<E> delegate = null;
+
+ NestedSetLazyIterator(NestedSet<E> nestedSet) {
+ this.nestedSet = nestedSet;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (delegate == null) {
+ return !nestedSet.isEmpty();
+ }
+ return delegate.hasNext();
+ }
+
+ @Override
+ public E next() {
+ if (delegate == null) {
+ delegate = nestedSet.toCollection().iterator();
+ nestedSet = null;
+ }
+ return delegate.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetVisitor.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetVisitor.java
new file mode 100644
index 0000000..ea8b810
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetVisitor.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+/**
+ * NestedSetVisitor facilitates a transitive visitation over a NestedSet, which must be in STABLE
+ * order. The callback may be called from multiple threads, and must be thread-safe.
+ *
+ * <p>The visitation is iterative: The caller may invoke a NestedSet within the top-level NestedSet
+ * in any order.
+ *
+ * <p>Currently this class is only used in Skyframe to facilitate iterative replay of transitive
+ * warnings/errors.
+ *
+ * @param <E> the data type
+ */
+// @ThreadSafety.ThreadSafe
+public final class NestedSetVisitor<E> {
+
+ /**
+ * For each element of the NestedSet the {@code Reciver} will receive one element during the
+ * visitation.
+ */
+ public interface Receiver<E> {
+ void accept(E arg);
+ }
+
+ private final Receiver<E> callback;
+
+ private final VisitedState<E> visited;
+
+ public NestedSetVisitor(Receiver<E> callback, VisitedState<E> visited) {
+ this.callback = Preconditions.checkNotNull(callback);
+ this.visited = Preconditions.checkNotNull(visited);
+ }
+
+ /**
+ * Transitively visit a nested set.
+ *
+ * @param nestedSet the nested set to visit transitively.
+ *
+ */
+ @SuppressWarnings("unchecked")
+ public void visit(NestedSet<E> nestedSet) {
+ // This method suppresses the unchecked warning so that it can access the internal NestedSet
+ // raw structure.
+ Preconditions.checkArgument(nestedSet.getOrder() == Order.STABLE_ORDER);
+ if (!visited.add(nestedSet)) {
+ return;
+ }
+
+ for (NestedSet<E> subset : nestedSet.transitiveSets()) {
+ visit(subset);
+ }
+ for (Object member : nestedSet.directMembers()) {
+ if (visited.add((E) member)) {
+ callback.accept((E) member);
+ }
+ }
+ }
+
+ /** A class that allows us to keep track of the seen nodes and transitive sets. */
+ public static class VisitedState<E> {
+ private final Set<NestedSet<E>> seenSets = Sets.newConcurrentHashSet();
+ private final Set<E> seenNodes = Sets.newConcurrentHashSet();
+
+ public void clear() {
+ seenSets.clear();
+ seenNodes.clear();
+ }
+
+ private boolean add(E node) {
+ return seenNodes.add(node);
+ }
+
+ private boolean add(NestedSet<E> set) {
+ return seenSets.add(set);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectManyTransitive.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectManyTransitive.java
new file mode 100644
index 0000000..a99883c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectManyTransitive.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-optimized NestedSet implementation for NestedSets with one direct element and
+ * many transitive dependencies.
+ */
+abstract class OneDirectManyTransitive<E> extends MemoizedUniquefierNestedSet<E> {
+
+ private final Object direct;
+ private final NestedSet[] transitives;
+ private Object memo;
+
+ OneDirectManyTransitive(Object direct, NestedSet[] transitives) {
+ this.direct = direct;
+ this.transitives = transitives;
+ }
+
+ @Override
+ Object getMemo() { return memo; }
+
+ @Override
+ void setMemo(Object memo) { this.memo = memo; }
+
+ @Override
+ Object[] directMembers() { return new Object[]{direct}; }
+
+ @Override
+ NestedSet[] transitiveSets() { return transitives; }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other != null
+ && getOrder().equals(other.getOrder())
+ && other instanceof OneDirectManyTransitive
+ && direct.equals(((OneDirectManyTransitive) other).direct)
+ && Arrays.equals(transitives, ((OneDirectManyTransitive) other).transitives);
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Objects.hash(getOrder(), direct, Arrays.hashCode(transitives)); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectOneTransitiveNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectOneTransitiveNestedSet.java
new file mode 100644
index 0000000..9acdba1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectOneTransitiveNestedSet.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-efficient implementation for the case where we have one direct element and one
+ * transitive NestedSet.
+ */
+abstract class OneDirectOneTransitiveNestedSet<E> extends MemoizedUniquefierNestedSet<E> {
+
+ private final E direct;
+ private final NestedSet<E> transitive;
+ private Object memo;
+
+ OneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) {
+ this.direct = direct;
+ this.transitive = transitive;
+ }
+
+ @Override
+ Object getMemo() { return memo; }
+
+ @Override
+ void setMemo(Object memo) { this.memo = memo; }
+
+ @Override
+ Object[] directMembers() { return new Object[]{direct}; }
+
+ @Override
+ NestedSet[] transitiveSets() { return new NestedSet[]{transitive}; }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other != null
+ && getOrder().equals(other.getOrder())
+ && other instanceof OneDirectOneTransitiveNestedSet
+ && direct.equals(((OneDirectOneTransitiveNestedSet) other).direct)
+ && transitive == ((OneDirectOneTransitiveNestedSet) other).transitive;
+ }
+
+ @Override
+ public int shallowHashCode() { return Objects.hash(getOrder(), direct, transitive); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyDirectsNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyDirectsNestedSet.java
new file mode 100644
index 0000000..9f7588d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyDirectsNestedSet.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-optimized NestedSet implementation for NestedSets without transitive dependencies.
+ *
+ */
+abstract class OnlyDirectsNestedSet<E> extends NestedSet<E> {
+
+ private static final NestedSet[] EMPTY = new NestedSet[0];
+ private final Object[] directDeps;
+
+ public OnlyDirectsNestedSet(Object[] directDeps) { this.directDeps = directDeps; }
+
+ @Override
+ public abstract Order getOrder();
+
+ @Override
+ Object[] directMembers() {
+ return directDeps;
+ }
+
+ @Override
+ NestedSet[] transitiveSets() {
+ return EMPTY;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+
+ /**
+ * Currently all the Order implementations return the direct elements in the same order if they
+ * do not have transitive elements. So we skip calling order.getExpander()... and return a view
+ * of the array.
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<E> toList() {
+ return (List<E>) ImmutableList.copyOf(directDeps);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Set<E> toSet() {
+ return (Set<E>) ImmutableSet.copyOf(directDeps);
+ }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null) {
+ return false;
+ }
+ return getOrder().equals(other.getOrder())
+ && other instanceof OnlyDirectsNestedSet
+ && Arrays.equals(directDeps, ((OnlyDirectsNestedSet) other).directDeps);
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Arrays.hashCode(directDeps);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyOneTransitiveNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyOneTransitiveNestedSet.java
new file mode 100644
index 0000000..a3e2d1d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyOneTransitiveNestedSet.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-efficient implementation for the case where we only have one transitive NestedSet.
+ *
+ * <p>Note that we cannot simply delegate to the inner NestedSet because the order could be
+ * different and the top-level order is the correct one.
+ */
+abstract class OnlyOneTransitiveNestedSet<E> extends MemoizedUniquefierNestedSet<E> {
+
+ private static final Object[] EMPTY = new Object[0];
+
+ private final NestedSet<E> transitive;
+ private Object memo;
+
+ public OnlyOneTransitiveNestedSet(NestedSet<E> transitive) {
+ this.transitive = transitive;
+ }
+
+ @Override
+ Object getMemo() { return memo; }
+
+ @Override
+ void setMemo(Object memo) { this.memo = memo; }
+
+ @Override
+ Object[] directMembers() {
+ return EMPTY;
+ }
+
+ @Override
+ NestedSet[] transitiveSets() {
+ return new NestedSet[]{transitive};
+ }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other != null
+ && getOrder().equals(other.getOrder())
+ && other instanceof OnlyOneTransitiveNestedSet
+ && transitive == ((OnlyOneTransitiveNestedSet) other).transitive;
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Objects.hash(getOrder(), transitive);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyTransitivesNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyTransitivesNestedSet.java
new file mode 100644
index 0000000..5a57053
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyTransitivesNestedSet.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-efficient implementation for the case where we have one direct element and one
+ * transitive NestedSet.
+ */
+abstract class OnlyTransitivesNestedSet<E> extends MemoizedUniquefierNestedSet<E> {
+
+ private static final NestedSet[] EMPTY = new NestedSet[0];
+
+ private final NestedSet[] transitives;
+ private Object memo;
+
+ OnlyTransitivesNestedSet(NestedSet[] transitives) {
+ this.transitives = transitives;
+ }
+
+ @Override
+ Object getMemo() { return memo; }
+
+ @Override
+ void setMemo(Object memo) { this.memo = memo; }
+
+ @Override
+ Object[] directMembers() { return EMPTY; }
+
+ @Override
+ NestedSet[] transitiveSets() { return transitives; }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other != null
+ && getOrder().equals(other.getOrder())
+ && other instanceof OnlyTransitivesNestedSet
+ && Arrays.equals(transitives, ((OnlyTransitivesNestedSet) other).transitives);
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return Objects.hash(getOrder(), Arrays.hashCode(transitives)); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java
new file mode 100644
index 0000000..38e6633
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+/**
+ * Type of a nested set (defines order).
+ */
+public enum Order {
+
+ STABLE_ORDER(new CompileOrderExpander<>(), new StableOrderNestedSetFactory()),
+ COMPILE_ORDER(new CompileOrderExpander<>(), new CompileOrderNestedSetFactory()),
+ LINK_ORDER(new LinkOrderExpander<>(), new LinkOrderNestedSetFactory()),
+ NAIVE_LINK_ORDER(new NaiveLinkOrderExpander<>(), new NaiveLinkOrderNestedSetFactory());
+
+ private final NestedSetExpander<?> expander;
+ final NestedSetFactory factory;
+ private final NestedSet<?> emptySet;
+
+ private Order(NestedSetExpander<?> expander, NestedSetFactory factory) {
+ this.expander = expander;
+ this.factory = factory;
+ this.emptySet = new EmptyNestedSet<Object>(this);
+ }
+
+ /**
+ * Returns an empty set of the given ordering.
+ */
+ @SuppressWarnings("unchecked") // Nested sets are immutable, so a downcast is fine.
+ <E> NestedSet<E> emptySet() {
+ return (NestedSet<E>) emptySet;
+ }
+
+ /**
+ * Returns an empty set of the given ordering.
+ */
+ @SuppressWarnings("unchecked") // Nested set expanders contain no data themselves.
+ <E> NestedSetExpander<E> expander() {
+ return (NestedSetExpander<E>) expander;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifier.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifier.java
new file mode 100644
index 0000000..c71f200
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifier.java
@@ -0,0 +1,140 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import java.util.BitSet;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A Uniqueifier that records a transcript of its interactions with the underlying Set. A memo can
+ * then be retrieved in the form of another Uniqueifier, and given the same sequence of isUnique
+ * calls, the Set interactions can be avoided.
+ */
+class RecordingUniqueifier implements Uniqueifier {
+ /**
+ * Unshared byte memos under this length are not constructed.
+ */
+ @VisibleForTesting static final int LENGTH_THRESHOLD = 4096 / 8; // bits -> bytes
+
+ /**
+ * Returned as a marker that memoization was not worthwhile.
+ */
+ private static final Object NO_MEMO = new Object();
+
+ /**
+ * Shared one-byte memos.
+ */
+ private static final byte[][] SHARED_SMALL_MEMOS_1;
+
+ /**
+ * Shared two-byte memos.
+ */
+ private static final byte[][] SHARED_SMALL_MEMOS_2;
+
+ static {
+ // Create interned arrays for one and two byte memos
+ // The memos always start with 0x3, so some one and two byte arrays can be skipped.
+
+ byte[][] memos1 = new byte[64][1];
+ for (int i = 0; i < 64; i++) {
+ memos1[i][0] = (byte) ((i << 2) | 0x3);
+ }
+ SHARED_SMALL_MEMOS_1 = memos1;
+
+ byte[][] memos2 = new byte[16384][2];
+ for (int i = 0; i < 64; i++) {
+ byte iAdj = (byte) (0x3 | (i << 2));
+ for (int j = 0; j < 256; j++) {
+ int idx = i | (j << 6);
+ memos2[idx][0] = iAdj;
+ memos2[idx][1] = (byte) j;
+ }
+ }
+ SHARED_SMALL_MEMOS_2 = memos2;
+ }
+
+ private final Set<Object> witnessed = new HashSet<>(256);
+ private final BitSet memo = new BitSet();
+ private int idx = 0;
+
+ static Uniqueifier createReplayUniqueifier(Object memo) {
+ if (memo == NO_MEMO) {
+ return new RecordingUniqueifier();
+ } else if (memo instanceof Integer) {
+ BitSet bs = new BitSet();
+ bs.set(0, (Integer) memo);
+ return new ReplayUniqueifier(bs);
+ }
+ return new ReplayUniqueifier(BitSet.valueOf((byte[]) memo));
+ }
+
+ @Override
+ public boolean isUnique(Object o) {
+ boolean firstInstance = witnessed.add(o);
+ memo.set(idx++, firstInstance);
+ return firstInstance;
+ }
+
+ /**
+ * Gets the memo of the set interactions. Do not call isUnique after this point.
+ */
+ Object getMemo() {
+ this.idx = -1; // will cause failures if isUnique is called after getMemo.
+
+ // If the bitset is just a contiguous block of ones, use a length memo
+ int length = memo.length();
+ if (memo.cardinality() == length) {
+ return length; // length-based memo
+ }
+
+ byte[] ba = memo.toByteArray();
+
+ Preconditions.checkState(
+ (length < 2) || ((ba[0] & 3) == 3),
+ "The memo machinery expects memos to always begin with two 1 bits, "
+ + "but instead, this memo starts with %X.", ba[0]);
+
+ // For short memos, use an interned array for the memo
+ if (ba.length == 1) {
+ return SHARED_SMALL_MEMOS_1[(0xFF & ba[0]) >>> 2]; // shared memo
+ } else if (ba.length == 2) {
+ return SHARED_SMALL_MEMOS_2[((ba[1] & 0xFF) << 6) | ((0xFF & ba[0]) >>> 2)];
+ }
+
+ // For mid-sized cases, skip the memo since it is not worthwhile
+ if (ba.length < LENGTH_THRESHOLD) {
+ return NO_MEMO; // skipped memo
+ }
+
+ return ba; // normal memo
+ }
+
+ private static final class ReplayUniqueifier implements Uniqueifier {
+ private final BitSet memo;
+ private int idx = 0;
+
+ ReplayUniqueifier(BitSet memo) {
+ this.memo = memo;
+ }
+
+ @Override
+ public boolean isUnique(Object o) {
+ return memo.get(idx++);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/SingleDirectNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/SingleDirectNestedSet.java
new file mode 100644
index 0000000..dc4a5fb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/SingleDirectNestedSet.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterators;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Memory-efficient implementation for nested sets with one element.
+ */
+public abstract class SingleDirectNestedSet<E> extends NestedSet<E> {
+
+ private static final NestedSet[] EMPTY = new NestedSet[0];
+ private final E e;
+
+ public SingleDirectNestedSet(E e) { this.e = Preconditions.checkNotNull(e); }
+
+ @Override
+ public Iterator<E> iterator() { return Iterators.singletonIterator(e); }
+
+ @Override
+ Object[] directMembers() { return new Object[]{e}; }
+
+ @Override
+ NestedSet[] transitiveSets() { return EMPTY; }
+
+ @Override
+ public boolean isEmpty() { return false; }
+
+ @Override
+ public List<E> toList() { return ImmutableList.of(e); }
+
+ @Override
+ public Set<E> toSet() { return ImmutableSet.of(e); }
+
+ @Override
+ public boolean shallowEquals(@Nullable NestedSet<? extends E> other) {
+ if (this == other) {
+ return true;
+ }
+ return other instanceof SingleDirectNestedSet
+ && e.equals(((SingleDirectNestedSet) other).e);
+ }
+
+ @Override
+ public int shallowHashCode() {
+ return e.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/StableOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/StableOrderNestedSetFactory.java
new file mode 100644
index 0000000..cd0b618
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/StableOrderNestedSetFactory.java
@@ -0,0 +1,152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Stable order {@code NestedSet} factory.
+ */
+final class StableOrderNestedSetFactory implements NestedSetFactory {
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(Object[] directs) {
+ return new StableOnlyDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) {
+ return new StableImmutableListDirectsNestedSet<>(directs);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) {
+ return new StableOneDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct,
+ NestedSet<E> transitive) {
+ return new StableManyDirectOneTransitiveNestedSet<>(direct, transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) {
+ return new StableOnlyOneTransitiveNestedSet<>(transitive);
+ }
+
+ @Override
+ public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) {
+ return new StableOnlyTransitivesNestedSet<>(transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) {
+ return new StableOneDirectManyTransitive<>(direct, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ return new StableManyDirectManyTransitive<>(directs, transitives);
+ }
+
+ @Override
+ public <E> NestedSet<E> oneDirect(final E element) {
+ return new StableSingleDirectNestedSet<>(element);
+ }
+
+ private static class StableOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> {
+
+ StableOnlyDirectsNestedSet(Object[] directs) { super(directs); }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableOneDirectOneTransitiveNestedSet<E> extends
+ OneDirectOneTransitiveNestedSet<E> {
+
+ private StableOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> {
+
+ private StableOneDirectManyTransitive(Object direct, NestedSet[] transitive) {
+ super(direct, transitive);
+ }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> {
+
+ private StableManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) {
+ super(directs, transitives);
+ }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableOnlyOneTransitiveNestedSet<E> extends OnlyOneTransitiveNestedSet<E> {
+
+ private StableOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableManyDirectOneTransitiveNestedSet<E> extends
+ ManyDirectOneTransitiveNestedSet<E> {
+
+ private StableManyDirectOneTransitiveNestedSet(Object[] direct,
+ NestedSet<E> transitive) { super(direct, transitive); }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> {
+
+ private StableOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+
+ private static class StableImmutableListDirectsNestedSet<E> extends
+ ImmutableListDirectsNestedSet<E> {
+
+ private StableImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); }
+
+ @Override
+ public Order getOrder() {
+ return Order.STABLE_ORDER;
+ }
+ }
+
+ private static class StableSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> {
+
+ private StableSingleDirectNestedSet(E element) { super(element); }
+
+ @Override
+ public Order getOrder() { return Order.STABLE_ORDER; }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Uniqueifier.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Uniqueifier.java
new file mode 100644
index 0000000..9421a57
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Uniqueifier.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+/**
+ * Helps reduce a sequence of potentially duplicated elements to a sequence of unique elements.
+ */
+interface Uniqueifier {
+
+ /**
+ * Returns true if-and-only-if this is the first time that this {@link Uniqueifier}'s method has
+ * been called with this Object. This uses Object.equals-type equality for the comparison.
+ */
+ public boolean isUnique(Object o);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitor.java b/src/main/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitor.java
new file mode 100644
index 0000000..a937d17
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitor.java
@@ -0,0 +1,578 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.concurrent;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * AbstractQueueVisitor is a wrapper around {@link ThreadPoolExecutor} which
+ * delays thread pool shutdown until entire visitation is complete.
+ * This is useful for cases in which worker tasks may submit additional tasks.
+ *
+ * <p>Consider the following example:
+ * <pre>
+ * ThreadPoolExecutor executor = <...>
+ * executor.submit(myRunnableTask);
+ * executor.shutdown();
+ * executor.awaitTermination();
+ * </pre>
+ *
+ * <p>This won't work properly if {@code myRunnableTask} submits additional
+ * tasks to the executor, because it may already have shut down
+ * by that point.
+ *
+ * <p>AbstractQueueVisitor supports interruption. If the main thread is
+ * interrupted, tasks will no longer be added to the queue, and the
+ * {@link #work(boolean)} method will throw {@link InterruptedException}.
+ */
+public class AbstractQueueVisitor {
+
+ /**
+ * The first unhandled exception thrown by a worker thread. We save it
+ * and re-throw it from the main thread to detect bugs faster;
+ * otherwise worker threads just quietly die.
+ *
+ * Field updates are synchronized; it's
+ * important to save the first one as it may be more informative than a
+ * subsequent one, and this is not a performance-critical path.
+ */
+ private Throwable unhandled = null;
+
+ /**
+ * An uncaught exception when submitting a job to the ThreadPool is catastrophic, and usually
+ * indicates a lack of stack space on which to allocate a native thread. The JDK
+ * ThreadPoolExecutor may reach an inconsistent state in such circumstances, so we avoid blocking
+ * on its termination when this field is non-null.
+ */
+ private volatile Throwable catastrophe;
+
+ /**
+ * Enables concurrency. For debugging or testing, set this to false
+ * to avoid thread creation and concurrency. Any deviation in observed
+ * behaviour is a bug.
+ */
+ private final boolean concurrent;
+
+ // Condition variable for remainingTasks==0, and a lock for it.
+ private final Object zeroRemainingTasks = new Object();
+ private long remainingTasks = 0;
+
+ // Map of thread ==> number of jobs executing in the thread.
+ // Currently used only for interrupt handling.
+ private final Map<Thread, Long> jobs = Maps.newConcurrentMap();
+
+ /**
+ * The thread pool. If !concurrent, always null. Created lazily on first
+ * call to {@link #enqueue(Runnable)}, and removed after call to
+ * {@link #work(boolean)}.
+ */
+ private final ThreadPoolExecutor pool;
+
+ /**
+ * Flag used to record when the main thread (the thread which called
+ * {@link #work(boolean)}) is interrupted.
+ *
+ * When this is true, adding tasks to the thread pool will
+ * fail quietly as a part of the process of shutting down the
+ * worker threads.
+ */
+ private volatile boolean threadInterrupted = false;
+
+ /**
+ * Latches used to signal when the visitor has been interrupted or
+ * seen an exception. Used only for testing.
+ */
+ private final CountDownLatch interruptedLatch = new CountDownLatch(1);
+ private final CountDownLatch exceptionLatch = new CountDownLatch(1);
+
+ /**
+ * If true, don't run new actions after an uncaught exception.
+ */
+ private final boolean failFastOnException;
+
+ /**
+ * If true, don't run new actions after an interrupt.
+ */
+ private final boolean failFastOnInterrupt;
+
+ /**
+ * If true, we must shut down the thread pool on completion.
+ */
+ private final boolean ownThreadPool;
+
+ /**
+ * Flag used to record when all threads were killed by failed action execution
+ */
+ private boolean jobsMustBeStopped = false;
+
+ /**
+ * Create the AbstractQueueVisitor.
+ *
+ * @param concurrent true if concurrency should be enabled. Only set to
+ * false for debugging.
+ * @param corePoolSize the core pool size of the thread pool. See
+ * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)}
+ * @param maxPoolSize the max number of threads in the pool.
+ * @param keepAliveTime the keep-alive time for the thread pool.
+ * @param units the time units of keepAliveTime.
+ * @param failFastOnException if true, don't run new actions after
+ * an uncaught exception.
+ * @param failFastOnInterrupt if true, don't run new actions after interrupt.
+ * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default
+ * thread naming will be used.
+ */
+ public AbstractQueueVisitor(boolean concurrent, int corePoolSize, int maxPoolSize,
+ long keepAliveTime, TimeUnit units, boolean failFastOnException,
+ boolean failFastOnInterrupt, String poolName) {
+ Preconditions.checkNotNull(poolName);
+
+ this.concurrent = concurrent;
+ this.failFastOnException = failFastOnException;
+ this.failFastOnInterrupt = failFastOnInterrupt;
+ this.ownThreadPool = true;
+ this.pool = concurrent
+ ? new ThreadPoolExecutor(corePoolSize, maxPoolSize, keepAliveTime, units, getWorkQueue(),
+ new ThreadFactoryBuilder().setNameFormat(poolName + " %d").build())
+ : null;
+ }
+
+ /**
+ * Create the AbstractQueueVisitor.
+ *
+ * @param concurrent true if concurrency should be enabled. Only set to
+ * false for debugging.
+ * @param corePoolSize the core pool size of the thread pool. See
+ * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)}
+ * @param maxPoolSize the max number of threads in the pool.
+ * @param keepAliveTime the keep-alive time for the thread pool.
+ * @param units the time units of keepAliveTime.
+ * @param failFastOnException if true, don't run new actions after
+ * an uncaught exception.
+ * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default
+ * thread naming will be used.
+ */
+ public AbstractQueueVisitor(boolean concurrent, int corePoolSize, int maxPoolSize,
+ long keepAliveTime, TimeUnit units, boolean failFastOnException, String poolName) {
+ this(concurrent, corePoolSize, maxPoolSize, keepAliveTime, units, failFastOnException, true,
+ poolName);
+ }
+
+ /**
+ * Create the AbstractQueueVisitor.
+ *
+ * @param executor The ThreadPool to use.
+ * @param shutdownOnCompletion If true, pass ownership of the Threadpool to
+ * this class. The pool will be shut down after a
+ * call to work(). Callers must not shutdown the
+ * threadpool while queue visitors use it.
+ * @param failFastOnException if true, don't run new actions after
+ * an uncaught exception.
+ * @param failFastOnInterrupt if true, don't run new actions after interrupt.
+ */
+ public AbstractQueueVisitor(ThreadPoolExecutor executor, boolean shutdownOnCompletion,
+ boolean failFastOnException, boolean failFastOnInterrupt) {
+ this(/*concurrent=*/true, executor, shutdownOnCompletion, failFastOnException,
+ failFastOnInterrupt);
+ }
+
+ /**
+ * Create the AbstractQueueVisitor.
+ *
+ * @param concurrent if false, run tasks inline instead of using the thread pool.
+ * @param executor The ThreadPool to use.
+ * @param shutdownOnCompletion If true, pass ownership of the Threadpool to
+ * this class. The pool will be shut down after a
+ * call to work(). Callers must not shut down the
+ * threadpool while queue visitors use it.
+ * @param failFastOnException if true, don't run new actions after
+ * an uncaught exception.
+ * @param failFastOnInterrupt if true, don't run new actions after interrupt.
+ */
+ public AbstractQueueVisitor(boolean concurrent, ThreadPoolExecutor executor,
+ boolean shutdownOnCompletion, boolean failFastOnException,
+ boolean failFastOnInterrupt) {
+ this.concurrent = concurrent;
+ this.failFastOnException = failFastOnException;
+ this.failFastOnInterrupt = failFastOnInterrupt;
+ this.pool = executor;
+ this.ownThreadPool = shutdownOnCompletion;
+ }
+
+ public AbstractQueueVisitor(ThreadPoolExecutor executor, boolean failFastOnException) {
+ this(executor, true, failFastOnException, true);
+ }
+
+ /**
+ * Create the AbstractQueueVisitor.
+ *
+ * @param concurrent true if concurrency should be enabled. Only set to
+ * false for debugging.
+ * @param corePoolSize the core pool size of the thread pool. See
+ * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)}
+ * @param maxPoolSize the max number of threads in the pool.
+ * @param keepAliveTime the keep-alive time for the thread pool.
+ * @param units the time units of keepAliveTime.
+ * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default
+ * thread naming will be used.
+ */
+ public AbstractQueueVisitor(boolean concurrent, int corePoolSize, int maxPoolSize,
+ long keepAliveTime, TimeUnit units, String poolName) {
+ this(concurrent, corePoolSize, maxPoolSize, keepAliveTime, units, false, poolName);
+ }
+
+ /**
+ * Create the AbstractQueueVisitor with concurrency enabled.
+ *
+ * @param corePoolSize the core pool size of the thread pool. See
+ * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)}
+ * @param maxPoolSize the max number of threads in the pool.
+ * @param keepAlive the keep-alive time for the thread pool.
+ * @param units the time units of keepAliveTime.
+ * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default
+ * thread naming will be used.
+ */
+ public AbstractQueueVisitor(int corePoolSize, int maxPoolSize, long keepAlive, TimeUnit units,
+ String poolName) {
+ this(true, corePoolSize, maxPoolSize, keepAlive, units, poolName);
+ }
+
+ protected BlockingQueue<Runnable> getWorkQueue() {
+ return new LinkedBlockingQueue<>();
+ }
+
+ /**
+ * Executes all tasks on the queue, and optionally shuts the pool down and deletes it.
+ *
+ * <p>Throws (the same) unchecked exception if any worker thread failed unexpectedly. If the pool
+ * is interrupted and a worker also throws an unchecked exception, the unchecked exception is
+ * rethrown, since it may indicate a programming bug. If callers handle the unchecked exception,
+ * they may check the interrupted bit to see if the pool was interrupted.
+ *
+ * @param interruptWorkers if true, interrupt worker threads when main thread gets an interrupt.
+ * If false, just wait for them to terminate normally.
+ */
+ protected void work(boolean interruptWorkers) throws InterruptedException {
+ if (concurrent) {
+ awaitTermination(interruptWorkers);
+ } else {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException();
+ }
+ }
+ }
+
+ /**
+ * Schedules a call.
+ * Called in a worker thread if concurrent.
+ */
+ protected void enqueue(Runnable runnable) {
+ if (concurrent) {
+ AtomicBoolean ranTask = new AtomicBoolean(false);
+ try {
+ pool.execute(wrapRunnable(runnable, ranTask));
+ } catch (Throwable e) {
+ if (!ranTask.get()) {
+ // Note that keeping track of ranTask is necessary to disambiguate the case where
+ // execute() itself failed, vs. a caller-runs policy on pool exhaustion, where the
+ // runnable threw. To be extra cautious, we decrement the task count in a finally
+ // block, even though the CountDownLatch is unlikely to throw.
+ recordError(e);
+ }
+ }
+ } else {
+ runnable.run();
+ }
+ }
+
+ private void recordError(Throwable e) {
+ catastrophe = e;
+ try {
+ synchronized (this) {
+ if (unhandled == null) { // save only the first one.
+ unhandled = e;
+ exceptionLatch.countDown();
+ }
+ }
+ } finally {
+ decrementRemainingTasks();
+ }
+ }
+
+ private Runnable wrapRunnable(final Runnable runnable, final AtomicBoolean ranTask) {
+ synchronized (zeroRemainingTasks) {
+ remainingTasks++;
+ }
+ return new Runnable() {
+ @Override
+ public void run() {
+ Thread thread = null;
+ boolean addedJob = false;
+ try {
+ ranTask.set(true);
+ thread = Thread.currentThread();
+ addJob(thread);
+ addedJob = true;
+ if (blockNewActions()) {
+ // Make any newly enqueued tasks quickly die. We check after adding to the jobs map so
+ // that if another thread is racing to kill this thread and didn't make it before this
+ // conditional, it will be able to find and kill this thread anyway.
+ return;
+ }
+ runnable.run();
+ } catch (Throwable e) {
+ synchronized (AbstractQueueVisitor.this) {
+ if (unhandled == null) { // save only the first one.
+ unhandled = e;
+ exceptionLatch.countDown();
+ }
+ markToStopAllJobsIfNeeded(e);
+ }
+ } finally {
+ try {
+ if (thread != null && addedJob) {
+ removeJob(thread);
+ }
+ } finally {
+ decrementRemainingTasks();
+ }
+ }
+ }
+ };
+ }
+
+ private final void addJob(Thread thread) {
+ // Note: this looks like a check-then-act race but it isn't, because each
+ // key implies thread-locality.
+ long count = jobs.containsKey(thread) ? jobs.get(thread) + 1 : 1;
+ jobs.put(thread, count);
+ }
+
+ private final void removeJob(Thread thread) {
+ Long boxedCount = Preconditions.checkNotNull(jobs.get(thread),
+ "Can't retrieve job after successfully adding it");
+ long count = boxedCount - 1;
+ if (count == 0) {
+ jobs.remove(thread);
+ } else {
+ jobs.put(thread, count);
+ }
+ }
+
+ /**
+ * Set an internal flag to show that an interrupt was detected.
+ */
+ private void setInterrupted() {
+ threadInterrupted = true;
+ setRejectedExecutionHandler();
+ }
+
+ private final void decrementRemainingTasks() {
+ synchronized (zeroRemainingTasks) {
+ if (--remainingTasks == 0) {
+ zeroRemainingTasks.notify();
+ }
+ }
+ }
+
+ /**
+ * If this returns true, don't enqueue new actions.
+ */
+ protected boolean blockNewActions() {
+ return (failFastOnInterrupt && isInterrupted()) || (unhandled != null && failFastOnException);
+ }
+
+ /**
+ * Await interruption. Used only in tests.
+ */
+ @VisibleForTesting
+ public boolean awaitInterruptionForTestingOnly(long timeout, TimeUnit units)
+ throws InterruptedException {
+ return interruptedLatch.await(timeout, units);
+ }
+
+ /** Get latch that is released when exception is received by visitor. Used only in tests. */
+ @VisibleForTesting
+ public CountDownLatch getExceptionLatchForTestingOnly() {
+ return exceptionLatch;
+ }
+
+ /** Get latch that is released when interruption is received by visitor. Used only in tests. */
+ @VisibleForTesting
+ public CountDownLatch getInterruptionLatchForTestingOnly() {
+ return interruptedLatch;
+ }
+
+ /**
+ * Get the value of the interrupted flag.
+ */
+ @ThreadSafety.ThreadSafe
+ protected boolean isInterrupted() {
+ return threadInterrupted;
+ }
+
+ /**
+ * Get number of jobs remaining. Note that this can increase in value
+ * if running tasks submit further jobs.
+ */
+ @VisibleForTesting
+ protected long getTaskCount() {
+ synchronized (zeroRemainingTasks) {
+ return remainingTasks;
+ }
+ }
+
+ /**
+ * Waits for the task queue to drain, then shuts down the thread pool and
+ * waits for it to terminate. Throws (the same) unchecked exception if any
+ * worker thread failed unexpectedly.
+ */
+ private void awaitTermination(boolean interruptWorkers) throws InterruptedException {
+ Preconditions.checkState(failFastOnInterrupt || !interruptWorkers);
+ Throwables.propagateIfPossible(catastrophe);
+ try {
+ synchronized (zeroRemainingTasks) {
+ while (remainingTasks != 0 && !jobsMustBeStopped) {
+ zeroRemainingTasks.wait();
+ }
+ }
+ } catch (InterruptedException e) {
+ // Mark the visitor, so that it's known to be interrupted, and
+ // then break out of here, stop the worker threads and return ASAP,
+ // sending the interruption to the parent thread.
+ setInterrupted();
+ }
+
+ reallyAwaitTermination(interruptWorkers);
+
+ if (isInterrupted()) {
+ // Set interrupted bit on current thread so that callers can see that it was interrupted. Note
+ // that if the thread was interrupted while awaiting termination, we might not hit this
+ // codepath, but then the current thread's interrupt bit is already set, so we are fine.
+ Thread.currentThread().interrupt();
+ }
+ // Throw the first unhandled (worker thread) exception in the main thread. We throw an unchecked
+ // exception instead of InterruptedException if both are present because an unchecked exception
+ // may indicate a catastrophic failure that should shut down the program. The caller can
+ // check the interrupted bit if they will handle the unchecked exception without crashing.
+ Throwables.propagateIfPossible(unhandled);
+
+ if (Thread.interrupted()) {
+ throw new InterruptedException();
+ }
+ }
+
+ private void reallyAwaitTermination(boolean interruptWorkers) {
+ // TODO(bazel-team): verify that interrupt() is safe for every use of
+ // AbstractQueueVisitor and remove the interruptWorkers flag.
+ if (interruptWorkers && !jobs.isEmpty()) {
+ interruptInFlightTasks();
+ }
+
+ if (isInterrupted()) {
+ interruptedLatch.countDown();
+ }
+
+ Throwables.propagateIfPossible(catastrophe);
+ synchronized (zeroRemainingTasks) {
+ while (remainingTasks != 0) {
+ try {
+ zeroRemainingTasks.wait();
+ } catch (InterruptedException e) {
+ setInterrupted();
+ }
+ }
+ }
+
+ if (ownThreadPool) {
+ pool.shutdown();
+ for (;;) {
+ try {
+ Throwables.propagateIfPossible(catastrophe);
+ pool.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
+ break;
+ } catch (InterruptedException e) {
+ setInterrupted();
+ }
+ }
+ }
+ }
+
+ private void interruptInFlightTasks() {
+ Thread thisThread = Thread.currentThread();
+ for (Thread thread : jobs.keySet()) {
+ if (thisThread != thread) {
+ thread.interrupt();
+ }
+ }
+ }
+
+ /**
+ * Makes the visitation terminate prematurely.
+ */
+ public void interrupt() {
+ setInterrupted();
+ reallyAwaitTermination(true);
+ }
+
+ /**
+ * If this returns true, that means the exception {@code e} is critical
+ * and all running actions should be stopped.
+ *
+ * <p>Default value - always false. If different behavior is needed
+ * then we should override this method in subclasses.
+ *
+ * @param e the exception object to check
+ */
+ protected boolean isCriticalError(Throwable e) {
+ return false;
+ }
+
+ private void setRejectedExecutionHandler() {
+ if (ownThreadPool) {
+ pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
+ @Override
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+ decrementRemainingTasks();
+ }
+ });
+ }
+ }
+
+ /**
+ * If exception is critical then set a flag which signals
+ * to stop all jobs inside {@link #awaitTermination(boolean)}.
+ */
+ private synchronized void markToStopAllJobsIfNeeded(Throwable e) {
+ if (isCriticalError(e) && !jobsMustBeStopped) {
+ jobsMustBeStopped = true;
+ synchronized (zeroRemainingTasks) {
+ zeroRemainingTasks.notify();
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/ExecutorShutdownUtil.java b/src/main/java/com/google/devtools/build/lib/concurrent/ExecutorShutdownUtil.java
new file mode 100644
index 0000000..95962b3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/concurrent/ExecutorShutdownUtil.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utilities for safely shutting down executors.
+ * TODO(bazel-team): Rename this class to something like "ExecutorUtil".
+ */
+public class ExecutorShutdownUtil {
+
+ private ExecutorShutdownUtil() {
+ }
+
+ /**
+ * Shutdown the executor. If an interrupt occurs, invoke shutdownNow(), but
+ * still block on the eventual termination of the pool.
+ *
+ * @param executor the executor service.
+ * @return true iff interrupted.
+ */
+ public static boolean interruptibleShutdown(ExecutorService executor) {
+ return shutdownImpl(executor, /*interruptible=*/true);
+ }
+
+ /**
+ * Shutdown the executor. If an interrupt occurs, ignore it and still block on the eventual
+ * termination of the pool. This way, all tasks are guaranteed to have completed normally.
+ *
+ * @param executor the executor service.
+ * @return true iff interrupted.
+ */
+ public static boolean uninterruptibleShutdown(ExecutorService executor) {
+ return shutdownImpl(executor, /*interruptible=*/false);
+ }
+
+ private static boolean shutdownImpl(ExecutorService executor, boolean interruptible) {
+ Preconditions.checkState(!executor.isShutdown());
+ executor.shutdown();
+
+ // Common pattern: check for interrupt, but don't return until all threads
+ // are finished.
+ boolean interrupted = false;
+ while (true) {
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
+ break;
+ } catch (InterruptedException e) {
+ if (interruptible) {
+ executor.shutdownNow();
+ }
+ interrupted = true;
+ }
+ }
+ return interrupted;
+ }
+
+ /**
+ * Create a "slack" thread pool which has the following properties:
+ * 1. the worker count shrinks as the threads go unused
+ * 2. the rejection policy is caller-runs
+ *
+ * @param threads maximum number of threads in the pool
+ * @param name name of the pool
+ * @return the new ThreadPoolExecutor
+ */
+ public static ThreadPoolExecutor newSlackPool(int threads, String name) {
+ // Using a synchronous queue with a bounded thread pool means we'll reject
+ // tasks after the pool size. The CallerRunsPolicy, however, implies that
+ // saturation is handled in the calling thread.
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 3L, TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>());
+ // Do not consume threads when not in use.
+ pool.allowCoreThreadTimeOut(true);
+ pool.setThreadFactory(new ThreadFactoryBuilder().setNameFormat(name + " %d").build());
+ pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
+ @Override
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+ r.run();
+ }
+ });
+ return pool;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/MoreFutures.java b/src/main/java/com/google/devtools/build/lib/concurrent/MoreFutures.java
new file mode 100644
index 0000000..ab84f99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/concurrent/MoreFutures.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Utility class for working with futures.
+ */
+public class MoreFutures {
+
+ private MoreFutures() {}
+
+ /**
+ * Creates a new {@code ListenableFuture} whose value is a list containing the
+ * values of all its input futures, if all succeed. If any input fails, the
+ * returned future fails. If any of the futures fails, it cancels all the other futures.
+ *
+ * <p> This method is similar to {@code Futures.allAsList} but additionally it cancels all the
+ * futures in case any of them fails.
+ */
+ public static <V> ListenableFuture<List<V>> allAsListOrCancelAll(
+ final Iterable<? extends ListenableFuture<? extends V>> futures) {
+ ListenableFuture<List<V>> combinedFuture = Futures.allAsList(futures);
+ Futures.addCallback(combinedFuture, new FutureCallback<List<V>>() {
+ @Override
+ public void onSuccess(@Nullable List<V> vs) {}
+
+ /**
+ * In case of a failure of any of the futures (that gets propagated to combinedFuture) we
+ * cancel all the futures in the list.
+ */
+ @Override
+ public void onFailure(Throwable ignore) {
+ for (ListenableFuture<? extends V> future : futures) {
+ future.cancel(true);
+ }
+ }
+ });
+ return combinedFuture;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/Sharder.java b/src/main/java/com/google/devtools/build/lib/concurrent/Sharder.java
new file mode 100644
index 0000000..67a63e0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/concurrent/Sharder.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A class to build shards (work queues) for a given task.
+ *
+ * <p>{@link #add}ed elements will be equally distributed among the shards.
+ *
+ * @param <T> the type of collection over which we're sharding
+ */
+public final class Sharder<T> implements Iterable<List<T>> {
+ private final List<List<T>> shards;
+ private int nextShard = 0;
+
+ public Sharder(int maxNumShards, int expectedTotalSize) {
+ Preconditions.checkArgument(maxNumShards > 0);
+ Preconditions.checkArgument(expectedTotalSize >= 0);
+ this.shards = immutableListOfLists(maxNumShards, expectedTotalSize / maxNumShards);
+ }
+
+ public void add(T item) {
+ shards.get(nextShard).add(item);
+ nextShard = (nextShard + 1) % shards.size();
+ }
+
+ /**
+ * Returns an immutable list of mutable lists.
+ *
+ * @param numLists the number of top-level lists.
+ * @param expectedSize the exepected size of each mutable list.
+ * @return a list of lists.
+ */
+ private static <T> List<List<T>> immutableListOfLists(int numLists, int expectedSize) {
+ List<List<T>> list = Lists.newArrayListWithCapacity(numLists);
+ for (int i = 0; i < numLists; i++) {
+ list.add(Lists.<T>newArrayListWithExpectedSize(expectedSize));
+ }
+ return Collections.unmodifiableList(list);
+ }
+
+ @Override
+ public Iterator<List<T>> iterator() {
+ return Iterables.filter(shards, new Predicate<List<T>>() {
+ @Override
+ public boolean apply(List<T> list) {
+ return !list.isEmpty();
+ }
+ }).iterator();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java b/src/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java
new file mode 100644
index 0000000..0c67fd9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java
@@ -0,0 +1,135 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Define some standard attributes for documenting thread safety properties.
+ *<p>
+ * The names used here are adapted from those used in Joshua Bloch's book
+ * "Effective Java", which are also described at
+ * <http://www-128.ibm.com/developerworks/java/library/j-jtp09263.html>.
+ *<p>
+ * These attributes are just documentation. They don't have any run-time
+ * effect. The main aim is mainly just to standardize the terminology.
+ * (However, if this catches on, I can also imagine in the future having
+ * a presubmit check that checks that all new classes have thread safety
+ * annotations :)
+ *<p>
+ * See ThreadSafetyTest for examples of how these attributes should be used.
+ */
+public class ThreadSafety {
+ /**
+ * The Immutable attribute indicates that instances of the class are
+ * immutable, or at least appear that way are far as their external API
+ * is concerned. Immutable classes are usually also ThreadSafe,
+ * but can be ThreadHostile if they perform unsynchronized access to
+ * mutable static data. (We deviate from Bloch's nomenclature by
+ * not assuming that Immutable implies ThreadSafe; developers should
+ * explicitly annotate classes as both Immutable and ThreadSafe when
+ * appropriate.)
+ */
+ @Documented
+ @Target(value = {ElementType.TYPE})
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Immutable {}
+
+ /**
+ * The ThreadSafe attribute marks a class or method which can safely be used
+ * from multiple threads without any need for external synchronization.
+ *
+ * When applied to a class, this attribute indicates that instances
+ * of the class can safely be used concurrently from multiple threads
+ * without any need for external synchronization, i.e. that all non-static methods
+ * are thread-safe (except any private methods that are explicitly
+ * annotated with a different thread safety annotation). In addition it
+ * also indicates that all non-static nested classes are thread-safe (except any private
+ * nested classes that are explicitly annotated with a different thread
+ * safety annotation). Note that no guarantees are made about static class methods or static
+ * nested classes - they should be annotated separately.
+ *
+ * When applied to a method, this attribute indicates that the
+ * method can safely be called concurrently from multiple threads.
+ * The implementation must synchronize any accesses to mutable data.
+ */
+ @Documented
+ @Target(value = {ElementType.CONSTRUCTOR, ElementType.METHOD, ElementType.TYPE})
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface ThreadSafe {}
+
+ /**
+ * The ThreadCompatible attribute marks a class or method that
+ * is thread-safe provided that only one thread attempts to
+ * access each object at a time.
+ *
+ * The implementation of such a class must synchronize accesses
+ * to mutable static data, but can assume that each instance will
+ * only be accessed from one thread at a time.
+ *
+ * The client must obtain an appropriate lock before calling ThreadCompatible
+ * methods, or must otherwise ensure that only one thread calls such methods.
+ * Unless otherwise specified, an appropriate lock means synchronizing on the
+ * instance.
+ *
+ * A ThreadCompatible class may contain private methods or private nested
+ * classes that are not ThreadCompatible provided that they are explicitly
+ * annotated with a different thread safety annotation.
+ */
+ @Documented
+ @Target(value = {ElementType.METHOD, ElementType.TYPE})
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface ThreadCompatible {}
+
+ /**
+ * The ThreadHostile attribute marks a class or method that
+ * can't safely be used by multiple threads, for example because
+ * it performs unsynchronized access to mutable static objects.
+ */
+ @Documented
+ @Target(value = {ElementType.CONSTRUCTOR, ElementType.METHOD, ElementType.TYPE})
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface ThreadHostile {}
+
+ /**
+ * The ConditionallyThreadSafe attribute marks a class that contains
+ * some methods (or nested classes) which are ThreadSafe but others which are
+ * only ThreadCompatible or ThreadHostile.
+ *
+ * The methods (and nested classes) of a ConditionallyThreadSafe class should
+ * each have their thread safety marked.
+ */
+ @Documented
+ @Target(value = {ElementType.METHOD, ElementType.TYPE})
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface ConditionallyThreadSafe {}
+
+ /**
+ * The ConditionallyThreadCompatible attribute marks a class that contains
+ * some methods (or nested classes) which are ThreadCompatible but others
+ * which are ThreadHostile.
+ *
+ * The methods (and nested classes) of a ConditionallyThreadCompatible class
+ * should each have their thread safety marked.
+ */
+ @Documented
+ @Target(value = {ElementType.METHOD, ElementType.TYPE})
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface ConditionallyThreadCompatible {}
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/ThrowableRecordingRunnableWrapper.java b/src/main/java/com/google/devtools/build/lib/concurrent/ThrowableRecordingRunnableWrapper.java
new file mode 100644
index 0000000..3f67d55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/concurrent/ThrowableRecordingRunnableWrapper.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import com.google.common.base.Preconditions;
+
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * A class that wraps Runnables and records the first Throwable thrown by the wrapped Runnables
+ * when they are run.
+ */
+public class ThrowableRecordingRunnableWrapper {
+
+ private final String name;
+ private AtomicReference<Throwable> errorRef = new AtomicReference<>();
+
+ private static final Logger LOG =
+ Logger.getLogger(ThrowableRecordingRunnableWrapper.class.getName());
+
+ public ThrowableRecordingRunnableWrapper(String name) {
+ this.name = Preconditions.checkNotNull(name);
+ }
+
+ @Nullable
+ public Throwable getFirstThrownError() {
+ return errorRef.get();
+ }
+
+ public Runnable wrap(final Runnable runnable) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ try {
+ runnable.run();
+ } catch (Throwable error) {
+ errorRef.compareAndSet(null, error);
+ LOG.log(Level.SEVERE, "Error thrown by runnable in " + name, error);
+ }
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/AbstractEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/AbstractEventHandler.java
new file mode 100644
index 0000000..39faf14
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/AbstractEventHandler.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import java.util.Set;
+
+/**
+ * An abstract event handler that keeps track of the event mask. Events
+ * matching the mask will be handled.
+ */
+public abstract class AbstractEventHandler implements EventHandler {
+
+ private final Set<EventKind> mask;
+
+ /**
+ * Events matching the mask will be handled.
+ */
+ public AbstractEventHandler(Set<EventKind> mask) {
+ this.mask = mask;
+ }
+
+ public Set<EventKind> getEventMask() {
+ return mask;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/DelegatingEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/DelegatingEventHandler.java
new file mode 100644
index 0000000..d26d70c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/DelegatingEventHandler.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * An ErrorEventListener which delegates to another ErrorEventListener.
+ * Primarily useful as a base class for extending behavior.
+ */
+public class DelegatingEventHandler implements EventHandler {
+ protected final EventHandler delegate;
+
+ public DelegatingEventHandler(EventHandler delegate) {
+ super();
+ this.delegate = Preconditions.checkNotNull(delegate);
+ }
+
+ @Override
+ public void handle(Event e) {
+ delegate.handle(e);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/DelegatingOnlyErrorsEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/DelegatingOnlyErrorsEventHandler.java
new file mode 100644
index 0000000..dec220d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/DelegatingOnlyErrorsEventHandler.java
@@ -0,0 +1,32 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+/**
+ * An {@link EventHandler} implementation that only
+ * passes through error messages.
+ */
+public class DelegatingOnlyErrorsEventHandler extends DelegatingEventHandler {
+
+ public DelegatingOnlyErrorsEventHandler(EventHandler eventHandler) {
+ super(eventHandler);
+ }
+
+ @Override
+ public void handle(Event e) {
+ if (e.getKind() == EventKind.ERROR) {
+ super.handle(e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/ErrorSensingEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/ErrorSensingEventHandler.java
new file mode 100644
index 0000000..705f7f4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/ErrorSensingEventHandler.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+/**
+ * Passes through any events, and keeps a flag if any of them were errors. It is thread-safe as long
+ * as the target eventHandler is thread-safe.
+ */
+public final class ErrorSensingEventHandler extends DelegatingEventHandler {
+
+ private volatile boolean hasErrors;
+
+ public ErrorSensingEventHandler(EventHandler eventHandler) {
+ super(eventHandler);
+ }
+
+ @Override
+ public void handle(Event e) {
+ hasErrors |= e.getKind() == EventKind.ERROR;
+ super.handle(e);
+ }
+
+ /**
+ * Returns whether any of the events on this objects were errors.
+ */
+ public boolean hasErrors() {
+ return hasErrors;
+ }
+
+ /**
+ * Reset the error flag. Don't call this while other threads are accessing the same object.
+ */
+ public void resetErrors() {
+ hasErrors = false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/Event.java b/src/main/java/com/google/devtools/build/lib/events/Event.java
new file mode 100644
index 0000000..db5bc5f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/Event.java
@@ -0,0 +1,183 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.base.Preconditions;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * An event is a situation encountered by the build system that's worth
+ * reporting: A 3-tuple of ({@link EventKind}, {@link Location}, message).
+ */
+@Immutable
+public final class Event {
+
+ private final EventKind kind;
+ private final Location location;
+ private final String message;
+ /**
+ * An alternative representation for message.
+ * Exactly one of message or messageBytes will be non-null.
+ * If messageBytes is non-null, then it contains the bytes
+ * of the message, encoded using the platform's default charset.
+ * We do this to avoid converting back and forth between Strings
+ * and bytes.
+ */
+ private final byte[] messageBytes;
+
+ @Nullable
+ private final String tag;
+
+ public Event withTag(String tag) {
+ if (this.message != null) {
+ return new Event(this.kind, this.location, this.message, tag);
+ } else {
+ return new Event(this.kind, this.location, this.messageBytes, tag);
+ }
+ }
+
+ public Event(EventKind kind, @Nullable Location location, String message) {
+ this(kind, location, message, null);
+ }
+
+ public Event(EventKind kind, @Nullable Location location, String message, @Nullable String tag) {
+ this.kind = kind;
+ this.location = location;
+ this.message = Preconditions.checkNotNull(message);
+ this.messageBytes = null;
+ this.tag = tag;
+ }
+
+ public Event(EventKind kind, @Nullable Location location, byte[] messageBytes) {
+ this(kind, location, messageBytes, null);
+ }
+
+ public Event(
+ EventKind kind, @Nullable Location location, byte[] messageBytes, @Nullable String tag) {
+ this.kind = kind;
+ this.location = location;
+ this.message = null;
+ this.messageBytes = Preconditions.checkNotNull(messageBytes);
+ this.tag = tag;
+ }
+
+ public String getMessage() {
+ return message != null ? message : new String(messageBytes);
+ }
+
+ public byte[] getMessageBytes() {
+ return messageBytes != null ? messageBytes : message.getBytes(ISO_8859_1);
+ }
+
+ public EventKind getKind() {
+ return kind;
+ }
+
+ /**
+ * the tag is typically the action that generated the event.
+ */
+ @Nullable
+ public String getTag() {
+ return tag;
+ }
+
+ /**
+ * Returns the location of this event, if any. Returns null iff the event
+ * wasn't associated with any particular location, for example, a progress
+ * message.
+ */
+ @Nullable public Location getLocation() {
+ return location;
+ }
+
+ /**
+ * Returns <i>some</i> moderately sane representation of the event. Should never be used in
+ * user-visible places, only for debugging and testing.
+ */
+ @Override
+ public String toString() {
+ return kind + " " + (location != null ? location.print() : "<no location>") + ": "
+ + getMessage();
+ }
+
+ /**
+ * Replay a sequence of events on an {@link EventHandler}.
+ */
+ public static void replayEventsOn(EventHandler eventHandler, Iterable<Event> events) {
+ for (Event event : events) {
+ eventHandler.handle(event);
+ }
+ }
+
+ /**
+ * Reports a warning.
+ */
+ public static Event warn(Location location, String message) {
+ return new Event(EventKind.WARNING, location, message);
+ }
+
+ /**
+ * Reports an error.
+ */
+ public static Event error(Location location, String message){
+ return new Event(EventKind.ERROR, location, message);
+ }
+
+ /**
+ * Reports atemporal statements about the build, i.e. they're true for the duration of execution.
+ */
+ public static Event info(Location location, String message) {
+ return new Event(EventKind.INFO, location, message);
+ }
+
+ /**
+ * Reports a temporal statement about the build.
+ */
+ public static Event progress(Location location, String message) {
+ return new Event(EventKind.PROGRESS, location, message);
+ }
+
+ /**
+ * Reports a warning.
+ */
+ public static Event warn(String message) {
+ return warn(null, message);
+ }
+
+ /**
+ * Reports an error.
+ */
+ public static Event error(String message){
+ return error(null, message);
+ }
+
+ /**
+ * Reports atemporal statements about the build, i.e. they're true for the duration of execution.
+ */
+ public static Event info(String message) {
+ return info(null, message);
+ }
+
+ /**
+ * Reports a temporal statement about the build.
+ */
+ public static Event progress(String message) {
+ return progress(null, message);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventCollector.java b/src/main/java/com/google/devtools/build/lib/events/EventCollector.java
new file mode 100644
index 0000000..774b323
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/EventCollector.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * An {@link EventHandler} that collects all events it encounters, and makes
+ * them available via the {@link Iterable} interface. The collected events
+ * contain not just the original event information but also the location
+ * context.
+ */
+public class EventCollector extends AbstractEventHandler implements Iterable<Event> {
+
+ private final Collection<Event> collected;
+
+ /**
+ * This collector will collect all events that match the event mask.
+ */
+ public EventCollector(Set<EventKind> mask) {
+ this(mask, new ArrayList<Event>());
+ }
+
+ /**
+ * This collector will save the Event instances in the provided
+ * collection.
+ */
+ public EventCollector(Set<EventKind> mask, Collection<Event> collected) {
+ super(mask);
+ this.collected = collected;
+ }
+
+ /**
+ * Implements {@link EventHandler#handle(Event)}.
+ */
+ @Override
+ public void handle(Event event) {
+ if (getEventMask().contains(event.getKind())) {
+ collected.add(event);
+ }
+ }
+
+ /**
+ * Returns an iterator over the collected events.
+ */
+ @Override
+ public Iterator<Event> iterator() {
+ return collected.iterator();
+ }
+
+ /**
+ * Returns the number of events collected.
+ */
+ public int count() {
+ return collected.size();
+ }
+
+ /*
+ * Clears the collected events
+ */
+ public void clear() {
+ collected.clear();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventHandler.java b/src/main/java/com/google/devtools/build/lib/events/EventHandler.java
new file mode 100644
index 0000000..28b6265
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/EventHandler.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+/**
+ * The ErrorEventListener is the primary means of reporting error and warning events. It is a subset
+ * of the functionality of the {@link Reporter}. In most cases, you should use this interface
+ * instead of the final {@code Reporter} class.
+ */
+public interface EventHandler {
+ /**
+ * Handles an event.
+ */
+ public void handle(Event event);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventKind.java b/src/main/java/com/google/devtools/build/lib/events/EventKind.java
new file mode 100644
index 0000000..eb58873
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/EventKind.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Indicates the kind of an {@link Event}.
+ */
+public enum EventKind {
+
+ /**
+ * For errors that will prevent a successful, correct build. In general, the
+ * build tool will not attempt to start or continue a build if an error is
+ * encountered (though the behaviour specified by --keep-going flag is a
+ * counterexample).
+ *
+ * Errors of a more severe nature in the input, such as those which might
+ * cause later passes of the analysis to fail catastrophically, should be
+ * handled by throwing an exception.
+ */
+ ERROR,
+
+ /**
+ * For warnings of minor problems that do not affect the integrity of a
+ * build.
+ */
+ WARNING,
+
+ /**
+ * For atemporal information that is true throughout the entire duration
+ * of a build. (e.g. the number of targets found)
+ */
+ INFO,
+
+ /**
+ * For temporal information that changes during the duration of a build.
+ * (e.g. what action is executing now)
+ */
+ PROGRESS,
+
+ /**
+ * For progress messages (temporal information) relating to the start
+ * and end of particular tasks.
+ * (e.g. "Loading package foo", "Compiling bar", etc.)
+ */
+ START,
+ FINISH,
+
+ /**
+ * For command lines of subcommands executed by the build tool (like make-dbg
+ * "-v").
+ */
+ SUBCOMMAND,
+
+ /**
+ * Output to stdout/stderr from subprocesses.
+ */
+ STDOUT,
+ STDERR,
+
+ /**
+ * Test result messages (similar to the INFO and ERROR, but test-specific).
+ */
+ PASS,
+ FAIL,
+ TIMEOUT,
+
+ /**
+ * For the reasoning of the dependency checker (like GNU Make "-d").
+ */
+ DEPCHECKER;
+
+ // Convenient predefined EnumSets. Clients should not mutate them!
+
+ public static final Set<EventKind> ALL_EVENTS =
+ EnumSet.allOf(EventKind.class);
+
+ public static final Set<EventKind> OUTPUT = EnumSet.of(
+ EventKind.STDOUT,
+ EventKind.STDERR
+ );
+
+ public static final Set<EventKind> ERRORS = EnumSet.of(
+ EventKind.ERROR,
+ EventKind.FAIL,
+ EventKind.TIMEOUT
+ );
+
+ public static final Set<EventKind> ERRORS_AND_WARNINGS = EnumSet.of(
+ EventKind.ERROR,
+ EventKind.WARNING,
+ EventKind.FAIL,
+ EventKind.TIMEOUT
+ );
+
+ public static final Set<EventKind> ERRORS_WARNINGS_AND_INFO = EnumSet.of(
+ EventKind.ERROR,
+ EventKind.WARNING,
+ EventKind.PASS,
+ EventKind.FAIL,
+ EventKind.TIMEOUT,
+ EventKind.INFO
+ );
+
+ public static final Set<EventKind> ERRORS_AND_OUTPUT = EnumSet.of(
+ EventKind.ERROR,
+ EventKind.FAIL,
+ EventKind.TIMEOUT,
+ EventKind.STDOUT,
+ EventKind.STDERR
+ );
+
+ public static final Set<EventKind> ERRORS_AND_WARNINGS_AND_OUTPUT = EnumSet.of(
+ EventKind.ERROR,
+ EventKind.WARNING,
+ EventKind.FAIL,
+ EventKind.TIMEOUT,
+ EventKind.STDOUT,
+ EventKind.STDERR
+ );
+
+ public static final Set<EventKind> ERRORS_WARNINGS_AND_INFO_AND_OUTPUT = EnumSet.of(
+ EventKind.ERROR,
+ EventKind.WARNING,
+ EventKind.PASS,
+ EventKind.FAIL,
+ EventKind.TIMEOUT,
+ EventKind.INFO,
+ EventKind.STDOUT,
+ EventKind.STDERR
+ );
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventSensor.java b/src/main/java/com/google/devtools/build/lib/events/EventSensor.java
new file mode 100644
index 0000000..5a31f13
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/EventSensor.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import java.util.Set;
+
+/**
+ * A "latch" that just detects whether or not a particular type of event has happened, based on its
+ * kind.
+ *
+ * <p>Be careful when using this class to track errors reported during some operation. Namely, this
+ * pattern is not thread-safe:
+ *
+ * <pre><code>
+ * EventSensor sensor = new EventSensor(EventKind.ERRORS);
+ * reporter.addHandler(sensor);
+ * someActionThatMightCreateErrors(reporter)
+ * reporter.removeHandler(sensor);
+ * boolean containsErrors = sensor.wasTriggered();
+ * </code></pre>
+ *
+ * <p>If other threads generate errors on the reporter, then containsErrors may be true even if
+ * someActionThatMightCreateErrors() did not cause any errors.
+ *
+ * <p>As a workaround, run someActionThatMightCreateErrors() with a local reporter, merging its
+ * events with those of the shared reporter.
+ */
+public class EventSensor extends AbstractEventHandler {
+
+ private int triggerCount;
+
+ /**
+ * Constructs a sensor that will register all events matching the mask.
+ */
+ public EventSensor(Set<EventKind> mask) {
+ super(mask);
+ }
+
+ /**
+ * Implements {@link EventHandler#handle(Event)}.
+ */
+ @Override
+ public void handle(Event event) {
+ if (getEventMask().contains(event.getKind())) {
+ triggerCount++;
+ }
+ }
+
+ /**
+ * Returns true iff a qualifying event was handled.
+ */
+ public boolean wasTriggered() {
+ return triggerCount > 0;
+ }
+
+ /**
+ * Returns the number of times the qualifying event was handled.
+ */
+ public int getTriggerCount() {
+ return triggerCount;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/ExceptionListener.java b/src/main/java/com/google/devtools/build/lib/events/ExceptionListener.java
new file mode 100644
index 0000000..174a5ca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/ExceptionListener.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+/**
+ * The ExceptionListener is the primary means of reporting exceptions. It is a subset of the
+ * functionality of the {@link Reporter}.
+ */
+public interface ExceptionListener {
+ /**
+ * Reports an error.
+ */
+ void error(Location location, String message, Throwable error);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/Location.java b/src/main/java/com/google/devtools/build/lib/events/Location.java
new file mode 100644
index 0000000..39508d1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/Location.java
@@ -0,0 +1,215 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.Serializable;
+
+/**
+ * A Location is a range of characters within a file.
+ *
+ * The start and end locations may be the same, in which case the Location
+ * denotes a point in the file, not a range. The path may be null, indicating
+ * an unknown file.
+ *
+ * Implementations of Location should be optimised for speed of construction,
+ * not speed of attribute access, as far more Locations are created during
+ * parsing than are ever used to display error messages.
+ */
+public abstract class Location implements Serializable {
+
+ @Immutable
+ private static final class LocationWithPathAndStartColumn extends Location {
+ private final PathFragment path;
+ private final LineAndColumn startLineAndColumn;
+
+ private LocationWithPathAndStartColumn(Path path, int startOffSet, int endOffSet,
+ LineAndColumn startLineAndColumn) {
+ super(startOffSet, endOffSet);
+ this.path = path != null ? path.asFragment() : null;
+ this.startLineAndColumn = startLineAndColumn;
+ }
+
+ @Override
+ public PathFragment getPath() { return path; }
+
+ @Override
+ public LineAndColumn getStartLineAndColumn() {
+ return startLineAndColumn;
+ }
+ }
+
+ protected final int startOffset;
+ protected final int endOffset;
+
+ /**
+ * Returns a Location with a given Path, start and end offset and start line and column info.
+ */
+ public static Location fromPathAndStartColumn(Path path, int startOffSet, int endOffSet,
+ LineAndColumn startLineAndColumn) {
+ return new LocationWithPathAndStartColumn(path, startOffSet, endOffSet, startLineAndColumn);
+ }
+
+ /**
+ * Returns a Location relating to file 'path', but not to any specific part
+ * region within the file. Try to use a more specific location if possible.
+ */
+ public static Location fromFile(Path path) {
+ return fromFileAndOffsets(path, 0, 0);
+ }
+
+ /**
+ * Returns a Location relating to the subset of file 'path', starting at
+ * 'startOffset' and ending at 'endOffset'.
+ */
+ public static Location fromFileAndOffsets(final Path path,
+ int startOffset,
+ int endOffset) {
+ return new LocationWithPathAndStartColumn(path, startOffset, endOffset, null);
+ }
+
+ protected Location(int startOffset, int endOffset) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ /**
+ * Returns the start offset relative to the beginning of the file the object
+ * resides in.
+ */
+ public final int getStartOffset() {
+ return startOffset;
+ }
+
+ /**
+ * Returns the end offset relative to the beginning of the file the object
+ * resides in.
+ *
+ * <p>The end offset is one position past the actual end position, making this method
+ * behave in a compatible fashion with {@link String#substring(int, int)}.
+ *
+ * <p>To compute the length of this location, use {@code getEndOffset() - getStartOffset()}.
+ */
+ public final int getEndOffset() {
+ return endOffset;
+ }
+
+ /**
+ * Returns the path of the file to which the start/end offsets refer. May be
+ * null if the file name information is not available.
+ *
+ * This method is intentionally abstract, as a space optimisation. Some
+ * subclass instances implement sharing of common data (e.g. tables for
+ * convering offsets into line numbers) and this enables them to share the
+ * Path value in the same way.
+ */
+ public abstract PathFragment getPath();
+
+ /**
+ * Returns a (line, column) pair corresponding to the position denoted by
+ * getStartOffset. Returns null if this information is not available.
+ */
+ public LineAndColumn getStartLineAndColumn() {
+ return null;
+ }
+
+ /**
+ * Returns a (line, column) pair corresponding to the position denoted by
+ * getEndOffset. Returns null if this information is not available.
+ */
+ public LineAndColumn getEndLineAndColumn() {
+ return null;
+ }
+
+ /**
+ * A default implementation of toString() that formats the location in the
+ * following ways based on the amount of information available:
+ * <pre>
+ * "foo.cc:23:2"
+ * "23:2"
+ * "foo.cc:char offsets 123--456"
+ * "char offsets 123--456"
+ * </pre>
+ */
+ public String print() {
+ StringBuilder buf = new StringBuilder();
+ if (getPath() != null) {
+ buf.append(getPath()).append(':');
+ }
+ LineAndColumn start = getStartLineAndColumn();
+ if (start == null) {
+ if (getStartOffset() == 0 && getEndOffset() == 0) {
+ buf.append("1"); // i.e. line 1 (special case: no information at all)
+ } else {
+ buf.append("char offsets ").
+ append(getStartOffset()).append("--").append(getEndOffset());
+ }
+ } else {
+ buf.append(start.getLine()).append(':').append(start.getColumn());
+ }
+ return buf.toString();
+ }
+
+ /**
+ * Prints the object in a sort of reasonable way. This should never be used in user-visible
+ * places, only for debugging and testing.
+ */
+ @Override
+ public String toString() {
+ return print();
+ }
+
+ /**
+ * A value class that describes the line and column of an offset in a file.
+ */
+ @Immutable
+ public static final class LineAndColumn {
+ private final int line;
+ private final int column;
+
+ public LineAndColumn(int line, int column) {
+ this.line = line;
+ this.column = column;
+ }
+
+ public int getLine() {
+ return line;
+ }
+
+ public int getColumn() {
+ return column;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof LineAndColumn)) {
+ return false;
+ }
+ LineAndColumn lac = (LineAndColumn) o;
+ return lac.line == line && lac.column == column;
+ }
+
+ @Override
+ public int hashCode() {
+ return line * 81 + column;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/NullEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/NullEventHandler.java
new file mode 100644
index 0000000..8bee1eb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/NullEventHandler.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+/**
+ * An ErrorEventListener which does nothing.
+ */
+public final class NullEventHandler implements EventHandler {
+ public static final EventHandler INSTANCE = new NullEventHandler();
+
+ private NullEventHandler() {} // Prevent instantiation
+
+ @Override
+ public void handle(Event e) {
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java b/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java
new file mode 100644
index 0000000..b5ca34d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+import java.util.regex.Pattern;
+
+/**
+ * An output filter for warnings.
+ */
+public interface OutputFilter {
+
+ /** An output filter that matches everything. */
+ public static final OutputFilter OUTPUT_EVERYTHING = new OutputFilter() {
+ @Override
+ public boolean showOutput(String tag) {
+ return true;
+ }
+ };
+
+ /** An output filter that matches nothing. */
+ public static final OutputFilter OUTPUT_NOTHING = new OutputFilter() {
+ @Override
+ public boolean showOutput(String tag) {
+ return false;
+ }
+ };
+
+ /**
+ * Returns true iff the given tag matches the output filter.
+ */
+ boolean showOutput(String tag);
+
+ /**
+ * An output filter using regular expression matching.
+ */
+ public static final class RegexOutputFilter implements OutputFilter {
+ /** Returns an output filter for the given regex (by compiling it). */
+ public static OutputFilter forRegex(String regex) {
+ return new RegexOutputFilter(Pattern.compile(regex));
+ }
+
+ /** Returns an output filter for the given pattern. */
+ public static OutputFilter forPattern(Pattern pattern) {
+ return new RegexOutputFilter(pattern);
+ }
+
+ private final Pattern pattern;
+
+ private RegexOutputFilter(Pattern pattern) {
+ this.pattern = pattern;
+ }
+
+ @Override
+ public boolean showOutput(String tag) {
+ return pattern.matcher(tag).find();
+ }
+
+ @Override
+ public String toString() {
+ return pattern.toString();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/PrintingEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/PrintingEventHandler.java
new file mode 100644
index 0000000..fa94d95
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/PrintingEventHandler.java
@@ -0,0 +1,119 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Set;
+
+/**
+ * An event handler that prints to an OutErr stream pair in a
+ * canonical format, for example:
+ * <pre>
+ * ERROR: /home/jrluser/src/workspace/x/BUILD:23:1: syntax error.
+ * </pre>
+ * This syntax is parseable by Emacs's compile.el.
+ *
+ * <p>
+ * By default, the output will go to SYSTEM_OUT_ERR,
+ * but this can be changed by calling the setOut() method.
+ *
+ * <p>
+ * This class is used only for tests.
+ */
+public class PrintingEventHandler extends AbstractEventHandler
+ implements EventHandler {
+
+ /**
+ * A convenient event-handler for terminal applications that prints all
+ * errors and warnings it encounters to the error stream.
+ * STDOUT and STDERR events pass their output directly
+ * through to the corresponding streams.
+ */
+ public static final PrintingEventHandler ERRORS_AND_WARNINGS_TO_STDERR =
+ new PrintingEventHandler(EventKind.ERRORS_AND_WARNINGS_AND_OUTPUT);
+
+ /**
+ * A convenient event-handler for terminal applications that prints all
+ * errors it encounters to the error stream.
+ * STDOUT and STDERR events pass their output directly
+ * through to the corresponding streams.
+ */
+ public static final PrintingEventHandler ERRORS_TO_STDERR =
+ new PrintingEventHandler(EventKind.ERRORS_AND_OUTPUT);
+
+ private OutErr outErr = OutErr.SYSTEM_OUT_ERR;
+
+ /**
+ * Setup a printing event handler that will handle events matching the mask.
+ * Events will be printed to the original System.out and System.err
+ * unless/until redirected by a call to setOutErr().
+ */
+ public PrintingEventHandler(Set<EventKind> mask) {
+ super(mask);
+ }
+
+ /**
+ * Redirect all output to the specified OutErr stream pair.
+ * Returns the previous OutErr.
+ */
+ public OutErr setOutErr(OutErr outErr) {
+ OutErr prev = this.outErr;
+ this.outErr = outErr;
+ return prev;
+ }
+
+ /**
+ * Print a description of the specified event to the appropriate
+ * output or error stream.
+ */
+ @Override
+ public void handle(Event event) {
+ if (!getEventMask().contains(event.getKind())) {
+ return;
+ }
+ try {
+ switch (event.getKind()) {
+ case STDOUT:
+ outErr.getOutputStream().write(event.getMessageBytes());
+ outErr.getOutputStream().flush();
+ break;
+ case STDERR:
+ outErr.getErrorStream().write(event.getMessageBytes());
+ outErr.getErrorStream().flush();
+ break;
+ default:
+ PrintWriter err = new PrintWriter(outErr.getErrorStream());
+ err.print(event.getKind());
+ err.print(": ");
+ if (event.getLocation() != null) {
+ err.print(event.getLocation().print());
+ err.print(": ");
+ }
+ err.println(event.getMessage());
+ err.flush();
+ }
+ } catch (IOException e) {
+ /*
+ * Note: we can't print to System.out or System.err here,
+ * because those will normally be set to streams which
+ * translate I/O to STDOUT and STDERR events,
+ * which would result in infinite recursion.
+ */
+ outErr.printErrLn(e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/Reporter.java b/src/main/java/com/google/devtools/build/lib/events/Reporter.java
new file mode 100644
index 0000000..e0c3925
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/Reporter.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The reporter is the primary means of reporting events such as errors,
+ * warnings, progress information and diagnostic information to the user. It
+ * is not intended as a logging mechanism for developer-only messages; use a
+ * Logger for that.
+ *
+ * The reporter instance is consumed by the build system, and passes events to
+ * {@link EventHandler} instances. These handlers are registered via {@link
+ * #addHandler(EventHandler)}.
+ *
+ * <p>Thread-safe: calls to {@code #report} may be made on any thread.
+ * Handlers may be run in an arbitary thread (but right now, they will not be
+ * run concurrently).
+ */
+public final class Reporter implements EventHandler, ExceptionListener {
+
+ private final List<EventHandler> handlers = new ArrayList<>();
+
+ /** An OutErr that sends all of its output to this Reporter.
+ * Each write will (when flushed) get mapped to an EventKind.STDOUT or EventKind.STDERR event.
+ */
+ private final OutErr outErrToReporter = outErrForReporter(this);
+ private volatile OutputFilter outputFilter = OutputFilter.OUTPUT_EVERYTHING;
+
+ public Reporter() {}
+
+ public static OutErr outErrForReporter(EventHandler rep) {
+ return OutErr.create(
+ // We don't use BufferedOutputStream here, because in general the Blaze
+ // code base assumes that the output streams are not buffered.
+ new ReporterStream(rep, EventKind.STDOUT),
+ new ReporterStream(rep, EventKind.STDERR));
+ }
+
+ /**
+ * A copy constructor, to make it convenient to replicate a reporter
+ * config for temporary configuration changes.
+ */
+ public Reporter(Reporter template) {
+ handlers.addAll(template.handlers);
+ }
+
+ /**
+ * Constructor which configures a reporter with the specified handlers.
+ */
+ public Reporter(EventHandler... handlers) {
+ for (EventHandler handler: handlers) {
+ addHandler(handler);
+ }
+ }
+
+ /**
+ * Returns an OutErr that sends all of its output to this Reporter.
+ * Each write to the OutErr will cause an EventKind.STDOUT or EventKind.STDERR event.
+ */
+ public OutErr getOutErr() {
+ return outErrToReporter;
+ }
+
+ /**
+ * Adds a handler to this reporter.
+ */
+ public synchronized void addHandler(EventHandler handler) {
+ handlers.add(handler);
+ }
+
+ /**
+ * Removes handler from this reporter.
+ */
+ public synchronized void removeHandler(EventHandler handler) {
+ handlers.remove(handler);
+ }
+
+ /**
+ * This method is called by the build system to report an event.
+ */
+ @Override
+ public synchronized void handle(Event e) {
+ if (e.getKind() != EventKind.ERROR && e.getTag() != null && !showOutput(e.getTag())) {
+ return;
+ }
+ for (EventHandler handler : handlers) {
+ handler.handle(e);
+ }
+ }
+
+ /**
+ * Reports the start of a particular task.
+ * Is a wrapper around report() with event kind START.
+ * Should always be matched by a corresponding call to finishTask()
+ * with the same message, except that the leading percentage
+ * progress indicator (if any) in the message may differ.
+ */
+ public void startTask(Location location, String message) {
+ handle(new Event(EventKind.START, location, message));
+ }
+
+ /**
+ * Reports the start of a particular task.
+ * Is a wrapper around report() with event kind FINISH.
+ * Should always be matched by a corresponding call to startTask()
+ * with the same message, except that the leading percentage
+ * progress indicator (if any) in the message may differ.
+ */
+ public void finishTask(Location location, String message) {
+ handle(new Event(EventKind.FINISH, location, message));
+ }
+
+ @Override
+ public void error(Location location, String message, Throwable error) {
+ handle(new Event(EventKind.ERROR, location, message));
+ error.printStackTrace(new PrintStream(getOutErr().getErrorStream()));
+ }
+
+ /**
+ * Returns true iff the given tag matches the output filter.
+ */
+ public boolean showOutput(String tag) {
+ return outputFilter.showOutput(tag);
+ }
+
+ public void setOutputFilter(OutputFilter outputFilter) {
+ this.outputFilter = outputFilter;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/ReporterStream.java b/src/main/java/com/google/devtools/build/lib/events/ReporterStream.java
new file mode 100644
index 0000000..9c625c8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/ReporterStream.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import java.io.OutputStream;
+
+/**
+ * An OutputStream that delegates all writes to a Reporter.
+ */
+public final class ReporterStream extends OutputStream {
+
+ private final EventHandler reporter;
+ private final EventKind eventKind;
+
+ public ReporterStream(EventHandler reporter, EventKind eventKind) {
+ this.reporter = reporter;
+ this.eventKind = eventKind;
+ }
+
+ @Override
+ public void close() {
+ // NOP.
+ }
+
+ @Override
+ public void flush() {
+ // NOP.
+ }
+
+ @Override
+ public void write(int b) {
+ reporter.handle(new Event(eventKind, null, new byte[] { (byte) b }));
+ }
+
+ @Override
+ public void write(byte[] bytes) {
+ reporter.handle(new Event(eventKind, null, bytes));
+ }
+
+ @Override
+ public void write(byte[] bytes, int offset, int len) {
+ reporter.handle(new Event(eventKind, null, new String(bytes, offset, len, ISO_8859_1)));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/StoredEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/StoredEventHandler.java
new file mode 100644
index 0000000..be8a627
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/StoredEventHandler.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Stores error and warning events, and later replays them. Thread-safe.
+ */
+public class StoredEventHandler implements EventHandler {
+
+ private final List<Event> events = new ArrayList<>();
+ private boolean hasErrors;
+
+ public synchronized ImmutableList<Event> getEvents() {
+ return ImmutableList.copyOf(events);
+ }
+
+ /** Returns true if there are no stored events. */
+ public synchronized boolean isEmpty() {
+ return events.isEmpty();
+ }
+
+
+ @Override
+ public synchronized void handle(Event e) {
+ hasErrors |= e.getKind() == EventKind.ERROR;
+ events.add(e);
+ }
+
+ /**
+ * Replay all events stored in this object on the given eventHandler, in the same order.
+ */
+ public synchronized void replayOn(EventHandler eventHandler) {
+ Event.replayEventsOn(eventHandler, events);
+ }
+
+ /**
+ * Returns whether any of the events on this objects were errors.
+ */
+ public synchronized boolean hasErrors() {
+ return hasErrors;
+ }
+
+ public synchronized void clear() {
+ events.clear();
+ hasErrors = false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandler.java
new file mode 100644
index 0000000..91a2150
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandler.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.events;
+
+/**
+ * Passes through any events, and converts any warnings to errors.
+ */
+public final class WarningsAsErrorsEventHandler extends DelegatingEventHandler {
+
+ boolean warningsEncountered = false;
+
+ public WarningsAsErrorsEventHandler(EventHandler eventHandler) {
+ super(eventHandler);
+ }
+
+ @Override
+ public synchronized void handle(Event e) {
+ if (e.getKind() == EventKind.WARNING) {
+ warningsEncountered = true;
+ super.handle(new Event(EventKind.ERROR, e.getLocation(), e.getMessage()));
+ } else {
+ super.handle(e);
+ }
+ }
+
+ public boolean warningsEncountered() {
+ return warningsEncountered;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java
new file mode 100644
index 0000000..0e484f7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java
@@ -0,0 +1,21 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+/**
+ * Marker interface for actions that must be run unconditionally.
+ */
+public interface AlwaysOutOfDateAction {
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java
new file mode 100644
index 0000000..84f3aef
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.rules.test.TestRunnerAction;
+
+/**
+ * Class implements --check_???_up_to_date execution filter predicate
+ * that prevents certain actions from being executed (thus aborting
+ * the build if action is not up-to-date).
+ */
+public final class CheckUpToDateFilter implements Predicate<Action> {
+
+ /**
+ * Determines an execution filter based on the --check_up_to_date and
+ * --check_tests_up_to_date options. Returns a singleton if possible.
+ */
+ public static Predicate<Action> fromOptions(ExecutionOptions options) {
+ if (!options.testCheckUpToDate && !options.checkUpToDate) {
+ return Predicates.alwaysTrue();
+ }
+ return new CheckUpToDateFilter(options);
+ }
+
+ private final boolean allowBuildActionExecution;
+ private final boolean allowTestActionExecution;
+
+ /**
+ * Creates new execution filter based on --check_up_to_date and
+ * --check_tests_up_to_date options.
+ */
+ private CheckUpToDateFilter(ExecutionOptions options) {
+ // If we want to check whether test is up-to-date, we should disallow
+ // test execution.
+ this.allowTestActionExecution = !options.testCheckUpToDate;
+
+ // Build action execution should be prohibited in two cases - if we are
+ // checking whether build is up-to-date or if we are checking that tests
+ // are up-to-date (and test execution is not allowed).
+ this.allowBuildActionExecution = allowTestActionExecution && !options.checkUpToDate;
+ }
+
+ /**
+ * @return true if actions' execution is allowed, false - otherwise
+ */
+ @Override
+ public boolean apply(Action action) {
+ if (action instanceof AlwaysOutOfDateAction) {
+ // Always allow fileset manifest action to execute because it identifies files included
+ // in the fileset during execution time.
+ return true;
+ } else if (action instanceof TestRunnerAction) {
+ return allowTestActionExecution;
+ } else {
+ return allowBuildActionExecution;
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/Digest.java b/src/main/java/com/google/devtools/build/lib/exec/Digest.java
new file mode 100644
index 0000000..4262711
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/Digest.java
@@ -0,0 +1,182 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.MessageLite;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * A utility class for obtaining MD5 digests.
+ * Digests are represented as 32 characters in lowercase ASCII.
+ */
+public class Digest {
+
+ public static final ByteString EMPTY_DIGEST = fromContent(new byte[]{});
+
+ private Digest() {
+ }
+
+ /**
+ * Get the digest from the given byte array.
+ * @param bytes the byte array.
+ * @return a digest.
+ */
+ public static ByteString fromContent(byte[] bytes) {
+ MessageDigest md = newBuilder();
+ md.update(bytes, 0, bytes.length);
+ return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest()));
+ }
+
+ /**
+ * Get the digest from the given ByteBuffer.
+ * @param buffer the ByteBuffer.
+ * @return a digest.
+ */
+ public static ByteString fromBuffer(ByteBuffer buffer) {
+ MessageDigest md = newBuilder();
+ md.update(buffer);
+ return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest()));
+ }
+
+ /**
+ * Gets the digest of the given proto.
+ *
+ * @param proto a protocol buffer.
+ * @return the digest.
+ */
+ public static ByteString fromProto(MessageLite proto) {
+ MD5OutputStream md5Stream = new MD5OutputStream();
+ try {
+ proto.writeTo(md5Stream);
+ } catch (IOException e) {
+ throw new IllegalStateException("Unexpected IOException: ", e);
+ }
+ return toByteString(md5Stream.getDigest());
+ }
+
+ /**
+ * Gets the digest and size of a given VirtualActionInput.
+ *
+ * @param input the VirtualActionInput.
+ * @return the digest and size.
+ */
+ public static Pair<ByteString, Long> fromVirtualActionInput(VirtualActionInput input)
+ throws IOException {
+ CountingMD5OutputStream md5Stream = new CountingMD5OutputStream();
+ input.writeTo(md5Stream);
+ ByteString digest = toByteString(md5Stream.getDigest());
+ return Pair.of(digest, md5Stream.getSize());
+ }
+
+ /**
+ * A Sink that does an online MD5 calculation, which avoids forcing us to keep the entire
+ * proto message in memory.
+ */
+ private static class MD5OutputStream extends OutputStream {
+ private final MessageDigest md = newBuilder();
+
+ @Override
+ public void write(int b) {
+ md.update((byte) b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ md.update(b, off, len);
+ }
+
+ public String getDigest() {
+ return BaseEncoding.base16().lowerCase().encode(md.digest());
+ }
+ }
+
+ private static final class CountingMD5OutputStream extends MD5OutputStream {
+ private long size;
+
+ @Override
+ public void write(int b) {
+ super.write(b);
+ size++;
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ super.write(b, off, len);
+ size += len;
+ }
+
+ public long getSize() {
+ return size;
+ }
+ }
+
+ /**
+ * @param digest the digest to check.
+ * @return true iff digest is a syntactically legal digest. It must be 32
+ * characters of hex with lowercase letters.
+ */
+ public static boolean isDigest(ByteString digest) {
+ if (digest == null || digest.size() != 32) {
+ return false;
+ }
+
+ for (byte b : digest) {
+ char c = (char) b;
+ if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
+ continue;
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * @param digest the digest.
+ * @return true iff the digest is that of an empty file.
+ */
+ public static boolean isEmpty(ByteString digest) {
+ return digest.equals(EMPTY_DIGEST);
+ }
+
+ /**
+ * @return a new MD5 digest builder.
+ */
+ public static MessageDigest newBuilder() {
+ try {
+ return MessageDigest.getInstance("md5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("MD5 not available");
+ }
+ }
+
+ /**
+ * Convert a String digest into a ByteString using ascii.
+ * @param digest the digest in ascii.
+ * @return the digest as a ByteString.
+ */
+ public static ByteString toByteString(String digest) {
+ return ByteString.copyFrom(digest.getBytes(US_ASCII));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
new file mode 100644
index 0000000..58e360b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
@@ -0,0 +1,195 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.rules.test.TestStrategy;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.Map;
+
+/**
+ * Options affecting the execution phase of a build.
+ *
+ * These options are interpreted by the BuildTool to choose an Executor to
+ * be used for the build.
+ *
+ * Note: from the user's point of view, the characteristic function of this
+ * set of options is indistinguishable from that of the BuildRequestOptions:
+ * they are all per-request. The difference is only apparent in the
+ * implementation: these options are used only by the lib.exec machinery, which
+ * affects how C++ and Java compilation occur. (The BuildRequestOptions
+ * contain a mixture of "semantic" options affecting the choice of targets to
+ * build, and "non-semantic" options affecting the lib.actions machinery.)
+ * Ideally, the user would be unaware of the difference. For now, the usage
+ * strings are identical modulo "part 1", "part 2".
+ */
+public class ExecutionOptions extends OptionsBase {
+
+ public static final ExecutionOptions DEFAULTS = Options.getDefaults(ExecutionOptions.class);
+
+ @Option(name = "verbose_failures",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "If a command fails, print out the full command line.")
+ public boolean verboseFailures;
+
+ @Option(name = "subcommands",
+ abbrev = 's',
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Display the subcommands executed during a build.")
+ public boolean showSubcommands;
+
+ @Option(name = "check_up_to_date",
+ defaultValue = "false",
+ category = "what",
+ help = "Don't perform the build, just check if it is up-to-date. If all targets are "
+ + "up-to-date, the build completes successfully. If any step needs to be executed "
+ + "an error is reported and the build fails.")
+ public boolean checkUpToDate;
+
+ @Option(name = "check_tests_up_to_date",
+ defaultValue = "false",
+ category = "testing",
+ implicitRequirements = { "--check_up_to_date" },
+ help = "Don't run tests, just check if they are up-to-date. If all tests results are "
+ + "up-to-date, the testing completes successfully. If any test needs to be built or "
+ + "executed, an error is reported and the testing fails. This option implies "
+ + "--check_up_to_date behavior."
+ )
+ public boolean testCheckUpToDate;
+
+ @Option(name = "test_strategy",
+ defaultValue = "",
+ category = "testing",
+ help = "Specifies which strategy to use when running tests.")
+ public String testStrategy;
+
+ @Option(name = "test_keep_going",
+ defaultValue = "true",
+ category = "testing",
+ help = "When disabled, any non-passing test will cause the entire build to stop. By default "
+ + "all tests are run, even if some do not pass.")
+ public boolean testKeepGoing;
+
+ @Option(name = "runs_per_test_detects_flakes",
+ defaultValue = "false",
+ category = "testing",
+ help = "If true, any shard in which at least one run/attempt passes and at least one "
+ + "run/attempt fails gets a FLAKY status.")
+ public boolean runsPerTestDetectsFlakes;
+
+ @Option(name = "flaky_test_attempts",
+ defaultValue = "default",
+ category = "testing",
+ converter = TestStrategy.TestAttemptsConverter.class,
+ help = "Each test will be retried up to the specified number of times in case of any test "
+ + "failure. Tests that required more than one attempt to pass would be marked as "
+ + "'FLAKY' in the test summary. If this option is set, it should specify an int N or the "
+ + "string 'default'. If it's an int, then all tests will be run up to N times. If it is "
+ + "not specified or its value is 'default', then only a single test attempt will be made "
+ + "for regular tests and three for tests marked explicitly as flaky by their rule "
+ + "(flaky=1 attribute).")
+ public int testAttempts;
+
+ @Option(name = "test_tmpdir",
+ defaultValue = "null",
+ category = "testing",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "Specifies the base temporary directory for 'blaze test' to use.")
+ public PathFragment testTmpDir;
+
+ @Option(name = "test_output",
+ defaultValue = "summary",
+ category = "testing",
+ converter = TestStrategy.TestOutputFormat.Converter.class,
+ help = "Specifies desired output mode. Valid values are 'summary' to "
+ + "output only test status summary, 'errors' to also print test logs "
+ + "for failed tests, 'all' to print logs for all tests and 'streamed' "
+ + "to output logs for all tests in real time (this will force tests "
+ + "to be executed locally one at a time regardless of --test_strategy "
+ + "value).")
+ public TestOutputFormat testOutput;
+
+ @Option(name = "test_summary",
+ defaultValue = "short",
+ category = "testing",
+ converter = TestStrategy.TestSummaryFormat.Converter.class,
+ help = "Specifies the desired format ot the test summary. Valid values "
+ + "are 'short' to print information only about tests executed, "
+ + "'terse', to print information only about unsuccessful tests,"
+ + "'detailed' to print detailed information about failed test cases, "
+ + "and 'none' to omit the summary.")
+ public TestSummaryFormat testSummary;
+
+ @Option(name = "test_timeout",
+ defaultValue = "-1",
+ category = "testing",
+ converter = TestTimeout.TestTimeoutConverter.class,
+ help = "Override the default test timeout values for test timeouts (in secs). If a single "
+ + "positive integer value is specified it will override all categories. If 4 comma-"
+ + "separated integers are specified, they will override the timeouts for short, "
+ + "moderate, long and eternal (in that order). In either form, a value of -1 tells blaze "
+ + "to use its default timeouts for that category.")
+ public Map<TestTimeout, Integer> testTimeout;
+
+
+ @Option(name = "resource_autosense",
+ defaultValue = "false",
+ category = "strategy",
+ help = "Periodically (every 3 seconds) poll system CPU load and available memory "
+ + "and allow execution of build commands if system has sufficient idle CPU and "
+ + "free RAM resources. By default this option is disabled, and Blaze will rely on "
+ + "approximation algorithms based on the total amount of available memory and number "
+ + "of CPU cores.")
+ public boolean useResourceAutoSense;
+
+ @Option(name = "ram_utilization_factor",
+ defaultValue = "67",
+ category = "strategy",
+ help = "Specify what percentage of the system's RAM Blaze should try to use for its "
+ + "subprocesses. "
+ + "This option affects how many processes Blaze will try to run in parallel. "
+ + "If you run several Blaze builds in parallel, using a lower value for "
+ + "this option may avoid thrashing and thus improve overall throughput. "
+ + "Using a value higher than the default is NOT recommended. "
+ + "Note that Blaze's estimates are very coarse, so the actual RAM usage may be much "
+ + "higher or much lower than specified. "
+ + "Note also that this option does not affect the amount of memory that the Blaze "
+ + "server itself will use. "
+ + "Also, this option has no effect if --resource_autosense is enabled."
+ )
+ public int ramUtilizationPercentage;
+
+ @Option(name = "local_resources",
+ defaultValue = "null",
+ category = "strategy",
+ help = "Explicitly set amount of local resources available to Blaze. "
+ + "By default, Blaze will query system configuration to estimate amount of RAM (in MB) "
+ + "and number of CPU cores available for the locally executed build actions. It would also "
+ + "assume default I/O capabilities of the local workstation (1.0). This options allows to "
+ + "explicitly set all 3 values. Note, that if this option is used, Blaze will ignore "
+ + "both --ram_utilization_factor and --resource_autosense values.",
+ converter = ResourceSet.ResourceSetConverter.class
+ )
+ public ResourceSet availableResources;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java
new file mode 100644
index 0000000..5dc9914
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A strategy for executing an {@link AbstractFileWriteAction}.
+ */
+@ExecutionStrategy(name = { "local" }, contextType = FileWriteActionContext.class)
+public final class FileWriteStrategy implements FileWriteActionContext {
+
+ public static final Class<FileWriteStrategy> TYPE = FileWriteStrategy.class;
+
+ public FileWriteStrategy() {
+ }
+
+ @Override
+ public void exec(Executor executor, AbstractFileWriteAction action, FileOutErr outErr,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
+ EventHandler reporter = executor == null ? null : executor.getEventHandler();
+ try {
+ Path outputPath = Iterables.getOnlyElement(action.getOutputs()).getPath();
+ try (OutputStream out = new BufferedOutputStream(outputPath.getOutputStream())) {
+ action.newDeterministicWriter(reporter, executor).writeOutputFile(out);
+ }
+ if (action.makeExecutable()) {
+ outputPath.setExecutable(true);
+ }
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("failed to create file '"
+ + Iterables.getOnlyElement(action.getOutputs()).prettyPrint()
+ + "' due to I/O error: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(AbstractFileWriteAction action) {
+ return action.estimateResourceConsumptionLocal();
+ }
+
+ @Override
+ public String strategyLocality(AbstractFileWriteAction action) {
+ return "local";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/OutputService.java b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java
new file mode 100644
index 0000000..88d9b94
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+
+/**
+ * An OutputService retains control over the Blaze output tree, and provides a higher level of
+ * abstraction compared to the VFS layer.
+ *
+ * <p>Higher-level facilities include batch statting, cleaning the output tree, creating symlink
+ * trees, and out-of-band insertion of metadata into the tree.
+ */
+public interface OutputService {
+
+ /**
+ * @return the name of filesystem, akin to what you might see in /proc/mounts
+ */
+ String getFilesSystemName();
+
+ /**
+ * @return true if the output service uses FUSE
+ */
+ boolean usesFuse();
+
+ /**
+ * @return a human-readable, one word name for the service
+ */
+ String getName();
+
+ /**
+ * Start the build.
+ *
+ * @throws BuildFailedException if build preparation failed
+ * @throws InterruptedException
+ */
+ void startBuild() throws BuildFailedException, AbruptExitException, InterruptedException;
+
+ /**
+ * Finish the build.
+ *
+ * @param buildSuccessful iff build was successful
+ * @throws BuildFailedException on failure
+ */
+ void finalizeBuild(boolean buildSuccessful) throws BuildFailedException, AbruptExitException;
+
+ /**
+ * Stages the given tool from the package path, possibly copying it to local disk.
+ *
+ * @param tool target representing the tool to stage
+ * @return a Path pointing to the staged target
+ */
+ Path stageTool(Target tool) throws IOException;
+
+ /**
+ * @return the name of the workspace this output service controls.
+ */
+ String getWorkspace();
+
+ /**
+ * @return the BatchStat instance or null.
+ */
+ BatchStat getBatchStatter();
+
+ /**
+ * @return true iff createSymlinkTree() is available.
+ */
+ boolean canCreateSymlinkTree();
+
+ /**
+ * Creates the symlink tree
+ *
+ * @param inputPath the input manifest
+ * @param outputPath the output manifest
+ * @param filesetTree is true iff we're constructing a Fileset
+ * @param symlinkTreeRoot the symlink tree root, relative to the execRoot
+ * @throws ExecException on failure
+ * @throws InterruptedException
+ */
+ void createSymlinkTree(Path inputPath, Path outputPath, boolean filesetTree,
+ PathFragment symlinkTreeRoot) throws ExecException, InterruptedException;
+
+ /**
+ * Cleans the entire output tree.
+ *
+ * @throws ExecException on failure
+ * @throws InterruptedException
+ */
+ void clean() throws ExecException, InterruptedException;
+
+ /**
+ * @param file the File
+ * @return true iff the file actually lives on a remote server
+ */
+ boolean isRemoteFile(Path file);
+
+ /**
+ * @param path a fully-resolved path
+ * @return true iff path is under this output service's control
+ */
+ boolean resolvedPathUnderTree(Path path);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java
new file mode 100644
index 0000000..8ec1e51
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Maps;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.DigestOfDirectoryException;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * An in-memory cache to ensure we do I/O for source files only once during a single build.
+ *
+ * <p>Simply maintains a two-way cached mapping from digest <--> filename that may be populated
+ * only once.
+ */
+@ThreadSafe
+public class SingleBuildFileCache implements ActionInputFileCache {
+
+ private final String cwd;
+ private final FileSystem fs;
+
+ public SingleBuildFileCache(String cwd, FileSystem fs) {
+ this.fs = Preconditions.checkNotNull(fs);
+ this.cwd = Preconditions.checkNotNull(cwd);
+ }
+
+ // If we can't get the digest, we store the exception. This avoids extra file IO for files
+ // that are allowed to be missing, as we first check a likely non-existent content file
+ // first. Further we won't need to unwrap the exception in getDigest().
+ private final LoadingCache<ActionInput, Pair<ByteString, IOException>> pathToDigest =
+ CacheBuilder.newBuilder()
+ // We default to 10 disk read threads, but we don't expect them all to edit the map
+ // simultaneously.
+ .concurrencyLevel(8)
+ // Even small-ish builds, as of 11/21/2011 typically have over 10k artifacts, so it's
+ // unlikely that this default will adversely affect memory in most cases.
+ .initialCapacity(10000)
+ .build(new CacheLoader<ActionInput, Pair<ByteString, IOException>>() {
+ @Override
+ public Pair<ByteString, IOException> load(ActionInput input) {
+ Path path = null;
+ try {
+ path = fs.getPath(fullPath(input));
+ BaseEncoding hex = BaseEncoding.base16().lowerCase();
+ ByteString digest = ByteString.copyFrom(
+ hex.encode(path.getMD5Digest())
+ .getBytes(US_ASCII));
+ pathToBytes.put(input, path.getFileSize());
+ // Inject reverse mapping. Doing this unconditionally in getDigest() showed up
+ // as a hotspot in CPU profiling.
+ digestToPath.put(digest, input);
+ return Pair.of(digest, null);
+ } catch (IOException e) {
+ if (path != null && path.isDirectory()) {
+ pathToBytes.put(input, 0L);
+ return Pair.<ByteString, IOException>of(null, new DigestOfDirectoryException(
+ "Input is a directory: " + input.getExecPathString()));
+ }
+
+ // Put value into size map to avoid trying to read file again later.
+ pathToBytes.put(input, 0L);
+ return Pair.of(null, e);
+ }
+ }
+ });
+
+ private final Map<ByteString, ActionInput> digestToPath = Maps.newConcurrentMap();
+
+ private final Map<ActionInput, Long> pathToBytes = Maps.newConcurrentMap();
+
+ @Nullable
+ @Override
+ public File getFileFromDigest(ByteString digest) {
+ ActionInput relPath = digestToPath.get(digest);
+ return relPath == null ? null : new File(fullPath(relPath));
+ }
+
+ @Override
+ public long getSizeInBytes(ActionInput input) throws IOException {
+ // TODO(bazel-team): this only works if pathToDigest has already been called.
+ Long sz = pathToBytes.get(input);
+ if (sz != null) {
+ return sz;
+ }
+ Path path = fs.getPath(fullPath(input));
+ sz = path.getFileSize();
+ pathToBytes.put(input, sz);
+ return sz;
+ }
+
+ @Override
+ public ByteString getDigest(ActionInput input) throws IOException {
+ Pair<ByteString, IOException> result = pathToDigest.getUnchecked(input);
+ if (result.second != null) {
+ throw result.second;
+ }
+ return result.first;
+ }
+
+ @Override
+ public boolean contentsAvailableLocally(ByteString digest) {
+ return digestToPath.containsKey(digest);
+ }
+
+ /**
+ * Creates a File object that refers to fileName, if fileName is an absolute path. Otherwise,
+ * returns a File object that refers to the fileName appended to the (absolute) current working
+ * directory.
+ */
+ private String fullPath(ActionInput input) {
+ String relPath = input.getExecPathString();
+ return relPath.startsWith("/") ? relPath : new File(cwd, relPath).getPath();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java
new file mode 100644
index 0000000..40fed77
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.analysis.SourceManifestAction;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A context for {@link SourceManifestAction} that uses the runtime to determine
+ * the workspace suffix.
+ */
+@ExecutionStrategy(contextType = SourceManifestAction.Context.class)
+public class SourceManifestActionContextImpl implements SourceManifestAction.Context {
+ private final PathFragment runfilesPrefix;
+
+ public SourceManifestActionContextImpl(PathFragment runfilesPrefix) {
+ this.runfilesPrefix = runfilesPrefix;
+ }
+
+ @Override
+ public PathFragment getRunfilesPrefix() {
+ return runfilesPrefix;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
new file mode 100644
index 0000000..6127cee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
@@ -0,0 +1,137 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.util.CommandBuilder;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * Helper class responsible for the symlink tree creation.
+ * Used to generate runfiles and fileset symlink farms.
+ */
+public final class SymlinkTreeHelper {
+
+ private static final String BUILD_RUNFILES = "build-runfiles" + OsUtils.executableExtension();
+
+ /**
+ * These actions run faster overall when serialized, because most of their
+ * cost is in the ext2 block allocator, and there's less seeking required if
+ * their directory creations get non-interleaved allocations. So we give them
+ * a huge resource cost.
+ */
+ public static final ResourceSet RESOURCE_SET = new ResourceSet(1000, 0.5, 0.75);
+
+ private final PathFragment inputManifest;
+ private final PathFragment symlinkTreeRoot;
+ private final boolean filesetTree;
+
+ /**
+ * Creates SymlinkTreeHelper instance. Can be used independently of
+ * SymlinkTreeAction.
+ *
+ * @param inputManifest exec path to the input runfiles manifest
+ * @param symlinkTreeRoot exec path to the symlink tree location
+ * @param filesetTree true if this is fileset symlink tree,
+ * false if this is a runfiles symlink tree.
+ */
+ public SymlinkTreeHelper(PathFragment inputManifest, PathFragment symlinkTreeRoot,
+ boolean filesetTree) {
+ this.inputManifest = inputManifest;
+ this.symlinkTreeRoot = symlinkTreeRoot;
+ this.filesetTree = filesetTree;
+ }
+
+ public PathFragment getSymlinkTreeRoot() { return symlinkTreeRoot; }
+
+ /**
+ * Creates a symlink tree using a CommandBuilder. This means that the symlink
+ * tree will always be present on the developer's workstation. Useful when
+ * running commands locally.
+ *
+ * Warning: this method REALLY executes the command on the box Blaze was
+ * run on, without any kind of synchronization, locking, or anything else.
+ *
+ * @param config the configuration that is used for creating the symlink tree.
+ * @throws CommandException
+ */
+ public void createSymlinksUsingCommand(Path execRoot,
+ BuildConfiguration config, BinTools binTools) throws CommandException {
+ List<String> argv = getSpawnArgumentList(execRoot, binTools);
+
+ CommandBuilder builder = new CommandBuilder();
+ builder.addArgs(argv);
+ builder.setWorkingDir(execRoot);
+ builder.build().execute();
+ }
+
+ /**
+ * Creates symlink tree using appropriate method. At this time tree
+ * always created using build-runfiles helper application.
+ *
+ * Note: method may try to acquire resources - meaning that it would
+ * block for undetermined period of time. If it is interrupted during
+ * that wait, ExecException will be thrown but interrupted bit will be
+ * preserved.
+ *
+ * @param action action instance that requested symlink tree creation
+ * @param actionExecutionContext Services that are in the scope of the action.
+ */
+ public void createSymlinks(AbstractAction action, ActionExecutionContext actionExecutionContext,
+ BinTools binTools) throws ExecException, InterruptedException {
+ List<String> args = getSpawnArgumentList(
+ actionExecutionContext.getExecutor().getExecRoot(), binTools);
+ try {
+ ResourceManager.instance().acquireResources(action, RESOURCE_SET);
+ actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()).exec(
+ new BaseSpawn.Local(args, ImmutableMap.<String, String>of(), action),
+ actionExecutionContext);
+ } finally {
+ ResourceManager.instance().releaseResources(action, RESOURCE_SET);
+ }
+ }
+
+ /**
+ * Returns the complete argument list build-runfiles has to be called with.
+ */
+ private List<String> getSpawnArgumentList(Path execRoot, BinTools binTools) {
+ List<String> args = Lists.newArrayList(
+ execRoot.getRelative(binTools.getExecPath(BUILD_RUNFILES))
+ .getPathString());
+
+ if (filesetTree) {
+ args.add("--allow_relative");
+ args.add("--use_metadata");
+ }
+
+ args.add(inputManifest.getPathString());
+ args.add(symlinkTreeRoot.getPathString());
+
+ return args;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
new file mode 100644
index 0000000..d9470e4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.SymlinkTreeAction;
+import com.google.devtools.build.lib.analysis.SymlinkTreeActionContext;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+
+/**
+ * Implements SymlinkTreeAction by using the output service or by running an embedded script to
+ * create the symlink tree.
+ */
+@ExecutionStrategy(contextType = SymlinkTreeActionContext.class)
+public final class SymlinkTreeStrategy implements SymlinkTreeActionContext {
+ private final OutputService outputService;
+ private final BinTools binTools;
+
+ public SymlinkTreeStrategy(OutputService outputService, BinTools binTools) {
+ this.outputService = outputService;
+ this.binTools = binTools;
+ }
+
+ @Override
+ public void createSymlinks(SymlinkTreeAction action,
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ try {
+ SymlinkTreeHelper helper = new SymlinkTreeHelper(
+ action.getInputManifest().getExecPath(),
+ action.getOutputManifest().getExecPath().getParentDirectory(), action.isFilesetTree());
+ if (outputService != null && outputService.canCreateSymlinkTree()) {
+ outputService.createSymlinkTree(action.getInputManifest().getPath(),
+ action.getOutputManifest().getPath(),
+ action.isFilesetTree(), helper.getSymlinkTreeRoot());
+ } else {
+ helper.createSymlinks(action, actionExecutionContext, binTools);
+ }
+ } catch (ExecException e) {
+ throw e.toActionExecutionException(
+ action.getProgressMessage(), executor.getVerboseFailures(), action);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/AbstractGraphVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/AbstractGraphVisitor.java
new file mode 100644
index 0000000..a08ed42
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/AbstractGraphVisitor.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.graph;
+
+/**
+ * <p> A stub implementation of GraphVisitor providing default behaviour (do
+ * nothing) for all its methods. </p>
+ */
+public class AbstractGraphVisitor<T> implements GraphVisitor<T> {
+ @Override
+ public void beginVisit() {}
+ @Override
+ public void endVisit() {}
+ @Override
+ public void visitEdge(Node<T> lhs, Node<T> rhs) {}
+ @Override
+ public void visitNode(Node<T> node) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/CollectingVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/CollectingVisitor.java
new file mode 100644
index 0000000..caeb07b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/CollectingVisitor.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.graph;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A graph visitor that collects the visited nodes in the order in which
+ * they were visited, and allows them to be accessed as a list.
+ */
+public class CollectingVisitor<T> extends AbstractGraphVisitor<T> {
+
+ private final List<Node<T>> order = new ArrayList<Node<T>>();
+
+ @Override
+ public void visitNode(Node<T> node) {
+ order.add(node);
+ }
+
+ /**
+ * Returns a reference to (not a copy of) the list of visited nodes in the
+ * order they were visited.
+ */
+ public List<Node<T>> getVisitedNodes() {
+ return order;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/DFS.java b/src/main/java/com/google/devtools/build/lib/graph/DFS.java
new file mode 100644
index 0000000..37cd30e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/DFS.java
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.graph;
+
+import com.google.common.collect.Lists;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * <p> The DFS class encapsulates a depth-first search visitation, including
+ * the order in which nodes are to be visited relative to their successors
+ * (PREORDER/POSTORDER), whether the forward or transposed graph is to be
+ * used, and which nodes have been seen already. </p>
+ *
+ * <p> A variety of common uses of DFS are offered through methods of
+ * Digraph; however clients can use this class directly for maximum
+ * flexibility. See the implementation of
+ * Digraph.getStronglyConnectedComponents() for an example. </p>
+ *
+ * <p> Clients should not modify the enclosing Digraph instance of a DFS
+ * while a traversal is in progress. </p>
+ */
+public class DFS<T> {
+
+ // (Preferred over a boolean to avoid parameter confusion.)
+ public enum Order {
+ PREORDER,
+ POSTORDER
+ }
+
+ private final Order order; // = (PREORDER|POSTORDER)
+
+ private final Comparator<Node<T>> edgeOrder;
+
+ private final boolean transpose;
+
+ private final Set<Node<T>> marked = new HashSet<Node<T>>();
+
+ /**
+ * Constructs a DFS instance for searching over the enclosing Digraph
+ * instance, using the specified visitation parameters.
+ *
+ * @param order PREORDER or POSTORDER, determines node visitation order
+ * @param edgeOrder an ordering in which the edges originating from the same
+ * node should be visited (if null, the order is unspecified)
+ * @param transpose iff true, the graph is implicitly transposed during
+ * visitation.
+ */
+ public DFS(Order order, final Comparator<T> edgeOrder, boolean transpose) {
+ this.order = order;
+ this.transpose = transpose;
+
+ if (edgeOrder == null) {
+ this.edgeOrder = null;
+ } else {
+ this.edgeOrder = new Comparator<Node<T>>() {
+ @Override
+ public int compare(Node<T> o1, Node<T> o2) {
+ return edgeOrder.compare(o1.getLabel(), o2.getLabel());
+ }
+ };
+ }
+ }
+
+ public DFS(Order order, boolean transpose) {
+ this(order, null, transpose);
+ }
+
+ /**
+ * Returns the (immutable) set of nodes visited so far.
+ */
+ public Set<Node<T>> getMarked() {
+ return Collections.unmodifiableSet(marked);
+ }
+
+ public void visit(Node<T> node, GraphVisitor<T> visitor) {
+ if (!marked.add(node)) {
+ return;
+ }
+
+ if (order == Order.PREORDER) {
+ visitor.visitNode(node);
+ }
+
+ Collection<Node<T>> edgeTargets = transpose
+ ? node.getPredecessors() : node.getSuccessors();
+ if (edgeOrder != null) {
+ List<Node<T>> mutableNodeList = Lists.newArrayList(edgeTargets);
+ Collections.sort(mutableNodeList, edgeOrder);
+ edgeTargets = mutableNodeList;
+ }
+
+ for (Node<T> v: edgeTargets) {
+ visit(v, visitor);
+ }
+
+ if (order == Order.POSTORDER) {
+ visitor.visitNode(node);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/Digraph.java b/src/main/java/com/google/devtools/build/lib/graph/Digraph.java
new file mode 100644
index 0000000..bea2f5a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/Digraph.java
@@ -0,0 +1,1063 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.graph;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+/**
+ * <p> {@code Digraph} a generic directed graph or "digraph", suitable for
+ * modeling asymmetric binary relations. </p>
+ *
+ * <p> An instance <code>G = <V,E></code> consists of a set of nodes or
+ * vertices <code>V</code>, and a set of directed edges <code>E</code>, which
+ * is a subset of <code>V × V</code>. This permits self-edges but does
+ * not represent multiple edges between the same pair of nodes. </p>
+ *
+ * <p> Nodes may be labeled with values of any type (type parameter
+ * T). All nodes within a graph have distinct labels. The null
+ * pointer is not a valid label.</p>
+ *
+ * <p> The package supports various operations for modeling partial order
+ * relations, and supports input/output in AT&T's 'dot' format. See
+ * http://www.research.att.com/sw/tools/graphviz/. </p>
+ *
+ * <p> Some invariants: </p>
+ * <ul>
+ *
+ * <li> Each graph instances "owns" the nodes is creates. The behaviour of
+ * operations on nodes a graph does not own is undefined.
+ *
+ * <li> {@code Digraph} assumes immutability of node labels, much like {@link
+ * HashMap} assumes it for keys.
+ *
+ * <li> Mutating the underlying graph invalidates any sets and iterators backed
+ * by it.
+ *
+ * </ul>
+ *
+ * <p>Each node stores successor and predecessor adjacency sets using a
+ * representation that dynamically changes with size: small sets are stored as
+ * arrays, large sets using hash tables. This representation provides
+ * significant space and time performance improvements upon two prior versions:
+ * the earliest used only HashSets; a later version used linked lists, as
+ * described in Cormen, Leiserson & Rivest.
+ */
+public final class Digraph<T> implements Cloneable {
+
+ /**
+ * Maps labels to nodes, which are in strict 1:1 correspondence.
+ */
+ private final HashMap<T, Node<T>> nodes = Maps.newHashMap();
+
+ /**
+ * A source of unique, deterministic hashCodes for {@link Node} instances.
+ */
+ private int nextHashCode = 0;
+
+ /**
+ * Construct an empty Digraph.
+ */
+ public Digraph() {}
+
+ /**
+ * Sanity-check: assert that a node is indeed a member of this graph and not
+ * another one. Perform this check whenever a function is supplied a node by
+ * the user.
+ */
+ private final void checkNode(Node<T> node) {
+ if (getNode(node.getLabel()) != node) {
+ throw new IllegalArgumentException("node " + node
+ + " is not a member of this graph");
+ }
+ }
+
+ /**
+ * Adds a directed edge between the nodes labelled 'from' and 'to', creating
+ * them if necessary.
+ *
+ * @return true iff the edge was not already present.
+ */
+ public boolean addEdge(T from, T to) {
+ Node<T> fromNode = createNode(from);
+ Node<T> toNode = createNode(to);
+ return addEdge(fromNode, toNode);
+ }
+
+ /**
+ * Adds a directed edge between the specified nodes, which must exist and
+ * belong to this graph.
+ *
+ * @return true iff the edge was not already present.
+ *
+ * Note: multi-edges are ignored. Self-edges are permitted.
+ */
+ public boolean addEdge(Node<T> fromNode, Node<T> toNode) {
+ checkNode(fromNode);
+ checkNode(toNode);
+ boolean isNewSuccessor = fromNode.addSuccessor(toNode);
+ boolean isNewPredecessor = toNode.addPredecessor(fromNode);
+ if (isNewPredecessor != isNewSuccessor) {
+ throw new IllegalStateException();
+ }
+ return isNewSuccessor;
+ }
+
+ /**
+ * Returns true iff the graph contains an edge between the
+ * specified nodes, which must exist and belong to this graph.
+ */
+ public boolean containsEdge(Node<T> fromNode, Node<T> toNode) {
+ checkNode(fromNode);
+ checkNode(toNode);
+ // TODO(bazel-team): (2009) iterate only over the shorter of from.succs, to.preds.
+ return fromNode.getSuccessors().contains(toNode);
+ }
+
+ /**
+ * Removes the edge between the specified nodes. Idempotent: attempts to
+ * remove non-existent edges have no effect.
+ *
+ * @return true iff graph changed.
+ */
+ public boolean removeEdge(Node<T> fromNode, Node<T> toNode) {
+ checkNode(fromNode);
+ checkNode(toNode);
+ boolean changed = fromNode.removeSuccessor(toNode);
+ if (changed) {
+ toNode.removePredecessor(fromNode);
+ }
+ return changed;
+ }
+
+ /**
+ * Remove all nodes and edges.
+ */
+ public void clear() {
+ nodes.clear();
+ }
+
+ @Override
+ public String toString() {
+ return "Digraph[" + getNodeCount() + " nodes]";
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException(); // avoid nondeterminism
+ }
+
+ /**
+ * Returns true iff the two graphs are equivalent, i.e. have the same set
+ * of node labels, with the same connectivity relation.
+ *
+ * O(n^2) in the worst case, i.e. equivalence. The algorithm could be speed up by
+ * close to a factor 2 in the worst case by a more direct implementation instead
+ * of using isSubgraph twice.
+ */
+ @Override
+ public boolean equals(Object thatObject) {
+ /* If this graph is a subgraph of thatObject, then we know that thatObject is of
+ * type Digraph<?> and thatObject can be cast to this type.
+ */
+ return isSubgraph(thatObject) && ((Digraph<?>) thatObject).isSubgraph(this);
+ }
+
+ /**
+ * Returns true iff this graph is a subgraph of the argument. This means that this graph's nodes
+ * are a subset of those of the argument; moreover, for each node of this graph the set of
+ * successors is a subset of those of the corresponding node in the argument graph.
+ *
+ * This algorithm is O(n^2), but linear in the total sizes of the graphs.
+ */
+ public boolean isSubgraph(Object thatObject) {
+ if (this == thatObject) {
+ return true;
+ }
+ if (!(thatObject instanceof Digraph)) {
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ Digraph<T> that = (Digraph<T>) thatObject;
+ if (this.getNodeCount() > that.getNodeCount()) {
+ return false;
+ }
+ for (Node<T> n1: nodes.values()) {
+ Node<T> n2 = that.getNodeMaybe(n1.getLabel());
+ if (n2 == null) {
+ return false; // 'that' is missing a node
+ }
+
+ // Now compare the successor relations.
+ // Careful:
+ // - We can't do simple equality on the succs-sets because the
+ // nodes belong to two different graphs!
+ // - There's no need to check both predecessor and successor
+ // relations, either one is sufficient.
+ Collection<Node<T>> n1succs = n1.getSuccessors();
+ Collection<Node<T>> n2succs = n2.getSuccessors();
+ if (n1succs.size() > n2succs.size()) {
+ return false;
+ }
+ // foreach successor of n1, ensure n2 has a similarly-labeled succ.
+ for (Node<T> succ1: n1succs) {
+ Node<T> succ2 = that.getNodeMaybe(succ1.getLabel());
+ if (succ2 == null) {
+ return false;
+ }
+ if (!n2succs.contains(succ2)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns a duplicate graph with the same set of node labels and the same
+ * connectivity relation. The labels themselves are not cloned.
+ */
+ @Override
+ public Digraph<T> clone() {
+ final Digraph<T> that = new Digraph<T>();
+ visitNodesBeforeEdges(new AbstractGraphVisitor<T>() {
+ @Override
+ public void visitEdge(Node<T> lhs, Node<T> rhs) {
+ that.addEdge(lhs.getLabel(), rhs.getLabel());
+ }
+ @Override
+ public void visitNode(Node<T> node) {
+ that.createNode(node.getLabel());
+ }
+ });
+ return that;
+ }
+
+ /**
+ * Returns a deterministic immutable view of the nodes of this graph.
+ */
+ public Collection<Node<T>> getNodes(final Comparator<T> comparator) {
+ Ordering<Node<T>> ordering = new Ordering<Node<T>>() {
+ @Override
+ public int compare(Node<T> o1, Node<T> o2) {
+ return comparator.compare(o1.getLabel(), o2.getLabel());
+ }
+ };
+ return ordering.immutableSortedCopy(nodes.values());
+ }
+
+ /**
+ * Returns an immutable view of the nodes of this graph.
+ *
+ * Note: we have to return Collection and not Set because values() returns
+ * one: the 'nodes' HashMap doesn't know that it is injective. :-(
+ */
+ public Collection<Node<T>> getNodes() {
+ return Collections.unmodifiableCollection(nodes.values());
+ }
+
+ /**
+ * @return the set of root nodes: those with no predecessors.
+ *
+ * NOTE: in a cyclic graph, there may be nodes that are not reachable from
+ * any "root".
+ */
+ public Set<Node<T>> getRoots() {
+ Set<Node<T>> roots = new HashSet<Node<T>>();
+ for (Node<T> node: nodes.values()) {
+ if (!node.hasPredecessors()) {
+ roots.add(node);
+ }
+ }
+ return roots;
+ }
+
+ /**
+ * @return the set of leaf nodes: those with no successors.
+ */
+ public Set<Node<T>> getLeaves() {
+ Set<Node<T>> leaves = new HashSet<Node<T>>();
+ for (Node<T> node: nodes.values()) {
+ if (!node.hasSuccessors()) {
+ leaves.add(node);
+ }
+ }
+ return leaves;
+ }
+
+ /**
+ * @return an immutable view of the set of labels of this graph's nodes.
+ */
+ public Set<T> getLabels() {
+ return Collections.unmodifiableSet(nodes.keySet());
+ }
+
+ /**
+ * Finds and returns the node with the specified label. If there is no such
+ * node, an exception is thrown. The null pointer is not a valid label.
+ *
+ * @return the node whose label is "label".
+ * @throws IllegalArgumentException if no node was found with the specified
+ * label.
+ */
+ public Node<T> getNode(T label) {
+ if (label == null) {
+ throw new NullPointerException();
+ }
+ Node<T> node = nodes.get(label);
+ if (node == null) {
+ throw new IllegalArgumentException("No such node label: " + label);
+ }
+ return node;
+ }
+
+ /**
+ * Find the node with the specified label. Returns null if it doesn't exist.
+ * The null pointer is not a valid label.
+ *
+ * @return the node whose label is "label", or null if it was not found.
+ */
+ public Node<T> getNodeMaybe(T label) {
+ if (label == null) {
+ throw new NullPointerException();
+ }
+ return nodes.get(label);
+ }
+
+ /**
+ * @return the number of nodes in the graph.
+ */
+ public int getNodeCount() {
+ return nodes.size();
+ }
+
+ /**
+ * @return the number of edges in the graph.
+ *
+ * Note: expensive! Useful when asserting against mutations though.
+ */
+ public int getEdgeCount() {
+ int edges = 0;
+ for (Node<T> node: nodes.values()) {
+ edges += node.getSuccessors().size();
+ }
+ return edges;
+ }
+
+ /**
+ * Find or create a node with the specified label. This is the <i>only</i>
+ * factory of Nodes. The null pointer is not a valid label.
+ */
+ public Node<T> createNode(T label) {
+ if (label == null) {
+ throw new NullPointerException();
+ }
+ Node<T> n = nodes.get(label);
+ if (n == null) {
+ nodes.put(label, n = new Node<T>(label, nextHashCode++));
+ }
+ return n;
+ }
+
+ /******************************************************************
+ * *
+ * Graph Algorithms *
+ * *
+ ******************************************************************/
+
+ /**
+ * These only manipulate the graph through methods defined above.
+ */
+
+ /**
+ * Returns true iff the graph is cyclic. Time: O(n).
+ */
+ public boolean isCyclic() {
+
+ // To detect cycles, we use a colored depth-first search. All nodes are
+ // initially marked white. When a node is encountered, it is marked grey,
+ // and when its descendants are completely visited, it is marked black.
+ // If a grey node is ever encountered, then there is a cycle.
+ final Object WHITE = null; // i.e. not present in nodeToColor, the default.
+ final Object GREY = new Object();
+ final Object BLACK = new Object();
+ final Map<Node<T>, Object> nodeToColor =
+ new HashMap<Node<T>, Object>(); // empty => all white
+
+ class CycleDetector { /* defining a class gives us lexical scope */
+ boolean visit(Node<T> node) {
+ nodeToColor.put(node, GREY);
+ for (Node<T> succ: node.getSuccessors()) {
+ if (nodeToColor.get(succ) == GREY) {
+ return true;
+ } else if (nodeToColor.get(succ) == WHITE) {
+ if (visit(succ)) {
+ return true;
+ }
+ }
+ }
+ nodeToColor.put(node, BLACK);
+ return false;
+ }
+ }
+
+ CycleDetector detector = new CycleDetector();
+ for (Node<T> node: nodes.values()) {
+ if (nodeToColor.get(node) == WHITE) {
+ if (detector.visit(node)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the strong component graph of "this". That is, returns a new
+ * acyclic graph in which all strongly-connected components in the original
+ * graph have been "fused" into a single node.
+ *
+ * @return a new graph, whose node labels are sets of nodes of the
+ * original graph. (Do not get confused as to which graph each
+ * set of Nodes belongs!)
+ */
+ public Digraph<Set<Node<T>>> getStrongComponentGraph() {
+ Collection<Set<Node<T>>> sccs = getStronglyConnectedComponents();
+ Digraph<Set<Node<T>>> scGraph = createImageUnderPartition(sccs);
+ scGraph.removeSelfEdges(); // scGraph should be acyclic: no self-edges
+ return scGraph;
+ }
+
+ /**
+ * Returns a partition of the nodes of this graph into sets, each set being
+ * one strongly-connected component of the graph.
+ */
+ public Collection<Set<Node<T>>> getStronglyConnectedComponents() {
+ final List<Set<Node<T>>> sccs = new ArrayList<Set<Node<T>>>();
+ NodeSetReceiver<T> r = new NodeSetReceiver<T>() {
+ @Override
+ public void accept(Set<Node<T>> scc) {
+ sccs.add(scc);
+ }
+ };
+ SccVisitor<T> v = new SccVisitor<T>();
+ for (Node<T> node : nodes.values()) {
+ v.visit(r, node);
+ }
+ return sccs;
+ }
+
+ /**
+ * <p> Given a partition of the graph into sets of nodes, returns the image
+ * of this graph under the function which maps each node to the
+ * partition-set in which it appears. The labels of the new graph are the
+ * (immutable) sets of the partition, and the edges of the new graph are the
+ * edges of the original graph, mapped via the same function. </p>
+ *
+ * <p> Note: the resulting graph may contain self-edges. If these are not
+ * wanted, call <code>removeSelfEdges()</code>> on the result. </p>
+ *
+ * <p> Interesting special case: if the partition is the set of
+ * strongly-connected components, the result of this function is the
+ * strong-component graph. </p>
+ */
+ public Digraph<Set<Node<T>>>
+ createImageUnderPartition(Collection<Set<Node<T>>> partition) {
+
+ // Build mapping function: each node label is mapped to its equiv class:
+ Map<T, Set<Node<T>>> labelToImage =
+ new HashMap<T, Set<Node<T>>>();
+ for (Set<Node<T>> set: partition) {
+ // It's important to use immutable sets of node labels when sets are keys
+ // in a map; see ImmutableSet class for explanation.
+ Set<Node<T>> imageSet = ImmutableSet.copyOf(set);
+ for (Node<T> node: imageSet) {
+ labelToImage.put(node.getLabel(), imageSet);
+ }
+ }
+
+ if (labelToImage.size() != getNodeCount()) {
+ throw new IllegalArgumentException(
+ "createImageUnderPartition(): argument is not a partition");
+ }
+
+ return createImageUnderMapping(labelToImage);
+ }
+
+ /**
+ * Returns the image of this graph in a given function, expressed as a
+ * mapping from labels to some other domain.
+ */
+ public <IMAGE> Digraph<IMAGE>
+ createImageUnderMapping(Map<T, IMAGE> map) {
+
+ Digraph<IMAGE> imageGraph = new Digraph<IMAGE>();
+
+ for (Node<T> fromNode: nodes.values()) {
+ T fromLabel = fromNode.getLabel();
+
+ IMAGE fromImage = map.get(fromLabel);
+ if (fromImage == null) {
+ throw new IllegalArgumentException(
+ "Incomplete function: undefined for " + fromLabel);
+ }
+ imageGraph.createNode(fromImage);
+
+ for (Node<T> toNode: fromNode.getSuccessors()) {
+ T toLabel = toNode.getLabel();
+
+ IMAGE toImage = map.get(toLabel);
+ if (toImage == null) {
+ throw new IllegalArgumentException(
+ "Incomplete function: undefined for " + toLabel);
+ }
+ imageGraph.addEdge(fromImage, toImage);
+ }
+ }
+
+ return imageGraph;
+ }
+
+ /**
+ * Removes any self-edges (x,x) in this graph.
+ */
+ public void removeSelfEdges() {
+ for (Node<T> node: nodes.values()) {
+ removeEdge(node, node);
+ }
+ }
+
+ /**
+ * Finds the shortest directed path from "fromNode" to "toNode". The path is
+ * returned as an ordered list of nodes, including both endpoints. Returns
+ * null if there is no path. Uses breadth-first search. Running time is
+ * O(n).
+ */
+ public List<Node<T>> getShortestPath(Node<T> fromNode,
+ Node<T> toNode) {
+ checkNode(fromNode);
+ checkNode(toNode);
+
+ if (fromNode == toNode) {
+ return Collections.singletonList(fromNode);
+ }
+
+ Map<Node<T>, Node<T>> pathPredecessor =
+ new HashMap<Node<T>, Node<T>>();
+
+ Set<Node<T>> marked = new HashSet<Node<T>>();
+
+ LinkedList<Node<T>> queue = new LinkedList<Node<T>>();
+ queue.addLast(fromNode);
+ marked.add(fromNode);
+
+ while (queue.size() > 0) {
+ Node<T> u = queue.removeFirst();
+ for (Node<T> v: u.getSuccessors()) {
+ if (marked.add(v)) {
+ pathPredecessor.put(v, u);
+ if (v == toNode) {
+ return getPathToTreeNode(pathPredecessor, v); // found a path
+ }
+ queue.addLast(v);
+ }
+ }
+ }
+ return null; // no path
+ }
+
+ /**
+ * Given a tree (expressed as a map from each node to its parent), and a
+ * starting node, returns the path from the root of the tree to 'node' as a
+ * list.
+ */
+ private static <X> List<X> getPathToTreeNode(Map<X, X> tree, X node) {
+ List<X> path = new ArrayList<X>();
+ while (node != null) {
+ path.add(node);
+ node = tree.get(node); // get parent
+ }
+ Collections.reverse(path);
+ return path;
+ }
+
+ /**
+ * Returns the nodes of an acyclic graph in topological order
+ * [a.k.a "reverse post-order" of depth-first search.]
+ *
+ * A topological order is one such that, if (u, v) is a path in
+ * acyclic graph G, then u is before v in the topological order.
+ * In other words "tails before heads" or "roots before leaves".
+ *
+ * @return The nodes of the graph, in a topological order
+ */
+ public List<Node<T>> getTopologicalOrder() {
+ List<Node<T>> order = getPostorder();
+ Collections.reverse(order);
+ return order;
+ }
+
+ /**
+ * Returns the nodes of an acyclic graph in topological order
+ * [a.k.a "reverse post-order" of depth-first search.]
+ *
+ * A topological order is one such that, if (u, v) is a path in
+ * acyclic graph G, then u is before v in the topological order.
+ * In other words "tails before heads" or "roots before leaves".
+ *
+ * If an ordering is given, returns a specific topological order from the set
+ * of all topological orders; if no ordering given, returns an arbitrary
+ * (nondeterministic) one, but is a bit faster because no sorting needs to be
+ * done for each node.
+ *
+ * @param edgeOrder the ordering in which edges originating from the same node
+ * are visited.
+ * @return The nodes of the graph, in a topological order
+ */
+ public List<Node<T>> getTopologicalOrder(
+ Comparator<T> edgeOrder) {
+ CollectingVisitor<T> visitor = new CollectingVisitor<T>();
+ DFS<T> visitation = new DFS<T>(DFS.Order.POSTORDER, edgeOrder, false);
+ visitor.beginVisit();
+ for (Node<T> node : getNodes(edgeOrder)) {
+ visitation.visit(node, visitor);
+ }
+ visitor.endVisit();
+
+ List<Node<T>> order = visitor.getVisitedNodes();
+ Collections.reverse(order);
+ return order;
+ }
+
+ /**
+ * Returns the nodes of an acyclic graph in post-order.
+ */
+ public List<Node<T>> getPostorder() {
+ CollectingVisitor<T> collectingVisitor = new CollectingVisitor<T>();
+ visitPostorder(collectingVisitor);
+ return collectingVisitor.getVisitedNodes();
+ }
+
+ /**
+ * Returns the (immutable) set of nodes reachable from node 'n' (reflexive
+ * transitive closure).
+ */
+ public Set<Node<T>> getFwdReachable(Node<T> n) {
+ return getFwdReachable(Collections.singleton(n));
+ }
+
+ /**
+ * Returns the (immutable) set of nodes reachable from any node in {@code
+ * startNodes} (reflexive transitive closure).
+ */
+ public Set<Node<T>> getFwdReachable(Collection<Node<T>> startNodes) {
+ // This method is intentionally not static, to permit future expansion.
+ DFS<T> dfs = new DFS<T>(DFS.Order.PREORDER, false);
+ for (Node<T> n : startNodes) {
+ dfs.visit(n, new AbstractGraphVisitor<T>());
+ }
+ return dfs.getMarked();
+ }
+
+ /**
+ * Returns the (immutable) set of nodes that reach node 'n' (reflexive
+ * transitive closure).
+ */
+ public Set<Node<T>> getBackReachable(Node<T> n) {
+ return getBackReachable(Collections.singleton(n));
+ }
+
+ /**
+ * Returns the (immutable) set of nodes that reach some node in {@code
+ * startNodes} (reflexive transitive closure).
+ */
+ public Set<Node<T>> getBackReachable(Collection<Node<T>> startNodes) {
+ // This method is intentionally not static, to permit future expansion.
+ DFS<T> dfs = new DFS<T>(DFS.Order.PREORDER, true);
+ for (Node<T> n : startNodes) {
+ dfs.visit(n, new AbstractGraphVisitor<T>());
+ }
+ return dfs.getMarked();
+ }
+
+ /**
+ * Removes the node in the graph specified by the given label. Optionally,
+ * preserves the graph order (by connecting up the broken edges) or drop them
+ * all. If the specified label is not the label of any node in the graph,
+ * does nothing.
+ *
+ * @param label the label of the node to remove.
+ * @param preserveOrder if true, adds edges between the neighbours
+ * of the removed node so as to maintain the graph ordering
+ * relation between all pairs of such nodes. If false, simply
+ * discards all edges from the deleted node to its neighbours.
+ * @return true iff 'label' identifies a node (i.e. the graph was changed).
+ */
+ public boolean removeNode(T label, boolean preserveOrder) {
+ Node<T> node = getNodeMaybe(label);
+ if (node != null) {
+ removeNode(node, preserveOrder);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Removes the specified node in the graph.
+ *
+ * @param n the node to remove (must be in the graph).
+ * @param preserveOrder see removeNode(T, boolean).
+ */
+ public void removeNode(Node<T> n, boolean preserveOrder) {
+ checkNode(n);
+ for (Node<T> b: n.getSuccessors()) { // edges from n
+ // exists: n -> b
+ if (preserveOrder) {
+ for (Node<T> a: n.getPredecessors()) { // edges to n
+ // exists: a -> n
+ // beware self edges: they prevent n's deletion!
+ if (a != n && b != n) {
+ addEdge(a, b); // concurrent mod?
+ }
+ }
+ }
+ b.removePredecessor(n); // remove edge n->b in b
+ }
+ for (Node<T> a: n.getPredecessors()) { // edges to n
+ a.removeSuccessor(n); // remove edge a->n in a
+ }
+
+ n.removeAllEdges(); // remove edges b->n and a->n in n
+ Object del = nodes.remove(n.getLabel());
+ if (del != n) {
+ throw new IllegalStateException(del + " " + n);
+ }
+ }
+
+ /**
+ * Extracts the subgraph G' of this graph G, containing exactly the nodes
+ * specified by the labels in V', and preserving the original
+ * <i>transitive</i> graph relation among those nodes. </p>
+ *
+ * @param subset a subset of the labels of this graph; the resulting graph
+ * will have only the nodes with these labels.
+ */
+ public Digraph<T> extractSubgraph(final Set<T> subset) {
+ Digraph<T> subgraph = this.clone();
+ subgraph.subgraph(subset);
+ return subgraph;
+ }
+
+ /**
+ * Removes all nodes from this graph except those whose label is an element of {@code keepLabels}.
+ * Edges are added so as to preserve the <i>transitive</i> closure relation.
+ *
+ * @param keepLabels a subset of the labels of this graph; the resulting graph
+ * will have only the nodes with these labels.
+ */
+ public void subgraph(final Set<T> keepLabels) {
+ // This algorithm does the following:
+ // Let keep = nodes that have labels in keepLabels.
+ // Let toRemove = nodes \ keep. reachables = successors and predecessors of keep in nodes.
+ // reachables is the subset of nodes of remove that are an immediate neighbor of some node in
+ // keep.
+ //
+ // Removes all nodes of reachables from keepLabels.
+ // Until reachables is empty:
+ // Takes n from reachables
+ // for all s in succ(n)
+ // for all p in pred(n)
+ // add the edge (p, s)
+ // add s to reachables
+ // for all p in pred(n)
+ // add p to reachables
+ // Remove n and its edges
+ //
+ // A few adjustments are needed to do the whole computation.
+
+ final Set<Node<T>> toRemove = new HashSet<>();
+ final Set<Node<T>> keepNeighbors = new HashSet<>();
+
+ // Look for all nodes if they are to be kept or removed
+ for (Node<T> node : nodes.values()) {
+ if (keepLabels.contains(node.getLabel())) {
+ // Node is to be kept
+ keepNeighbors.addAll(node.getPredecessors());
+ keepNeighbors.addAll(node.getSuccessors());
+ } else {
+ // node is to be removed.
+ toRemove.add(node);
+ }
+ }
+
+ if (toRemove.isEmpty()) {
+ // This premature return is needed to avoid 0-size priority queue creation.
+ return;
+ }
+
+ // We use a priority queue to look for low-order nodes first so we don't propagate the high
+ // number of paths of high-order nodes making the time consumption explode.
+ // For perfect results we should reorder the set each time we add a new edge but this would
+ // be too expensive, so this is a good enough approximation.
+ final PriorityQueue<Node<T>> reachables = new PriorityQueue<>(toRemove.size(),
+ new Comparator<Node<T>>() {
+ @Override
+ public int compare(Node<T> o1, Node<T> o2) {
+ return Long.compare((long) o1.numPredecessors() * (long) o1.numSuccessors(),
+ (long) o2.numPredecessors() * (long) o2.numSuccessors());
+ }
+ });
+
+ // Construct the reachables queue with the list of successors and predecessors of keep in
+ // toRemove.
+ keepNeighbors.retainAll(toRemove);
+ reachables.addAll(keepNeighbors);
+ toRemove.removeAll(reachables);
+
+ // Remove nodes, least connected first, preserving reachability.
+ while (!reachables.isEmpty()) {
+ Node<T> node = reachables.poll();
+ for (Node<T> s : node.getSuccessors()) {
+ if (s == node) { continue; } // ignore self-edge
+
+ for (Node<T> p : node.getPredecessors()) {
+ if (p == node) { continue; } // ignore self-edge
+ addEdge(p, s);
+ }
+
+ // removes n -> s
+ s.removePredecessor(node);
+ if (toRemove.remove(s)) {
+ reachables.add(s);
+ }
+ }
+
+ for (Node<T> p : node.getPredecessors()) {
+ if (p == node) { continue; } // ignore self-edge
+ p.removeSuccessor(node);
+ if (toRemove.remove(p)) {
+ reachables.add(p);
+ }
+ }
+
+ // After the node deletion, the graph is again well-formed and the original topological order
+ // is preserved.
+ nodes.remove(node.getLabel());
+ }
+
+ // Final cleanup for non-reachable nodes.
+ for (Node<T> node : toRemove) {
+ removeNode(node, false);
+ }
+ }
+
+ private interface NodeSetReceiver<T> {
+ void accept(Set<Node<T>> nodes);
+ }
+
+ /**
+ * Find strongly connected components using path-based strong component
+ * algorithm. This has the advantage over the default method of returning
+ * the components in postorder.
+ *
+ * We visit nodes depth-first, keeping track of the order that
+ * we visit them in (preorder). Our goal is to find the smallest node (in
+ * this preorder of visitation) reachable from a given node. We keep track of the
+ * smallest node pointed to so far at the top of a stack. If we ever find an
+ * already-visited node, then if it is not already part of a component, we
+ * pop nodes from that stack until we reach this already-visited node's number
+ * or an even smaller one.
+ *
+ * Once the depth-first visitation of a node is complete, if this node's
+ * number is at the top of the stack, then it is the "first" element visited
+ * in its strongly connected component. Hence we pop all elements that were
+ * pushed onto the visitation stack and put them in a strongly connected
+ * component with this one, then send a passed-in {@link Digraph.NodeSetReceiver} this component.
+ */
+ private class SccVisitor<T> {
+ // Nodes already assigned to a strongly connected component.
+ private final Set<Node<T>> assigned = new HashSet<Node<T>>();
+ // The order each node was visited in.
+ private final Map<Node<T>, Integer> preorder = new HashMap<Node<T>, Integer>();
+ // Stack of all nodes visited whose SCC has not yet been determined. When an SCC is found,
+ // that SCC is an initial segment of this stack, and is popped off. Every time a new node is
+ // visited, it is put on this stack.
+ private final List<Node<T>> stack = new ArrayList<Node<T>>();
+ // Stack of visited indices for the first-visited nodes in each of their known-so-far
+ // strongly connected components. A node pushes its index on when it is visited. If any of
+ // its successors have already been visited and are not in an already-found strongly connected
+ // component, then, since the successor was already visited, it and this node must be part of a
+ // cycle. So every node visited since the successor is actually in the same strongly connected
+ // component. In this case, preorderStack is popped until the top is at most the successor's
+ // index.
+ //
+ // After all descendants of a node have been visited, if the top element of preorderStack is
+ // still the current node's index, then it was the first element visited of the current strongly
+ // connected component. So all nodes on {@code stack} down to the current node are in its
+ // strongly connected component. And the node's index is popped from preorderStack.
+ private final List<Integer> preorderStack = new ArrayList<Integer>();
+ // Index of node being visited.
+ private int counter = 0;
+
+ private void visit(NodeSetReceiver<T> visitor, Node<T> node) {
+ if (preorder.containsKey(node)) {
+ // This can only happen if this was a non-recursive call, and a previous
+ // visit call had already visited node.
+ return;
+ }
+ preorder.put(node, counter);
+ stack.add(node);
+ preorderStack.add(counter++);
+ int preorderLength = preorderStack.size();
+ for (Node<T> succ : node.getSuccessors()) {
+ Integer succPreorder = preorder.get(succ);
+ if (succPreorder == null) {
+ visit(visitor, succ);
+ } else {
+ // Does succ not already belong to an SCC? If it doesn't, then it
+ // must be in the same SCC as node. The "starting node" of this SCC
+ // must have been visited before succ (or is succ itself).
+ if (!assigned.contains(succ)) {
+ while (preorderStack.get(preorderStack.size() - 1) > succPreorder) {
+ preorderStack.remove(preorderStack.size() - 1);
+ }
+ }
+ }
+ }
+ if (preorderLength == preorderStack.size()) {
+ // If the length of the preorderStack is unchanged, we did not find any earlier-visited
+ // nodes that were part of a cycle with this node. So this node is the first-visited
+ // element in its strongly connected component, and we collect the component.
+ preorderStack.remove(preorderStack.size() - 1);
+ Set<Node<T>> scc = new HashSet<Node<T>>();
+ Node<T> compNode;
+ do {
+ compNode = stack.remove(stack.size() - 1);
+ assigned.add(compNode);
+ scc.add(compNode);
+ } while (!node.equals(compNode));
+ visitor.accept(scc);
+ }
+ }
+ }
+
+ /********************************************************************
+ * *
+ * Orders, traversals and visitors *
+ * *
+ ********************************************************************/
+
+ /**
+ * A visitation over all the nodes in the graph that invokes
+ * <code>visitor.visitNode()</code> for each node in a depth-first
+ * post-order: each node is visited <i>after</i> each of its successors; the
+ * order in which edges are traversed is the order in which they were added
+ * to the graph. <code>visitor.visitEdge()</code> is not called.
+ *
+ * @param startNodes the set of nodes from which to begin the visitation.
+ */
+ public void visitPostorder(GraphVisitor<T> visitor,
+ Iterable<Node<T>> startNodes) {
+ visitDepthFirst(visitor, DFS.Order.POSTORDER, false, startNodes);
+ }
+
+ /**
+ * Equivalent to {@code visitPostorder(visitor, getNodes())}.
+ */
+ public void visitPostorder(GraphVisitor<T> visitor) {
+ visitPostorder(visitor, nodes.values());
+ }
+
+ /**
+ * A visitation over all the nodes in the graph that invokes
+ * <code>visitor.visitNode()</code> for each node in a depth-first
+ * pre-order: each node is visited <i>before</i> each of its successors; the
+ * order in which edges are traversed is the order in which they were added
+ * to the graph. <code>visitor.visitEdge()</code> is not called.
+ *
+ * @param startNodes the set of nodes from which to begin the visitation.
+ */
+ public void visitPreorder(GraphVisitor<T> visitor,
+ Iterable<Node<T>> startNodes) {
+ visitDepthFirst(visitor, DFS.Order.PREORDER, false, startNodes);
+ }
+
+ /**
+ * Equivalent to {@code visitPreorder(visitor, getNodes())}.
+ */
+ public void visitPreorder(GraphVisitor<T> visitor) {
+ visitPreorder(visitor, nodes.values());
+ }
+
+ /**
+ * A visitation over all the nodes in the graph in depth-first order. See
+ * DFS constructor for meaning of 'order' and 'transpose' parameters.
+ *
+ * @param startNodes the set of nodes from which to begin the visitation.
+ */
+ public void visitDepthFirst(GraphVisitor<T> visitor,
+ DFS.Order order,
+ boolean transpose,
+ Iterable<Node<T>> startNodes) {
+ DFS<T> visitation = new DFS<T>(order, transpose);
+ visitor.beginVisit();
+ for (Node<T> node: startNodes) {
+ visitation.visit(node, visitor);
+ }
+ visitor.endVisit();
+ }
+
+ /**
+ * A visitation over the graph that visits all nodes and edges in some order
+ * such that each node is visited before any edge coming out of that node;
+ * the order is otherwise unspecified.
+ *
+ * @param startNodes the set of nodes from which to begin the visitation.
+ */
+ public void visitNodesBeforeEdges(GraphVisitor<T> visitor,
+ Iterable<Node<T>> startNodes) {
+ visitor.beginVisit();
+ for (Node<T> fromNode: startNodes) {
+ visitor.visitNode(fromNode);
+ for (Node<T> toNode: fromNode.getSuccessors()) {
+ visitor.visitEdge(fromNode, toNode);
+ }
+ }
+ visitor.endVisit();
+ }
+
+ /**
+ * Equivalent to {@code visitNodesBeforeEdges(visitor, getNodes())}.
+ */
+ public void visitNodesBeforeEdges(GraphVisitor<T> visitor) {
+ visitNodesBeforeEdges(visitor, nodes.values());
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/DotOutputVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/DotOutputVisitor.java
new file mode 100644
index 0000000..2d18ac2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/DotOutputVisitor.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.graph;
+
+import java.io.PrintWriter;
+
+/**
+ * <p> An implementation of GraphVisitor for displaying graphs in dot
+ * format. </p>
+ */
+public class DotOutputVisitor<T> implements GraphVisitor<T> {
+
+ /**
+ * Constructs a dot output visitor.
+ *
+ * The visitor writes to writer 'out', and rendering node labels as
+ * strings using the specified displayer, 'disp'.
+ */
+ public DotOutputVisitor(PrintWriter out, LabelSerializer<T> disp) {
+ // assert disp != null;
+ // assert out != null;
+ this.out = out;
+ this.disp = disp;
+ }
+
+ private final LabelSerializer<T> disp;
+ protected final PrintWriter out;
+ private boolean closeAtEnd = false;
+
+ @Override
+ public void beginVisit() {
+ out.println("digraph mygraph {");
+ }
+
+ @Override
+ public void endVisit() {
+ out.println("}");
+ out.flush();
+ if (closeAtEnd) {
+ out.close();
+ }
+ }
+
+ @Override
+ public void visitEdge(Node<T> lhs, Node<T> rhs) {
+ String s_lhs = disp.serialize(lhs);
+ String s_rhs = disp.serialize(rhs);
+ out.println("\"" + s_lhs + "\" -> \"" + s_rhs + "\"");
+ }
+
+ @Override
+ public void visitNode(Node<T> node) {
+ out.println("\"" + disp.serialize(node) + "\"");
+ }
+
+ /******************************************************************
+ * *
+ * Factories *
+ * *
+ ******************************************************************/
+
+ /**
+ * Create a DotOutputVisitor for output to a writer; uses default
+ * LabelSerializer.
+ */
+ public static <U> DotOutputVisitor<U> create(PrintWriter writer) {
+ return new DotOutputVisitor<U>(writer, new DefaultLabelSerializer<U>());
+ }
+
+ /**
+ * The default implementation of LabelSerializer simply serializes
+ * each node using its toString method.
+ */
+ private static class DefaultLabelSerializer<T> implements LabelSerializer<T> {
+ @Override
+ public String serialize(Node<T> node) {
+ return node.getLabel().toString();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/DotSyntaxException.java b/src/main/java/com/google/devtools/build/lib/graph/DotSyntaxException.java
new file mode 100644
index 0000000..adf70aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/DotSyntaxException.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.graph;
+
+import java.io.File;
+
+/**
+ * <p> A DotSyntaxException represents a syntax error encountered while
+ * parsing a dot-format fule. Thrown by createFromDotFile if syntax errors
+ * are encountered. May also be thrown by implementations of
+ * LabelDeserializer. </p>
+ *
+ * <p> The 'file' and 'lineNumber' fields indicate location of syntax error,
+ * and are populated externally by Digraph.createFromDotFile(). </p>
+ */
+public class DotSyntaxException extends Exception {
+
+ public DotSyntaxException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/GraphVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/GraphVisitor.java
new file mode 100644
index 0000000..f7e0f62
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/GraphVisitor.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.graph;
+
+/**
+ * <p> An graph visitor interface; particularly useful for allowing subclasses
+ * to specify how to output a graph. The order in which node and edge
+ * callbacks are made (DFS, BFS, etc) is defined by the choice of Digraph
+ * visitation method used. </p>
+ */
+public interface GraphVisitor<T> {
+
+ /**
+ * Called before visitation commences.
+ */
+ void beginVisit();
+
+ /**
+ * Called after visitation is complete.
+ */
+ void endVisit();
+
+ /**
+ * <p> Called for each edge. </p>
+ *
+ * TODO(bazel-team): This method is not essential, and in all known cases so
+ * far, the visitEdge code can always be placed within visitNode. Perhaps
+ * we should remove it, and the begin/end methods, and make this just a
+ * NodeVisitor? Are there any algorithms for which edge-visitation order is
+ * important?
+ */
+ void visitEdge(Node<T> lhs, Node<T> rhs);
+ /**
+ * Called for each node.
+ */
+ void visitNode(Node<T> node);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/LabelDeserializer.java b/src/main/java/com/google/devtools/build/lib/graph/LabelDeserializer.java
new file mode 100644
index 0000000..2ae9f96
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/LabelDeserializer.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.graph;
+
+/**
+ * <p> An interface for specifying a graph label de-serialization
+ * function. </p>
+ *
+ * <p> Implementations should provide a means of mapping from the string
+ * representation to an instance of the graph label type T. </p>
+ *
+ * <p> e.g. to construct Digraph{Integer} from a String representation, the
+ * LabelDeserializer{Integer} implementation would return
+ * Integer.parseInt(rep). </p>
+ */
+public interface LabelDeserializer<T> {
+
+ /**
+ * Returns an instance of the label object (of type T)
+ * corresponding to serialized representation 'rep'.
+ *
+ * @throws DotSyntaxException if 'rep' is invalid.
+ */
+ T deserialize(String rep) throws DotSyntaxException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/LabelSerializer.java b/src/main/java/com/google/devtools/build/lib/graph/LabelSerializer.java
new file mode 100644
index 0000000..7386583
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/LabelSerializer.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.graph;
+
+/**
+ * <p> An interface for specifying a user-defined serialization of graph node
+ * labels as strings. </p>
+ */
+public interface LabelSerializer<T> {
+
+ /**
+ * Returns the serialized form of the label of the specified node.
+ */
+ String serialize(Node<T> node);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/Matrix.java b/src/main/java/com/google/devtools/build/lib/graph/Matrix.java
new file mode 100644
index 0000000..225d3d2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/Matrix.java
@@ -0,0 +1,105 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.graph;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * <p>A simple and inefficient directed graph with the adjacency
+ * relation represented as a 2-D bit-matrix. </p>
+ *
+ * <p> Used as an adjunct to Digraph for performing certain algorithms
+ * which are more naturally implemented on this representation,
+ * e.g. transitive closure and reduction. </p>
+ *
+ * <p> Not many operations are supported. </p>
+ */
+final class Matrix<T> {
+
+ /**
+ * Constructs a square bit-matrix, initially empty, with the ith row/column
+ * corresponding to the ith element of 'labels', in iteration order.
+ *
+ * Does not retain a references to 'labels'.
+ */
+ public Matrix(Set<T> labels) {
+ this.N = labels.size();
+ this.values = new ArrayList<T>(N);
+ this.indices = new HashMap<T, Integer>();
+ this.m = new boolean[N][N];
+
+ for (T label: labels) {
+ int idx = values.size();
+ values.add(label);
+ indices.put(label, idx);
+ }
+ }
+
+ /**
+ * Constructs a matrix from the set of logical values specified. There is
+ * one row/column for each node in the graph, and the entry matrix[i,j] is
+ * set iff there is an edge in 'graph' from the node labelled values[i] to
+ * the node labelled values[j].
+ */
+ public Matrix(Digraph<T> graph) {
+ this(graph.getLabels());
+
+ for (Node<T> nfrom: graph.getNodes()) {
+ Integer ifrom = indices.get(nfrom.getLabel());
+ for (Node<T> nto: nfrom.getSuccessors()) {
+ Integer ito = indices.get(nto.getLabel());
+ m[ifrom][ito] = true;
+ }
+ }
+ }
+
+ /**
+ * The size of one side of the matrix.
+ */
+ private final int N;
+
+ /**
+ * The logical values associated with each row/column.
+ */
+ private final List<T> values;
+
+ /**
+ * The mapping from logical values to row/column index.
+ */
+ private final Map<T, Integer> indices;
+
+ /**
+ * The bit-matrix itself.
+ * m[from][to] indicates an edge from-->to.
+ */
+ private final boolean[][] m;
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (int ii = 0; ii < N; ++ii) {
+ for (int jj = 0; jj < N; ++jj) {
+ sb.append(m[ii][jj] ? '1' : '0');
+ }
+ sb.append(' ').append(values.get(ii)).append('\n');
+ }
+ return sb.toString();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/Node.java b/src/main/java/com/google/devtools/build/lib/graph/Node.java
new file mode 100644
index 0000000..9db2a4c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/graph/Node.java
@@ -0,0 +1,294 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.graph;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * <p>A generic directed-graph Node class. Type parameter T is the type
+ * of the node's label.
+ *
+ * <p>Each node is identified by a label, which is unique within the graph
+ * owning the node.
+ *
+ * <p>Nodes are immutable, that is, their labels cannot be changed. However,
+ * their predecessor/successor lists are mutable.
+ *
+ * <p>Nodes cannot be created directly by clients.
+ *
+ * <p>Clients should not confuse nodes belonging to two different graphs! (Use
+ * Digraph.checkNode() to catch such errors.) There is no way to find the
+ * graph to which a node belongs; it is intentionally not represented, to save
+ * space.
+ */
+public final class Node<T> {
+
+ private static final int ARRAYLIST_THRESHOLD = 6;
+ private static final int INITIAL_HASHSET_CAPACITY = 12;
+
+ // The succs and preds set representation changes depending on its size.
+ // It is implemented using the following collections:
+ // - null for size = 0.
+ // - Collections$SingletonList for size = 1.
+ // - ArrayList(6) for size = [2..6].
+ // - HashSet(12) for size > 6.
+ // These numbers were chosen based on profiling.
+
+ private final T label;
+
+ /**
+ * A duplicate-free collection of edges from this node. May be null,
+ * indicating the empty set.
+ */
+ private Collection<Node<T>> succs = null;
+
+ /**
+ * A duplicate-free collection of edges to this node. May be null,
+ * indicating the empty set.
+ */
+ private Collection<Node<T>> preds = null;
+
+ private final int hashCode;
+
+ /**
+ * Only Digraph.createNode() can call this!
+ */
+ Node(T label, int hashCode) {
+ if (label == null) { throw new NullPointerException("label"); }
+ this.label = label;
+ this.hashCode = hashCode;
+ }
+
+ /**
+ * Returns the label for this node.
+ */
+ public T getLabel() {
+ return label;
+ }
+
+ /**
+ * Returns a duplicate-free collection of the nodes that this node links to.
+ */
+ public Collection<Node<T>> getSuccessors() {
+ if (succs == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.unmodifiableCollection(succs);
+ }
+ }
+
+ /**
+ * Equivalent to {@code !getSuccessors().isEmpty()} but possibly more
+ * efficient.
+ */
+ public boolean hasSuccessors() {
+ return succs != null;
+ }
+
+ /**
+ * Equivalent to {@code getSuccessors().size()} but possibly more efficient.
+ */
+ public int numSuccessors() {
+ return succs == null ? 0 : succs.size();
+ }
+
+ /**
+ * Removes all edges to/from this node.
+ * Private: breaks graph invariant!
+ */
+ void removeAllEdges() {
+ this.succs = null;
+ this.preds = null;
+ }
+
+ /**
+ * Returns an (unordered, possibly immutable) set of the nodes that link to
+ * this node.
+ */
+ public Collection<Node<T>> getPredecessors() {
+ if (preds == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.unmodifiableCollection(preds);
+ }
+ }
+
+ /**
+ * Equivalent to {@code getPredecessors().size()} but possibly more
+ * efficient.
+ */
+ public int numPredecessors() {
+ return preds == null ? 0 : preds.size();
+ }
+
+ /**
+ * Equivalent to {@code !getPredecessors().isEmpty()} but possibly more
+ * efficient.
+ */
+ public boolean hasPredecessors() {
+ return preds != null;
+ }
+
+ /**
+ * Adds 'value' to either the predecessor or successor set, updating the
+ * appropriate field as necessary.
+ * @return {@code true} if the set was modified; {@code false} if the set
+ * was not modified
+ */
+ private boolean add(boolean predecessorSet, Node<T> value) {
+ final Collection<Node<T>> set = predecessorSet ? preds : succs;
+ if (set == null) {
+ // null -> SingletonList
+ return updateField(predecessorSet, Collections.singletonList(value));
+ }
+ if (set.contains(value)) {
+ // already exists in this set
+ return false;
+ }
+ int previousSize = set.size();
+ if (previousSize == 1) {
+ // SingletonList -> ArrayList
+ Collection<Node<T>> newSet =
+ new ArrayList<Node<T>>(ARRAYLIST_THRESHOLD);
+ newSet.addAll(set);
+ newSet.add(value);
+ return updateField(predecessorSet, newSet);
+ } else if (previousSize < ARRAYLIST_THRESHOLD) {
+ // ArrayList
+ set.add(value);
+ return true;
+ } else if (previousSize == ARRAYLIST_THRESHOLD) {
+ // ArrayList -> HashSet
+ Collection<Node<T>> newSet =
+ new HashSet<Node<T>>(INITIAL_HASHSET_CAPACITY);
+ newSet.addAll(set);
+ newSet.add(value);
+ return updateField(predecessorSet, newSet);
+ } else {
+ // HashSet
+ set.add(value);
+ return true;
+ }
+ }
+
+ /**
+ * Removes 'value' from either 'preds' or 'succs', updating the appropriate
+ * field as necessary.
+ * @return {@code true} if the set was modified; {@code false} if the set
+ * was not modified
+ */
+ private boolean remove(boolean predecessorSet, Node<T> value) {
+ final Collection<Node<T>> set = predecessorSet ? preds : succs;
+ if (set == null) {
+ // null
+ return false;
+ }
+
+ int previousSize = set.size();
+ if (previousSize == 1) {
+ if (set.contains(value)) {
+ // -> null
+ return updateField(predecessorSet, null);
+ } else {
+ return false;
+ }
+ }
+ // now remove the value
+ if (set.remove(value)) {
+ // may need to change representation
+ if (previousSize == 2) {
+ // -> SingletonList
+ List<Node<T>> list =
+ Collections.singletonList(set.iterator().next());
+ return updateField(predecessorSet, list);
+
+ } else if (previousSize == 1 + ARRAYLIST_THRESHOLD) {
+ // -> ArrayList
+ Collection<Node<T>> newSet =
+ new ArrayList<Node<T>>(ARRAYLIST_THRESHOLD);
+ newSet.addAll(set);
+ return updateField(predecessorSet, newSet);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Update either the {@link #preds} or {@link #succs} field to point to the
+ * new set.
+ * @return {@code true}, because the set must have been updated
+ */
+ private boolean updateField(boolean predecessorSet,
+ Collection<Node<T>> newSet) {
+ if (predecessorSet) {
+ preds = newSet;
+ } else {
+ succs = newSet;
+ }
+ return true;
+ }
+
+
+ /**
+ * Add 'to' as a successor of 'this' node. Returns true iff
+ * the graph changed. Private: breaks graph invariant!
+ */
+ boolean addSuccessor(Node<T> to) {
+ return add(false, to);
+ }
+
+ /**
+ * Add 'from' as a predecessor of 'this' node. Returns true iff
+ * the graph changed. Private: breaks graph invariant!
+ */
+ boolean addPredecessor(Node<T> from) {
+ return add(true, from);
+ }
+
+ /**
+ * Remove edge: fromNode.succs = {n | n in fromNode.succs && n != toNode}
+ * Private: breaks graph invariant!
+ */
+ boolean removeSuccessor(Node<T> to) {
+ return remove(false, to);
+ }
+
+ /**
+ * Remove edge: toNode.preds = {n | n in toNode.preds && n != fromNode}
+ * Private: breaks graph invariant!
+ */
+ boolean removePredecessor(Node<T> from) {
+ return remove(true, from);
+ }
+
+ @Override
+ public String toString() {
+ return "node:" + label;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode; // Fast, deterministic.
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ return this == that; // Nodes are unique for a given label
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AbstractAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/AbstractAttributeMapper.java
new file mode 100644
index 0000000..20b9304
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AbstractAttributeMapper.java
@@ -0,0 +1,212 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.Label;
+
+import javax.annotation.Nullable;
+
+/**
+ * Base {@link AttributeMap} implementation providing direct, unmanipulated access to
+ * underlying attribute data as stored within the Rule.
+ *
+ * <p>Any instantiable subclass should define a clear policy of what it does with this
+ * data before exposing it to consumers.
+ */
+public abstract class AbstractAttributeMapper implements AttributeMap {
+
+ private final Package pkg;
+ private final RuleClass ruleClass;
+ private final Label ruleLabel;
+ private final AttributeContainer attributes;
+
+ public AbstractAttributeMapper(Package pkg, RuleClass ruleClass, Label ruleLabel,
+ AttributeContainer attributes) {
+ this.pkg = pkg;
+ this.ruleClass = ruleClass;
+ this.ruleLabel = ruleLabel;
+ this.attributes = attributes;
+ }
+
+ @Override
+ public String getName() {
+ return ruleLabel.getName();
+ }
+
+ @Override
+ public Label getLabel() {
+ return ruleLabel;
+ }
+
+ @Nullable
+ @Override
+ public <T> T get(String attributeName, Type<T> type) {
+ int index = getIndexWithTypeCheck(attributeName, type);
+ Object value = attributes.getAttributeValue(index);
+ if (value instanceof Attribute.ComputedDefault) {
+ value = ((Attribute.ComputedDefault) value).getDefault(this);
+ }
+ return type.cast(value);
+ }
+
+ /**
+ * Returns the given attribute if it's a computed default, null otherwise.
+ *
+ * @throws IllegalArgumentException if the given attribute doesn't exist with the specified
+ * type. This happens whether or not it's a computed default.
+ */
+ protected <T> Attribute.ComputedDefault getComputedDefault(String attributeName, Type<T> type) {
+ int index = getIndexWithTypeCheck(attributeName, type);
+ Object value = attributes.getAttributeValue(index);
+ if (value instanceof Attribute.ComputedDefault) {
+ return (Attribute.ComputedDefault) value;
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Iterable<String> getAttributeNames() {
+ ImmutableList.Builder<String> names = ImmutableList.builder();
+ for (Attribute a : ruleClass.getAttributes()) {
+ names.add(a.getName());
+ }
+ return names.build();
+ }
+
+ @Nullable
+ @Override
+ public Type<?> getAttributeType(String attrName) {
+ Attribute attr = getAttributeDefinition(attrName);
+ return attr == null ? null : attr.getType();
+ }
+
+ @Nullable
+ @Override
+ public Attribute getAttributeDefinition(String attrName) {
+ return ruleClass.getAttributeByNameMaybe(attrName);
+ }
+
+ @Override
+ public boolean isAttributeValueExplicitlySpecified(String attributeName) {
+ return attributes.isAttributeValueExplicitlySpecified(attributeName);
+ }
+
+ @Override
+ public String getPackageDefaultHdrsCheck() {
+ return pkg.getDefaultHdrsCheck();
+ }
+
+ @Override
+ public Boolean getPackageDefaultObsolete() {
+ return pkg.getDefaultObsolete();
+ }
+
+ @Override
+ public Boolean getPackageDefaultTestOnly() {
+ return pkg.getDefaultTestOnly();
+ }
+
+ @Override
+ public String getPackageDefaultDeprecation() {
+ return pkg.getDefaultDeprecation();
+ }
+
+ @Override
+ public ImmutableList<String> getPackageDefaultCopts() {
+ return pkg.getDefaultCopts();
+ }
+
+ @Override
+ public void visitLabels(AcceptsLabelAttribute observer) {
+ for (Attribute attribute : ruleClass.getAttributes()) {
+ Type<?> type = attribute.getType();
+ // TODO(bazel-team): This is incoherent: we shouldn't have to special-case these types
+ // for our visitation policy (e.g., why is Type.NODEP_LABEL_LIST excluded but not
+ // Type.NODEP_LABEL?). But this is the semantics the calling code requires. Audit
+ // exactly which calling code expects what and clean up this interface.
+ if (type == Type.OUTPUT || type == Type.OUTPUT_LIST || type == Type.NODEP_LABEL_LIST) {
+ continue;
+ }
+ for (Object value : visitAttribute(attribute.getName(), type)) {
+ if (value == null) {
+ // This is particularly possible for computed defaults.
+ continue;
+ }
+ for (Label label : type.getLabels(value)) {
+ observer.acceptLabelAttribute(label, attribute);
+ }
+ }
+ }
+ }
+
+ /**
+ * Implementations should provide policy-appropriate mappings when an attribute is requested in
+ * the context of a rule visitation.
+ */
+ protected abstract <T> Iterable<T> visitAttribute(String attributeName, Type<T> type);
+
+ /**
+ * Returns a {@link Type.Selector} for the given attribute if the attribute is configurable
+ * for this rule, null otherwise.
+ *
+ * @return a {@link Type.Selector} if the attribute takes the form
+ * "attrName = { 'a': value1_of_type_T, 'b': value2_of_type_T }") for this rule, null
+ * if it takes the form "attrName = value_of_type_T", null if it doesn't exist
+ * @throws IllegalArgumentException if the attribute is configurable but of the wrong type
+ */
+ @Nullable
+ protected <T> Type.Selector<T> getSelector(String attributeName, Type<T> type) {
+ Integer index = ruleClass.getAttributeIndex(attributeName);
+ if (index == null) {
+ return null;
+ }
+ Object attrValue = attributes.getAttributeValue(index);
+ if (!(attrValue instanceof Type.Selector<?>)) {
+ return null;
+ }
+ if (((Type.Selector<?>) attrValue).getOriginalType() != type) {
+ throw new IllegalArgumentException("Attribute " + attributeName
+ + " is not of type " + type + " in rule " + ruleLabel.getName());
+ }
+ return (Type.Selector<T>) attrValue;
+ }
+
+ /**
+ * Returns the index of the specified attribute, if its type is 'type'. Throws
+ * an exception otherwise.
+ */
+ private int getIndexWithTypeCheck(String attrName, Type<?> type) {
+ Integer index = ruleClass.getAttributeIndex(attrName);
+ if (index == null) {
+ throw new IllegalArgumentException("No such attribute " + attrName
+ + " in rule " + ruleLabel.getName());
+ }
+ Attribute attr = ruleClass.getAttribute(index);
+ if (attr.getType() != type) {
+ throw new IllegalArgumentException("Attribute " + attrName
+ + " is not of type " + type + " in rule " + ruleLabel.getName());
+ }
+ return index;
+ }
+
+ /**
+ * Helper routine that just checks the given attribute has the given type for this rule and
+ * throws an IllegalException if not.
+ */
+ protected void checkType(String attrName, Type<?> type) {
+ getIndexWithTypeCheck(attrName, type);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AggregatingAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/AggregatingAttributeMapper.java
new file mode 100644
index 0000000..2aef224
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AggregatingAttributeMapper.java
@@ -0,0 +1,218 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * {@link AttributeMap} implementation that provides the ability to retrieve *all possible*
+ * values an attribute might take.
+ */
+public class AggregatingAttributeMapper extends AbstractAttributeMapper {
+
+ /**
+ * Store for all of this rule's attributes that are non-configurable. These are
+ * unconditionally available to computed defaults no matter what dependencies
+ * they've declared.
+ */
+ private final List<String> nonconfigurableAttributes;
+
+ private AggregatingAttributeMapper(Rule rule) {
+ super(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(),
+ rule.getAttributeContainer());
+
+ ImmutableList.Builder<String> nonconfigurableAttributesBuilder = ImmutableList.builder();
+ for (Attribute attr : rule.getAttributes()) {
+ if (!attr.isConfigurable()) {
+ nonconfigurableAttributesBuilder.add(attr.getName());
+ }
+ }
+ nonconfigurableAttributes = nonconfigurableAttributesBuilder.build();
+ }
+
+ public static AggregatingAttributeMapper of(Rule rule) {
+ return new AggregatingAttributeMapper(rule);
+ }
+
+ /**
+ * Override that also visits the rule's configurable attribute keys (which are
+ * themselves labels).
+ */
+ @Override
+ public void visitLabels(AcceptsLabelAttribute observer) {
+ super.visitLabels(observer);
+ for (String attrName : getAttributeNames()) {
+ Attribute attribute = getAttributeDefinition(attrName);
+ Type.Selector<?> selector = getSelector(attrName, attribute.getType());
+ if (selector != null) {
+ for (Label configLabel : selector.getEntries().keySet()) {
+ if (!Type.Selector.isReservedLabel(configLabel)) {
+ observer.acceptLabelAttribute(configLabel, attribute);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns a list of all possible values an attribute can take for this rule.
+ */
+ @Override
+ public <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) {
+ // If this attribute value is configurable, visit all possible values.
+ Type.Selector<T> selector = getSelector(attributeName, type);
+ if (selector != null) {
+ ImmutableList.Builder<T> builder = ImmutableList.builder();
+ for (Map.Entry<Label, T> entry : selector.getEntries().entrySet()) {
+ builder.add(entry.getValue());
+ }
+ return builder.build();
+ }
+
+ // If this attribute is a computed default, feed it all possible value combinations of
+ // its declared dependencies and return all computed results. For example, if this default
+ // uses attributes x and y, x can configurably be x1 or x2, and y can configurably be y1
+ // or y1, then compute default values for the (x1,y1), (x1,y2), (x2,y1), and (x2,y2) cases.
+ Attribute.ComputedDefault computedDefault = getComputedDefault(attributeName, type);
+ if (computedDefault != null) {
+ // This will hold every (value1, value2, ..) combination of the declared dependencies.
+ List<Map<String, Object>> depMaps = new LinkedList<>();
+ // Collect those combinations.
+ mapDepsForComputedDefault(computedDefault.dependencies(), depMaps,
+ ImmutableMap.<String, Object>of());
+ List<T> possibleValues = new ArrayList<>(); // Not ImmutableList.Builder: values may be null.
+ // For each combination, call getDefault on a specialized AttributeMap providing those values.
+ for (Map<String, Object> depMap : depMaps) {
+ possibleValues.add(type.cast(computedDefault.getDefault(mapBackedAttributeMap(depMap))));
+ }
+ return possibleValues;
+ }
+
+ // For any other attribute, just return its direct value.
+ T value = get(attributeName, type);
+ return value == null ? ImmutableList.<T>of() : ImmutableList.of(value);
+ }
+
+ /**
+ * Given (possibly configurable) attributes that a computed default depends on, creates an
+ * {attrName -> attrValue} map for every possible combination of those attribute values and
+ * returns a list of all the maps. This defines the complete dependency space that can affect
+ * the computed default's values.
+ *
+ * <p>For example, given dependencies x and y, which might respectively have values x1, x2 and
+ * y1, y2, this returns:
+ * <pre>
+ * [
+ * {x: x1, y: y1},
+ * {x: x1, y: y2},
+ * {x: x2, y: y1},
+ * {x: x2, y: y2}
+ * ]
+ * </pre>
+ *
+ * @param depAttributes the names of the attributes this computed default depends on
+ * @param mappings the list of {attrName --> attrValue} maps defining the computed default's
+ * dependency space. This is where this method's results are written.
+ * @param currentMap a (possibly non-empty) map to add {attrName --> attrValue}
+ * entries to. Outside callers can just pass in an empty map.
+ */
+ private void mapDepsForComputedDefault(List<String> depAttributes,
+ List<Map<String, Object>> mappings, Map<String, Object> currentMap) {
+ // Because this method uses exponential time/space on the number of inputs, keep the
+ // maximum number of inputs conservatively small.
+ Preconditions.checkState(depAttributes.size() <= 2);
+
+ if (depAttributes.isEmpty()) {
+ // Recursive base case: store whatever's already been populated in currentMap.
+ mappings.add(currentMap);
+ return;
+ }
+
+ // Take the first attribute in the dependency list and iterate over all its values. For each
+ // value x, copy currentMap with the additional entry { firstAttrName: x }, then feed
+ // this recursively into a subcall over all remaining dependencies. This recursively
+ // continues until we run out of values.
+ String firstAttribute = depAttributes.get(0);
+ for (Object value : visitAttribute(firstAttribute, getAttributeType(firstAttribute))) {
+ Map<String, Object> newMap = new HashMap<>();
+ newMap.putAll(currentMap);
+ newMap.put(firstAttribute, value);
+ mapDepsForComputedDefault(depAttributes.subList(1, depAttributes.size()), mappings, newMap);
+ }
+ }
+
+ /**
+ * A custom {@link AttributeMap} that reads attribute values from the given Map. All
+ * non-configurable attributes are also readable. Any attempt to read an attribute
+ * that's not in one of these two cases triggers an IllegalArgumentException.
+ */
+ private AttributeMap mapBackedAttributeMap(final Map<String, Object> directMap) {
+ final AggregatingAttributeMapper owner = AggregatingAttributeMapper.this;
+ return new AttributeMap() {
+
+ @Override
+ public <T> T get(String attributeName, Type<T> type) {
+ owner.checkType(attributeName, type);
+ if (nonconfigurableAttributes.contains(attributeName)) {
+ return owner.get(attributeName, type);
+ }
+ if (!directMap.containsKey(attributeName)) {
+ throw new IllegalArgumentException("attribute \"" + attributeName
+ + "\" isn't available in this computed default context");
+ }
+ return type.cast(directMap.get(attributeName));
+ }
+
+ @Override public String getName() { return owner.getName(); }
+ @Override public Label getLabel() { return owner.getLabel(); }
+ @Override public Iterable<String> getAttributeNames() {
+ return ImmutableList.<String>builder()
+ .addAll(directMap.keySet()).addAll(nonconfigurableAttributes).build();
+ }
+ @Override
+ public void visitLabels(AcceptsLabelAttribute observer) { owner.visitLabels(observer); }
+ @Override
+ public String getPackageDefaultHdrsCheck() { return owner.getPackageDefaultHdrsCheck(); }
+ @Override
+ public Boolean getPackageDefaultObsolete() { return owner.getPackageDefaultObsolete(); }
+ @Override
+ public Boolean getPackageDefaultTestOnly() { return owner.getPackageDefaultTestOnly(); }
+ @Override
+ public String getPackageDefaultDeprecation() { return owner.getPackageDefaultDeprecation(); }
+ @Override
+ public ImmutableList<String> getPackageDefaultCopts() {
+ return owner.getPackageDefaultCopts();
+ }
+ @Nullable @Override
+ public Type<?> getAttributeType(String attrName) { return owner.getAttributeType(attrName); }
+ @Nullable @Override public Attribute getAttributeDefinition(String attrName) {
+ return owner.getAttributeDefinition(attrName);
+ }
+ @Override public boolean isAttributeValueExplicitlySpecified(String attributeName) {
+ return owner.isAttributeValueExplicitlySpecified(attributeName);
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AnalysisIssues.java b/src/main/java/com/google/devtools/build/lib/packages/AnalysisIssues.java
new file mode 100644
index 0000000..22c02b5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AnalysisIssues.java
@@ -0,0 +1,109 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Checked exception for analysis-time errors, which can store the errors for later reporting.
+ *
+ * <p>It's more robust for a method to throw this exception than expecting a
+ * {@link RuleErrorConsumer} object (which may be null).
+ */
+public final class AnalysisIssues extends Exception {
+
+ /**
+ * An error entry.
+ *
+ * <p>{@link AnalysisIssues} can accumulate multiple of these, and report all of them at once.
+ */
+ public static final class Entry {
+ private final String attribute;
+ private final String messageTemplate;
+ private final Object[] arguments;
+
+ private Entry(@Nullable String attribute, String messageTemplate, Object... arguments) {
+ this.attribute = attribute;
+ this.messageTemplate = messageTemplate;
+ this.arguments = arguments;
+ }
+
+ private void reportTo(RuleErrorConsumer errors) {
+ String msg = String.format(messageTemplate, arguments);
+ if (attribute == null) {
+ errors.ruleError(msg);
+ } else {
+ errors.attributeError(attribute, msg);
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (attribute == null) {
+ return String.format("ERROR: " + messageTemplate, arguments);
+ } else {
+ List<Object> args = new ArrayList<>();
+ args.add(attribute);
+ args.addAll(Arrays.asList(arguments));
+ return String.format("ERROR in '%s': " + messageTemplate, args.toArray());
+ }
+ }
+ }
+
+ private final ImmutableList<Entry> entries;
+
+ public AnalysisIssues(Entry entry) {
+ this.entries = ImmutableList.of(Preconditions.checkNotNull(entry));
+ }
+
+ public AnalysisIssues(Collection<Entry> entries) {
+ this.entries = ImmutableList.copyOf(Preconditions.checkNotNull(entries));
+ }
+
+ /**
+ * Creates a attribute error entry that will be added to a {@link AnalysisIssues} later.
+ */
+ public static Entry attributeError(String attribute, String messageTemplate,
+ Object... arguments) {
+ return new Entry(attribute, messageTemplate, arguments);
+ }
+
+ public static Entry ruleError(String messageTemplate, Object... arguments) {
+ return new Entry(null, messageTemplate, arguments);
+ }
+
+ /**
+ * Report all accumulated errors and warnings to the given consumer object.
+ */
+ public void reportTo(RuleErrorConsumer errors) {
+ Preconditions.checkNotNull(errors);
+ for (Entry e : entries) {
+ e.reportTo(errors);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Errors during analysis:\n" + Joiner.on("\n").join(entries);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AspectDefinition.java b/src/main/java/com/google/devtools/build/lib/packages/AspectDefinition.java
new file mode 100644
index 0000000..e1b5f05
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AspectDefinition.java
@@ -0,0 +1,167 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The definition of an aspect (see {@link com.google.devtools.build.lib.analysis.Aspect} for more
+ * information.)
+ *
+ * <p>Contains enough information to build up the configured target graph except for the actual way
+ * to build the Skyframe node (that is the territory of
+ * {@link com.google.devtools.build.lib.view AspectFactory}). In particular:
+ * <ul>
+ * <li>The condition that must be fulfilled for an aspect to be able to operate on a configured
+ * target
+ * <li>The (implicit or late-bound) attributes of the aspect that denote dependencies the aspect
+ * itself needs (e.g. runtime libraries for a new language for protocol buffers)
+ * <li>The aspects this aspect requires from its direct dependencies
+ * </ul>
+ *
+ * <p>The way to build the Skyframe node is not here because this data needs to be accessible from
+ * the {@code .packages} package and that one requires references to the {@code .view} package.
+ */
+@Immutable
+public final class AspectDefinition {
+
+ private final String name;
+ private final ImmutableSet<Class<?>> requiredProviders;
+ private final ImmutableMap<String, Attribute> attributes;
+ private final ImmutableMultimap<String, Class<? extends AspectFactory<?, ?, ?>>> attributeAspects;
+
+ private AspectDefinition(
+ String name,
+ ImmutableSet<Class<?>> requiredProviders,
+ ImmutableMap<String, Attribute> attributes,
+ ImmutableMultimap<String, Class<? extends AspectFactory<?, ?, ?>>> attributeAspects) {
+ this.name = name;
+ this.requiredProviders = requiredProviders;
+ this.attributes = attributes;
+ this.attributeAspects = attributeAspects;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns the attributes of the aspect in the form of a String -> {@link Attribute} map.
+ *
+ * <p>All attributes are either implicit or late-bound.
+ */
+ public ImmutableMap<String, Attribute> getAttributes() {
+ return attributes;
+ }
+
+ /**
+ * Returns the set of {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider} instances
+ * that must be present on a configured target so that this aspect can be applied to it.
+ *
+ * <p>We cannot refer to that class here due to our dependency structure, so this returns a set
+ * of unconstrained class objects.
+ *
+ * <p>If a configured target does not have a required provider, the aspect is silently not created
+ * for it.
+ */
+ public ImmutableSet<Class<?>> getRequiredProviders() {
+ return requiredProviders;
+ }
+
+ /**
+ * Returns the attribute -> set of required aspects map.
+ *
+ * <p>Note that the map actually contains {@link AspectFactory}
+ * instances, except that we cannot reference that class here.
+ */
+ public ImmutableMultimap<String, Class<? extends AspectFactory<?, ?, ?>>> getAttributeAspects() {
+ return attributeAspects;
+ }
+
+ /**
+ * Builder class for {@link AspectDefinition}.
+ */
+ public static final class Builder {
+ private final String name;
+ private final Map<String, Attribute> attributes = new LinkedHashMap<>();
+ private final Set<Class<?>> requiredProviders = new LinkedHashSet<>();
+ private final Multimap<String, Class<? extends AspectFactory<?, ?, ?>>> attributeAspects =
+ LinkedHashMultimap.create();
+
+ public Builder(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Asserts that this aspect can only be evaluated for rules that supply the specified provider.
+ */
+ public Builder requireProvider(Class<?> requiredProvider) {
+ this.requiredProviders.add(requiredProvider);
+ return this;
+ }
+
+ /**
+ * Tells that in order for this aspect to work, the given aspect must be computed for the
+ * direct dependencies in the attribute with the specified name on the associated configured
+ * target.
+ *
+ * <p>Note that {@code AspectFactory} instances are expected in the second argument, but we
+ * cannot reference that interface here.
+ */
+ public Builder attributeAspect(
+ String attribute, Class<? extends AspectFactory<?, ?, ?>> aspectFactory) {
+ this.attributeAspects.put(
+ Preconditions.checkNotNull(attribute), Preconditions.checkNotNull(aspectFactory));
+ return this;
+ }
+
+ /**
+ * Adds an attribute to the aspect.
+ *
+ * <p>Since aspects do not appear in BUILD files, the attribute must be either implicit
+ * (not available in the BUILD file, starting with '$') or late-bound (determined after the
+ * configuration is available, starting with ':')
+ */
+ public <TYPE> Builder add(Attribute.Builder<TYPE> attr) {
+ Attribute attribute = attr.build();
+ Preconditions.checkState(attribute.isImplicit() || attribute.isLateBound());
+ Preconditions.checkState(!attributes.containsKey(attribute.getName()),
+ "An attribute with the name '%s' already exists.", attribute.getName());
+ attributes.put(attribute.getName(), attribute);
+ return this;
+ }
+
+ /**
+ * Builds the aspect definition.
+ *
+ * <p>The builder object is reusable afterwards.
+ */
+ public AspectDefinition build() {
+ return new AspectDefinition(name, ImmutableSet.copyOf(requiredProviders),
+ ImmutableMap.copyOf(attributes), ImmutableMultimap.copyOf(attributeAspects));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AspectFactory.java b/src/main/java/com/google/devtools/build/lib/packages/AspectFactory.java
new file mode 100644
index 0000000..2283f1b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AspectFactory.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Creates the Skyframe node of an aspect.
+ *
+ * <p>Also has a reference to the definition of the aspect.
+ */
+public interface AspectFactory<TConfiguredTarget, TRuleContext, TAspect> {
+ /**
+ * Creates the aspect based on the configured target of the associated rule.
+ *
+ * @param base the configured target of the associated rule
+ * @param context the context of the associated configured target plus all the attributes the
+ * aspect itself has defined
+ */
+ TAspect create(TConfiguredTarget base, TRuleContext context);
+
+ /**
+ * Returns the definition of the aspect.
+ */
+ AspectDefinition getDefinition();
+
+ /**
+ * Dummy wrapper class for utility methods because interfaces cannot even have static ones.
+ */
+ public static final class Util {
+ private Util() {
+ // Should never be instantiated
+ }
+
+ public static AspectFactory create(Class<? extends AspectFactory<?, ?, ?>> clazz) {
+ // TODO(bazel-team): This should be cached somehow, because this method is invoked quite often
+ try {
+ return clazz.newInstance();
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Attribute.java b/src/main/java/com/google/devtools/build/lib/packages/Attribute.java
new file mode 100644
index 0000000..9a8ae61
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/Attribute.java
@@ -0,0 +1,1343 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.util.StringUtil;
+
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Metadata of a rule attribute. Contains the attribute name and type, and an
+ * default value to be used if none is provided in a rule declaration in a BUILD
+ * file. Attributes are immutable, and may be shared by more than one rule (for
+ * example, <code>foo_binary</code> and <code>foo_library</code> may share many
+ * attributes in common).
+ */
+@Immutable
+public final class Attribute implements Comparable<Attribute> {
+
+ public static final Predicate<RuleClass> ANY_RULE = Predicates.alwaysTrue();
+
+ public static final Predicate<RuleClass> NO_RULE = Predicates.alwaysFalse();
+
+ /**
+ * A configuration transition.
+ */
+ public interface Transition {
+ /**
+ * Usually, a non-existent entry in the configuration transition table indicates an error.
+ * Unfortunately, that means that we need to always build the full table. This method allows a
+ * transition to indicate that a non-existent entry indicates a self transition, i.e., that the
+ * resulting configuration is the same as the current configuration. This can simplify the code
+ * needed to set up the transition table.
+ */
+ boolean defaultsToSelf();
+ }
+
+ /**
+ * A configuration split transition; this should be used to transition to multiple configurations
+ * simultaneously. Note that the corresponding rule implementations must have special support to
+ * handle this.
+ */
+ // TODO(bazel-team): Serializability constraints?
+ public interface SplitTransition<T> extends Transition {
+ /**
+ * Return the list of {@code BuildOptions} after splitting; empty if not applicable.
+ */
+ List<T> split(T buildOptions);
+ }
+
+ /**
+ * Declaration how the configuration should change when following a label or
+ * label list attribute.
+ */
+ public enum ConfigurationTransition implements Transition {
+ /** No transition, i.e., the same configuration as the current. */
+ NONE,
+
+ /** Transition to the host configuration. */
+ HOST,
+
+ /** Transition from the target configuration to the data configuration. */
+ // TODO(bazel-team): Move this elsewhere.
+ DATA;
+
+ @Override
+ public boolean defaultsToSelf() {
+ return false;
+ }
+ }
+
+ private enum PropertyFlag {
+ MANDATORY,
+ EXECUTABLE,
+ UNDOCUMENTED,
+ TAGGABLE,
+
+ /**
+ * Whether the list attribute is order-independent and can be sorted.
+ */
+ ORDER_INDEPENDENT,
+
+ /**
+ * Whether the allowedRuleClassesForLabels or allowedFileTypesForLabels are
+ * set to custom values. If so, and the attribute is called "deps", the
+ * legacy deps checking is skipped, and the new stricter checks are used
+ * instead. For non-"deps" attributes, this allows skipping the check if it
+ * would pass anyway, as the default setting allows any rule classes and
+ * file types.
+ */
+ STRICT_LABEL_CHECKING,
+
+ /**
+ * Set for things that would cause the a compile or lint-like action to
+ * be executed when the input changes. Used by compile_one_dependency.
+ * Set for attributes like hdrs and srcs on cc_ rules or srcs on java_
+ * or py_rules. Generally not set on data/resource attributes.
+ */
+ DIRECT_COMPILE_TIME_INPUT,
+
+ /**
+ * Whether the value of the list type attribute must not be an empty list.
+ */
+ NON_EMPTY,
+
+ /**
+ * Verifies that the referenced rule produces a single artifact. Note that this check happens
+ * on a per label basis, i.e. the check happens separately for every label in a label list.
+ */
+ SINGLE_ARTIFACT,
+
+ /**
+ * Whether we perform silent ruleclass filtering of the dependencies of the label type
+ * attribute according to their rule classes. I.e. elements of the list which don't match the
+ * allowedRuleClasses predicate or not rules will be filtered out without throwing any errors.
+ * This flag is introduced to handle plugins, do not use it in other cases.
+ */
+ SILENT_RULECLASS_FILTER,
+
+ // TODO(bazel-team): This is a hack introduced because of the bad design of the original rules.
+ // Depot cleanup would be too expensive, but don't migrate this to Skylark.
+ /**
+ * Whether to perform analysis time filetype check on this label-type attribute or not.
+ * If the flag is set, we skip the check that applies the allowedFileTypes filter
+ * to generated files. Do not use this if avoidable.
+ */
+ SKIP_ANALYSIS_TIME_FILETYPE_CHECK,
+
+ /**
+ * Whether the value of the attribute should come from a given set of values.
+ */
+ CHECK_ALLOWED_VALUES,
+
+ /**
+ * Whether this attribute is opted out of "configurability", i.e. the ability to determine
+ * its value based on properties of the build configuration.
+ */
+ NONCONFIGURABLE,
+ }
+
+ // TODO(bazel-team): modify this interface to extend Predicate and have an extra error
+ // message function like AllowedValues does
+ /**
+ * A predicate-like class that determines whether an edge between two rules is valid or not.
+ */
+ public interface ValidityPredicate {
+ /**
+ * This method should return null if the edge is valid, or a suitable error message
+ * if it is not. Note that warnings are not supported.
+ */
+ String checkValid(Rule from, Rule to);
+ }
+
+ public static final ValidityPredicate ANY_EDGE =
+ new ValidityPredicate() {
+ @Override
+ public String checkValid(Rule from, Rule to) {
+ return null;
+ }
+ };
+
+ /**
+ * Using this callback function, rules can set the configuration of their dependencies during the
+ * analysis phase.
+ */
+ public interface Configurator<TConfig, TRule> {
+ TConfig apply(TRule fromRule, TConfig fromConfiguration, Attribute attribute, Target toTarget);
+ }
+
+ /**
+ * A predicate class to check if the value of the attribute comes from a predefined set.
+ */
+ public static class AllowedValueSet implements PredicateWithMessage<Object> {
+
+ private final Set<Object> allowedValues;
+
+ public AllowedValueSet(Iterable<?> values) {
+ Preconditions.checkNotNull(values);
+ Preconditions.checkArgument(!Iterables.isEmpty(values));
+ allowedValues = ImmutableSet.copyOf(values);
+ }
+
+ @Override
+ public boolean apply(Object input) {
+ return allowedValues.contains(input);
+ }
+
+ @Override
+ public String getErrorReason(Object value) {
+ return String.format("has to be one of %s instead of '%s'",
+ StringUtil.joinEnglishList(allowedValues, "or", "'"), value);
+ }
+
+ @VisibleForTesting
+ public Collection<Object> getAllowedValues() {
+ return allowedValues;
+ }
+ }
+
+ /**
+ * Creates a new attribute builder.
+ *
+ * @param name attribute name
+ * @param type attribute type
+ * @return attribute builder
+ *
+ * @param <TYPE> attribute type class
+ */
+ public static <TYPE> Attribute.Builder<TYPE> attr(String name, Type<TYPE> type) {
+ return new Builder<>(name, type);
+ }
+
+ /**
+ * A fluent builder for the {@code Attribute} instances.
+ *
+ * <p>All methods could be called only once per builder. The attribute
+ * already undocumented based on its name cannot be marked as undocumented.
+ */
+ public static class Builder <TYPE> {
+ private String name;
+ private final Type<TYPE> type;
+ private Transition configTransition = ConfigurationTransition.NONE;
+ private Predicate<RuleClass> allowedRuleClassesForLabels = Predicates.alwaysTrue();
+ private Predicate<RuleClass> allowedRuleClassesForLabelsWarning = Predicates.alwaysFalse();
+ private Configurator<?, ?> configurator = null;
+ private boolean allowedFileTypesForLabelsSet;
+ private FileTypeSet allowedFileTypesForLabels = FileTypeSet.ANY_FILE;
+ private ValidityPredicate validityPredicate = ANY_EDGE;
+ private Object value;
+ private boolean valueSet;
+ private Predicate<AttributeMap> condition;
+ private Set<PropertyFlag> propertyFlags = EnumSet.noneOf(PropertyFlag.class);
+ private PredicateWithMessage<Object> allowedValues = null;
+ private ImmutableSet<String> mandatoryProviders = ImmutableSet.<String>of();
+ private Set<Class<? extends AspectFactory<?, ?, ?>>> aspects = new LinkedHashSet<>();
+
+ /**
+ * Creates an attribute builder with given name and type. This attribute is optional, uses
+ * target configuration and has a default value the same as its type default value. This
+ * attribute will be marked as undocumented if its name starts with the dollar sign ({@code $})
+ * or colon ({@code :}).
+ *
+ * @param name attribute name
+ * @param type attribute type
+ */
+ public Builder(String name, Type<TYPE> type) {
+ this.name = Preconditions.checkNotNull(name);
+ this.type = Preconditions.checkNotNull(type);
+ if (isImplicit(name) || isLateBound(name)) {
+ setPropertyFlag(PropertyFlag.UNDOCUMENTED, "undocumented");
+ }
+ }
+
+ private Builder<TYPE> setPropertyFlag(PropertyFlag flag, String propertyName) {
+ Preconditions.checkState(!propertyFlags.contains(flag),
+ propertyName + " flag is already set");
+ propertyFlags.add(flag);
+ return this;
+ }
+
+ /**
+ * Sets the property flag of the corresponding name if exists, otherwise throws an Exception.
+ * Only meant to use from Skylark, do not use from Java.
+ */
+ public Builder<TYPE> setPropertyFlag(String propertyName) {
+ PropertyFlag flag = null;
+ try {
+ flag = PropertyFlag.valueOf(propertyName);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("unknown attribute flag " + propertyName);
+ }
+ setPropertyFlag(flag, propertyName);
+ return this;
+ }
+
+ /**
+ * Makes the built attribute mandatory.
+ */
+ public Builder<TYPE> mandatory() {
+ return setPropertyFlag(PropertyFlag.MANDATORY, "mandatory");
+ }
+
+ /**
+ * Makes the built attribute non empty, meaning the attribute cannot have an empty list value.
+ * Only applicable for list type attributes.
+ */
+ public Builder<TYPE> nonEmpty() {
+ Preconditions.checkNotNull(type.getListElementType(),
+ "attribute '" + name + "' must be a list");
+ return setPropertyFlag(PropertyFlag.NON_EMPTY, "non_empty");
+ }
+
+ /**
+ * Makes the built attribute producing a single artifact.
+ */
+ public Builder<TYPE> singleArtifact() {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "attribute '" + name + "' must be a label-valued type");
+ return setPropertyFlag(PropertyFlag.SINGLE_ARTIFACT, "single_artifact");
+ }
+
+ /**
+ * Forces silent ruleclass filtering on the label type attribute.
+ * This flag is introduced to handle plugins, do not use it in other cases.
+ */
+ public Builder<TYPE> silentRuleClassFilter() {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "must be a label-valued type");
+ return setPropertyFlag(PropertyFlag.SILENT_RULECLASS_FILTER, "silent_ruleclass_filter");
+ }
+
+ /**
+ * Skip analysis time filetype check. Don't use it if avoidable.
+ */
+ public Builder<TYPE> skipAnalysisTimeFileTypeCheck() {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "must be a label-valued type");
+ return setPropertyFlag(PropertyFlag.SKIP_ANALYSIS_TIME_FILETYPE_CHECK,
+ "skip_analysis_time_filetype_check");
+ }
+
+ /**
+ * Mark the built attribute as order-independent.
+ */
+ public Builder<TYPE> orderIndependent() {
+ Preconditions.checkNotNull(type.getListElementType(),
+ "attribute '" + name + "' must be a list");
+ return setPropertyFlag(PropertyFlag.ORDER_INDEPENDENT, "order-independent");
+ }
+
+ /**
+ * Defines the configuration transition for this attribute. Defaults to
+ * {@code NONE}.
+ */
+ public Builder<TYPE> cfg(Transition configTransition) {
+ Preconditions.checkState(this.configTransition == ConfigurationTransition.NONE,
+ "the configuration transition is already set");
+ this.configTransition = configTransition;
+ return this;
+ }
+
+ public Builder<TYPE> cfg(Configurator<?, ?> configurator) {
+ this.configurator = configurator;
+ return this;
+ }
+
+ /**
+ * Requires the attribute target to be executable; only for label or label
+ * list attributes. Defaults to {@code false}.
+ */
+ public Builder<TYPE> exec() {
+ return setPropertyFlag(PropertyFlag.EXECUTABLE, "executable");
+ }
+
+ /**
+ * Indicates that the attribute (like srcs or hdrs) should be used as an input when calculating
+ * compile_one_dependency.
+ */
+ public Builder<TYPE> direct_compile_time_input() {
+ return setPropertyFlag(PropertyFlag.DIRECT_COMPILE_TIME_INPUT,
+ "direct_compile_time_input");
+ }
+
+ /**
+ * Makes the built attribute undocumented.
+ *
+ * @param reason explanation why the attribute is undocumented. This is not
+ * used but required for documentation
+ */
+ public Builder<TYPE> undocumented(String reason) {
+ return setPropertyFlag(PropertyFlag.UNDOCUMENTED, "undocumented");
+ }
+
+ /**
+ * Sets the attribute default value. The type of the default value must
+ * match the type parameter. (e.g. list=[], integer=0, string="",
+ * label=null). The {@code defaultValue} must be immutable.
+ *
+ * <p>If defaultValue is of type Label and is a target, that target will
+ * become an implicit dependency of the Rule; we will load the target
+ * (and its dependencies) if it encounters the Rule and build the target
+ * if needs to apply the Rule.
+ */
+ public Builder<TYPE> value(TYPE defaultValue) {
+ Preconditions.checkState(!valueSet, "the default value is already set");
+ value = defaultValue;
+ valueSet = true;
+ return this;
+ }
+
+ /**
+ * See value(TYPE) above. This method is only meant for Skylark usage.
+ */
+ public Builder<TYPE> defaultValue(Object defaultValue) throws ConversionException {
+ Preconditions.checkState(!valueSet, "the default value is already set");
+ value = type.convert(defaultValue, "attribute " + name);
+ valueSet = true;
+ return this;
+ }
+
+ /**
+ * Sets the attribute default value to a computed default value - use
+ * this when the default value is a function of other attributes of the
+ * Rule. The type of the computed default value for a mandatory attribute
+ * must match the type parameter: (e.g. list=[], integer=0, string="",
+ * label=null). The {@code defaultValue} implementation must be immutable.
+ *
+ * <p>If computedDefault returns a Label that is a target, that target will
+ * become an implicit dependency of this Rule; we will load the target
+ * (and its dependencies) if it encounters the Rule and build the target if
+ * needs to apply the Rule.
+ */
+ public Builder<TYPE> value(ComputedDefault defaultValue) {
+ Preconditions.checkState(!valueSet, "the default value is already set");
+ value = defaultValue;
+ valueSet = true;
+ return this;
+ }
+
+ /**
+ * Sets the attribute default value to be late-bound, i.e., it is derived from the build
+ * configuration.
+ */
+ public Builder<TYPE> value(LateBoundDefault<?> defaultValue) {
+ Preconditions.checkState(!valueSet, "the default value is already set");
+ Preconditions.checkState(name.isEmpty() || isLateBound(name));
+ value = defaultValue;
+ valueSet = true;
+ return this;
+ }
+
+ /**
+ * Returns true if a late-bound value has been set. Useful only for Skylark.
+ */
+ public boolean hasLateBoundValue() {
+ return value != null && value instanceof LateBoundDefault;
+ }
+
+ /**
+ * Sets a condition predicate. The default value of the attribute only applies if the condition
+ * evaluates to true. If the value is explicitly provided, then this condition is ignored.
+ *
+ * <p>The condition is only evaluated if the attribute is not explicitly set, and after all
+ * explicit attributes have been set. It can generally not access default values of other
+ * attributes.
+ */
+ public Builder<TYPE> condition(Predicate<AttributeMap> condition) {
+ Preconditions.checkState(this.condition == null, "the condition is already set");
+ this.condition = condition;
+ return this;
+ }
+
+ /**
+ * Switches on the capability of an attribute to be published to the rule's
+ * tag set.
+ */
+ public Builder<TYPE> taggable() {
+ return setPropertyFlag(PropertyFlag.TAGGABLE, "taggable");
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * rule types for the labels occurring in the attribute. If the attribute
+ * contains Labels of any other rule type, then an error is produced during
+ * the analysis phase. Defaults to allow any types.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedRuleClasses(Iterable<String> allowedRuleClasses) {
+ return allowedRuleClasses(
+ new RuleClass.Builder.RuleClassNamePredicate(allowedRuleClasses));
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * rule types for the labels occurring in the attribute. If the attribute
+ * contains Labels of any other rule type, then an error is produced during
+ * the analysis phase. Defaults to allow any types.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedRuleClasses(Predicate<RuleClass> allowedRuleClasses) {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "must be a label-valued type");
+ propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING);
+ allowedRuleClassesForLabels = allowedRuleClasses;
+ return this;
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * rule types for the labels occurring in the attribute. If the attribute
+ * contains Labels of any other rule type, then an error is produced during
+ * the analysis phase. Defaults to allow any types.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedRuleClasses(String... allowedRuleClasses) {
+ return allowedRuleClasses(ImmutableSet.copyOf(allowedRuleClasses));
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * file types for file labels occurring in the attribute. If the attribute
+ * contains labels that correspond to files of any other type, then an error
+ * is produced during the analysis phase.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedFileTypes(FileTypeSet allowedFileTypes) {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "must be a label-valued type");
+ propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING);
+ allowedFileTypesForLabelsSet = true;
+ allowedFileTypesForLabels = allowedFileTypes;
+ return this;
+ }
+
+ /**
+ * Allow all files for legacy compatibility. All uses of this method should be audited and then
+ * removed. In some cases, it's correct to allow any file, but mostly the set of files should be
+ * restricted to a reasonable set.
+ */
+ public Builder<TYPE> legacyAllowAnyFileType() {
+ return allowedFileTypes(FileTypeSet.ANY_FILE);
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * file types for file labels occurring in the attribute. If the attribute
+ * contains labels that correspond to files of any other type, then an error
+ * is produced during the analysis phase.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedFileTypes(FileType... allowedFileTypes) {
+ return allowedFileTypes(FileTypeSet.of(allowedFileTypes));
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * rule types with warning for the labels occurring in the attribute. If the attribute
+ * contains Labels of any other rule type (other than this or those set in
+ * allowedRuleClasses()), then a warning is produced during
+ * the analysis phase. Defaults to deny any types.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedRuleClassesWithWarning(Collection<String> allowedRuleClasses) {
+ return allowedRuleClassesWithWarning(
+ new RuleClass.Builder.RuleClassNamePredicate(allowedRuleClasses));
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * rule types for the labels occurring in the attribute. If the attribute
+ * contains Labels of any other rule type (other than this or those set in
+ * allowedRuleClasses()), then a warning is produced during
+ * the analysis phase. Defaults to deny any types.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedRuleClassesWithWarning(Predicate<RuleClass> allowedRuleClasses) {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "must be a label-valued type");
+ propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING);
+ allowedRuleClassesForLabelsWarning = allowedRuleClasses;
+ return this;
+ }
+
+ /**
+ * If this is a label or label-list attribute, then this sets the allowed
+ * rule types for the labels occurring in the attribute. If the attribute
+ * contains Labels of any other rule type (other than this or those set in
+ * allowedRuleClasses()), then a warning is produced during
+ * the analysis phase. Defaults to deny any types.
+ *
+ * <p>This only works on a per-target basis, not on a per-file basis; with
+ * other words, it works for 'deps' attributes, but not 'srcs' attributes.
+ */
+ public Builder<TYPE> allowedRuleClassesWithWarning(String... allowedRuleClasses) {
+ return allowedRuleClassesWithWarning(ImmutableSet.copyOf(allowedRuleClasses));
+ }
+
+ /**
+ * Sets a set of mandatory Skylark providers. Every configured target occurring in
+ * this label type attribute has to provide all of these providers, otherwise an
+ * error is produces during the analysis phase for every missing provider.
+ */
+ public Builder<TYPE> mandatoryProviders(Iterable<String> providers) {
+ Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST),
+ "must be a label-valued type");
+ this.mandatoryProviders = ImmutableSet.copyOf(providers);
+ return this;
+ }
+
+ /**
+ * Asserts that a particular aspect needs to be computed for all direct dependencies through
+ * this attribute.
+ */
+ public Builder<TYPE> aspect(Class<? extends AspectFactory<?, ?, ?>> aspect) {
+ this.aspects.add(aspect);
+ return this;
+ }
+ /**
+ * Sets the predicate-like edge validity checker.
+ */
+ public Builder<TYPE> validityPredicate(ValidityPredicate validityPredicate) {
+ propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING);
+ this.validityPredicate = validityPredicate;
+ return this;
+ }
+
+ /**
+ * The value of the attribute must be one of allowedValues.
+ */
+ public Builder<TYPE> allowedValues(PredicateWithMessage<Object> allowedValues) {
+ this.allowedValues = allowedValues;
+ propertyFlags.add(PropertyFlag.CHECK_ALLOWED_VALUES);
+ return this;
+ }
+
+ /**
+ * Makes the built attribute "non-configurable", i.e. its value cannot be influenced by
+ * the build configuration. Attributes are "configurable" unless explicitly opted out here.
+ *
+ * <p>Non-configurability indicates an exceptional state: there exists Blaze logic that needs
+ * the attribute's value, has no access to configurations, and can't apply a workaround
+ * through an appropriate {@link AbstractAttributeMapper} implementation. Scenarios like
+ * this should be as uncommon as possible, so it's important we maintain clear documentation
+ * on what causes them and why users consequently can't configure certain attributes.
+ *
+ * @param reason why this attribute can't be configurable. This isn't used by Blaze - it's
+ * solely a documentation mechanism.
+ */
+ public Builder<TYPE> nonconfigurable(String reason) {
+ Preconditions.checkState(!reason.isEmpty());
+ return setPropertyFlag(PropertyFlag.NONCONFIGURABLE, "nonconfigurable");
+ }
+
+ /**
+ * Creates the attribute. Uses name, type, optionality, configuration type
+ * and the default value configured by the builder.
+ */
+ public Attribute build() {
+ return build(this.name);
+ }
+
+ /**
+ * Creates the attribute. Uses type, optionality, configuration type
+ * and the default value configured by the builder. Use the name
+ * passed as an argument. This function is used by Skylark where the
+ * name is provided only when we build. We don't want to modify the
+ * builder, as it is shared in a multithreaded environment.
+ */
+ public Attribute build(String name) {
+ Preconditions.checkState(!name.isEmpty(), "name has not been set");
+ // TODO(bazel-team): Remove this check again, and remove all allowedFileTypes() calls.
+ if ((type == Type.LABEL) || (type == Type.LABEL_LIST)) {
+ if ((name.startsWith("$") || name.startsWith(":")) && !allowedFileTypesForLabelsSet) {
+ allowedFileTypesForLabelsSet = true;
+ allowedFileTypesForLabels = FileTypeSet.ANY_FILE;
+ }
+ if (!allowedFileTypesForLabelsSet) {
+ throw new IllegalStateException(name);
+ }
+ }
+ return new Attribute(name, type, Sets.immutableEnumSet(propertyFlags),
+ valueSet ? value : type.getDefaultValue(), configTransition, configurator,
+ allowedRuleClassesForLabels, allowedRuleClassesForLabelsWarning,
+ allowedFileTypesForLabels, allowedFileTypesForLabelsSet, validityPredicate, condition,
+ allowedValues, mandatoryProviders, ImmutableSet.copyOf(aspects));
+ }
+ }
+
+ /**
+ * A computed default is a default value for a Rule attribute that is a
+ * function of other attributes of the rule.
+ *
+ * <p>Attributes whose defaults are computed are first initialized to the default
+ * for their type, and then the computed defaults are evaluated after all
+ * non-computed defaults have been initialized. There is no defined order
+ * among computed defaults, so they must not depend on each other.
+ *
+ * <p>If a computed default reads the value of another attribute, at least one of
+ * the following must be true:
+ *
+ * <ol>
+ * <li>The other attribute must be declared in the computed default's constructor</li>
+ * <li>The other attribute must be non-configurable ({@link Builder#nonconfigurable()}</li>
+ * </ol>
+ *
+ * <p>The reason for enforced declarations is that, since attribute values might be
+ * configurable, a computed default that depends on them may itself take multiple
+ * values. Since we have no access to a target's configuration at the time these values
+ * are computed, we need the ability to probe the default's *complete* dependency space.
+ * Declared dependencies allow us to do so sanely. Non-configurable attributes don't have
+ * this problem because their value is fixed and known even without configuration information.
+ *
+ * <p>Implementations of this interface must be immutable.
+ */
+ public abstract static class ComputedDefault {
+ private final List<String> dependencies;
+ List<String> dependencies() { return dependencies; }
+
+ /**
+ * Create a computed default that can read all non-configurable attribute values and no
+ * configurable attribute values.
+ */
+ public ComputedDefault() {
+ dependencies = ImmutableList.of();
+ }
+
+ /**
+ * Create a computed default that can read all non-configurable attributes values and one
+ * explicitly specified configurable attribute value
+ */
+ public ComputedDefault(String depAttribute) {
+ dependencies = ImmutableList.of(depAttribute);
+ }
+
+ /**
+ * Create a computed default that can read all non-configurable attributes values and two
+ * explicitly specified configurable attribute values.
+ */
+ public ComputedDefault(String depAttribute1, String depAttribute2) {
+ dependencies = ImmutableList.of(depAttribute1, depAttribute2);
+ }
+
+ public abstract Object getDefault(AttributeMap rule);
+ }
+
+ /**
+ * Marker interface for late-bound values. Unfortunately, we can't refer to BuildConfiguration
+ * right now, since that is in a separate compilation unit.
+ *
+ * <p>Implementations of this interface must be immutable.
+ *
+ * <p>Use sparingly - having different values for attributes during loading and analysis can
+ * confuse users.
+ */
+ public interface LateBoundDefault<T> {
+ /**
+ * Whether to look up the label in the host configuration. This is only here for the host JDK -
+ * we usually need to look up labels in the target configuration.
+ */
+ boolean useHostConfiguration();
+
+ /**
+ * Returns the set of required configuration fragments, i.e., fragments that will be accessed by
+ * the code.
+ */
+ Set<Class<?>> getRequiredConfigurationFragments();
+
+ /**
+ * The default value for the attribute that is set during the loading phase.
+ */
+ Object getDefault();
+
+ /**
+ * The actual value for the attribute for the analysis phase, which depends on the build
+ * configuration. Note that configurations transitions are applied after the late-bound
+ * attribute was evaluated.
+ */
+ Object getDefault(Rule rule, T o) throws EvalException;
+ }
+
+ /**
+ * Abstract super class for label-typed {@link LateBoundDefault} implementations that simplifies
+ * the client code a little and makes it a bit more type-safe.
+ */
+ public abstract static class LateBoundLabel<T> implements LateBoundDefault<T> {
+ private final Label label;
+ private final ImmutableSet<Class<?>> requiredConfigurationFragments;
+
+ public LateBoundLabel() {
+ this((Label) null);
+ }
+
+ public LateBoundLabel(Label label) {
+ this.label = label;
+ this.requiredConfigurationFragments = ImmutableSet.of();
+ }
+
+ public LateBoundLabel(Label label, Class<?>... requiredConfigurationFragments) {
+ this.label = label;
+ this.requiredConfigurationFragments = ImmutableSet.copyOf(requiredConfigurationFragments);
+ }
+
+ public LateBoundLabel(String label) {
+ this(Label.parseAbsoluteUnchecked(label));
+ }
+
+ public LateBoundLabel(String label, Class<?>... requiredConfigurationFragments) {
+ this(Label.parseAbsoluteUnchecked(label), requiredConfigurationFragments);
+ }
+
+ @Override
+ public boolean useHostConfiguration() {
+ return false;
+ }
+
+ @Override
+ public ImmutableSet<Class<?>> getRequiredConfigurationFragments() {
+ return requiredConfigurationFragments;
+ }
+
+ @Override
+ public final Label getDefault() {
+ return label;
+ }
+
+ @Override
+ public abstract Label getDefault(Rule rule, T configuration);
+ }
+
+ /**
+ * Abstract super class for label-list-typed {@link LateBoundDefault} implementations that
+ * simplifies the client code a little and makes it a bit more type-safe.
+ */
+ public abstract static class LateBoundLabelList<T> implements LateBoundDefault<T> {
+ private final ImmutableList<Label> labels;
+
+ public LateBoundLabelList() {
+ this.labels = ImmutableList.of();
+ }
+
+ public LateBoundLabelList(List<Label> labels) {
+ this.labels = ImmutableList.copyOf(labels);
+ }
+
+ @Override
+ public boolean useHostConfiguration() {
+ return false;
+ }
+
+ @Override
+ public ImmutableSet<Class<?>> getRequiredConfigurationFragments() {
+ return ImmutableSet.of();
+ }
+
+ @Override
+ public final List<Label> getDefault() {
+ return labels;
+ }
+
+ @Override
+ public abstract List<Label> getDefault(Rule rule, T configuration);
+ }
+
+ /**
+ * A class for late bound attributes defined in Skylark.
+ */
+ public static final class SkylarkLateBound implements LateBoundDefault<Object> {
+
+ private final SkylarkCallbackFunction callback;
+
+ public SkylarkLateBound(SkylarkCallbackFunction callback) {
+ this.callback = callback;
+ }
+
+ @Override
+ public boolean useHostConfiguration() {
+ return false;
+ }
+
+ @Override
+ public ImmutableSet<Class<?>> getRequiredConfigurationFragments() {
+ return ImmutableSet.of();
+ }
+
+ @Override
+ public Object getDefault() {
+ return null;
+ }
+
+ @Override
+ public Object getDefault(Rule rule, Object o) throws EvalException {
+ Map<String, Object> attrValues = new HashMap<>();
+ // TODO(bazel-team): support configurable attributes here. RawAttributeMapper will throw
+ // an exception on any instance of configurable attributes.
+ AttributeMap attributes = RawAttributeMapper.of(rule);
+ for (Attribute attr : rule.getAttributes()) {
+ if (!attr.isLateBound()) {
+ Object value = attributes.get(attr.getName(), attr.getType());
+ if (value != null) {
+ attrValues.put(attr.getName(), value);
+ }
+ }
+ }
+ ClassObject attrs = new SkylarkClassObject(attrValues,
+ "No such regular (non late-bound) attribute '%s'.");
+ return callback.call(attrs, o);
+ }
+ }
+
+ private final String name;
+
+ private final Type<?> type;
+
+ private final Set<PropertyFlag> propertyFlags;
+
+ // Exactly one of these conditions is true:
+ // 1. defaultValue == null.
+ // 2. defaultValue instanceof ComputedDefault &&
+ // type.isValid(defaultValue.getDefault())
+ // 3. type.isValid(defaultValue).
+ // 4. defaultValue instanceof LateBoundDefault &&
+ // type.isValid(defaultValue.getDefault(configuration))
+ // (We assume a hypothetical Type.isValid(Object) predicate.)
+ private final Object defaultValue;
+
+ private final Transition configTransition;
+
+ private final Configurator<?, ?> configurator;
+
+ /**
+ * For label or label-list attributes, this predicate returns which rule
+ * classes are allowed for the targets in the attribute.
+ */
+ private final Predicate<RuleClass> allowedRuleClassesForLabels;
+
+ /**
+ * For label or label-list attributes, this predicate returns which rule
+ * classes are allowed for the targets in the attribute with warning.
+ */
+ private final Predicate<RuleClass> allowedRuleClassesForLabelsWarning;
+
+ /**
+ * For label or label-list attributes, this predicate returns which file
+ * types are allowed for targets in the attribute that happen to be file
+ * targets (rather than rules).
+ */
+ private final FileTypeSet allowedFileTypesForLabels;
+ private final boolean allowedFileTypesForLabelsSet;
+
+ /**
+ * This predicate-like object checks
+ * if the edge between two rules using this attribute is valid
+ * in the dependency graph. Returns null if valid, otherwise an error message.
+ */
+ private final ValidityPredicate validityPredicate;
+
+ private final Predicate<AttributeMap> condition;
+
+ private final PredicateWithMessage<Object> allowedValues;
+
+ private final ImmutableSet<String> mandatoryProviders;
+
+ private final ImmutableSet<Class<? extends AspectFactory<?, ?, ?>>> aspects;
+
+ /**
+ * Constructs a rule attribute with the specified name, type and default
+ * value.
+ *
+ * @param name the name of the attribute
+ * @param type the type of the attribute
+ * @param defaultValue the default value to use for this attribute if none is
+ * specified in rule declaration in the BUILD file. Must be null, or of
+ * type "type". May be an instance of ComputedDefault, in which case
+ * its getDefault() method must return an instance of "type", or null.
+ * Must be immutable.
+ * @param configTransition the configuration transition for this attribute
+ * (which must be of type LABEL, LABEL_LIST, NODEP_LABEL or
+ * NODEP_LABEL_LIST).
+ */
+ private Attribute(String name, Type<?> type, Set<PropertyFlag> propertyFlags,
+ Object defaultValue, Transition configTransition,
+ Configurator<?, ?> configurator,
+ Predicate<RuleClass> allowedRuleClassesForLabels,
+ Predicate<RuleClass> allowedRuleClassesForLabelsWarning,
+ FileTypeSet allowedFileTypesForLabels,
+ boolean allowedFileTypesForLabelsSet,
+ ValidityPredicate validityPredicate,
+ Predicate<AttributeMap> condition,
+ PredicateWithMessage<Object> allowedValues,
+ ImmutableSet<String> mandatoryProviders,
+ ImmutableSet<Class<? extends AspectFactory<?, ?, ?>>> aspects) {
+ Preconditions.checkNotNull(configTransition);
+ Preconditions.checkArgument(
+ (configTransition == ConfigurationTransition.NONE && configurator == null)
+ || type == Type.LABEL || type == Type.LABEL_LIST
+ || type == Type.NODEP_LABEL || type == Type.NODEP_LABEL_LIST,
+ "Configuration transitions can only be specified for label or label list attributes");
+ Preconditions.checkArgument(isLateBound(name) == (defaultValue instanceof LateBoundDefault),
+ "late bound attributes require a default value that is late bound (and vice versa): "
+ + name);
+ if (isLateBound(name)) {
+ LateBoundDefault<?> lateBoundDefault = (LateBoundDefault<?>) defaultValue;
+ Preconditions.checkArgument((configurator == null),
+ "a late bound attribute cannot specify a configurator");
+ Preconditions.checkArgument(!lateBoundDefault.useHostConfiguration()
+ || (configTransition == ConfigurationTransition.HOST),
+ "a late bound default value using the host configuration must use the host transition");
+ }
+
+ this.name = name;
+ this.type = type;
+ this.propertyFlags = propertyFlags;
+ this.defaultValue = defaultValue;
+ this.configTransition = configTransition;
+ this.configurator = configurator;
+ this.allowedRuleClassesForLabels = allowedRuleClassesForLabels;
+ this.allowedRuleClassesForLabelsWarning = allowedRuleClassesForLabelsWarning;
+ this.allowedFileTypesForLabels = allowedFileTypesForLabels;
+ this.allowedFileTypesForLabelsSet = allowedFileTypesForLabelsSet;
+ this.validityPredicate = validityPredicate;
+ this.condition = condition;
+ this.allowedValues = allowedValues;
+ this.mandatoryProviders = mandatoryProviders;
+ this.aspects = aspects;
+ }
+
+ /**
+ * Returns the name of this attribute.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns the logical type of this attribute. (May differ from the actual
+ * representation as a value in the build interpreter; for example, an
+ * attribute may logically be a list of labels, but be represented as a list
+ * of strings.)
+ */
+ public Type<?> getType() {
+ return type;
+ }
+
+ private boolean getPropertyFlag(PropertyFlag flag) {
+ return propertyFlags.contains(flag);
+ }
+
+ /**
+ * Returns true if this parameter is mandatory.
+ */
+ public boolean isMandatory() {
+ return getPropertyFlag(PropertyFlag.MANDATORY);
+ }
+
+ /**
+ * Returns true if this list parameter cannot have an empty list as a value.
+ */
+ public boolean isNonEmpty() {
+ return getPropertyFlag(PropertyFlag.NON_EMPTY);
+ }
+
+ /**
+ * Returns true if this label parameter must produce a single artifact.
+ */
+ public boolean isSingleArtifact() {
+ return getPropertyFlag(PropertyFlag.SINGLE_ARTIFACT);
+ }
+
+ /**
+ * Returns true if this label type parameter is checked by silent ruleclass filtering.
+ */
+ public boolean isSilentRuleClassFilter() {
+ return getPropertyFlag(PropertyFlag.SILENT_RULECLASS_FILTER);
+ }
+
+ /**
+ * Returns true if this label type parameter skips the analysis time filetype check.
+ */
+ public boolean isSkipAnalysisTimeFileTypeCheck() {
+ return getPropertyFlag(PropertyFlag.SKIP_ANALYSIS_TIME_FILETYPE_CHECK);
+ }
+
+ /**
+ * Returns true if this parameter is order-independent.
+ */
+ public boolean isOrderIndependent() {
+ return getPropertyFlag(PropertyFlag.ORDER_INDEPENDENT);
+ }
+
+ /**
+ * Returns the configuration transition for this attribute for label or label
+ * list attributes. For other attributes it will always return {@code NONE}.
+ */
+ public Transition getConfigurationTransition() {
+ return configTransition;
+ }
+
+ /**
+ * Returns the configurator instance for this attribute for label or label list attributes.
+ * For other attributes it will always return {@code null}.
+ */
+ public Configurator<?, ?> getConfigurator() {
+ return configurator;
+ }
+
+ /**
+ * Returns whether the target is required to be executable for label or label
+ * list attributes. For other attributes it always returns {@code false}.
+ */
+ public boolean isExecutable() {
+ return getPropertyFlag(PropertyFlag.EXECUTABLE);
+ }
+
+ /**
+ * Returns {@code true} iff the rule is a direct input for an action.
+ */
+ public boolean isDirectCompileTimeInput() {
+ return getPropertyFlag(PropertyFlag.DIRECT_COMPILE_TIME_INPUT);
+ }
+
+ /**
+ * Returns {@code true} iff this attribute requires documentation.
+ */
+ public boolean isDocumented() {
+ return !getPropertyFlag(PropertyFlag.UNDOCUMENTED);
+ }
+
+ /**
+ * Returns {@code true} iff this attribute should be published to the rule's
+ * tag set. Note that not all Type classes support tag conversion.
+ */
+ public boolean isTaggable() {
+ return getPropertyFlag(PropertyFlag.TAGGABLE);
+ }
+
+ public boolean isStrictLabelCheckingEnabled() {
+ return getPropertyFlag(PropertyFlag.STRICT_LABEL_CHECKING);
+ }
+
+ /**
+ * Returns true if the value of this attribute should be a part of a given set.
+ */
+ public boolean checkAllowedValues() {
+ return getPropertyFlag(PropertyFlag.CHECK_ALLOWED_VALUES);
+ }
+
+ /**
+ * Returns true if this attribute's value can be influenced by the build configuration.
+ */
+ public boolean isConfigurable() {
+ return !(type == Type.OUTPUT // Excluded because of Rule#populateExplicitOutputFiles.
+ || type == Type.OUTPUT_LIST
+ || getPropertyFlag(PropertyFlag.NONCONFIGURABLE));
+ }
+
+ /**
+ * Returns a predicate that evaluates to true for rule classes that are
+ * allowed labels in this attribute. If this is not a label or label-list
+ * attribute, the returned predicate always evaluates to true.
+ */
+ public Predicate<RuleClass> getAllowedRuleClassesPredicate() {
+ return allowedRuleClassesForLabels;
+ }
+
+ /**
+ * Returns a predicate that evaluates to true for rule classes that are
+ * allowed labels in this attribute with warning. If this is not a label or label-list
+ * attribute, the returned predicate always evaluates to true.
+ */
+ public Predicate<RuleClass> getAllowedRuleClassesWarningPredicate() {
+ return allowedRuleClassesForLabelsWarning;
+ }
+
+ /**
+ * Returns the set of mandatory Skylark providers.
+ */
+ public ImmutableSet<String> getMandatoryProviders() {
+ return mandatoryProviders;
+ }
+
+ public FileTypeSet getAllowedFileTypesPredicate() {
+ return allowedFileTypesForLabels;
+ }
+
+ public ValidityPredicate getValidityPredicate() {
+ return validityPredicate;
+ }
+
+ public Predicate<AttributeMap> getCondition() {
+ return condition == null ? Predicates.<AttributeMap>alwaysTrue() : condition;
+ }
+
+ public PredicateWithMessage<Object> getAllowedValues() {
+ return allowedValues;
+ }
+
+ /**
+ * Returns the set of aspects required for dependencies through this attribute.
+ */
+ public ImmutableSet<Class<? extends AspectFactory<?, ?, ?>>> getAspects() {
+ return aspects;
+ }
+
+ /**
+ * Returns the default value of this attribute in the context of the
+ * specified Rule. For attributes with a computed default, i.e. {@code
+ * hasComputedDefault()}, {@code rule} must be non-null since the result may
+ * depend on the values of its other attributes.
+ *
+ * <p>The result may be null (although this is not a value in the build
+ * language).
+ *
+ * <p>During population of the rule's attribute dictionary, all non-computed
+ * defaults must be set before all computed ones.
+ *
+ * @param rule the rule to which this attribute belongs; non-null if
+ * {@code hasComputedDefault()}; ignored otherwise.
+ */
+ public Object getDefaultValue(Rule rule) {
+ if (!getCondition().apply(rule == null ? null : NonconfigurableAttributeMapper.of(rule))) {
+ return null;
+ } else if (defaultValue instanceof LateBoundDefault<?>) {
+ return ((LateBoundDefault<?>) defaultValue).getDefault();
+ } else {
+ return defaultValue;
+ }
+ }
+
+ /**
+ * Returns the default value of this attribute, even if it has a condition, is a computed default,
+ * or a late-bound default.
+ */
+ @VisibleForTesting
+ public Object getDefaultValueForTesting() {
+ return defaultValue;
+ }
+
+ public LateBoundDefault<?> getLateBoundDefault() {
+ Preconditions.checkState(isLateBound());
+ return (LateBoundDefault<?>) defaultValue;
+ }
+
+ /**
+ * Returns true iff this attribute has a computed default or a condition.
+ *
+ * @see #getDefaultValue(Rule)
+ */
+ boolean hasComputedDefault() {
+ return (defaultValue instanceof ComputedDefault) || (condition != null);
+ }
+
+ /**
+ * Returns if this attribute is an implicit dependency according to the naming policy that
+ * designates implicit attributes.
+ */
+ public boolean isImplicit() {
+ return isImplicit(getName());
+ }
+
+ /**
+ * Returns if an attribute with the given name is an implicit dependency according to the
+ * naming policy that designates implicit attributes.
+ */
+ public static boolean isImplicit(String name) {
+ return name.startsWith("$");
+ }
+
+ /**
+ * Returns if this attribute is late-bound according to the naming policy that designates
+ * late-bound attributes.
+ */
+ public boolean isLateBound() {
+ return isLateBound(getName());
+ }
+
+ /**
+ * Returns if an attribute with the given name is late-bound according to the naming policy
+ * that designates late-bound attributes.
+ */
+ public static boolean isLateBound(String name) {
+ return name.startsWith(":");
+ }
+
+ @Override
+ public String toString() {
+ return "Attribute(" + name + ", " + type + ")";
+ }
+
+ @Override
+ public int compareTo(Attribute other) {
+ return name.compareTo(other.name);
+ }
+
+ /**
+ * Returns a replica builder of this Attribute.
+ */
+ public Attribute.Builder<?> cloneBuilder() {
+ Builder<?> builder = new Builder<>(name, this.type);
+ builder.allowedFileTypesForLabels = allowedFileTypesForLabels;
+ builder.allowedFileTypesForLabelsSet = allowedFileTypesForLabelsSet;
+ builder.allowedRuleClassesForLabels = allowedRuleClassesForLabels;
+ builder.allowedRuleClassesForLabelsWarning = allowedRuleClassesForLabelsWarning;
+ builder.validityPredicate = validityPredicate;
+ builder.condition = condition;
+ builder.configTransition = configTransition;
+ builder.propertyFlags = propertyFlags.isEmpty() ?
+ EnumSet.noneOf(PropertyFlag.class) : EnumSet.copyOf(propertyFlags);
+ builder.value = defaultValue;
+ builder.valueSet = false;
+ builder.allowedValues = allowedValues;
+
+ return builder;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java
new file mode 100644
index 0000000..be7584a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java
@@ -0,0 +1,116 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.BitSet;
+
+/**
+ * Provides attribute setting and retrieval for a Rule. Encapsulating attribute access
+ * here means it can be passed around independently of the Rule itself. In particular,
+ * it can be consumed by independent {@link AttributeMap} instances that can apply
+ * varying kinds of logic for determining the "value" of an attribute. For example,
+ * a configurable attribute's "value" may be a { config --> value } dictionary
+ * or a configuration-bound lookup on that dictionary, depending on the context in
+ * which it's requested.
+ *
+ * <p>This class provides the lowest-level access to attribute information. It is *not*
+ * intended to be a robust public interface, but rather just an input to {@link AttributeMap}
+ * instances. Use those instances for all domain-level attribute access.
+ */
+public class AttributeContainer {
+
+ private final RuleClass ruleClass;
+
+ // Attribute values, keyed by attribute index:
+ private final Object[] attributeValues;
+
+ // Whether an attribute value has been set explicitly in the BUILD file, keyed by attribute index.
+ private final BitSet attributeValueExplicitlySpecified;
+
+ // Attribute locations, keyed by attribute index:
+ private final Location[] attributeLocations;
+
+ /**
+ * Create a container for a rule of the given rule class.
+ */
+ AttributeContainer(RuleClass ruleClass) {
+ this.ruleClass = ruleClass;
+ this.attributeValues = new Object[ruleClass.getAttributeCount()];
+ this.attributeValueExplicitlySpecified = new BitSet(ruleClass.getAttributeCount());
+ this.attributeLocations = new Location[ruleClass.getAttributeCount()];
+ }
+
+ /**
+ * Returns an attribute value by instance, or null on no match.
+ */
+ public Object getAttr(Attribute attribute) {
+ return getAttr(attribute.getName());
+ }
+
+ /**
+ * Returns an attribute value by name, or null on no match.
+ */
+ public Object getAttr(String attrName) {
+ Integer idx = ruleClass.getAttributeIndex(attrName);
+ return idx != null ? attributeValues[idx] : null;
+ }
+
+ /**
+ * Returns true iff the given attribute exists for this rule and its value
+ * is explicitly set in the BUILD file (as opposed to its default value).
+ */
+ public boolean isAttributeValueExplicitlySpecified(Attribute attribute) {
+ return isAttributeValueExplicitlySpecified(attribute.getName());
+ }
+
+ public boolean isAttributeValueExplicitlySpecified(String attributeName) {
+ Integer idx = ruleClass.getAttributeIndex(attributeName);
+ return idx != null ? attributeValueExplicitlySpecified.get(idx) : false;
+ }
+
+ /**
+ * Returns the location of the attribute definition for this rule, or null if not found.
+ */
+ public Location getAttributeLocation(String attrName) {
+ Integer idx = ruleClass.getAttributeIndex(attrName);
+ return idx != null ? attributeLocations[idx] : null;
+ }
+
+ Object getAttributeValue(int index) {
+ return attributeValues[index];
+ }
+
+ void setAttributeValue(Attribute attribute, Object value, boolean explicit) {
+ Integer index = ruleClass.getAttributeIndex(attribute.getName());
+ attributeValues[index] = value;
+ attributeValueExplicitlySpecified.set(index, explicit);
+ }
+
+ void setAttributeValueByName(String attrName, Object value) {
+ Integer index = ruleClass.getAttributeIndex(attrName);
+ attributeValues[index] = value;
+ attributeValueExplicitlySpecified.set(index);
+ }
+
+ void setAttributeLocation(int attrIndex, Location location) {
+ attributeLocations[attrIndex] = location;
+ }
+
+ void setAttributeLocation(Attribute attribute, Location location) {
+ Integer index = ruleClass.getAttributeIndex(attribute.getName());
+ attributeLocations[index] = location;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AttributeMap.java b/src/main/java/com/google/devtools/build/lib/packages/AttributeMap.java
new file mode 100644
index 0000000..eeb6951
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/AttributeMap.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.Label;
+
+import javax.annotation.Nullable;
+
+/**
+ * The interface for accessing a {@link Rule}'s attributes.
+ *
+ * <p>Since what an attribute lookup should return can be ambiguous (e.g. for configurable
+ * attributes, should we return a configuration-resolved value or the original, unresolved
+ * selection expression?), different implementations can apply different policies for how to
+ * fulfill these methods. Calling code can then use the appropriate implementation for whatever
+ * its particular needs are.
+ */
+public interface AttributeMap {
+ /**
+ * Returns the name of the rule; this is equivalent to {@code getLabel().getName()}.
+ */
+ String getName();
+
+ /**
+ * Returns the label of the rule.
+ */
+ Label getLabel();
+
+ /**
+ * Returns the value of the named rule attribute, which must be of the given type. If it does not
+ * exist or has the wrong type, throws an {@link IllegalArgumentException}.
+ */
+ <T> T get(String attributeName, Type<T> type);
+
+ /**
+ * Returns the names of all attributes covered by this map.
+ */
+ Iterable<String> getAttributeNames();
+
+ /**
+ * Returns the type of the given attribute, if it exists. Otherwise returns null.
+ */
+ @Nullable Type<?> getAttributeType(String attrName);
+
+ /**
+ * Returns the attribute definition whose name is {@code attrName}, or null
+ * if not found.
+ */
+ @Nullable Attribute getAttributeDefinition(String attrName);
+
+ /**
+ * Returns true iff the value of the specified attribute is explicitly set in the BUILD file (as
+ * opposed to its default value). This also returns true if the value from the BUILD file is the
+ * same as the default value.
+ *
+ * <p>It is probably a good idea to avoid this method in default value and implicit outputs
+ * computation, because it is confusing that setting an attribute to an empty list (for example)
+ * is different from not setting it at all.
+ */
+ boolean isAttributeValueExplicitlySpecified(String attributeName);
+
+ /**
+ * An interface which accepts {@link Attribute}s, used by {@link #visitLabels}.
+ */
+ interface AcceptsLabelAttribute {
+ /**
+ * Accept a (Label, Attribute) pair describing a dependency edge.
+ *
+ * @param label the target node of the (Rule, Label) edge.
+ * The source node should already be known.
+ * @param attribute the attribute.
+ */
+ void acceptLabelAttribute(Label label, Attribute attribute);
+ }
+
+ /**
+ * For all attributes that contain labels in their values (either by *being* a label or
+ * being a collection that includes labels), visits every label and notifies the
+ * specified observer at each visit.
+ */
+ void visitLabels(AcceptsLabelAttribute observer);
+
+ // TODO(bazel-team): These methods are here to support computed defaults that inherit
+ // package-level default values. Instead, we should auto-inherit and remove the computed
+ // defaults. If we really need to give access to package-level defaults, we should come up with
+ // a more generic interface.
+ String getPackageDefaultHdrsCheck();
+
+ Boolean getPackageDefaultObsolete();
+
+ Boolean getPackageDefaultTestOnly();
+
+ String getPackageDefaultDeprecation();
+
+ ImmutableList<String> getPackageDefaultCopts();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/BuildFileContainsErrorsException.java b/src/main/java/com/google/devtools/build/lib/packages/BuildFileContainsErrorsException.java
new file mode 100644
index 0000000..d9283c3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/BuildFileContainsErrorsException.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import javax.annotation.Nullable;
+
+/**
+ * Exception indicating a failed attempt to access a package that could not
+ * be read or had syntax errors.
+ */
+public class BuildFileContainsErrorsException extends NoSuchPackageException {
+
+ private Package pkg;
+
+ public BuildFileContainsErrorsException(String packageName, String message) {
+ super(packageName, "error loading package", message);
+ }
+
+ public BuildFileContainsErrorsException(String packageName, String message,
+ Throwable cause) {
+ super(packageName, "error loading package", message, cause);
+ }
+
+ public BuildFileContainsErrorsException(Package pkg, String msg) {
+ this(pkg.getName(), msg);
+ this.pkg = pkg;
+ }
+
+ @Override
+ @Nullable
+ public Package getPackage() {
+ return pkg;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/BuildFileNotFoundException.java b/src/main/java/com/google/devtools/build/lib/packages/BuildFileNotFoundException.java
new file mode 100644
index 0000000..2c70a4e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/BuildFileNotFoundException.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Exception indicating an attempt to access a package which is not found or
+ * does not exist.
+ */
+public class BuildFileNotFoundException extends NoSuchPackageException {
+
+ public BuildFileNotFoundException(String packageName, String message) {
+ super(packageName, message);
+ }
+
+ public BuildFileNotFoundException(String packageName, String message,
+ Throwable cause) {
+ super(packageName, message, cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/CachingPackageLocator.java b/src/main/java/com/google/devtools/build/lib/packages/CachingPackageLocator.java
new file mode 100644
index 0000000..f6badad
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/CachingPackageLocator.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Path;
+
+/**
+ * CachingPackageLocator implementations locate a package by its name.
+ *
+ * <p> Similar to #pkgcache.PathPackageLocator, but implementations are required
+ * to cache results and handle deleted packages.
+ *
+ * <p> This interface exists for two reasons: (1) to avoid creating a bad dependency edge from the
+ * PythonPreprocessor to lib.pkgcache ("dependency injection") and (2) to allow Skyframe to use
+ * pieces of legacy code while still updating the Skyframe node graph.
+ */
+public interface CachingPackageLocator {
+
+ /**
+ * Returns path of BUILD file for specified package iff the specified package exists, null
+ * otherwise (e.g. invalid package name, no build file, or package has been deleted via
+ * --deleted_packages)..
+ *
+ * <p> The package's root directory may be computed by calling getParentFile().
+ *
+ * <p> Instances of this interface are required to cache the results.
+ *
+ * <p> This method must be thread-safe.
+ */
+ @ThreadSafe
+ Path getBuildFileForPackage(String packageName);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ConstantRuleVisibility.java b/src/main/java/com/google/devtools/build/lib/packages/ConstantRuleVisibility.java
new file mode 100644
index 0000000..bb64015
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/ConstantRuleVisibility.java
@@ -0,0 +1,94 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A rule visibility that simply says yes or no. It corresponds to public,
+ * legacy_public and private visibilities.
+ */
+@Immutable @ThreadSafe
+public class ConstantRuleVisibility implements RuleVisibility, Serializable {
+ static final Label LEGACY_PUBLIC_LABEL; // same as "public"; used for automated depot cleanup
+ private static final Label PUBLIC_LABEL;
+ private static final Label PRIVATE_LABEL;
+
+ public static final ConstantRuleVisibility PUBLIC =
+ new ConstantRuleVisibility(true);
+
+ public static final ConstantRuleVisibility PRIVATE =
+ new ConstantRuleVisibility(false);
+
+ static {
+ try {
+ PUBLIC_LABEL = Label.parseAbsolute("//visibility:public");
+ LEGACY_PUBLIC_LABEL = Label.parseAbsolute("//visibility:legacy_public");
+ PRIVATE_LABEL = Label.parseAbsolute("//visibility:private");
+ } catch (Label.SyntaxException e) {
+ throw new IllegalStateException();
+ }
+ }
+
+ private final boolean result;
+
+ public ConstantRuleVisibility(boolean result) {
+ this.result = result;
+ }
+
+ public boolean isPubliclyVisible() {
+ return result;
+ }
+
+ @Override
+ public List<Label> getDependencyLabels() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<Label> getDeclaredLabels() {
+ return ImmutableList.of(result ? PUBLIC_LABEL : PRIVATE_LABEL);
+ }
+
+ /**
+ * Tries to parse a list of labels into a {@link ConstantRuleVisibility}.
+ *
+ * @param labels the list of labels to parse
+ * @return The resulting visibility object, or null if the list of labels
+ * could not be parsed.
+ */
+ public static ConstantRuleVisibility tryParse(List<Label> labels) {
+ if (labels.size() != 1) {
+ return null;
+ }
+ return tryParse(labels.get(0));
+ }
+
+ public static ConstantRuleVisibility tryParse(Label label) {
+ if (PUBLIC_LABEL.equals(label) || LEGACY_PUBLIC_LABEL.equals(label)) {
+ return PUBLIC;
+ } else if (PRIVATE_LABEL.equals(label)) {
+ return PRIVATE;
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/DefaultSetting.java b/src/main/java/com/google/devtools/build/lib/packages/DefaultSetting.java
new file mode 100644
index 0000000..1c89b4e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/DefaultSetting.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * A feature of the build that can be switched on and off on a per-package
+ * basis.
+ *
+ * <p>This interface is only for marking targets as being affected by a feature;
+ * their implementation can be anywhere.
+ *
+ * Implementations of this interface must be immutable.
+ */
+public interface DefaultSetting {
+ String getSettingName();
+
+ /**
+ * Returns if the default setting in question affects the specific target.
+ */
+ boolean appliesTo(Target target);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/DuplicatePackageException.java b/src/main/java/com/google/devtools/build/lib/packages/DuplicatePackageException.java
new file mode 100644
index 0000000..dc21cba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/DuplicatePackageException.java
@@ -0,0 +1,32 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Exception indicating that the same package (i.e. BUILD file) can be loaded
+ * via different package paths.
+ */
+// TODO(bazel-team): (2009) Change exception hierarchy so that DuplicatePackageException is no
+// longer a child of NoSuchPackageException.
+public class DuplicatePackageException extends NoSuchPackageException {
+
+ public DuplicatePackageException(String packageName, String message) {
+ super(packageName, message);
+ }
+
+ public DuplicatePackageException(String packageName, String message, Throwable cause) {
+ super(packageName, message, cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/EnumFilterConverter.java b/src/main/java/com/google/devtools/build/lib/packages/EnumFilterConverter.java
new file mode 100644
index 0000000..31242e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/EnumFilterConverter.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Converter that translates a string of the form "value1,value2,-value3,value4"
+ * into a corresponding set of allowed Enum values.
+ *
+ * <p>Values preceded by '-' are excluded from this set. So "value1,-value2,value3"
+ * translates to the set [EnumType.value1, EnumType.value3].
+ *
+ * <p>If *all* values are exclusions (e.g. "-value1,-value2,-value3"), the returned
+ * set contains all values for the Enum type *except* those specified.
+ */
+class EnumFilterConverter<E extends Enum<E>> implements Converter<Set<E>> {
+
+ private final Set<String> allowedValues = new LinkedHashSet<>();
+ private final Class<E> typeClass;
+ private final String prettyEnumName;
+
+ /**
+ * Constructor.
+ *
+ * @param typeClass this should be E.class (Java generics can't infer that directly)
+ * @param userFriendlyName a user-friendly description of this enum type
+ */
+ public EnumFilterConverter(Class<E> typeClass, String userFriendlyName) {
+ this.typeClass = typeClass;
+ this.prettyEnumName = userFriendlyName;
+ for (E value : EnumSet.allOf(typeClass)) {
+ allowedValues.add(value.name());
+ }
+ }
+
+ /**
+ * Returns the set of allowed values for the option.
+ *
+ * Implements {@link #convert(String)}.
+ */
+ @Override
+ public Set<E> convert(String input) throws OptionsParsingException {
+ if (input.equals("")) {
+ return Collections.emptySet();
+ }
+ EnumSet<E> includedSet = EnumSet.noneOf(typeClass);
+ EnumSet<E> excludedSet = EnumSet.noneOf(typeClass);
+ for (String value : input.split(",", -1)) {
+ boolean excludeFlag = value.startsWith("-");
+ String s = (excludeFlag ? value.substring(1) : value).toUpperCase();
+ if (!allowedValues.contains(s)) {
+ throw new OptionsParsingException("Invalid " + prettyEnumName + " filter '" + value +
+ "' in the input '" + input + "'");
+ }
+ (excludeFlag ? excludedSet : includedSet).add(E.valueOf(typeClass, s));
+ }
+ if (includedSet.isEmpty()) {
+ includedSet = EnumSet.complementOf(excludedSet);
+ } else {
+ includedSet.removeAll(excludedSet);
+ }
+ if (includedSet.isEmpty()) {
+ throw new OptionsParsingException(
+ Character.toUpperCase(prettyEnumName.charAt(0)) + prettyEnumName.substring(1) +
+ " filter '" + input + "' definition cannot match any tests");
+ }
+ return includedSet;
+ }
+
+ /**
+ * Implements {@link #getTypeDescription()}.
+ */
+ @Override
+ public final String getTypeDescription() {
+ return "comma-separated list of values: "
+ + StringUtil.joinEnglishList(allowedValues).toLowerCase();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/EnvironmentGroup.java b/src/main/java/com/google/devtools/build/lib/packages/EnvironmentGroup.java
new file mode 100644
index 0000000..07da227
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/EnvironmentGroup.java
@@ -0,0 +1,241 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Model for the "environment_group' rule: the piece of Bazel's rule constraint system that binds
+ * thematically related environments together and determines which environments a rule supports
+ * by default. See {@link com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics}
+ * for precise semantic details of how this information is used.
+ *
+ * <p>Note that "environment_group" is implemented as a loading-time function, not a rule. This is
+ * to support proper discovery of defaults: Say rule A has no explicit constraints and depends
+ * on rule B, which is explicitly constrained to environment ":bar". Since A declares nothing
+ * explicitly, it's implicitly constrained to DEFAULTS (whatever that is). Therefore, the
+ * dependency is only allowed if DEFAULTS doesn't include environments beyond ":bar". To figure
+ * that out, we need to be able to look up the environment group for ":bar", which is what this
+ * class provides.
+ *
+ * <p>If we implemented this as a rule, we'd have to provide that lookup via rule dependencies,
+ * e.g. something like:
+ *
+ * <code>
+ * environment(
+ * name = 'bar',
+ * group = [':sample_environments'],
+ * is_default = 1
+ * )
+ * </code>
+ *
+ * <p>But this won't work. This would let us find the environment group for ":bar", but the only way
+ * to determine what other environments belong to the group is to have the group somehow reference
+ * them. That would produce circular dependencies in the build graph, which is no good.
+ */
+@Immutable
+public class EnvironmentGroup implements Target {
+ private final Label label;
+ private final Location location;
+ private final Package containingPackage;
+ private final Set<Label> environments;
+ private final Set<Label> defaults;
+
+ /**
+ * Predicate that matches labels from a different package than the initialized package.
+ */
+ private static final class DifferentPackage implements Predicate<Label> {
+ private final Package containingPackage;
+
+ private DifferentPackage(Package containingPackage) {
+ this.containingPackage = containingPackage;
+ }
+
+ @Override
+ public boolean apply(Label environment) {
+ return !environment.getPackageName().equals(containingPackage.getName());
+ }
+ }
+
+ /**
+ * Instantiates a new group without verifying the soundness of its contents. See the validation
+ * methods below for appropriate checks.
+ *
+ * @param label the build label identifying this group
+ * @param pkg the package this group belongs to
+ * @param environments the set of environments that belong to this group
+ * @param defaults the environments a rule implicitly supports unless otherwise specified
+ * @param location location in the BUILD file of this group
+ */
+ EnvironmentGroup(Label label, Package pkg, final List<Label> environments, List<Label> defaults,
+ Location location) {
+ this.label = label;
+ this.location = location;
+ this.containingPackage = pkg;
+ this.environments = ImmutableSet.copyOf(environments);
+ this.defaults = ImmutableSet.copyOf(defaults);
+ }
+
+ /**
+ * Checks that all environments declared by this group are in the same package as the group (so
+ * we can perform an environment --> environment_group lookup and know the package is available)
+ * and checks that all defaults are legitimate members of the group.
+ *
+ * <p>Does <b>not</b> check that the referenced environments exist (see
+ * {@link #checkEnvironmentsExist).
+ *
+ * @return a list of validation errors that occurred
+ */
+ List<Event> validateMembership() {
+ List<Event> events = new ArrayList<>();
+
+ // All environments should belong to the same package as this group.
+ for (Label environment :
+ Iterables.filter(environments, new DifferentPackage(containingPackage))) {
+ events.add(Event.error(location,
+ environment + " is not in the same package as group " + label));
+ }
+
+ // The defaults must be a subset of the member environments.
+ for (Label unknownDefault : Sets.difference(defaults, environments)) {
+ events.add(Event.error(location, "default " + unknownDefault + " is not a "
+ + "declared environment for group " + getLabel()));
+ }
+
+ return events;
+ }
+
+ /**
+ * Given the set of targets in this group's package, checks that all of the group's declared
+ * environments are part of that set (i.e. the group doesn't reference non-existant labels).
+ *
+ * @param pkgTargets mapping from label name to target instance for this group's package
+ * @return a list of validation errors that occurred
+ */
+ List<Event> checkEnvironmentsExist(Map<String, Target> pkgTargets) {
+ List<Event> events = new ArrayList<>();
+ for (Label envName : environments) {
+ Target env = pkgTargets.get(envName.getName());
+ if (env == null) {
+ events.add(Event.error(location, "environment " + envName + " does not exist"));
+ } else if (!env.getTargetKind().equals("environment rule")) {
+ events.add(Event.error(location, env.getLabel() + " is not a valid environment"));
+ }
+ }
+ return events;
+ }
+
+ /**
+ * Returns the environments that belong to this group.
+ */
+ public Set<Label> getEnvironments() {
+ return environments;
+ }
+
+ /**
+ * Returns the environments a rule supports by default, i.e. if it has no explicit references to
+ * environments in this group.
+ */
+ public Set<Label> getDefaults() {
+ return defaults;
+ }
+
+ /**
+ * Determines whether or not an environment is a default. Returns false if the environment
+ * doesn't belong to this group.
+ */
+ public boolean isDefault(Label environment) {
+ return defaults.contains(environment);
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public String getName() {
+ return label.getName();
+ }
+
+ @Override
+ public Package getPackage() {
+ return containingPackage;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return targetKind();
+ }
+
+ @Override
+ public Rule getAssociatedRule() {
+ return null;
+ }
+
+ @Override
+ public License getLicense() {
+ return License.NO_LICENSE;
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return targetKind() + " " + getLabel();
+ }
+
+ @Override
+ public Set<License.DistributionType> getDistributions() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public RuleVisibility getVisibility() {
+ return ConstantRuleVisibility.PRIVATE; // No rule should be referencing an environment_group.
+ }
+
+ public static String targetKind() {
+ return "environment group";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ // In a distributed implementation these may not be the same object.
+ if (o == this) {
+ return true;
+ } else if (!(o instanceof EnvironmentGroup)) {
+ return false;
+ } else {
+ return ((EnvironmentGroup) o).getLabel().equals(getLabel());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ExternalPackage.java b/src/main/java/com/google/devtools/build/lib/packages/ExternalPackage.java
new file mode 100644
index 0000000..9c92b33
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/ExternalPackage.java
@@ -0,0 +1,193 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * This creates the //external package, where targets not homed in this repository can be bound.
+ */
+public class ExternalPackage extends Package {
+
+ private Map<RepositoryName, Rule> repositoryMap;
+
+ ExternalPackage() {
+ super(PackageIdentifier.createInDefaultRepo("external"));
+ }
+
+ /**
+ * Returns a description of the repository with the given name, or null if there's no such
+ * repository.
+ */
+ public Rule getRepositoryInfo(RepositoryName repositoryName) {
+ return repositoryMap.get(repositoryName);
+ }
+
+ /**
+ * Holder for a binding's actual label and location.
+ */
+ public static class Binding {
+ private final Label actual;
+ private final Location location;
+
+ public Binding(Label actual, Location location) {
+ this.actual = actual;
+ this.location = location;
+ }
+
+ public Label getActual() {
+ return actual;
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+
+ /**
+ * Checks if the label is bound, i.e., starts with //external:.
+ */
+ public static boolean isBoundLabel(Label label) {
+ return label.getPackageName().equals("external");
+ }
+ }
+
+ /**
+ * Given a workspace file path, creates an ExternalPackage.
+ */
+ public static class ExternalPackageBuilder
+ extends AbstractBuilder<ExternalPackage, ExternalPackageBuilder> {
+ private Map<Label, Binding> bindMap = Maps.newHashMap();
+ private Map<RepositoryName, Rule> repositoryMap = Maps.newHashMap();
+
+ public ExternalPackageBuilder(Path workspacePath) {
+ super(new ExternalPackage());
+ setFilename(workspacePath);
+ setMakeEnv(new MakeEnvironment.Builder());
+ }
+
+ @Override
+ protected ExternalPackageBuilder self() {
+ return this;
+ }
+
+ @Override
+ public ExternalPackage build() {
+ pkg.repositoryMap = ImmutableMap.copyOf(repositoryMap);
+ return super.build();
+ }
+
+ public void addBinding(Label label, Binding binding) {
+ bindMap.put(label, binding);
+ }
+
+ public void resolveBindTargets(RuleClass ruleClass)
+ throws EvalException, NoSuchBindingException {
+ for (Entry<Label, Binding> entry : bindMap.entrySet()) {
+ resolveLabel(entry.getKey(), entry.getValue());
+ }
+
+ for (Entry<Label, Binding> entry : bindMap.entrySet()) {
+ try {
+ addRule(ruleClass, entry);
+ } catch (NameConflictException | InvalidRuleException e) {
+ throw new EvalException(entry.getValue().location, e.getMessage());
+ }
+ }
+ }
+
+ // Uses tortoise and the hare algorithm to detect cycles.
+ private void resolveLabel(final Label virtual, Binding binding)
+ throws NoSuchBindingException {
+ Label actual = binding.getActual();
+ Label tortoise = virtual;
+ Label hare = actual;
+ boolean moveTortoise = true;
+ while (Binding.isBoundLabel(actual)) {
+ if (tortoise == hare) {
+ throw new NoSuchBindingException("cycle detected resolving " + virtual + " binding");
+ }
+
+ Label previous = actual; // For the exception.
+ binding = bindMap.get(actual);
+ if (binding == null) {
+ throw new NoSuchBindingException("no binding found for target " + previous + " (via "
+ + virtual + ")");
+ }
+ actual = binding.getActual();
+ hare = actual;
+ moveTortoise = !moveTortoise;
+ if (moveTortoise) {
+ tortoise = bindMap.get(tortoise).getActual();
+ }
+ }
+ bindMap.put(virtual, binding);
+ }
+
+ private void addRule(RuleClass klass, Map.Entry<Label, Binding> bindingEntry)
+ throws InvalidRuleException, NameConflictException {
+ Label virtual = bindingEntry.getKey();
+ Label actual = bindingEntry.getValue().actual;
+ Location location = bindingEntry.getValue().location;
+
+ Map<String, Object> attributes = Maps.newHashMap();
+ // Bound rules don't have a name field, but this works because we don't want more than one
+ // with the same virtual name.
+ attributes.put("name", virtual.getName());
+ attributes.put("actual", actual);
+ StoredEventHandler handler = new StoredEventHandler();
+ Rule rule = RuleFactory.createAndAddRule(this, klass, attributes, handler, null, location);
+ rule.setVisibility(ConstantRuleVisibility.PUBLIC);
+ }
+
+ /**
+ * This is used when a binding is invalid, either because one of the targets is malformed,
+ * refers to a package that does not exist, or creates a circular dependency.
+ */
+ public class NoSuchBindingException extends NoSuchThingException {
+ public NoSuchBindingException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Creates an external repository rule.
+ * @throws SyntaxException if the repository name is invalid.
+ */
+ public ExternalPackageBuilder createAndAddRepositoryRule(RuleClass ruleClass,
+ Map<String, Object> kwargs, FuncallExpression ast)
+ throws InvalidRuleException, NameConflictException, SyntaxException {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ Rule rule = RuleFactory.createAndAddRule(this, ruleClass, kwargs, eventHandler, ast,
+ ast.getLocation());
+ // Propagate Rule errors to the builder.
+ addEvents(eventHandler.getEvents());
+ repositoryMap.put(RepositoryName.create("@" + rule.getName()), rule);
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/FileTarget.java b/src/main/java/com/google/devtools/build/lib/packages/FileTarget.java
new file mode 100644
index 0000000..60f30ce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/FileTarget.java
@@ -0,0 +1,92 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType.HasFilename;
+
+import java.util.Set;
+
+/**
+ * Common superclass for InputFile and OutputFile which provides implementation
+ * for the file operations in common.
+ */
+public abstract class FileTarget implements Target, HasFilename {
+ protected final Package pkg;
+ protected final Label label;
+
+ /**
+ * Constructs a file with the given label, which must be in the given package.
+ */
+ protected FileTarget(Package pkg, Label label) {
+ Preconditions.checkArgument(label.getPackageFragment().equals(pkg.getNameFragment()));
+ this.pkg = pkg;
+ this.label = label;
+ }
+
+ @Override
+ public String getFilename() {
+ return label.getName();
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public String getName() {
+ return label.getName();
+ }
+
+ @Override
+ public Package getPackage() {
+ return pkg;
+ }
+
+ @Override
+ public String toString() {
+ return getTargetKind() + "(" + getLabel() + ")"; // Just for debugging
+ }
+
+ @Override
+ public int hashCode() {
+ return label.hashCode();
+ }
+
+ @Override
+ public Set<DistributionType> getDistributions() {
+ return getPackage().getDefaultDistribs();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>File licenses are strange, and require some special handling. When
+ * you ask "What license covers this file?" in a query, the answer should
+ * be the license declared for the enclosing package. On the other hand,
+ * if the file is a source for a rule target, and the rule's license declares
+ * more exceptions than the default inherited by the file, the rule's
+ * more liberal target should override the stricter license of the file. In
+ * other words, the license of the rule always overrides the license of
+ * the non-rule file targets that are inputs to that rule.
+ */
+ @Override
+ public License getLicense() {
+ return getPackage().getDefaultLicense();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
new file mode 100644
index 0000000..3e150b4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
@@ -0,0 +1,347 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Caches the results of glob expansion for a package.
+ */
+@ThreadSafety.ThreadCompatible
+public class GlobCache {
+ public static class BadGlobException extends Exception {
+ BadGlobException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * A mapping from glob expressions (e.g. "*.java") to the list of files it
+ * matched (in the order returned by VFS) at the time the package was
+ * constructed. Required for sound dependency analysis.
+ *
+ * We don't use a Multimap because it provides no way to distinguish "key not
+ * present" from (key -> {}).
+ */
+ private final Map<Pair<String, Boolean>, Future<List<Path>>> globCache = new HashMap<>();
+
+ /**
+ * The directory in which our package's BUILD file resides.
+ */
+ private final Path packageDirectory;
+
+ /**
+ * The name of the package we belong to.
+ */
+ private final PackageIdentifier packageId;
+
+ /**
+ * The package locator-based directory traversal predicate.
+ */
+ private final Predicate<Path> childDirectoryPredicate;
+
+ /**
+ * System call caching layer.
+ */
+ private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls;
+
+ /**
+ * The thread pool for glob evaluation.
+ */
+ private final ThreadPoolExecutor globExecutor;
+
+ /**
+ * Create a glob expansion cache.
+ * @param packageDirectory globs will be expanded relatively to this
+ * directory.
+ * @param packageId the name of the package this cache belongs to.
+ * @param locator the package locator.
+ * @param globExecutor thread pool for glob evaluation.
+ */
+ public GlobCache(final Path packageDirectory,
+ final PackageIdentifier packageId,
+ final CachingPackageLocator locator,
+ AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls,
+ ThreadPoolExecutor globExecutor) {
+ this.packageDirectory = Preconditions.checkNotNull(packageDirectory);
+ this.packageId = Preconditions.checkNotNull(packageId);
+ this.globExecutor = Preconditions.checkNotNull(globExecutor);
+ this.syscalls = syscalls == null ? new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS) : syscalls;
+
+ Preconditions.checkNotNull(locator);
+ final PathFragment pkgNameFrag = packageId.getPackageFragment();
+ childDirectoryPredicate = new Predicate<Path>() {
+ @Override
+ public boolean apply(Path directory) {
+ if (directory.equals(packageDirectory)) {
+ return true;
+ }
+
+ PathFragment pkgName = pkgNameFrag.getRelative(directory.relativeTo(packageDirectory));
+ UnixGlob.FilesystemCalls syscalls = GlobCache.this.syscalls.get();
+ if (syscalls != UnixGlob.DEFAULT_SYSCALLS) {
+ // This is needed because in case the BUILD file exists, we do not call readdir() on its
+ // directory. However, the package needs to be re-evaluated if the BUILD file is removed.
+ // Therefore, we add this BUILD file to our dependencies by statting it through the
+ // recording syscall object so that the BUILD file itself is added to the dependencies of
+ // this package.
+ //
+ // The stat() call issued by locator.getBuildFileForPackage() does not quite cut it
+ // because 1. it is cached so there may not be a stat() call at all and 2. even if there
+ // is, it does not go through the proxy in GlobCache.this.syscalls.
+ //
+ // Note that this does not cause any significant slowdown; the BUILD file cache will have
+ // already evaluated the very same stat, so we don't pay any I/O cost, only a cache
+ // lookup.
+ syscalls.statNullable(directory.getChild("BUILD"), Symlinks.FOLLOW);
+ }
+
+ return locator.getBuildFileForPackage(pkgName.getPathString()) == null;
+ }
+ };
+ }
+
+ /**
+ * Returns the future result of evaluating glob "pattern" against this
+ * package's directory, using the package's cache of previously-started
+ * globs if possible.
+ *
+ * @return the list of paths matching the pattern, relative to the package's
+ * directory.
+ * @throws BadGlobException if the glob was syntactically invalid, or
+ * contained uplevel references.
+ */
+ Future<List<Path>> getGlobAsync(String pattern, boolean excludeDirs)
+ throws BadGlobException {
+ Future<List<Path>> cached = globCache.get(Pair.of(pattern, excludeDirs));
+ if (cached == null) {
+ cached = safeGlob(pattern, excludeDirs);
+ setGlobPaths(pattern, excludeDirs, cached);
+ }
+ return cached;
+ }
+
+ @VisibleForTesting
+ List<String> getGlob(String pattern)
+ throws IOException, BadGlobException, InterruptedException {
+ return getGlob(pattern, false);
+ }
+
+ @VisibleForTesting
+ protected List<String> getGlob(String pattern, boolean excludeDirs)
+ throws IOException, BadGlobException, InterruptedException {
+ Future<List<Path>> futureResult = getGlobAsync(pattern, excludeDirs);
+ List<Path> globPaths = fromFuture(futureResult);
+ // Replace the UnixGlob.GlobFuture with a completed future object, to allow
+ // garbage collection of the GlobFuture and GlobVisitor objects.
+ if (!(futureResult instanceof SettableFuture<?>)) {
+ SettableFuture<List<Path>> completedFuture = SettableFuture.create();
+ completedFuture.set(globPaths);
+ globCache.put(Pair.of(pattern, excludeDirs), completedFuture);
+ }
+
+ List<String> result = Lists.newArrayListWithCapacity(globPaths.size());
+ for (Path path : globPaths) {
+ String relative = path.relativeTo(packageDirectory).getPathString();
+ // Don't permit "" (meaning ".") in the glob expansion, since it's
+ // invalid as a label, plus users should say explicitly if they
+ // really want to name the package directory.
+ if (!relative.isEmpty()) {
+ result.add(relative);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Adds glob entries to the cache, making sure they are sorted first.
+ */
+ @VisibleForTesting
+ void setGlobPaths(String pattern, boolean excludeDirectories, Future<List<Path>> result) {
+ globCache.put(Pair.of(pattern, excludeDirectories), result);
+ }
+
+ /**
+ * Actually execute a glob against the filesystem. Otherwise similar to
+ * getGlob().
+ */
+ @VisibleForTesting
+ Future<List<Path>> safeGlob(String pattern, boolean excludeDirs) throws BadGlobException {
+ // Forbidden patterns:
+ if (pattern.indexOf('?') != -1) {
+ throw new BadGlobException("glob pattern '" + pattern + "' contains forbidden '?' wildcard");
+ }
+ // Patterns forbidden by UnixGlob library:
+ String error = UnixGlob.checkPatternForError(pattern);
+ if (error != null) {
+ throw new BadGlobException(error + " (in glob pattern '" + pattern + "')");
+ }
+ return UnixGlob.forPath(packageDirectory)
+ .addPattern(pattern)
+ .setExcludeDirectories(excludeDirs)
+ .setDirectoryFilter(childDirectoryPredicate)
+ .setThreadPool(globExecutor)
+ .setFilesystemCalls(syscalls)
+ .globAsync(true);
+ }
+
+ /**
+ * Sanitize the future exceptions - the only expected checked exception
+ * is IOException.
+ */
+ private static List<Path> fromFuture(Future<List<Path>> future)
+ throws IOException, InterruptedException {
+ try {
+ return future.get();
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Throwables.propagateIfPossible(cause,
+ IOException.class, InterruptedException.class);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Returns true iff all this package's globs are up-to-date. That is,
+ * re-evaluating the package's BUILD file at this moment would yield an
+ * equivalent Package instance. (This call requires filesystem I/O to
+ * re-evaluate the globs.)
+ */
+ public boolean globsUpToDate() throws InterruptedException {
+ // Start all globs in parallel.
+ Map<Pair<String, Boolean>, Future<List<Path>>> newGlobs = new HashMap<>();
+ try {
+ for (Map.Entry<Pair<String, Boolean>, Future<List<Path>>> entry : globCache.entrySet()) {
+ Pair<String, Boolean> key = entry.getKey();
+ try {
+ newGlobs.put(key, safeGlob(key.first, key.second));
+ } catch (BadGlobException e) {
+ return false;
+ }
+ }
+
+ for (Map.Entry<Pair<String, Boolean>, Future<List<Path>>> entry : globCache.entrySet()) {
+ try {
+ Pair<String, Boolean> key = entry.getKey();
+ List<Path> newGlob = fromFuture(newGlobs.get(key));
+ List<Path> oldGlob = fromFuture(entry.getValue());
+ if (!oldGlob.equals(newGlob)) {
+ return false;
+ }
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ return true;
+ } finally {
+ finishBackgroundTasks(newGlobs.values());
+ }
+ }
+
+ /**
+ * Evaluate the build language expression "glob(includes, excludes)" in the
+ * context of this package.
+ *
+ * <p>Called by PackageFactory via Package.
+ */
+ public List<String> glob(List<String> includes, List<String> excludes, boolean excludeDirs)
+ throws IOException, BadGlobException, InterruptedException {
+ // Start globbing all patterns in parallel. The getGlob() calls below will
+ // block on an individual pattern's results, but the other globs can
+ // continue in the background.
+ for (String pattern : Iterables.concat(includes, excludes)) {
+ getGlobAsync(pattern, excludeDirs);
+ }
+
+ Set<String> results = new LinkedHashSet<>();
+ for (String pattern : includes) {
+ results.addAll(getGlob(pattern, excludeDirs));
+ }
+ for (String pattern : excludes) {
+ results.removeAll(getGlob(pattern, excludeDirs));
+ }
+
+ Preconditions.checkState(!results.contains(null), "glob returned null");
+ return new ArrayList<>(results);
+ }
+
+ public Set<Pair<String, Boolean>> getKeySet() {
+ return globCache.keySet();
+ }
+
+ /**
+ * Block on the completion of all potentially-abandoned background tasks.
+ */
+ public void finishBackgroundTasks() {
+ finishBackgroundTasks(globCache.values());
+ }
+
+ public void cancelBackgroundTasks() {
+ cancelBackgroundTasks(globCache.values());
+ }
+
+ private static void finishBackgroundTasks(Collection<Future<List<Path>>> tasks) {
+ for (Future<List<Path>> task : tasks) {
+ try {
+ fromFuture(task);
+ } catch (IOException | InterruptedException e) {
+ // Ignore: If this was still going on in the background, some other
+ // failure already occurred.
+ }
+ }
+ }
+
+ private static void cancelBackgroundTasks(Collection<Future<List<Path>>> tasks) {
+ for (Future<List<Path>> task : tasks) {
+ task.cancel(true);
+
+ try {
+ task.get();
+ } catch (ExecutionException | InterruptedException e) {
+ // We don't care. Point is, the task does not bother us anymore.
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "GlobCache for " + packageId + " in " + packageDirectory;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ImplicitOutputsFunction.java b/src/main/java/com/google/devtools/build/lib/packages/ImplicitOutputsFunction.java
new file mode 100644
index 0000000..9f72fd0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/ImplicitOutputsFunction.java
@@ -0,0 +1,421 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import static com.google.devtools.build.lib.syntax.SkylarkFunction.castMap;
+import static java.util.Collections.singleton;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.escape.Escaper;
+import com.google.common.escape.Escapers;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction;
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A function interface allowing rules to specify their set of implicit outputs
+ * in a more dynamic way than just simple template-substitution. For example,
+ * the set of implicit outputs may be a function of rule attributes.
+ */
+public abstract class ImplicitOutputsFunction {
+
+ /**
+ * Implicit output functions for Skylark supporting key value access of expanded implicit outputs.
+ */
+ public abstract static class SkylarkImplicitOutputsFunction extends ImplicitOutputsFunction {
+
+ public abstract ImmutableMap<String, String> calculateOutputs(AttributeMap map)
+ throws EvalException;
+
+ @Override
+ public Iterable<String> getImplicitOutputs(AttributeMap map) throws EvalException {
+ return calculateOutputs(map).values();
+ }
+ }
+
+ /**
+ * Implicit output functions executing Skylark code.
+ */
+ public static final class SkylarkImplicitOutputsFunctionWithCallback
+ extends SkylarkImplicitOutputsFunction {
+
+ private final SkylarkCallbackFunction callback;
+ private final Location loc;
+
+ public SkylarkImplicitOutputsFunctionWithCallback(
+ SkylarkCallbackFunction callback, Location loc) {
+ this.callback = callback;
+ this.loc = loc;
+ }
+
+ @Override
+ public ImmutableMap<String, String> calculateOutputs(AttributeMap map) throws EvalException {
+ Map<String, Object> attrValues = new HashMap<>();
+ for (String attrName : map.getAttributeNames()) {
+ // TODO(bazel-team): support configurable attributes - which value would we want to
+ // pass on to the child outputs function? Maybe implicit output functions shouldn't
+ // have access to configurable values (makes them too complicated?). Maybe they
+ // should have *full* access (gives them the most power?).
+ Object value = map.get(attrName, map.getAttributeType(attrName));
+ if (value != null) {
+ attrValues.put(attrName, value);
+ }
+ }
+ ClassObject attrs = new SkylarkClassObject(attrValues, "No such attribute '%s'");
+ try {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ for (Map.Entry<String, String> entry : castMap(callback.call(attrs),
+ String.class, String.class, "implicit outputs function return value")) {
+ Iterable<String> substitutions = fromTemplates(entry.getValue()).getImplicitOutputs(map);
+ if (!Iterables.isEmpty(substitutions)) {
+ builder.put(entry.getKey(), Iterables.getOnlyElement(substitutions));
+ }
+ }
+ return builder.build();
+ } catch (IllegalArgumentException e) {
+ throw new EvalException(loc, e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Implicit output functions using a simple an output map.
+ */
+ public static final class SkylarkImplicitOutputsFunctionWithMap
+ extends SkylarkImplicitOutputsFunction {
+
+ private final ImmutableMap<String, String> outputMap;
+
+ public SkylarkImplicitOutputsFunctionWithMap(ImmutableMap<String, String> outputMap) {
+ this.outputMap = outputMap;
+ }
+
+ @Override
+ public ImmutableMap<String, String> calculateOutputs(AttributeMap map) {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ for (Map.Entry<String, String> entry : outputMap.entrySet()) {
+ Iterable<String> substitutions = fromTemplates(entry.getValue()).getImplicitOutputs(map);
+ if (!Iterables.isEmpty(substitutions)) {
+ builder.put(entry.getKey(), Iterables.getOnlyElement(substitutions));
+ }
+ }
+ return builder.build();
+ }
+ }
+
+ /**
+ * Implicit output functions which can not throw an EvalException.
+ */
+ public abstract static class SafeImplicitOutputsFunction extends ImplicitOutputsFunction {
+ @Override
+ public abstract Iterable<String> getImplicitOutputs(AttributeMap map);
+ }
+
+ /**
+ * An interface to objects that can retrieve rule attributes.
+ */
+ public interface AttributeValueGetter {
+ /**
+ * Returns the value(s) of attribute "attr" in "rule", or empty set if attribute unknown.
+ */
+ Set<String> get(AttributeMap rule, String attr);
+ }
+
+ /**
+ * The default rule attribute retriever.
+ *
+ * <p>Custom {@link AttributeValueGetter} implementations may delegate to this object as a
+ * fallback mechanism.
+ */
+ public static final AttributeValueGetter DEFAULT_RULE_ATTRIBUTE_GETTER =
+ new AttributeValueGetter() {
+ @Override
+ public Set<String> get(AttributeMap rule, String attr) {
+ return attributeValues(rule, attr);
+ }
+ };
+
+ private static final Escaper PERCENT_ESCAPER = Escapers.builder().addEscape('%', "%%").build();
+
+ /**
+ * Given a newly-constructed Rule instance (with attributes populated),
+ * returns the list of output files that this rule produces implicitly.
+ */
+ public abstract Iterable<String> getImplicitOutputs(AttributeMap rule) throws EvalException;
+
+ /**
+ * The implicit output function that returns no files.
+ */
+ public static final SafeImplicitOutputsFunction NONE = new SafeImplicitOutputsFunction() {
+ @Override public Iterable<String> getImplicitOutputs(AttributeMap rule) {
+ return Collections.emptyList();
+ }
+ };
+
+ /**
+ * A convenience wrapper for {@link #fromTemplates(Iterable)}.
+ */
+ public static SafeImplicitOutputsFunction fromTemplates(String... templates) {
+ return fromTemplates(Arrays.asList(templates));
+ }
+
+ /**
+ * The implicit output function that generates files based on a set of
+ * template substitutions using rule attribute values.
+ *
+ * @param templates The templates used to construct the name of the implicit
+ * output file target. The substring "%{name}" will be replaced by the
+ * actual name of the rule, the substring "%{srcs}" will be replaced by the
+ * name of each source file without its extension. If multiple %{}
+ * substrings exist, the cross-product of them is generated.
+ */
+ public static SafeImplicitOutputsFunction fromTemplates(final Iterable<String> templates) {
+ return new SafeImplicitOutputsFunction() {
+ // TODO(bazel-team): parse the templates already here
+ @Override
+ public Iterable<String> getImplicitOutputs(AttributeMap rule) {
+ Iterable<String> result = null;
+ for (String template : templates) {
+ List<String> substitutions = substitutePlaceholderIntoTemplate(template, rule);
+ if (substitutions.isEmpty()) {
+ continue;
+ }
+ if (result == null) {
+ result = substitutions;
+ } else {
+ result = Iterables.concat(result, substitutions);
+ }
+ }
+ if (result == null) {
+ return ImmutableList.<String>of();
+ } else {
+ return Sets.newLinkedHashSet(result);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return StringUtil.joinEnglishList(templates);
+ }
+ };
+ }
+
+ /**
+ * A convenience wrapper for {@link #fromFunctions(Iterable)}.
+ */
+ public static SafeImplicitOutputsFunction fromFunctions(
+ SafeImplicitOutputsFunction... functions) {
+ return fromFunctions(Arrays.asList(functions));
+ }
+
+ /**
+ * The implicit output function that generates files based on a set of
+ * template substitutions using rule attribute values.
+ *
+ * @param functions The functions used to construct the name of the implicit
+ * output file target. The substring "%{name}" will be replaced by the
+ * actual name of the rule, the substring "%{srcs}" will be replaced by the
+ * name of each source file without its extension. If multiple %{}
+ * substrings exist, the cross-product of them is generated.
+ */
+ public static SafeImplicitOutputsFunction fromFunctions(
+ final Iterable<SafeImplicitOutputsFunction> functions) {
+ return new SafeImplicitOutputsFunction() {
+ @Override
+ public Iterable<String> getImplicitOutputs(AttributeMap rule) {
+ Collection<String> result = new LinkedHashSet<>();
+ for (SafeImplicitOutputsFunction function : functions) {
+ Iterables.addAll(result, function.getImplicitOutputs(rule));
+ }
+ return result;
+ }
+ @Override
+ public String toString() {
+ return StringUtil.joinEnglishList(functions);
+ }
+ };
+ }
+
+ /**
+ * Coerces attribute "attrName" of the specified rule into a sequence of
+ * strings. Helper function for {@link #fromTemplates(Iterable)}.
+ */
+ private static Set<String> attributeValues(AttributeMap rule, String attrName) {
+ // Special case "name" since it's not treated as an attribute.
+ if (attrName.equals("name")) {
+ return singleton(rule.getName());
+ } else if (attrName.equals("dirname")) {
+ PathFragment dir = new PathFragment(rule.getName()).getParentDirectory();
+ return (dir.segmentCount() == 0) ? singleton("") : singleton(dir.getPathString() + "/");
+ } else if (attrName.equals("basename")) {
+ return singleton(new PathFragment(rule.getName()).getBaseName());
+ }
+
+ Type<?> attrType = rule.getAttributeType(attrName);
+ if (attrType == null) { return Collections.emptySet(); }
+ // String attributes and lists are easy.
+ if (Type.STRING == attrType) {
+ return singleton(rule.get(attrName, Type.STRING));
+ } else if (Type.STRING_LIST == attrType) {
+ Iterable<String> values = rule.get(attrName, Type.STRING_LIST);
+ // TODO(bazel-team): extract this for modularization
+ if ("locales".equals(attrName)) {
+ // Locales have to be lowercased before used in a file name for
+ // consistency with file naming guidelines, and convert dash-style
+ // (en-US-pseudo) to underscore-style (en_US_pseudo).
+ values = Iterables.transform(values,
+ new Function<String, String>() {
+ @Override
+ public String apply(String s) {
+ return s.toLowerCase().replace('-', '_');
+ }
+ });
+ }
+ return Sets.newLinkedHashSet(values);
+ } else if (Type.LABEL_LIST == attrType) {
+ // Labels are most often used to change the extension,
+ // e.g. %.foo -> %.java, so we return the basename w/o extension.
+ return Sets.newLinkedHashSet(
+ Iterables.transform(rule.get(attrName, Type.LABEL_LIST),
+ new Function<Label, String>() {
+ @Override
+ public String apply(Label label) {
+ return FileSystemUtils.removeExtension(label.getName());
+ }
+ }));
+ } else if (Type.OUTPUT == attrType) {
+ Label out = rule.get(attrName, Type.OUTPUT);
+ return singleton(out.getName());
+ } else if (Type.OUTPUT_LIST == attrType) {
+ return Sets.newLinkedHashSet(
+ Iterables.transform(rule.get(attrName, Type.OUTPUT_LIST),
+ new Function<Label, String>() {
+ @Override
+ public String apply(Label label) {
+ return label.getName();
+ }
+ }));
+ }
+ throw new IllegalArgumentException(
+ "Don't know how to handle " + attrName + " : " + attrType);
+ }
+
+ /**
+ * Collects all named placeholders from the template while replacing them with %s.
+ *
+ * <p>Example: for {@code template} "%{name}_%{locales}.foo", it will return "%s_%s.foo" and
+ * store "name" and "locales" in {@code placeholders}.
+ *
+ * <p>Incomplete placeholders are treated like text: for "a-%{x}-%{y" this method returns
+ * "a-%s-%%{y" and stores "x" in {@code placeholders}.
+ *
+ * @param template a string with placeholders of the format %{...}
+ * @param placeholders a collection to collect placeholders into; may contain duplicates if not a
+ * Set
+ * @return a format string for {@link String#format}, created from the template string with every
+ * placeholder replaced by %s
+ */
+ public static String createPlaceholderSubstitutionFormatString(String template,
+ Collection<String> placeholders) {
+ return createPlaceholderSubstitutionFormatStringRecursive(template, placeholders,
+ new StringBuilder());
+ }
+
+ private static String createPlaceholderSubstitutionFormatStringRecursive(String template,
+ Collection<String> placeholders, StringBuilder formatBuilder) {
+ int start = template.indexOf("%{");
+ if (start < 0) {
+ return formatBuilder.append(PERCENT_ESCAPER.escape(template)).toString();
+ }
+
+ int end = template.indexOf("}", start + 2);
+ if (end < 0) {
+ return formatBuilder.append(PERCENT_ESCAPER.escape(template)).toString();
+ }
+
+ formatBuilder.append(PERCENT_ESCAPER.escape(template.substring(0, start))).append("%s");
+ placeholders.add(template.substring(start + 2, end));
+ return createPlaceholderSubstitutionFormatStringRecursive(template.substring(end + 1),
+ placeholders, formatBuilder);
+ }
+
+ /**
+ * Given a template string, replaces all placeholders of the form %{...} with
+ * the values from attributeSource. If there are multiple placeholders, then
+ * the output is the cross product of substitutions.
+ */
+ public static ImmutableList<String> substitutePlaceholderIntoTemplate(String template,
+ AttributeMap rule) {
+ return substitutePlaceholderIntoTemplate(template, rule, DEFAULT_RULE_ATTRIBUTE_GETTER, null);
+ }
+
+ /**
+ * Substitutes attribute-placeholders in a template string, producing all possible combinations.
+ *
+ * @param template the template string, may contain named placeholders for rule attributes, like
+ * <code>%{name}</code> or <code>%{deps}</code>
+ * @param rule the rule whose attributes the placeholders correspond to
+ * @param placeholdersInTemplate if specified, will contain all placeholders found in the
+ * template; may contain duplicates
+ * @return all possible combinations of the attributes referenced by the placeholders,
+ * substituted into the template; empty if any of the placeholders expands to no values
+ */
+ public static ImmutableList<String> substitutePlaceholderIntoTemplate(String template,
+ AttributeMap rule, AttributeValueGetter attributeGetter,
+ @Nullable List<String> placeholdersInTemplate) {
+ List<String> placeholders = (placeholdersInTemplate == null)
+ ? Lists.<String>newArrayList()
+ : placeholdersInTemplate;
+ String formatStr = createPlaceholderSubstitutionFormatString(template, placeholders);
+ if (placeholders.isEmpty()) {
+ return ImmutableList.of(template);
+ }
+
+ List<Set<String>> values = Lists.newArrayListWithCapacity(placeholders.size());
+ for (String placeholder : placeholders) {
+ Set<String> attrValues = attributeGetter.get(rule, placeholder);
+ if (attrValues.isEmpty()) {
+ return ImmutableList.<String>of();
+ }
+ values.add(attrValues);
+ }
+ ImmutableList.Builder<String> out = new ImmutableList.Builder<>();
+ for (List<String> combination : Sets.cartesianProduct(values)) {
+ out.add(String.format(formatStr, combination.toArray()));
+ }
+ return out.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/InputFile.java b/src/main/java/com/google/devtools/build/lib/packages/InputFile.java
new file mode 100644
index 0000000..130e2b7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/InputFile.java
@@ -0,0 +1,124 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A file that is an input to the build system.
+ *
+ * <p>In the build system, a file is considered an <i>input</i> file iff it is
+ * not generated by the build system (e.g. it's maintained under version
+ * control, or created by the test harness). It has nothing to do with the
+ * type of the file; a generated file containing <code>Java</code> source code
+ * is an OutputFile, not an InputFile.
+ */
+@Immutable @ThreadSafe
+public final class InputFile extends FileTarget {
+ private final Location location;
+ private final RuleVisibility visibility;
+ private final License license;
+
+ /**
+ * Constructs an input file with the given label, which must be a label for
+ * the given package, and package-default visibility.
+ */
+ InputFile(Package pkg, Label label, Location location) {
+ this(pkg, label, location, null, License.NO_LICENSE);
+ }
+
+ /**
+ * Constructs an input file with the given label, which must be a label for the given package
+ * that was parsed from the specified location, and has the specified visibility.
+ */
+ InputFile(Package pkg, Label label, Location location, RuleVisibility visibility,
+ License license) {
+ super(pkg, label);
+ Preconditions.checkNotNull(location);
+ this.location = location;
+ this.visibility = visibility;
+ this.license = license;
+ }
+
+ public boolean isVisibilitySpecified() {
+ return visibility != null;
+ }
+
+ @Override
+ public RuleVisibility getVisibility() {
+ if (visibility != null) {
+ return visibility;
+ } else {
+ return pkg.getDefaultVisibility();
+ }
+ }
+
+ public boolean isLicenseSpecified() {
+ return license != null && license != License.NO_LICENSE;
+ }
+
+ @Override
+ public License getLicense() {
+ if (license != null) {
+ return license;
+ } else {
+ return pkg.getDefaultLicense();
+ }
+ }
+
+ /**
+ * Returns the path to the location of the input file (which is necessarily
+ * within the source tree, not beneath <code>bin</code> or
+ * <code>genfiles</code>.
+ *
+ * <p>Prefer {@link #getExecPath} if possible.
+ */
+ public Path getPath() {
+ return pkg.getPackageDirectory().getRelative(label.getName());
+ }
+
+ /**
+ * Returns the exec path of the file, i.e. the path relative to the package source root.
+ */
+ public PathFragment getExecPath() {
+ return label.toPathFragment();
+ }
+
+ @Override
+ public int hashCode() {
+ return label.hashCode();
+ }
+
+ @Override
+ public String getTargetKind() {
+ return "source file";
+ }
+
+ @Override
+ public Rule getAssociatedRule() {
+ return null;
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/InvalidPackageNameException.java b/src/main/java/com/google/devtools/build/lib/packages/InvalidPackageNameException.java
new file mode 100644
index 0000000..69561b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/InvalidPackageNameException.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Exception indicating that a package name was invalid.
+ */
+public class InvalidPackageNameException extends NoSuchPackageException {
+
+ public InvalidPackageNameException(String packageName, String message) {
+ super(packageName, message);
+ }
+
+ public InvalidPackageNameException(String packageName, String message, Throwable cause) {
+ super(packageName, message, cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/License.java b/src/main/java/com/google/devtools/build/lib/packages/License.java
new file mode 100644
index 0000000..fe63c9c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/License.java
@@ -0,0 +1,327 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Support for license and distribution checking.
+ */
+@Immutable @ThreadSafe
+public final class License {
+
+ private final Set<LicenseType> licenseTypes;
+ private final Set<Label> exceptions;
+
+ /**
+ * The error that's thrown if a build file contains an invalid license string.
+ */
+ public static class LicenseParsingException extends Exception {
+ public LicenseParsingException(String s) {
+ super(s);
+ }
+ }
+
+ /**
+ * LicenseType is the basis of the License lattice - stricter licenses should
+ * be declared before less-strict licenses in the enum.
+ *
+ * <p>Note that the order is important for the purposes of finding the least
+ * restrictive license.
+ */
+ public enum LicenseType {
+ BY_EXCEPTION_ONLY,
+ RESTRICTED,
+ RESTRICTED_IF_STATICALLY_LINKED,
+ RECIPROCAL,
+ NOTICE,
+ PERMISSIVE,
+ UNENCUMBERED,
+ NONE
+ }
+
+ /**
+ * Gets the least restrictive license type from the list of licenses declared
+ * for a target. For the purposes of license checking, the license type set of
+ * a declared license can be reduced to its least restrictive member.
+ *
+ * @param types a collection of license types
+ * @return the least restrictive license type
+ */
+ @VisibleForTesting
+ static LicenseType leastRestrictive(Collection<LicenseType> types) {
+ return types.isEmpty() ? LicenseType.BY_EXCEPTION_ONLY : Collections.max(types);
+ }
+
+ /**
+ * An instance of LicenseType.None with no exceptions, used for packages
+ * outside of third_party which have no license clause in their BUILD files.
+ */
+ public static final License NO_LICENSE =
+ new License(ImmutableSet.of(LicenseType.NONE), Collections.<Label>emptySet());
+
+ /**
+ * A default instance of Distributions which is used for packages which
+ * have no "distribs" declaration. If nothing is declared, we opt for the
+ * most permissive kind of distribution, which is the internal-only distrib.
+ */
+ public static final Set<DistributionType> DEFAULT_DISTRIB =
+ Collections.singleton(DistributionType.INTERNAL);
+
+ /**
+ * The types of distribution that are supported.
+ */
+ public enum DistributionType {
+ INTERNAL,
+ WEB,
+ CLIENT,
+ EMBEDDED
+ }
+
+ /**
+ * Parses a set of strings declaring distribution types.
+ *
+ * @param distStrings strings containing distribution declarations from BUILD
+ * files
+ * @return a new, unmodifiable set of DistributionTypes
+ * @throws LicenseParsingException
+ */
+ public static Set<DistributionType> parseDistributions(Collection<String> distStrings)
+ throws LicenseParsingException {
+ if (distStrings.isEmpty()) {
+ return Collections.unmodifiableSet(EnumSet.of(DistributionType.INTERNAL));
+ } else {
+ Set<DistributionType> result = EnumSet.noneOf(DistributionType.class);
+ for (String distStr : distStrings) {
+ try {
+ DistributionType dist = Enum.valueOf(DistributionType.class, distStr.toUpperCase());
+ result.add(dist);
+ } catch (IllegalArgumentException e) {
+ throw new LicenseParsingException("Invalid distribution type '" + distStr + "'");
+ }
+ }
+ return Collections.unmodifiableSet(result);
+ }
+ }
+
+ private static final Object MARKER = new Object();
+
+ /**
+ * The license incompatibility set. This contains the set of
+ * (Distribution,License) pairs that should generate errors.
+ */
+ private static Table<DistributionType, LicenseType, Object> LICENSE_INCOMPATIBILIES =
+ createLicenseIncompatibilitySet();
+
+ private static Table<DistributionType, LicenseType, Object> createLicenseIncompatibilitySet() {
+ Table<DistributionType, LicenseType, Object> result = HashBasedTable.create();
+ result.put(DistributionType.CLIENT, LicenseType.RESTRICTED, MARKER);
+ result.put(DistributionType.EMBEDDED, LicenseType.RESTRICTED, MARKER);
+ result.put(DistributionType.INTERNAL, LicenseType.BY_EXCEPTION_ONLY, MARKER);
+ result.put(DistributionType.CLIENT, LicenseType.BY_EXCEPTION_ONLY, MARKER);
+ result.put(DistributionType.WEB, LicenseType.BY_EXCEPTION_ONLY, MARKER);
+ result.put(DistributionType.EMBEDDED, LicenseType.BY_EXCEPTION_ONLY, MARKER);
+ return ImmutableTable.copyOf(result);
+ }
+
+ /**
+ * The license warning set. This contains the set of
+ * (Distribution,License) pairs that should generate warnings when the user
+ * requests verbose license checking.
+ */
+ private static Table<DistributionType, LicenseType, Object> LICENSE_WARNINGS =
+ createLicenseWarningsSet();
+
+ private static Table<DistributionType, LicenseType, Object> createLicenseWarningsSet() {
+ Table<DistributionType, LicenseType, Object> result = HashBasedTable.create();
+ result.put(DistributionType.CLIENT, LicenseType.RECIPROCAL, MARKER);
+ result.put(DistributionType.EMBEDDED, LicenseType.RECIPROCAL, MARKER);
+ result.put(DistributionType.CLIENT, LicenseType.NOTICE, MARKER);
+ result.put(DistributionType.EMBEDDED, LicenseType.NOTICE, MARKER);
+ return ImmutableTable.copyOf(result);
+ }
+
+ private License(Set<LicenseType> licenseTypes, Set<Label> exceptions) {
+ // Defensive copy is done in .of()
+ this.licenseTypes = licenseTypes;
+ this.exceptions = exceptions;
+ }
+
+ public static License of(Collection<LicenseType> licenses, Collection<Label> exceptions) {
+ Set<LicenseType> licenseSet = ImmutableSet.copyOf(licenses);
+ Set<Label> exceptionSet = ImmutableSet.copyOf(exceptions);
+
+ if (exceptionSet.isEmpty() && licenseSet.equals(ImmutableSet.of(LicenseType.NONE))) {
+ return License.NO_LICENSE;
+ }
+
+ return new License(licenseSet, exceptionSet);
+ }
+ /**
+ * Computes a license which can be used to check if a package is compatible
+ * with some kinds of distribution. The list of licenses is scanned for the
+ * least restrictive, and the exceptions are added.
+ *
+ * @param licStrings the list of license strings declared for the package
+ * @throws LicenseParsingException if there are any parsing problems
+ */
+ public static License parseLicense(List<String> licStrings) throws LicenseParsingException {
+ /*
+ * The semantics of comparison for licenses depends on a stable iteration
+ * order for both license types and exceptions. For licenseTypes, it will be
+ * the comparison order from the enumerated types; for exceptions, it will
+ * be lexicographic order achieved using TreeSets.
+ */
+ Set<LicenseType> licenseTypes = EnumSet.noneOf(LicenseType.class);
+ Set<Label> exceptions = Sets.newTreeSet();
+ for (String str : licStrings) {
+ if (str.startsWith("exception=")) {
+ try {
+ Label label = Label.parseAbsolute(str.substring("exception=".length()));
+ exceptions.add(label);
+ } catch (SyntaxException e) {
+ throw new LicenseParsingException(e.getMessage());
+ }
+ } else {
+ try {
+ licenseTypes.add(LicenseType.valueOf(str.toUpperCase()));
+ } catch (IllegalArgumentException e) {
+ throw new LicenseParsingException("invalid license type: '" + str + "'");
+ }
+ }
+ }
+
+ return License.of(licenseTypes, exceptions);
+ }
+
+ /**
+ * Checks if this license is compatible with distributing a particular target
+ * in some set of distribution modes.
+ *
+ * @param dists the modes of distribution
+ * @param target the target which is being checked, and which will be used for
+ * checking exceptions
+ * @param licensedTarget the target which declared the license being checked.
+ * @param eventHandler a reporter where any licensing issues discovered should be
+ * reported
+ * @param staticallyLinked whether the target is statically linked under this command
+ * @return true if the license is compatible with the distributions
+ */
+ public boolean checkCompatibility(Set<DistributionType> dists,
+ Target target, Label licensedTarget, EventHandler eventHandler,
+ boolean staticallyLinked) {
+ Location location = (target instanceof Rule) ? ((Rule) target).getLocation() : null;
+
+ LicenseType leastRestrictiveLicense;
+ if (licenseTypes.contains(LicenseType.RESTRICTED_IF_STATICALLY_LINKED)) {
+ Set<LicenseType> tempLicenses = EnumSet.copyOf(licenseTypes);
+ tempLicenses.remove(LicenseType.RESTRICTED_IF_STATICALLY_LINKED);
+ if (staticallyLinked) {
+ tempLicenses.add(LicenseType.RESTRICTED);
+ } else {
+ tempLicenses.add(LicenseType.UNENCUMBERED);
+ }
+ leastRestrictiveLicense = leastRestrictive(tempLicenses);
+ } else {
+ leastRestrictiveLicense = leastRestrictive(licenseTypes);
+ }
+ for (DistributionType dt : dists) {
+ if (LICENSE_INCOMPATIBILIES.contains(dt, leastRestrictiveLicense)) {
+ if (!exceptions.contains(target.getLabel())) {
+ eventHandler.handle(Event.error(location, "Build target '" + target.getLabel()
+ + "' is not compatible with license '" + this + "' from target '"
+ + licensedTarget + "'"));
+ return false;
+ }
+ } else if (LICENSE_WARNINGS.contains(dt, leastRestrictiveLicense)) {
+ eventHandler.handle(
+ Event.warn(location, "Build target '" + target
+ + "' has a potential licensing issue "
+ + "with a '" + this + "' license from target '" + licensedTarget + "'"));
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @return an immutable set of {@link LicenseType}s contained in this {@code
+ * License}
+ */
+ public Set<LicenseType> getLicenseTypes() {
+ return licenseTypes;
+ }
+
+ /**
+ * @return an immutable set of {@link Label}s that describe exceptions to the
+ * {@code License}
+ */
+ public Set<Label> getExceptions() {
+ return exceptions;
+ }
+
+ /**
+ * A simple toString implementation which generates a canonical form of the
+ * license. (The order of license types is guaranteed to be canonical by
+ * EnumSet, and the order of exceptions is guaranteed to be lexicographic
+ * order by TreeSet.)
+ */
+ @Override
+ public String toString() {
+ if (exceptions.isEmpty()) {
+ return licenseTypes.toString().toLowerCase();
+ } else {
+ return licenseTypes.toString().toLowerCase() + " with exceptions " + exceptions.toString();
+ }
+ }
+
+ /**
+ * A simple equals implementation leveraging the support built into Set that
+ * delegates to its contents.
+ */
+ @Override
+ public boolean equals(Object o) {
+ return o == this ||
+ o instanceof License &&
+ ((License) o).licenseTypes.equals(this.licenseTypes) &&
+ ((License) o).exceptions.equals(this.exceptions);
+ }
+
+ /**
+ * A simple hashCode implementation leveraging the support built into Set that
+ * delegates to its contents.
+ */
+ @Override
+ public int hashCode() {
+ return licenseTypes.hashCode() * 43 + exceptions.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/MakeEnvironment.java b/src/main/java/com/google/devtools/build/lib/packages/MakeEnvironment.java
new file mode 100644
index 0000000..6c0d849
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/MakeEnvironment.java
@@ -0,0 +1,184 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Environment for varref variables (formerly called "Makefile
+ * variables").
+ *
+ * <p><code>update</code> emulates a very restricted subset of the behaviour of
+ * GNU Make's environment. In particular, does not attempt to simulate Make's
+ * complex range of assigment operators.
+ */
+@Immutable @ThreadSafe
+public class MakeEnvironment {
+ /**
+ * The platform set regexp that matches all platforms. Canonical.
+ */
+ public static final String MATCH_ANY = ".*";
+
+ // A platform-specific binding of a value for a given variable.
+ static class Binding {
+ private final String value;
+ private final String platformSetRegexp;
+
+ Binding(String value, String platformSetRegexp) {
+ this.value = value;
+ this.platformSetRegexp = platformSetRegexp;
+ }
+
+ @Override
+ public String toString() {
+ return value + " (" + platformSetRegexp + ")";
+ }
+
+ String getValue() {
+ return value;
+ }
+
+ String getPlatformSetRegexp() {
+ return platformSetRegexp;
+ }
+ }
+
+ // Maps each variable name to the [short] list of platform-specific bindings
+ // for it. The first matching binding is definitive.
+ private final ImmutableMap<String, ImmutableList<Binding>> env;
+
+ private MakeEnvironment(ImmutableMap<String, ImmutableList<Binding>> env) {
+ this.env = env;
+ }
+
+ /**
+ * @return the "Make" value from the environment whose name is "varname", or
+ * null iff the variable is not defined in the environment.
+ */
+ public String lookup(String varname, String platform) {
+ List<Binding> bindings = env.get(varname);
+ if (bindings == null) {
+ return null;
+ }
+ // First, look for a matching non-default binding.
+ // (The order in 'bindings' is the reverse of the order of vardefs in the BUILD file, so
+ // the first match in this for loop selects the last matching definition in the BUILD file.)
+ for (Binding binding : bindings) {
+ if (!binding.platformSetRegexp.equals(MATCH_ANY) &&
+ platform.matches(binding.platformSetRegexp)) {
+ return binding.value;
+ }
+ }
+ // If we didn't find a matching non-default binding,
+ // try using the last default binding.
+ for (Binding binding : bindings) {
+ if (binding.platformSetRegexp.equals(MATCH_ANY)) {
+ return binding.value;
+ }
+ }
+ return null;
+ }
+
+ Map<String, ImmutableList<Binding>> getBindings() {
+ return env;
+ }
+
+ /**
+ * Interface for creating a MakeEnvironment, settings its environment values,
+ * and exposing it in immutable state.
+ */
+ public static class Builder {
+ private final Map<String, LinkedList<Binding>> env = new HashMap<>();
+ private Map<String, String> platformSets = ImmutableMap.<String, String>of("any", MATCH_ANY);
+
+ /**
+ * Performs an update of Makefile variable 'var' to value 'value' for all
+ * platforms belonging to the specified 'platformSetRegexp'. Corresponds to
+ * vardef. We explicitly do not support the various complex nuances of
+ * Make's assignment operator.
+ *
+ * <p>The most recent binding for a particular variable takes precedence, even if
+ * a more specific binding came earlier.
+ *
+ * @param varname the name of the Makefile variable;
+ * @param value the string value to assign;
+ * @param platformSetRegexp a set of platforms for which this variable definition
+ * should take effect. This is expressed as a regexp over gplatform
+ * strings.
+ */
+ public void update(String varname, String value, String platformSetRegexp) {
+ if (varname == null || value == null || platformSetRegexp == null) {
+ throw new NullPointerException();
+ }
+ LinkedList<Binding> bindings = env.get(varname);
+ if (bindings == null) {
+ bindings = new LinkedList<Binding>();
+ env.put(varname, bindings);
+ }
+ // push new bindings onto head of list (=> most recent binding is
+ // definitive):
+ bindings.addFirst(new Binding(value, platformSetRegexp));
+ }
+
+ /**
+ * Sets the nickname to regexp mapping for <tt>vardef</tt>.
+ */
+ public void setPlatformSetRegexps(Map<String, String> sets) {
+ this.platformSets = sets;
+ }
+
+ @Nullable
+ public String getPlatformSetRegexp(String nickname) {
+ return this.platformSets.get(nickname);
+ }
+
+ /**
+ * Returns a new MakeEnvironment with environment settings corresponding
+ * to the cumulative results of this builder's {@link #update} calls.
+ */
+ public MakeEnvironment build() {
+ Map<String, ImmutableList<Binding>> newMap = new HashMap<>();
+ for (Map.Entry<String, LinkedList<Binding>> entry : env.entrySet()) {
+ newMap.put(entry.getKey(), ImmutableList.copyOf(entry.getValue()));
+ }
+ return new MakeEnvironment(ImmutableMap.copyOf(newMap));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String toString() {
+ return "MakeEnvironment=" + env;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/MethodLibrary.java b/src/main/java/com/google/devtools/build/lib/packages/MethodLibrary.java
new file mode 100644
index 0000000..5969697
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/MethodLibrary.java
@@ -0,0 +1,1053 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import static com.google.devtools.build.lib.syntax.SkylarkFunction.cast;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.AbstractFunction;
+import com.google.devtools.build.lib.syntax.AbstractFunction.NoArgFunction;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.syntax.DotExpression;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.MixedModeFunction;
+import com.google.devtools.build.lib.syntax.SelectorValue;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+import com.google.devtools.build.lib.syntax.SkylarkType;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A helper class containing built in functions for the Build and the Build Extension Language.
+ */
+public class MethodLibrary {
+
+ private MethodLibrary() {}
+
+ // Convert string index in the same way Python does.
+ // If index is negative, starts from the end.
+ // If index is outside bounds, it is restricted to the valid range.
+ private static int getPythonStringIndex(int index, int stringLength) {
+ if (index < 0) {
+ index += stringLength;
+ }
+ return Math.max(Math.min(index, stringLength), 0);
+ }
+
+ // Emulate Python substring function
+ // It converts out of range indices, and never fails
+ private static String getPythonSubstring(String str, int start, int end) {
+ start = getPythonStringIndex(start, str.length());
+ end = getPythonStringIndex(end, str.length());
+ if (start > end) {
+ return "";
+ } else {
+ return str.substring(start, end);
+ }
+ }
+
+ public static int getListIndex(Object key, int listSize, FuncallExpression ast)
+ throws ConversionException, EvalException {
+ // Get the nth element in the list
+ int index = Type.INTEGER.convert(key, "index operand");
+ if (index < 0) {
+ index += listSize;
+ }
+ if (index < 0 || index >= listSize) {
+ throw new EvalException(ast.getLocation(), "List index out of range (index is "
+ + index + ", but list has " + listSize + " elements)");
+ }
+ return index;
+ }
+
+ // supported string methods
+
+ @SkylarkBuiltin(name = "join", objectType = StringModule.class, returnType = String.class,
+ doc = "Returns a string in which the string elements of the argument have been "
+ + "joined by this string as a separator. Example:<br>"
+ + "<pre class=language-python>\"|\".join([\"a\", \"b\", \"c\"]) == \"a|b|c\"</pre>",
+ mandatoryParams = {
+ @Param(name = "elements", type = SkylarkList.class, doc = "The objects to join.")})
+ private static Function join = new MixedModeFunction("join",
+ ImmutableList.of("this", "elements"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'join' operand");
+ List<?> seq = Type.OBJECT_LIST.convert(args[1], "'join' argument");
+ StringBuilder sb = new StringBuilder();
+ for (Iterator<?> i = seq.iterator(); i.hasNext();) {
+ sb.append(i.next().toString());
+ if (i.hasNext()) {
+ sb.append(thiz);
+ }
+ }
+ return sb.toString();
+ }
+ };
+
+ @SkylarkBuiltin(name = "lower", objectType = StringModule.class, returnType = String.class,
+ doc = "Returns the lower case version of this string.")
+ private static Function lower = new MixedModeFunction("lower",
+ ImmutableList.of("this"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'lower' operand");
+ return thiz.toLowerCase();
+ }
+ };
+
+ @SkylarkBuiltin(name = "upper", objectType = StringModule.class, returnType = String.class,
+ doc = "Returns the upper case version of this string.")
+ private static Function upper = new MixedModeFunction("upper",
+ ImmutableList.of("this"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'upper' operand");
+ return thiz.toUpperCase();
+ }
+ };
+
+ @SkylarkBuiltin(name = "replace", objectType = StringModule.class, returnType = String.class,
+ doc = "Returns a copy of the string in which the occurrences "
+ + "of <code>old</code> have been replaced with <code>new</code>, optionally restricting "
+ + "the number of replacements to <code>maxsplit</code>.",
+ mandatoryParams = {
+ @Param(name = "old", type = String.class, doc = "The string to be replaced."),
+ @Param(name = "new", type = String.class, doc = "The string to replace with.")},
+ optionalParams = {
+ @Param(name = "maxsplit", type = Integer.class, doc = "The maximum number of replacements.")})
+ private static Function replace =
+ new MixedModeFunction("replace", ImmutableList.of("this", "old", "new", "maxsplit"), 3, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'replace' operand");
+ String old = Type.STRING.convert(args[1], "'replace' argument");
+ String neww = Type.STRING.convert(args[2], "'replace' argument");
+ int maxsplit =
+ args[3] != null ? Type.INTEGER.convert(args[3], "'replace' argument")
+ : Integer.MAX_VALUE;
+ StringBuffer sb = new StringBuffer();
+ try {
+ Matcher m = Pattern.compile(old, Pattern.LITERAL).matcher(thiz);
+ for (int i = 0; i < maxsplit && m.find(); i++) {
+ m.appendReplacement(sb, Matcher.quoteReplacement(neww));
+ }
+ m.appendTail(sb);
+ } catch (IllegalStateException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage() + " in call to replace");
+ }
+ return sb.toString();
+ }
+ };
+
+ @SkylarkBuiltin(name = "split", objectType = StringModule.class, returnType = SkylarkList.class,
+ doc = "Returns a list of all the words in the string, using <code>sep</code> "
+ + "as the separator, optionally limiting the number of splits to <code>maxsplit</code>.",
+ optionalParams = {
+ @Param(name = "sep", type = String.class,
+ doc = "The string to split on, default is space (\" \")."),
+ @Param(name = "maxsplit", type = Integer.class, doc = "The maximum number of splits.")})
+ private static Function split = new MixedModeFunction("split",
+ ImmutableList.of("this", "sep", "maxsplit"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'split' operand");
+ String sep = args[1] != null
+ ? Type.STRING.convert(args[1], "'split' argument")
+ : " ";
+ int maxsplit = args[2] != null
+ ? Type.INTEGER.convert(args[2], "'split' argument") + 1 // last is remainder
+ : -1;
+ String[] ss = Pattern.compile(sep, Pattern.LITERAL).split(thiz,
+ maxsplit);
+ List<String> result = java.util.Arrays.asList(ss);
+ return env.isSkylarkEnabled() ? SkylarkList.list(result, String.class) : result;
+ }
+ };
+
+ @SkylarkBuiltin(name = "rfind", objectType = StringModule.class, returnType = Integer.class,
+ doc = "Returns the last index where <code>sub</code> is found, "
+ + "or -1 if no such index exists, optionally restricting to "
+ + "[<code>start</code>:<code>end</code>], "
+ + "<code>start</code> being inclusive and <code>end</code> being exclusive.",
+ mandatoryParams = {
+ @Param(name = "sub", type = String.class, doc = "The substring to find.")},
+ optionalParams = {
+ @Param(name = "start", type = Integer.class, doc = "Restrict to search from this position."),
+ @Param(name = "end", type = Integer.class, doc = "Restrict to search before this position.")})
+ private static Function rfind =
+ new MixedModeFunction("rfind", ImmutableList.of("this", "sub", "start", "end"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'rfind' operand");
+ String sub = Type.STRING.convert(args[1], "'rfind' argument");
+ int start = 0;
+ if (args[2] != null) {
+ start = Type.INTEGER.convert(args[2], "'rfind' argument");
+ }
+ int end = thiz.length();
+ if (args[3] != null) {
+ end = Type.INTEGER.convert(args[3], "'rfind' argument");
+ }
+ int subpos = getPythonSubstring(thiz, start, end).lastIndexOf(sub);
+ start = getPythonStringIndex(start, thiz.length());
+ return subpos < 0 ? subpos : subpos + start;
+ }
+ };
+
+ @SkylarkBuiltin(name = "find", objectType = StringModule.class, returnType = Integer.class,
+ doc = "Returns the first index where <code>sub</code> is found, "
+ + "or -1 if no such index exists, optionally restricting to "
+ + "[<code>start</code>:<code>end]</code>, "
+ + "<code>start</code> being inclusive and <code>end</code> being exclusive.",
+ mandatoryParams = {
+ @Param(name = "sub", type = String.class, doc = "The substring to find.")},
+ optionalParams = {
+ @Param(name = "start", type = Integer.class, doc = "Restrict to search from this position."),
+ @Param(name = "end", type = Integer.class, doc = "Restrict to search before this position.")})
+ private static Function find =
+ new MixedModeFunction("find", ImmutableList.of("this", "sub", "start", "end"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'find' operand");
+ String sub = Type.STRING.convert(args[1], "'find' argument");
+ int start = 0;
+ if (args[2] != null) {
+ start = Type.INTEGER.convert(args[2], "'find' argument");
+ }
+ int end = thiz.length();
+ if (args[3] != null) {
+ end = Type.INTEGER.convert(args[3], "'find' argument");
+ }
+ int subpos = getPythonSubstring(thiz, start, end).indexOf(sub);
+ start = getPythonStringIndex(start, thiz.length());
+ return subpos < 0 ? subpos : subpos + start;
+ }
+ };
+
+ @SkylarkBuiltin(name = "count", objectType = StringModule.class, returnType = Integer.class,
+ doc = "Returns the number of (non-overlapping) occurrences of substring <code>sub</code> in "
+ + "string, optionally restricting to [<code>start</code>:<code>end</code>], "
+ + "<code>start</code> being inclusive and <code>end</code> being exclusive.",
+ mandatoryParams = {
+ @Param(name = "sub", type = String.class, doc = "The substring to count.")},
+ optionalParams = {
+ @Param(name = "start", type = Integer.class, doc = "Restrict to search from this position."),
+ @Param(name = "end", type = Integer.class, doc = "Restrict to search before this position.")})
+ private static Function count =
+ new MixedModeFunction("count", ImmutableList.of("this", "sub", "start", "end"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'count' operand");
+ String sub = Type.STRING.convert(args[1], "'count' argument");
+ int start = 0;
+ if (args[2] != null) {
+ start = Type.INTEGER.convert(args[2], "'count' argument");
+ }
+ int end = thiz.length();
+ if (args[3] != null) {
+ end = Type.INTEGER.convert(args[3], "'count' argument");
+ }
+ String str = getPythonSubstring(thiz, start, end);
+ if (sub.equals("")) {
+ return str.length() + 1;
+ }
+ int count = 0;
+ int index = -1;
+ while ((index = str.indexOf(sub)) >= 0) {
+ count++;
+ str = str.substring(index + sub.length());
+ }
+ return count;
+ }
+ };
+
+ @SkylarkBuiltin(name = "endswith", objectType = StringModule.class, returnType = Boolean.class,
+ doc = "Returns True if the string ends with <code>sub</code>, "
+ + "otherwise False, optionally restricting to [<code>start</code>:<code>end</code>], "
+ + "<code>start</code> being inclusive and <code>end</code> being exclusive.",
+ mandatoryParams = {
+ @Param(name = "sub", type = String.class, doc = "The substring to check.")},
+ optionalParams = {
+ @Param(name = "start", type = Integer.class, doc = "Test beginning at this position."),
+ @Param(name = "end", type = Integer.class, doc = "Stop comparing at this position.")})
+ private static Function endswith =
+ new MixedModeFunction("endswith", ImmutableList.of("this", "sub", "start", "end"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'endswith' operand");
+ String sub = Type.STRING.convert(args[1], "'endswith' argument");
+ int start = 0;
+ if (args[2] != null) {
+ start = Type.INTEGER.convert(args[2], "'endswith' argument");
+ }
+ int end = thiz.length();
+ if (args[3] != null) {
+ end = Type.INTEGER.convert(args[3], "");
+ }
+
+ return getPythonSubstring(thiz, start, end).endsWith(sub);
+ }
+ };
+
+ @SkylarkBuiltin(name = "startswith", objectType = StringModule.class, returnType = Boolean.class,
+ doc = "Returns True if the string starts with <code>sub</code>, "
+ + "otherwise False, optionally restricting to [<code>start</code>:<code>end</code>], "
+ + "<code>start</code> being inclusive and <code>end</code> being exclusive.",
+ mandatoryParams = {
+ @Param(name = "sub", type = String.class, doc = "The substring to check.")},
+ optionalParams = {
+ @Param(name = "start", type = Integer.class, doc = "Test beginning at this position."),
+ @Param(name = "end", type = Integer.class, doc = "Stop comparing at this position.")})
+ private static Function startswith =
+ new MixedModeFunction("startswith", ImmutableList.of("this", "sub", "start", "end"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "'startswith' operand");
+ String sub = Type.STRING.convert(args[1], "'startswith' argument");
+ int start = 0;
+ if (args[2] != null) {
+ start = Type.INTEGER.convert(args[2], "'startswith' argument");
+ }
+ int end = thiz.length();
+ if (args[3] != null) {
+ end = Type.INTEGER.convert(args[3], "'startswith' argument");
+ }
+ return getPythonSubstring(thiz, start, end).startsWith(sub);
+ }
+ };
+
+ // TODO(bazel-team): Maybe support an argument to tell the type of the whitespace.
+ @SkylarkBuiltin(name = "strip", objectType = StringModule.class, returnType = String.class,
+ doc = "Returns a copy of the string in which all whitespace characters "
+ + "have been stripped from the beginning and the end of the string.")
+ private static Function strip =
+ new MixedModeFunction("strip", ImmutableList.of("this"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws ConversionException {
+ String operand = Type.STRING.convert(args[0], "'strip' operand");
+ return operand.trim();
+ }
+ };
+
+ // substring operator
+ @SkylarkBuiltin(name = "$substring", hidden = true,
+ doc = "String[<code>start</code>:<code>end</code>] returns a substring.")
+ private static Function substring = new MixedModeFunction("$substring",
+ ImmutableList.of("this", "start", "end"), 3, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws ConversionException {
+ String thiz = Type.STRING.convert(args[0], "substring operand");
+ int left = Type.INTEGER.convert(args[1], "substring operand");
+ int right = Type.INTEGER.convert(args[2], "substring operand");
+ return getPythonSubstring(thiz, left, right);
+ }
+ };
+
+ // supported list methods
+ @SkylarkBuiltin(name = "append", hidden = true,
+ doc = "Adds an item to the end of the list.")
+ private static Function append = new MixedModeFunction("append",
+ ImmutableList.of("this", "x"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ List<Object> thiz = Type.OBJECT_LIST.convert(args[0], "'append' operand");
+ thiz.add(args[1]);
+ return Environment.NONE;
+ }
+ };
+
+ @SkylarkBuiltin(name = "extend", hidden = true,
+ doc = "Adds all items to the end of the list.")
+ private static Function extend = new MixedModeFunction("extend",
+ ImmutableList.of("this", "x"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ List<Object> thiz = Type.OBJECT_LIST.convert(args[0], "'extend' operand");
+ List<Object> l = Type.OBJECT_LIST.convert(args[1], "'extend' argument");
+ thiz.addAll(l);
+ return Environment.NONE;
+ }
+ };
+
+ // dictionary access operator
+ @SkylarkBuiltin(name = "$index", hidden = true,
+ doc = "Returns the nth element of a list or string, "
+ + "or looks up a value in a dictionary.")
+ private static Function index = new MixedModeFunction("$index",
+ ImmutableList.of("this", "index"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ Object collectionCandidate = args[0];
+ Object key = args[1];
+
+ if (collectionCandidate instanceof Map<?, ?>) {
+ Map<?, ?> dictionary = (Map<?, ?>) collectionCandidate;
+ if (!dictionary.containsKey(key)) {
+ throw new EvalException(ast.getLocation(), "Key '" + key + "' not found in dictionary");
+ }
+ return dictionary.get(key);
+ } else if (collectionCandidate instanceof List<?>) {
+
+ List<Object> list = Type.OBJECT_LIST.convert(collectionCandidate, "index operand");
+
+ if (!list.isEmpty()) {
+ int index = getListIndex(key, list.size(), ast);
+ return list.get(index);
+ }
+
+ throw new EvalException(ast.getLocation(), "List is empty");
+ } else if (collectionCandidate instanceof SkylarkList) {
+ SkylarkList list = (SkylarkList) collectionCandidate;
+
+ if (!list.isEmpty()) {
+ int index = getListIndex(key, list.size(), ast);
+ return list.get(index);
+ }
+
+ throw new EvalException(ast.getLocation(), "List is empty");
+ } else if (collectionCandidate instanceof String) {
+ String str = (String) collectionCandidate;
+ int index = getListIndex(key, str.length(), ast);
+ return str.substring(index, index + 1);
+
+ } else {
+ // TODO(bazel-team): This is dead code, get rid of it.
+ throw new EvalException(ast.getLocation(), String.format(
+ "Unsupported datatype (%s) for indexing, only works for dict and list",
+ EvalUtils.getDatatypeName(collectionCandidate)));
+ }
+ }
+ };
+
+ @SkylarkBuiltin(name = "values", objectType = DictModule.class, returnType = SkylarkList.class,
+ doc = "Return the list of values.")
+ private static Function values = new NoArgFunction("values") {
+ @Override
+ public Object call(Object self, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException {
+ Map<?, ?> dict = (Map<?, ?>) self;
+ return convert(dict.values(), env, ast.getLocation());
+ }
+ };
+
+ @SkylarkBuiltin(name = "items", objectType = DictModule.class, returnType = SkylarkList.class,
+ doc = "Return the list of key-value tuples.")
+ private static Function items = new NoArgFunction("items") {
+ @Override
+ public Object call(Object self, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException {
+ Map<?, ?> dict = (Map<?, ?>) self;
+ List<Object> list = Lists.newArrayListWithCapacity(dict.size());
+ for (Map.Entry<?, ?> entries : dict.entrySet()) {
+ List<?> item = ImmutableList.of(entries.getKey(), entries.getValue());
+ list.add(env.isSkylarkEnabled() ? SkylarkList.tuple(item) : item);
+ }
+ return convert(list, env, ast.getLocation());
+ }
+ };
+
+ @SkylarkBuiltin(name = "keys", objectType = DictModule.class, returnType = SkylarkList.class,
+ doc = "Return the list of keys.")
+ private static Function keys = new NoArgFunction("keys") {
+ @Override
+ public Object call(Object self, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException {
+ Map<?, ?> dict = (Map<?, ?>) self;
+ return convert(dict.keySet(), env, ast.getLocation());
+ }
+ };
+
+ @SuppressWarnings("unchecked")
+ private static Iterable<Object> convert(Collection<?> list, Environment env, Location loc)
+ throws EvalException {
+ if (env.isSkylarkEnabled()) {
+ return SkylarkList.list(list, loc);
+ } else {
+ return Lists.newArrayList(list);
+ }
+ }
+
+ // unary minus
+ @SkylarkBuiltin(name = "-", hidden = true, doc = "Unary minus operator.")
+ private static Function minus = new MixedModeFunction("-", ImmutableList.of("this"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws ConversionException {
+ int num = Type.INTEGER.convert(args[0], "'unary minus' argument");
+ return -num;
+ }
+ };
+
+ @SkylarkBuiltin(name = "list", returnType = SkylarkList.class,
+ doc = "Converts a collection (e.g. set or dictionary) to a list.",
+ mandatoryParams = {@Param(name = "x", doc = "The object to convert.")})
+ private static Function list = new MixedModeFunction("list",
+ ImmutableList.of("list"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException {
+ Location loc = ast.getLocation();
+ return SkylarkList.list(EvalUtils.toCollection(args[0], loc), loc);
+ }
+ };
+
+ @SkylarkBuiltin(name = "len", returnType = Integer.class, doc =
+ "Returns the length of a string, list, tuple, set, or dictionary.",
+ mandatoryParams = {@Param(name = "x", doc = "The object to check length of.")})
+ private static Function len = new MixedModeFunction("len",
+ ImmutableList.of("list"), 1, false) {
+
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException {
+ Object arg = args[0];
+ int l = EvalUtils.size(arg);
+ if (l == -1) {
+ throw new EvalException(ast.getLocation(),
+ EvalUtils.getDatatypeName(arg) + " is not iterable");
+ }
+ return l;
+ }
+ };
+
+ @SkylarkBuiltin(name = "str", returnType = String.class, doc =
+ "Converts any object to string. This is useful for debugging.",
+ mandatoryParams = {@Param(name = "x", doc = "The object to convert.")})
+ private static Function str = new MixedModeFunction("str", ImmutableList.of("this"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException {
+ return EvalUtils.printValue(args[0]);
+ }
+ };
+
+ @SkylarkBuiltin(name = "bool", returnType = Boolean.class, doc = "Converts an object to boolean. "
+ + "It returns False if the object is None, False, an empty string, the number 0, or an "
+ + "empty collection. Otherwise, it returns True.",
+ mandatoryParams = {@Param(name = "x", doc = "The variable to convert.")})
+ private static Function bool = new MixedModeFunction("bool",
+ ImmutableList.of("this"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException {
+ return EvalUtils.toBoolean(args[0]);
+ }
+ };
+
+ @SkylarkBuiltin(name = "struct", returnType = SkylarkClassObject.class, doc =
+ "Creates an immutable struct using the keyword arguments as fields. It is used to group "
+ + "multiple values together.Example:<br>"
+ + "<pre class=language-python>s = struct(x = 2, y = 3)\n"
+ + "return s.x + s.y # returns 5</pre>")
+ private static Function struct = new AbstractFunction("struct") {
+
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ Environment env) throws EvalException, InterruptedException {
+ if (args.size() > 0) {
+ throw new EvalException(ast.getLocation(), "struct only supports keyword arguments");
+ }
+ return new SkylarkClassObject(kwargs, ast.getLocation());
+ }
+ };
+
+ @SkylarkBuiltin(name = "set", returnType = SkylarkNestedSet.class,
+ doc = "Creates a set from the <code>items</code>, that supports nesting. "
+ + "The nesting is applied to other nested sets among <code>items</code>.<br>"
+ + "Examples:<br>"
+ + "<pre class=language-python>set([1, set([2, 3]), 2])\n"
+ + "set([1, 2, 3], order=\"compile\")</pre>",
+ optionalParams = {
+ @Param(name = "items", type = SkylarkList.class,
+ doc = "The items to initialize the set with."),
+ @Param(name = "order", type = String.class,
+ doc = "The ordering strategy for the set if it's nested, "
+ + "possible values are: <code>stable</code> (default), <code>compile</code>, "
+ + "<code>link</code> or <code>naive_link</code>.")})
+ private static final Function set =
+ new MixedModeFunction("set", ImmutableList.of("items", "order"), 0, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ Order order;
+ if (args[1] == null || args[1].equals("stable")) {
+ order = Order.STABLE_ORDER;
+ } else if (args[1].equals("compile")) {
+ order = Order.COMPILE_ORDER;
+ } else if (args[1].equals("link")) {
+ order = Order.LINK_ORDER;
+ } else if (args[1].equals("naive_link")) {
+ order = Order.NAIVE_LINK_ORDER;
+ } else {
+ throw new EvalException(ast.getLocation(), "Invalid order: " + args[1]);
+ }
+
+ if (args[0] == null) {
+ return new SkylarkNestedSet(order, SkylarkList.EMPTY_LIST, ast.getLocation());
+ }
+ return new SkylarkNestedSet(order, args[0], ast.getLocation());
+ }
+ };
+
+ @SkylarkBuiltin(name = "enumerate", returnType = SkylarkList.class,
+ doc = "Return a list of pairs, with the index (int) and the item from the input list.\n"
+ + "<pre class=language-python>"
+ + "enumerate([24, 21, 84]) == [[0, 24], [1, 21], [2, 84]]</pre>\n",
+ mandatoryParams = {
+ @Param(name = "list", type = SkylarkList.class,
+ doc = "input list"),
+ })
+ private static Function enumerate = new MixedModeFunction("enumerate",
+ ImmutableList.of("list"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ List<Object> input = Type.OBJECT_LIST.convert(args[0], "'enumerate' operand");
+ List<List<Object>> result = Lists.newArrayList();
+ int count = 0;
+ for (Object obj : input) {
+ result.add(Lists.newArrayList(count, obj));
+ count++;
+ }
+ return result;
+ }
+ };
+
+ @SkylarkBuiltin(name = "range", returnType = SkylarkList.class,
+ doc = "Creates a list where items go from <code>start</code> to <end>, using a "
+ + "<code>step</code> increment. If a single argument is provided, items will "
+ + "range from 0 to that element."
+ + "<pre class=language-python>range(4) == [0, 1, 2, 3]\n"
+ + "range(3, 9, 2) == [3, 5, 7]\n"
+ + "range(3, 0, -1) == [3, 2, 1]</pre>",
+ mandatoryParams = {
+ @Param(name = "start", type = Integer.class,
+ doc = "Value of the first element"),
+ },
+ optionalParams = {
+ @Param(name = "end", type = SkylarkList.class,
+ doc = "Generation of the list stops before <code>end</code> is reached."),
+ @Param(name = "step", type = String.class,
+ doc = "The increment (default is 1). It may be negative.")})
+ private static final Function range =
+ new MixedModeFunction("range", ImmutableList.of("start", "stop", "step"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException,
+ ConversionException {
+ int start;
+ int stop;
+ if (args[1] == null) {
+ start = 0;
+ stop = Type.INTEGER.convert(args[0], "stop");
+ } else {
+ start = Type.INTEGER.convert(args[0], "start");
+ stop = Type.INTEGER.convert(args[1], "stop");
+ }
+ int step = args[2] == null ? 1 : Type.INTEGER.convert(args[2], "step");
+ if (step == 0) {
+ throw new EvalException(ast.getLocation(), "step cannot be 0");
+ }
+ List<Integer> result = Lists.newArrayList();
+ if (step > 0) {
+ while (start < stop) {
+ result.add(start);
+ start += step;
+ }
+ } else {
+ while (start > stop) {
+ result.add(start);
+ start += step;
+ }
+ }
+ return SkylarkList.list(result, Integer.class);
+ }
+ };
+
+ /**
+ * Returns a function-value implementing "select" (i.e. configurable attributes)
+ * in the specified package context.
+ */
+ @SkylarkBuiltin(name = "select",
+ doc = "Creates a SelectorValue from the dict parameter.",
+ mandatoryParams = {@Param(name = "x", type = Map.class, doc = "The parameter to convert.")})
+ private static final Function select = new MixedModeFunction("select",
+ ImmutableList.of("x"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws EvalException, ConversionException {
+ Object dict = args[0];
+ if (!(dict instanceof Map<?, ?>)) {
+ throw new EvalException(ast.getLocation(),
+ "select({...}) argument isn't a dictionary");
+ }
+ return new SelectorValue((Map<?, ?>) dict);
+ }
+ };
+
+ /**
+ * Returns true if the object has a field of the given name, otherwise false.
+ */
+ @SkylarkBuiltin(name = "hasattr", returnType = Boolean.class,
+ doc = "Returns True if the object <code>x</code> has a field of the given <code>name</code>, "
+ + "otherwise False. Example:<br>"
+ + "<pre class=language-python>hasattr(ctx.attr, \"myattr\")</pre>",
+ mandatoryParams = {
+ @Param(name = "object", doc = "The object to check."),
+ @Param(name = "name", type = String.class, doc = "The name of the field.")})
+ private static final Function hasattr =
+ new MixedModeFunction("hasattr", ImmutableList.of("object", "name"), 2, false) {
+
+ @Override
+ public Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws EvalException, ConversionException {
+ Object obj = args[0];
+ String name = cast(args[1], String.class, "name", ast.getLocation());
+
+ if (obj instanceof ClassObject && ((ClassObject) obj).getValue(name) != null) {
+ return true;
+ }
+
+ if (env.getFunctionNames(obj.getClass()).contains(name)) {
+ return true;
+ }
+
+ try {
+ return FuncallExpression.getMethodNames(obj.getClass()).contains(name);
+ } catch (ExecutionException e) {
+ // This shouldn't happen
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+ };
+
+ @SkylarkBuiltin(name = "getattr",
+ doc = "Returns the struct's field of the given name if exists, otherwise <code>default</code>"
+ + " if specified, otherwise rasies an error. For example, <code>getattr(x, \"foobar\")"
+ + "</code> is equivalent to <code>x.foobar</code>."
+ + "Example:<br>"
+ + "<pre class=language-python>getattr(ctx.attr, \"myattr\")\n"
+ + "getattr(ctx.attr, \"myattr\", \"mydefault\")</pre>",
+ mandatoryParams = {
+ @Param(name = "object", doc = "The struct which's field is accessed."),
+ @Param(name = "name", doc = "The name of the struct field.")},
+ optionalParams = {
+ @Param(name = "default", doc = "The default value to return in case the struct "
+ + "doesn't have a field of the given name.")})
+ private static final Function getattr = new MixedModeFunction(
+ "getattr", ImmutableList.of("object", "name", "default"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws EvalException {
+ Object obj = args[0];
+ String name = cast(args[1], String.class, "name", ast.getLocation());
+ Object result = DotExpression.eval(obj, name, ast.getLocation());
+ if (result == null) {
+ if (args[2] != null) {
+ return args[2];
+ } else {
+ throw new EvalException(ast.getLocation(), "Object of type '"
+ + EvalUtils.getDatatypeName(obj) + "' has no field '" + name + "'");
+ }
+ }
+ return result;
+ }
+ };
+
+ @SkylarkBuiltin(name = "dir", returnType = SkylarkList.class,
+ doc = "Returns the list of the names (list of strings) of the fields and "
+ + "methods of the parameter object.",
+ mandatoryParams = {@Param(name = "object", doc = "The object to check.")})
+ private static final Function dir = new MixedModeFunction(
+ "dir", ImmutableList.of("object"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws EvalException {
+ Object obj = args[0];
+ // Order the fields alphabetically.
+ Set<String> fields = new TreeSet<>();
+ if (obj instanceof ClassObject) {
+ fields.addAll(((ClassObject) obj).getKeys());
+ }
+ fields.addAll(env.getFunctionNames(obj.getClass()));
+ try {
+ fields.addAll(FuncallExpression.getMethodNames(obj.getClass()));
+ } catch (ExecutionException e) {
+ // This shouldn't happen
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ return SkylarkList.list(fields, String.class);
+ }
+ };
+
+ @SkylarkBuiltin(name = "type", returnType = String.class,
+ doc = "Returns the type name of its argument.",
+ mandatoryParams = {@Param(name = "object", doc = "The object to check type of.")})
+ private static final Function type = new MixedModeFunction("type",
+ ImmutableList.of("object"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) throws EvalException {
+ // There is no 'type' type in Skylark, so we return a string with the type name.
+ return EvalUtils.getDatatypeName(args[0]);
+ }
+ };
+
+ @SkylarkBuiltin(name = "fail",
+ doc = "Raises an error (the execution stops), except if the <code>when</code> condition "
+ + "is False.",
+ returnType = Environment.NoneType.class,
+ mandatoryParams = {
+ @Param(name = "msg", type = String.class, doc = "Error message to display for the user")},
+ optionalParams = {
+ @Param(name = "attr", type = String.class,
+ doc = "The name of the attribute that caused the error"),
+ @Param(name = "when", type = Boolean.class,
+ doc = "When False, the function does nothing. Default is True.")})
+ private static final Function fail = new MixedModeFunction(
+ "fail", ImmutableList.of("msg", "attr", "when"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws EvalException {
+ if (args[2] != null) {
+ if (!EvalUtils.toBoolean(args[2])) {
+ return Environment.NONE;
+ }
+ }
+ String msg = cast(args[0], String.class, "msg", ast.getLocation());
+ if (args[1] != null) {
+ msg = "attribute " + cast(args[1], String.class, "attr", ast.getLocation())
+ + ": " + msg;
+ }
+ throw new EvalException(ast.getLocation(), msg);
+ }
+ };
+
+ @SkylarkBuiltin(name = "print", returnType = Environment.NoneType.class,
+ doc = "Prints <code>msg</code> to the console.",
+ mandatoryParams = {
+ @Param(name = "*args", doc = "The objects to print.")},
+ optionalParams = {
+ @Param(name = "sep", type = String.class,
+ doc = "The separator string between the objects, default is space (\" \").")})
+ private static final Function print = new AbstractFunction("print") {
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ Environment env) throws EvalException, InterruptedException {
+ String sep = " ";
+ if (kwargs.containsKey("sep")) {
+ sep = cast(kwargs.remove("sep"), String.class, "sep", ast.getLocation());
+ }
+ if (kwargs.size() > 0) {
+ throw new EvalException(ast.getLocation(),
+ "unexpected keywords: '" + kwargs.keySet() + "'");
+ }
+ String msg = Joiner.on(sep).join(Iterables.transform(args,
+ new com.google.common.base.Function<Object, String>() {
+ @Override
+ public String apply(Object input) {
+ return EvalUtils.printValue(input);
+ }
+ }));
+ ((SkylarkEnvironment) env).handleEvent(Event.warn(ast.getLocation(), msg));
+ return Environment.NONE;
+ }
+ };
+
+ /**
+ * Skylark String module.
+ */
+ @SkylarkModule(name = "string", doc =
+ "A language built-in type to support strings. "
+ + "Example of string literals:<br>"
+ + "<pre class=language-python>a = 'abc\\ndef'\n"
+ + "b = \"ab'cd\"\n"
+ + "c = \"\"\"multiline string\"\"\"</pre>"
+ + "Strings are iterable and support the <code>in</code> operator. Examples:<br>"
+ + "<pre class=language-python>\"a\" in \"abc\" # evaluates as True\n"
+ + "l = []\n"
+ + "for s in \"abc\":\n"
+ + " l += [s] # l == [\"a\", \"b\", \"c\"]</pre>")
+ public static final class StringModule {}
+
+ /**
+ * Skylark Dict module.
+ */
+ @SkylarkModule(name = "dict", doc =
+ "A language built-in type to support dicts. "
+ + "Example of dict literal:<br>"
+ + "<pre class=language-python>d = {\"a\": 2, \"b\": 5}</pre>"
+ + "Accessing elements works just like in Python:<br>"
+ + "<pre class=language-python>e = d[\"a\"] # e == 2</pre>"
+ + "Dicts support the <code>+</code> operator to concatenate two dicts. In case of multiple "
+ + "keys the second one overrides the first one. Examples:<br>"
+ + "<pre class=language-python>"
+ + "d = {\"a\" : 1} + {\"b\" : 2} # d == {\"a\" : 1, \"b\" : 2}\n"
+ + "d += {\"c\" : 3} # d == {\"a\" : 1, \"b\" : 2, \"c\" : 3}\n"
+ + "d = d + {\"c\" : 5} # d == {\"a\" : 1, \"b\" : 2, \"c\" : 5}</pre>"
+ + "Since the language doesn't have mutable objects <code>d[\"a\"] = 5</code> automatically "
+ + "translates to <code>d = d + {\"a\" : 5}</code>.<br>"
+ + "Dicts are iterable, the iteration works on their keyset.<br>"
+ + "Dicts support the <code>in</code> operator, testing membership in the keyset of the dict. "
+ + "Example:<br>"
+ + "<pre class=language-python>\"a\" in {\"a\" : 2, \"b\" : 5} # evaluates as True</pre>")
+ public static final class DictModule {}
+
+ public static final Map<Function, SkylarkType> stringFunctions = ImmutableMap
+ .<Function, SkylarkType>builder()
+ .put(join, SkylarkType.STRING)
+ .put(lower, SkylarkType.STRING)
+ .put(upper, SkylarkType.STRING)
+ .put(replace, SkylarkType.STRING)
+ .put(split, SkylarkType.of(List.class, String.class))
+ .put(rfind, SkylarkType.INT)
+ .put(find, SkylarkType.INT)
+ .put(endswith, SkylarkType.BOOL)
+ .put(startswith, SkylarkType.BOOL)
+ .put(strip, SkylarkType.STRING)
+ .put(substring, SkylarkType.STRING)
+ .put(count, SkylarkType.INT)
+ .build();
+
+ public static final List<Function> listFunctions = ImmutableList
+ .<Function>builder()
+ .add(append)
+ .add(extend)
+ .build();
+
+ public static final Map<Function, SkylarkType> dictFunctions = ImmutableMap
+ .<Function, SkylarkType>builder()
+ .put(items, SkylarkType.of(List.class))
+ .put(keys, SkylarkType.of(Set.class))
+ .put(values, SkylarkType.of(List.class))
+ .build();
+
+ private static final Map<Function, SkylarkType> pureGlobalFunctions = ImmutableMap
+ .<Function, SkylarkType>builder()
+ // TODO(bazel-team): String methods are added two times, because there are
+ // a lot of cases when they are used as global functions in the depot. Those
+ // should be cleaned up first.
+ .put(minus, SkylarkType.INT)
+ .put(select, SkylarkType.of(SelectorValue.class))
+ .put(len, SkylarkType.INT)
+ .put(str, SkylarkType.STRING)
+ .put(bool, SkylarkType.BOOL)
+ .build();
+
+ private static final Map<Function, SkylarkType> skylarkGlobalFunctions = ImmutableMap
+ .<Function, SkylarkType>builder()
+ .putAll(pureGlobalFunctions)
+ .put(list, SkylarkType.of(SkylarkList.class))
+ .put(struct, SkylarkType.of(ClassObject.class))
+ .put(hasattr, SkylarkType.BOOL)
+ .put(getattr, SkylarkType.UNKNOWN)
+ .put(set, SkylarkType.of(SkylarkNestedSet.class))
+ .put(dir, SkylarkType.of(SkylarkList.class, String.class))
+ .put(enumerate, SkylarkType.of(SkylarkList.class))
+ .put(range, SkylarkType.of(SkylarkList.class, Integer.class))
+ .put(type, SkylarkType.of(String.class))
+ .put(fail, SkylarkType.NONE)
+ .put(print, SkylarkType.NONE)
+ .build();
+
+ /**
+ * Set up a given environment for supported class methods.
+ */
+ public static void setupMethodEnvironment(Environment env) {
+ env.registerFunction(Map.class, index.getName(), index);
+ setupMethodEnvironment(env, Map.class, dictFunctions.keySet());
+ env.registerFunction(String.class, index.getName(), index);
+ setupMethodEnvironment(env, String.class, stringFunctions.keySet());
+ if (env.isSkylarkEnabled()) {
+ env.registerFunction(SkylarkList.class, index.getName(), index);
+ setupMethodEnvironment(env, skylarkGlobalFunctions.keySet());
+ } else {
+ env.registerFunction(List.class, index.getName(), index);
+ env.registerFunction(ImmutableList.class, index.getName(), index);
+ // TODO(bazel-team): listFunctions are not allowed in Skylark extensions (use += instead).
+ // It is allowed in BUILD files only for backward-compatibility.
+ setupMethodEnvironment(env, List.class, listFunctions);
+ setupMethodEnvironment(env, stringFunctions.keySet());
+ setupMethodEnvironment(env, pureGlobalFunctions.keySet());
+ }
+ }
+
+ private static void setupMethodEnvironment(
+ Environment env, Class<?> nameSpace, Iterable<Function> functions) {
+ for (Function function : functions) {
+ env.registerFunction(nameSpace, function.getName(), function);
+ }
+ }
+
+ private static void setupMethodEnvironment(Environment env, Iterable<Function> functions) {
+ for (Function function : functions) {
+ env.update(function.getName(), function);
+ }
+ }
+
+ private static void setupValidationEnvironment(
+ Map<Function, SkylarkType> functions, Map<String, SkylarkType> result) {
+ for (Map.Entry<Function, SkylarkType> function : functions.entrySet()) {
+ String name = function.getKey().getName();
+ result.put(name, SkylarkFunctionType.of(name, function.getValue()));
+ }
+ }
+
+ public static void setupValidationEnvironment(
+ Map<SkylarkType, Map<String, SkylarkType>> builtIn) {
+ Map<String, SkylarkType> global = builtIn.get(SkylarkType.GLOBAL);
+ setupValidationEnvironment(skylarkGlobalFunctions, global);
+
+ Map<String, SkylarkType> dict = new HashMap<>();
+ setupValidationEnvironment(dictFunctions, dict);
+ builtIn.put(SkylarkType.of(Map.class), dict);
+
+ Map<String, SkylarkType> string = new HashMap<>();
+ setupValidationEnvironment(stringFunctions, string);
+ builtIn.put(SkylarkType.STRING, string);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NoSuchPackageException.java b/src/main/java/com/google/devtools/build/lib/packages/NoSuchPackageException.java
new file mode 100644
index 0000000..720d3d5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/NoSuchPackageException.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import javax.annotation.Nullable;
+
+/**
+ * Exception indicating an attempt to access a package which is not found, does
+ * not exist, or can't be parsed into a package.
+ */
+public abstract class NoSuchPackageException extends NoSuchThingException {
+
+ private final String packageName;
+
+ public NoSuchPackageException(String packageName, String message) {
+ this(packageName, "no such package", message);
+ }
+
+ public NoSuchPackageException(String packageName, String message,
+ Throwable cause) {
+ this(packageName, "no such package", message, cause);
+ }
+
+ protected NoSuchPackageException(String packageName, String messagePrefix, String message) {
+ super(messagePrefix + " '" + packageName + "': " + message);
+ this.packageName = packageName;
+ }
+
+ protected NoSuchPackageException(String packageName, String messagePrefix, String message,
+ Throwable cause) {
+ super(messagePrefix + " '" + packageName + "': " + message, cause);
+ this.packageName = packageName;
+ }
+
+ public String getPackageName() {
+ return packageName;
+ }
+
+ /**
+ * Return the package if parsing completed enough to construct it. May return null.
+ */
+ @Nullable
+ public Package getPackage() {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NoSuchTargetException.java b/src/main/java/com/google/devtools/build/lib/packages/NoSuchTargetException.java
new file mode 100644
index 0000000..fa180fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/NoSuchTargetException.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.syntax.Label;
+
+import javax.annotation.Nullable;
+
+/**
+ * Exception indicating an attempt to access a target which is not found or does
+ * not exist.
+ */
+public class NoSuchTargetException extends NoSuchThingException {
+
+ @Nullable private final Label label;
+ // TODO(bazel-team): rename/refactor this class and NoSuchPackageException since it's confusing
+ // that they embed Target/Package instances.
+ @Nullable private final Target target;
+ private final boolean packageLoadedSuccessfully;
+
+ public NoSuchTargetException(String message) {
+ this(null, message);
+ }
+
+ public NoSuchTargetException(@Nullable Label label, String message) {
+ this((label != null ? "no such target '" + label + "': " : "") + message, label, null, null);
+ }
+
+ public NoSuchTargetException(Target targetInError, NoSuchPackageException nspe) {
+ this(String.format("Target '%s' contains an error and its package is in error",
+ targetInError.getLabel()), targetInError.getLabel(), targetInError, nspe);
+ }
+
+ private NoSuchTargetException(String message, @Nullable Label label, @Nullable Target target,
+ @Nullable NoSuchPackageException nspe) {
+ super(message, nspe);
+ this.label = label;
+ this.target = target;
+ this.packageLoadedSuccessfully = nspe != null ? false : true;
+ }
+
+ @Nullable
+ public Label getLabel() {
+ return label;
+ }
+
+ /**
+ * Return the target (in error) if parsing completed enough to construct it. May return null.
+ */
+ @Nullable
+ public Target getTarget() {
+ return target;
+ }
+
+ public boolean getPackageLoadedSuccessfully() {
+ return packageLoadedSuccessfully;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NoSuchThingException.java b/src/main/java/com/google/devtools/build/lib/packages/NoSuchThingException.java
new file mode 100644
index 0000000..49e703e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/NoSuchThingException.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Exception indicating an attempt to access something which is not found or
+ * does not exist.
+ */
+public class NoSuchThingException extends Exception {
+
+ public NoSuchThingException(String message) {
+ super(message);
+ }
+
+ public NoSuchThingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NonconfigurableAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/NonconfigurableAttributeMapper.java
new file mode 100644
index 0000000..d54c847
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/NonconfigurableAttributeMapper.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * {@link AttributeMap} implementation that triggers an {@link IllegalStateException} if called
+ * on any attribute that supports configurable values, as determined by
+ * {@link Attribute#isConfigurable()}.
+ *
+ * <p>This is particularly useful for logic that doesn't have access to configurations - it
+ * protects against undefined behavior in response to unexpected configuration-dependent inputs.
+ */
+public class NonconfigurableAttributeMapper extends AbstractAttributeMapper {
+ private NonconfigurableAttributeMapper(Rule rule) {
+ super(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(),
+ rule.getAttributeContainer());
+ }
+
+ /**
+ * Example usage:
+ *
+ * <pre>
+ * Label fooLabel = NonconfigurableAttributeMapper.of(rule).get("foo", Type.LABEL);
+ * </pre>
+ */
+ public static NonconfigurableAttributeMapper of (Rule rule) {
+ return new NonconfigurableAttributeMapper(rule);
+ }
+
+ @Override
+ public <T> T get(String attributeName, Type<T> type) {
+ Preconditions.checkState(!getAttributeDefinition(attributeName).isConfigurable(),
+ "Attribute '" + attributeName + "' is potentially configurable - not allowed here");
+ return super.get(attributeName, type);
+ }
+
+ @Override
+ protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) {
+ T value = get(attributeName, type);
+ return value == null ? ImmutableList.<T>of() : ImmutableList.of(value);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/OutputFile.java b/src/main/java/com/google/devtools/build/lib/packages/OutputFile.java
new file mode 100644
index 0000000..9c91afb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/OutputFile.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A generated file that is the output of a rule.
+ */
+public final class OutputFile extends FileTarget {
+
+ private final Rule generatingRule;
+
+ /**
+ * Constructs an output file with the given label, which must be in the given
+ * package.
+ */
+ OutputFile(Package pkg, Label label, Rule generatingRule) {
+ super(pkg, label);
+ this.generatingRule = generatingRule;
+ }
+
+ @Override
+ public RuleVisibility getVisibility() {
+ return generatingRule.getVisibility();
+ }
+
+ /**
+ * Returns the rule which generates this output file.
+ */
+ public Rule getGeneratingRule() {
+ return generatingRule;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return "generated file";
+ }
+
+ @Override
+ public Rule getAssociatedRule() {
+ return getGeneratingRule();
+ }
+
+ @Override
+ public Location getLocation() {
+ return generatingRule.getLocation();
+ }
+
+ @Override
+ public int hashCode() {
+ return label.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Package.java b/src/main/java/com/google/devtools/build/lib/packages/Package.java
new file mode 100644
index 0000000..a2216e0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/Package.java
@@ -0,0 +1,1516 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.ImmutableSortedKeyMap;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.AttributeMap.AcceptsLabelAttribute;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.packages.PackageDeserializer.PackageDeserializationException;
+import com.google.devtools.build.lib.packages.PackageFactory.Globber;
+
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Canonicalizer;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A package, which is a container of {@link Rule}s, each of
+ * which contains a dictionary of named attributes.
+ *
+ * <p>Package instances are intended to be immutable and for all practical
+ * purposes can be treated as such. Note, however, that some member variables
+ * exposed via the public interface are not strictly immutable, so until their
+ * types are guaranteed immutable we're not applying the {@code @Immutable}
+ * annotation here.
+ */
+public class Package implements Serializable {
+
+ /**
+ * Common superclass for all name-conflict exceptions.
+ */
+ public static class NameConflictException extends Exception {
+ protected NameConflictException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * The repository identifier for this package.
+ */
+ private final PackageIdentifier packageIdentifier;
+
+ /**
+ * The name of the package, e.g. "foo/bar".
+ */
+ protected final String name;
+
+ /**
+ * Like name, but in the form of a PathFragment.
+ */
+ private final PathFragment nameFragment;
+
+ /**
+ * The filename of this package's BUILD file.
+ */
+ protected Path filename;
+
+ /**
+ * The directory in which this package's BUILD file resides. All InputFile
+ * members of the packages are located relative to this directory.
+ */
+ private Path packageDirectory;
+
+ /**
+ * The root of the source tree in which this package was found. It is an invariant that
+ * {@code sourceRoot.getRelative(name).equals(packageDirectory)}.
+ */
+ private Path sourceRoot;
+
+ /**
+ * The "Make" environment of this package, containing package-local
+ * definitions of "Make" variables.
+ */
+ private MakeEnvironment makeEnv;
+
+ /**
+ * The collection of all targets defined in this package, indexed by name.
+ */
+ protected Map<String, Target> targets;
+
+ /**
+ * Default visibility for rules that do not specify it. null is interpreted
+ * as VISIBILITY_PRIVATE.
+ */
+ private RuleVisibility defaultVisibility;
+ private boolean defaultVisibilitySet;
+
+ /**
+ * Default package-level 'obsolete' value for rules that do not specify it.
+ */
+ private boolean defaultObsolete = false;
+
+ /**
+ * Default package-level 'testonly' value for rules that do not specify it.
+ */
+ private boolean defaultTestOnly = false;
+
+ /**
+ * Default package-level 'deprecation' value for rules that do not specify it.
+ */
+ private String defaultDeprecation;
+
+ /**
+ * Default header strictness checking for rules that do not specify it.
+ */
+ private String defaultHdrsCheck;
+
+ /**
+ * Default copts for cc_* rules. The rules' individual copts will append to
+ * this value.
+ */
+ private ImmutableList<String> defaultCopts;
+
+ /**
+ * The InputFile target corresponding to this package's BUILD file.
+ */
+ private InputFile buildFile;
+
+ /**
+ * True iff this package's BUILD files contained lexical or grammatical
+ * errors, or experienced errors during evaluation, or semantic errors during
+ * the construction of any rule.
+ *
+ * <p>Note: A package containing errors does not necessarily prevent a build;
+ * if all the rules needed for a given build were constructed prior to the
+ * first error, the build may proceed.
+ */
+ private boolean containsErrors;
+
+ /**
+ * True iff this package contains errors that were caused by temporary conditions (e.g. an I/O
+ * error). If this is true, {@link #containsErrors} is also true.
+ */
+ private boolean containsTemporaryErrors;
+
+ /**
+ * The set of labels subincluded by this package.
+ */
+ private Set<Label> subincludes;
+
+ /**
+ * The list of transitive closure of the Skylark file dependencies.
+ */
+ private ImmutableList<Label> skylarkFileDependencies;
+
+ /**
+ * The package's default "licenses" and "distribs" attributes, as specified
+ * in calls to licenses() and distribs() in the BUILD file.
+ */
+ // These sets contain the values specified by the most recent licenses() or
+ // distribs() declarations encountered during package parsing:
+ private License defaultLicense;
+ private Set<License.DistributionType> defaultDistributionSet;
+
+
+ /**
+ * The names of the package() attributes that declare default values for rule
+ * {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} and {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR}
+ * values when not explicitly specified.
+ */
+ public static final String DEFAULT_COMPATIBLE_WITH_ATTRIBUTE = "default_compatible_with";
+ public static final String DEFAULT_RESTRICTED_TO_ATTRIBUTE = "default_restricted_to";
+
+ private Set<Label> defaultCompatibleWith = ImmutableSet.of();
+ private Set<Label> defaultRestrictedTo = ImmutableSet.of();
+
+ private ImmutableSet<String> features;
+
+ private ImmutableList<Event> events;
+
+ // Hack to avoid having to copy every attribute. See #readObject and #readResolve.
+ // This will always be null for externally observable instances.
+ private Package deserializedPkg = null;
+
+ /**
+ * Package initialization, part 1 of 3: instantiates a new package with the
+ * given name.
+ *
+ * <p>As part of initialization, {@link Builder} constructs {@link InputFile}
+ * and {@link PackageGroup} instances that require a valid Package instance where
+ * {@link Package#getNameFragment()} is accessible. That's why these settings are
+ * applied here at the start.
+ *
+ * @precondition {@code name} must be a suffix of
+ * {@code filename.getParentDirectory())}.
+ */
+ protected Package(PackageIdentifier packageId) {
+ this.packageIdentifier = packageId;
+ this.nameFragment = Canonicalizer.fragments().intern(packageId.getPackageFragment());
+ this.name = nameFragment.getPathString();
+ }
+
+ private void writeObject(ObjectOutputStream out) {
+ com.google.devtools.build.lib.query2.proto.proto2api.Build.Package pb =
+ PackageSerializer.serializePackage(this);
+ try {
+ pb.writeDelimitedTo(out);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException {
+ com.google.devtools.build.lib.query2.proto.proto2api.Build.Package pb =
+ com.google.devtools.build.lib.query2.proto.proto2api.Build.Package.parseDelimitedFrom(in);
+ Package pkg;
+ try {
+ pkg = new PackageDeserializer(null, null).deserialize(pb);
+ } catch (PackageDeserializationException e) {
+ throw new IllegalStateException(e);
+ }
+ deserializedPkg = pkg;
+ }
+
+ protected Object readResolve() {
+ // This method needs to be protected so serialization works for subclasses.
+ return deserializedPkg;
+ }
+
+ // See: http://docs.oracle.com/javase/6/docs/platform/serialization/spec/input.html#6053
+ @SuppressWarnings("unused")
+ private void readObjectNoData() {
+ throw new IllegalStateException();
+ }
+
+ /** Returns this packages' identifier. */
+ public PackageIdentifier getPackageIdentifier() {
+ return packageIdentifier;
+ }
+
+ /**
+ * Package initialization: part 2 of 3: sets this package's default header
+ * strictness checking.
+ *
+ * <p>This is needed to support C++-related rule classes
+ * which accesses {@link #getDefaultHdrsCheck} from the still-under-construction
+ * package.
+ */
+ protected void setDefaultHdrsCheck(String defaultHdrsCheck) {
+ this.defaultHdrsCheck = defaultHdrsCheck;
+ }
+
+ /**
+ * Set the default 'obsolete' value for this package.
+ */
+ protected void setDefaultObsolete(boolean obsolete) {
+ defaultObsolete = obsolete;
+ }
+
+ /**
+ * Set the default 'testonly' value for this package.
+ */
+ protected void setDefaultTestOnly(boolean testOnly) {
+ defaultTestOnly = testOnly;
+ }
+
+ /**
+ * Set the default 'deprecation' value for this package.
+ */
+ protected void setDefaultDeprecation(String deprecation) {
+ defaultDeprecation = deprecation;
+ }
+
+ /**
+ * Sets the default value to use for a rule's {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR}
+ * attribute when not explicitly specified by the rule.
+ */
+ protected void setDefaultCompatibleWith(Set<Label> environments) {
+ defaultCompatibleWith = environments;
+ }
+
+ /**
+ * Sets the default value to use for a rule's {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR}
+ * attribute when not explicitly specified by the rule.
+ */
+ protected void setDefaultRestrictedTo(Set<Label> environments) {
+ defaultRestrictedTo = environments;
+ }
+
+ public static Path getSourceRoot(Path buildFile, PathFragment nameFragment) {
+ Path current = buildFile.getParentDirectory();
+ for (int i = 0, len = nameFragment.segmentCount(); i < len && current != null; i++) {
+ current = current.getParentDirectory();
+ }
+ return current;
+ }
+
+ /**
+ * Package initialization: part 3 of 3: applies all other settings and completes
+ * initialization of the package.
+ *
+ * <p>Only after this method is called can this package be considered "complete"
+ * and be shared publicly.
+ */
+ protected void finishInit(AbstractBuilder<?, ?> builder) {
+ // If any error occurred during evaluation of this package, consider all
+ // rules in the package to be "in error" also (even if they were evaluated
+ // prior to the error). This behaviour is arguably stricter than need be,
+ // but stopping a build only for some errors but not others creates user
+ // confusion.
+ if (builder.containsErrors) {
+ for (Rule rule : builder.getTargets(Rule.class)) {
+ rule.setContainsErrors();
+ }
+ }
+ this.filename = builder.filename;
+ this.packageDirectory = filename.getParentDirectory();
+
+ this.sourceRoot = getSourceRoot(filename, nameFragment);
+ if ((sourceRoot == null
+ || !sourceRoot.getRelative(nameFragment).equals(packageDirectory))
+ && !filename.getBaseName().equals("WORKSPACE")) {
+ throw new IllegalArgumentException(
+ "Invalid BUILD file name for package '" + name + "': " + filename);
+ }
+
+ this.makeEnv = builder.makeEnv.build();
+ this.targets = ImmutableSortedKeyMap.copyOf(builder.targets);
+ this.defaultVisibility = builder.defaultVisibility;
+ this.defaultVisibilitySet = builder.defaultVisibilitySet;
+ if (builder.defaultCopts == null) {
+ this.defaultCopts = ImmutableList.of();
+ } else {
+ this.defaultCopts = ImmutableList.copyOf(builder.defaultCopts);
+ }
+ this.buildFile = builder.buildFile;
+ this.containsErrors = builder.containsErrors;
+ this.containsTemporaryErrors = builder.containsTemporaryErrors;
+ this.subincludes = builder.subincludes.keySet();
+ this.skylarkFileDependencies = builder.skylarkFileDependencies;
+ this.defaultLicense = builder.defaultLicense;
+ this.defaultDistributionSet = builder.defaultDistributionSet;
+ this.features = ImmutableSortedSet.copyOf(builder.features);
+ this.events = ImmutableList.copyOf(builder.events);
+ }
+
+ /**
+ * Returns the list of subincluded labels on which the validity of this package depends.
+ */
+ public Set<Label> getSubincludeLabels() {
+ return subincludes;
+ }
+
+ /**
+ * Returns the list of transitive closure of the Skylark file dependencies of this package.
+ */
+ public ImmutableList<Label> getSkylarkFileDependencies() {
+ return skylarkFileDependencies;
+ }
+
+ /**
+ * Returns the filename of the BUILD file which defines this package. The
+ * parent directory of the BUILD file is the package directory.
+ */
+ public Path getFilename() {
+ return filename;
+ }
+
+ /**
+ * Returns the source root (a directory) beneath which this package's BUILD file was found.
+ *
+ * Assumes invariant:
+ * {@code getSourceRoot().getRelative(getName()).equals(getPackageDirectory())}
+ */
+ public Path getSourceRoot() {
+ return sourceRoot;
+ }
+
+ /**
+ * Returns the directory containing the package's BUILD file.
+ */
+ public Path getPackageDirectory() {
+ return packageDirectory;
+ }
+
+ /**
+ * Returns the name of this package. If this build is using external repositories then this name
+ * may not be unique!
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Like {@link #getName}, but has type {@code PathFragment}.
+ */
+ public PathFragment getNameFragment() {
+ return nameFragment;
+ }
+
+ /**
+ * Returns the "Make" value from the package's make environment whose name
+ * is "varname", or null iff the variable is not defined in the environment.
+ */
+ public String lookupMakeVariable(String varname, String platform) {
+ return makeEnv.lookup(varname, platform);
+ }
+
+ /**
+ * Returns the make environment. This should only ever be used for serialization -- how the
+ * make variables are implemented is an implementation detail.
+ */
+ MakeEnvironment getMakeEnvironment() {
+ return makeEnv;
+ }
+
+ /**
+ * Returns the label of this package's BUILD file.
+ *
+ * Typically <code>getBuildFileLabel().getName().equals("BUILD")</code> --
+ * though not necessarily: data in a subdirectory of a test package may use a
+ * different filename to avoid inadvertently creating a new package.
+ */
+ Label getBuildFileLabel() {
+ return buildFile.getLabel();
+ }
+
+ /**
+ * Returns the InputFile target for this package's BUILD file.
+ */
+ public InputFile getBuildFile() {
+ return buildFile;
+ }
+
+ /**
+ * Returns true if errors were encountered during evaluation of this package.
+ * (The package may be incomplete and its contents should not be relied upon
+ * for critical operations. However, any Rules belonging to the package are
+ * guaranteed to be intact, unless their <code>containsErrors()</code> flag
+ * is set.)
+ */
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ /**
+ * True iff this package contains errors that were caused by temporary conditions (e.g. an I/O
+ * error). If this is true, {@link #containsErrors()} also returns true.
+ */
+ public boolean containsTemporaryErrors() {
+ return containsTemporaryErrors;
+ }
+
+ public List<Event> getEvents() {
+ return events;
+ }
+
+ /**
+ * Returns an (immutable, unordered) view of all the targets belonging to this package.
+ */
+ public Collection<Target> getTargets() {
+ return getTargets(targets);
+ }
+
+ /**
+ * Common getTargets implementation, accessible by both {@link Package} and
+ * {@link Package.AbstractBuilder}.
+ */
+ private static Collection<Target> getTargets(Map<String, Target> targetMap) {
+ return Collections.unmodifiableCollection(targetMap.values());
+ }
+
+ /**
+ * Returns a (read-only, unordered) iterator of all the targets belonging
+ * to this package which are instances of the specified class.
+ */
+ public <T extends Target> Iterable<T> getTargets(Class<T> targetClass) {
+ return getTargets(targets, targetClass);
+ }
+
+ /**
+ * Common getTargets implementation, accessible by both {@link Package} and
+ * {@link Package.AbstractBuilder}.
+ */
+ private static <T extends Target> Iterable<T> getTargets(Map<String, Target> targetMap,
+ Class<T> targetClass) {
+ return Iterables.filter(targetMap.values(), targetClass);
+ }
+
+ /**
+ * Returns a (read-only, unordered) iterator over the rules in this package.
+ */
+ @VisibleForTesting // Legacy. Production code should use getTargets(Class) instead
+ Iterable<? extends Rule> getRules() {
+ return getTargets(Rule.class);
+ }
+
+ /**
+ * Returns a (read-only, unordered) iterator over the files in this package.
+ */
+ @VisibleForTesting // Legacy. Production code should use getTargets(Class) instead
+ Iterable<? extends FileTarget> getFiles() {
+ return getTargets(FileTarget.class);
+ }
+
+ /**
+ * Returns the rule that corresponds to a particular BUILD target name. Useful
+ * for walking through the dependency graph of a target.
+ * Fails if the target is not a Rule.
+ */
+ @VisibleForTesting
+ Rule getRule(String targetName) {
+ return (Rule) targets.get(targetName);
+ }
+
+ /**
+ * Returns the features specified in the <code>package()</code> declaration.
+ */
+ public ImmutableSet<String> getFeatures() {
+ return features;
+ }
+
+ /**
+ * Returns the target (a member of this package) whose name is "targetName".
+ * First rules are searched, then output files, then input files. The target
+ * name must be valid, as defined by {@code LabelValidator#validateTargetName}.
+ *
+ * @throws NoSuchTargetException if the specified target was not found.
+ */
+ public Target getTarget(String targetName) throws NoSuchTargetException {
+ Target target = targets.get(targetName);
+ if (target != null) {
+ return target;
+ }
+
+ // No such target.
+
+ // If there's a file on the disk that's not mentioned in the BUILD file,
+ // produce a more informative error. NOTE! this code path is only executed
+ // on failure, which is (relatively) very rare. In the common case no
+ // stat(2) is executed.
+ Path filename = getPackageDirectory().getRelative(targetName);
+ String suffix;
+ if (!new PathFragment(targetName).isNormalized()) {
+ // Don't check for file existence in this case because the error message
+ // would be confusing and wrong. If the targetName is "foo/bar/.", and
+ // there is a directory "foo/bar", it doesn't mean that "//pkg:foo/bar/."
+ // is a valid label.
+ suffix = "";
+ } else if (filename.isDirectory()) {
+ suffix = "; however, a source directory of this name exists. (Perhaps add "
+ + "'exports_files([\"" + targetName + "\"])' to " + name + "/BUILD, or define a "
+ + "filegroup?)";
+ } else if (filename.exists()) {
+ suffix = "; however, a source file of this name exists. (Perhaps add "
+ + "'exports_files([\"" + targetName + "\"])' to " + name + "/BUILD?)";
+ } else {
+ suffix = "";
+ }
+
+ try {
+ throw new NoSuchTargetException(createLabel(targetName), "target '" + targetName
+ + "' not declared in package '" + name + "'" + suffix + " defined by "
+ + this.filename);
+ } catch (Label.SyntaxException e) {
+ throw new IllegalArgumentException(targetName);
+ }
+ }
+
+ /**
+ * Creates a label for a target inside this package.
+ *
+ * @throws SyntaxException if the {@code targetName} is invalid
+ */
+ public Label createLabel(String targetName) throws SyntaxException {
+ return Label.create(packageIdentifier, targetName);
+ }
+
+ /**
+ * Returns the default visibility for this package.
+ */
+ public RuleVisibility getDefaultVisibility() {
+ if (defaultVisibility != null) {
+ return defaultVisibility;
+ } else {
+ return ConstantRuleVisibility.PRIVATE;
+ }
+ }
+
+ /**
+ * Returns the default obsolete value.
+ */
+ public Boolean getDefaultObsolete() {
+ return defaultObsolete;
+ }
+
+ /**
+ * Returns the default testonly value.
+ */
+ public Boolean getDefaultTestOnly() {
+ return defaultTestOnly;
+ }
+
+ /**
+ * Returns the default obsolete value.
+ */
+ public String getDefaultDeprecation() {
+ return defaultDeprecation;
+ }
+
+ /**
+ * Gets the default header checking mode.
+ */
+ public String getDefaultHdrsCheck() {
+ return defaultHdrsCheck != null ? defaultHdrsCheck : "loose";
+ }
+
+ /**
+ * Returns the default copts value, to which rules should append their
+ * specific copts.
+ */
+ public ImmutableList<String> getDefaultCopts() {
+ return defaultCopts;
+ }
+
+ /**
+ * Returns whether the default header checking mode has been set or it is the
+ * default value.
+ */
+ public boolean isDefaultHdrsCheckSet() {
+ return defaultHdrsCheck != null;
+ }
+
+ public boolean isDefaultVisibilitySet() {
+ return defaultVisibilitySet;
+ }
+
+ /**
+ * Gets the parsed license object for the default license
+ * declared by this package.
+ */
+ public License getDefaultLicense() {
+ return defaultLicense;
+ }
+
+ /**
+ * Returns the parsed set of distributions declared as the default for this
+ * package.
+ */
+ public Set<License.DistributionType> getDefaultDistribs() {
+ return defaultDistributionSet;
+ }
+
+ /**
+ * Returns the default value to use for a rule's {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR}
+ * attribute when not explicitly specified by the rule.
+ */
+ public Set<Label> getDefaultCompatibleWith() {
+ return defaultCompatibleWith;
+ }
+
+ /**
+ * Returns the default value to use for a rule's {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR}
+ * attribute when not explicitly specified by the rule.
+ */
+ public Set<Label> getDefaultRestrictedTo() {
+ return defaultRestrictedTo;
+ }
+
+ @Override
+ public String toString() {
+ return "Package(" + name + ")=" + (targets != null ? getRules() : "initializing...");
+ }
+
+ /**
+ * Dumps the package for debugging. Do not depend on the exact format/contents of this debugging
+ * output.
+ */
+ public void dump(PrintStream out) {
+ out.println(" Package " + getName() + " (" + getFilename() + ")");
+
+ // Rules:
+ out.println(" Rules");
+ for (Rule rule : getTargets(Rule.class)) {
+ out.println(" " + rule.getTargetKind() + " " + rule.getLabel());
+ for (Attribute attr : rule.getAttributes()) {
+ for (Object possibleValue : AggregatingAttributeMapper.of(rule)
+ .visitAttribute(attr.getName(), attr.getType())) {
+ out.println(" " + attr.getName() + " = " + possibleValue);
+ }
+ }
+ }
+
+ // Files:
+ out.println(" Files");
+ for (FileTarget file : getTargets(FileTarget.class)) {
+ out.print(" " + file.getTargetKind() + " " + file.getLabel());
+ if (file instanceof OutputFile) {
+ out.println(" (generated by " + ((OutputFile) file).getGeneratingRule().getLabel() + ")");
+ } else {
+ out.println();
+ }
+ }
+
+ // TODO(bazel-team): (2009) perhaps dump also:
+ // - subincludes
+ // - globs
+ // - containsErrors
+ // - makeEnv
+ }
+
+ /**
+ * Builder class for {@link Package}.
+ *
+ * <p>Should only be used by the package loading and the package deserialization machineries.
+ */
+ static class Builder extends AbstractBuilder<Package, Builder> {
+ Builder(PackageIdentifier packageId) {
+ super(new Package(packageId));
+ }
+
+ @Override
+ protected Builder self() {
+ return this;
+ }
+ }
+
+ /** Builder class for {@link Package} that does its own globbing. */
+ public static class LegacyBuilder extends AbstractBuilder<Package, LegacyBuilder> {
+
+ private Globber globber = null;
+
+ LegacyBuilder(PackageIdentifier packageId) {
+ super(AbstractBuilder.newPackage(packageId));
+ }
+
+ @Override
+ protected LegacyBuilder self() {
+ return this;
+ }
+
+ /**
+ * Sets the globber used for this package's glob expansions.
+ */
+ LegacyBuilder setGlobber(Globber globber) {
+ this.globber = globber;
+ return this;
+ }
+
+ /**
+ * Removes a target from the {@link Package} under construction. Intended to be used only by
+ * {@link PackageFunction} to remove targets whose labels cross subpackage boundaries.
+ */
+ public void removeTarget(Target target) {
+ if (target.getPackage() == pkg) {
+ this.targets.remove(target.getName());
+ }
+ }
+
+ /**
+ * Returns the glob patterns requested by {@link PackageFactory} during evaluation of this
+ * package's BUILD file. Intended to be used only by {@link PackageFunction} to mark the
+ * appropriate Skyframe dependencies after the fact.
+ */
+ public Set<Pair<String, Boolean>> getGlobPatterns() {
+ return globber.getGlobPatterns();
+ }
+ }
+
+ abstract static class AbstractBuilder<P extends Package, B extends AbstractBuilder<P, B>> {
+ /**
+ * The output instance for this builder. Needs to be instantiated and
+ * available with name info throughout initialization. All other settings
+ * are applied during {@link #build}. See {@link Package#Package(String)}
+ * and {@link Package#finishInit} for details.
+ */
+ protected P pkg;
+
+ protected Path filename = null;
+ private Label buildFileLabel = null;
+ private InputFile buildFile = null;
+ private MakeEnvironment.Builder makeEnv = null;
+ private RuleVisibility defaultVisibility = null;
+ private boolean defaultVisibilitySet;
+ private List<String> defaultCopts = null;
+ private List<String> features = new ArrayList<>();
+ private List<Event> events = Lists.newArrayList();
+ private boolean containsErrors = false;
+ private boolean containsTemporaryErrors = false;
+
+ private License defaultLicense = License.NO_LICENSE;
+ private Set<License.DistributionType> defaultDistributionSet = License.DEFAULT_DISTRIB;
+
+ protected Map<String, Target> targets = new HashMap<>();
+ protected Map<Label, EnvironmentGroup> environmentGroups = new HashMap<>();
+
+ protected Map<Label, Path> subincludes = null;
+ protected ImmutableList<Label> skylarkFileDependencies = null;
+
+ /**
+ * True iff the "package" function has already been called in this package.
+ */
+ private boolean packageFunctionUsed;
+
+ /**
+ * The collection of the prefixes of every output file. Maps every prefix
+ * to an output file whose prefix it is.
+ *
+ * <p>This is needed to make the output file prefix conflict check be
+ * reasonably fast. However, since it can potentially take a lot of memory and
+ * is useless after the package has been loaded, it isn't passed to the
+ * package itself.
+ */
+ private Map<String, OutputFile> outputFilePrefixes = new HashMap<>();
+
+ private boolean alreadyBuilt = false;
+
+ private EventHandler builderEventHandler = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ addEvent(event);
+ }
+ };
+
+ protected AbstractBuilder(P pkg) {
+ this.pkg = pkg;
+ if (pkg.getName().startsWith("javatests/")) {
+ setDefaultTestonly(true);
+ }
+ }
+
+ protected static Package newPackage(PackageIdentifier packageId) {
+ return new Package(packageId);
+ }
+
+ protected abstract B self();
+
+ protected PackageIdentifier getPackageIdentifier() {
+ return pkg.getPackageIdentifier();
+ }
+
+ /**
+ * Sets the name of this package's BUILD file.
+ */
+ B setFilename(Path filename) {
+ this.filename = filename;
+ try {
+ buildFileLabel = createLabel(filename.getBaseName());
+ addInputFile(buildFileLabel, Location.fromFile(filename));
+ } catch (Label.SyntaxException e) {
+ // This can't actually happen.
+ throw new AssertionError("Package BUILD file has an illegal name: " + filename);
+ }
+ return self();
+ }
+
+ public Label getBuildFileLabel() {
+ return buildFileLabel;
+ }
+
+ Path getFilename() {
+ return filename;
+ }
+
+ /**
+ * Sets this package's Make environment.
+ */
+ B setMakeEnv(MakeEnvironment.Builder makeEnv) {
+ this.makeEnv = makeEnv;
+ return self();
+ }
+
+ /**
+ * Sets the default visibility for this package. Called at most once per
+ * package from PackageFactory.
+ */
+ B setDefaultVisibility(RuleVisibility visibility) {
+ this.defaultVisibility = visibility;
+ this.defaultVisibilitySet = true;
+ return self();
+ }
+
+ /**
+ * Sets whether the default visibility is set in the BUILD file.
+ */
+ B setDefaultVisibilitySet(boolean defaultVisibilitySet) {
+ this.defaultVisibilitySet = defaultVisibilitySet;
+ return self();
+ }
+
+ /**
+ * Sets the default value of 'obsolete'. Rule-level 'obsolete' will override this.
+ */
+ B setDefaultObsolete(boolean defaultObsolete) {
+ pkg.setDefaultObsolete(defaultObsolete);
+ return self();
+ }
+
+ /** Sets the default value of 'testonly'. Rule-level 'testonly' will override this. */
+ B setDefaultTestonly(boolean defaultTestonly) {
+ pkg.setDefaultTestOnly(defaultTestonly);
+ return self();
+ }
+
+ /**
+ * Sets the default value of 'deprecation'. Rule-level 'deprecation' will append to this.
+ */
+ B setDefaultDeprecation(String defaultDeprecation) {
+ pkg.setDefaultDeprecation(defaultDeprecation);
+ return self();
+ }
+
+ /**
+ * Returns whether the "package" function has been called yet
+ */
+ public boolean isPackageFunctionUsed() {
+ return packageFunctionUsed;
+ }
+
+ public void setPackageFunctionUsed() {
+ packageFunctionUsed = true;
+ }
+
+ /**
+ * Sets the default header checking mode.
+ */
+ public B setDefaultHdrsCheck(String hdrsCheck) {
+ // Note that this setting is propagated directly to the package because
+ // other code needs the ability to read this info directly from the
+ // under-construction package. See {@link Package#setDefaultHdrsCheck}.
+ pkg.setDefaultHdrsCheck(hdrsCheck);
+ return self();
+ }
+
+ /**
+ * Sets the default value of copts. Rule-level copts will append to this.
+ */
+ public B setDefaultCopts(List<String> defaultCopts) {
+ this.defaultCopts = defaultCopts;
+ return self();
+ }
+
+ public B addFeatures(Iterable<String> features) {
+ Iterables.addAll(this.features, features);
+ return self();
+ }
+
+ /**
+ * Declares that errors were encountering while loading this package.
+ */
+ public B setContainsErrors() {
+ containsErrors = true;
+ return self();
+ }
+
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ B setContainsTemporaryErrors() {
+ setContainsErrors();
+ containsTemporaryErrors = true;
+ return self();
+ }
+
+ public B addEvents(Iterable<Event> events) {
+ for (Event event : events) {
+ addEvent(event);
+ }
+ return self();
+ }
+
+ public B addEvent(Event event) {
+ this.events.add(event);
+ return self();
+ }
+
+ B setSkylarkFileDependencies(ImmutableList<Label> skylarkFileDependencies) {
+ this.skylarkFileDependencies = skylarkFileDependencies;
+ return self();
+ }
+
+ /**
+ * Sets the default license for this package.
+ */
+ void setDefaultLicense(License license) {
+ this.defaultLicense = license;
+ }
+
+ License getDefaultLicense() {
+ return defaultLicense;
+ }
+
+ /**
+ * Initializes the default set of distributions for targets in this package.
+ *
+ * TODO(bazel-team): (2011) consider moving the license & distribs info into Metadata--maybe
+ * even in the Build language.
+ */
+ void setDefaultDistribs(Set<DistributionType> dists) {
+ this.defaultDistributionSet = dists;
+ }
+
+ Set<DistributionType> getDefaultDistribs() {
+ return defaultDistributionSet;
+ }
+
+ /**
+ * Sets the default value to use for a rule's {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR}
+ * attribute when not explicitly specified by the rule. Records a package error if
+ * any labels are duplicated.
+ */
+ void setDefaultCompatibleWith(List<Label> environments, String attrName, Location location) {
+ if (!checkForDuplicateLabels(environments, "package " + pkg.getName(), attrName, location,
+ builderEventHandler)) {
+ setContainsErrors();
+ }
+ pkg.setDefaultCompatibleWith(ImmutableSet.copyOf(environments));
+ }
+
+ /**
+ * Sets the default value to use for a rule's {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR}
+ * attribute when not explicitly specified by the rule. Records a package error if
+ * any labels are duplicated.
+ */
+ void setDefaultRestrictedTo(List<Label> environments, String attrName, Location location) {
+ if (!checkForDuplicateLabels(environments, "package " + pkg.getName(), attrName, location,
+ builderEventHandler)) {
+ setContainsErrors();
+ }
+
+ pkg.setDefaultRestrictedTo(ImmutableSet.copyOf(environments));
+ }
+
+ /**
+ * Returns a new Rule belonging to this package instance, and uses the given Label.
+ *
+ * <p>Useful for RuleClass instantiation, where the rule name is checked by trying to create a
+ * Label. This label can then be used again here.
+ */
+ Rule newRuleWithLabel(Label label, RuleClass ruleClass, FuncallExpression ast,
+ Location location) {
+ return new Rule(pkg, label, ruleClass, ast, location);
+ }
+
+ /**
+ * Called by the parser when a "mocksubinclude" is encountered, to record the
+ * mappings from labels to absolute paths upon which that the validity of
+ * this package depends.
+ */
+ void addSubinclude(Label label, Path resolvedPath) {
+ if (subincludes == null) {
+ // This is a TreeMap because the order needs to be deterministic.
+ subincludes = Maps.newTreeMap();
+ }
+
+ Path oldResolvedPath = subincludes.put(label, resolvedPath);
+ if (oldResolvedPath != null && !oldResolvedPath.equals(resolvedPath)){
+ // The same label should have been resolved to the same path
+ throw new IllegalStateException("Ambiguous subinclude path");
+ }
+ }
+
+ public Set<Label> getSubincludeLabels() {
+ return subincludes == null ? Sets.<Label>newHashSet() : subincludes.keySet();
+ }
+
+ public Map<Label, Path> getSubincludes() {
+ return subincludes == null ? Maps.<Label, Path>newHashMap() : subincludes;
+ }
+
+ public Collection<Target> getTargets() {
+ return Package.getTargets(targets);
+ }
+
+ /**
+ * Returns an (immutable, unordered) view of all the targets belonging to
+ * this package which are instances of the specified class.
+ */
+ <T extends Target> Iterable<T> getTargets(Class<T> targetClass) {
+ return Package.getTargets(targets, targetClass);
+ }
+
+ /**
+ * An input file name conflicts with an existing package member.
+ */
+ static class GeneratedLabelConflict extends NameConflictException {
+ private GeneratedLabelConflict(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Creates an input file target in this package with the specified name.
+ *
+ * @param targetName name of the input file. This must be a valid target
+ * name as defined by {@link
+ * com.google.devtools.build.lib.cmdline.LabelValidator#validateTargetName}.
+ * @return the newly-created InputFile, or the old one if it already existed.
+ * @throws GeneratedLabelConflict if the name was already taken by a Rule or
+ * an OutputFile target.
+ * @throws IllegalArgumentException if the name is not a valid label
+ */
+ InputFile createInputFile(String targetName, Location location)
+ throws GeneratedLabelConflict {
+ Target existing = targets.get(targetName);
+ if (existing == null) {
+ try {
+ return addInputFile(createLabel(targetName), location);
+ } catch (Label.SyntaxException e) {
+ throw new IllegalArgumentException("FileTarget in package " + pkg.getName()
+ + " has illegal name: " + targetName);
+ }
+ } else if (existing instanceof InputFile) {
+ return (InputFile) existing; // idempotent
+ } else {
+ throw new GeneratedLabelConflict("generated label '//" + pkg.getName() + ":"
+ + targetName + "' conflicts with existing "
+ + existing.getTargetKind());
+ }
+ }
+
+ /**
+ * Sets the visibility and license for an input file. The input file must already exist as
+ * a member of this package.
+ * @throws IllegalArgumentException if the input file doesn't exist in this
+ * package's target map.
+ */
+ void setVisibilityAndLicense(InputFile inputFile, RuleVisibility visibility, License license) {
+ String filename = inputFile.getName();
+ Target cacheInstance = targets.get(filename);
+ if (cacheInstance == null || !(cacheInstance instanceof InputFile)) {
+ throw new IllegalArgumentException("Can't set visibility for nonexistent FileTarget "
+ + filename + " in package " + pkg.getName() + ".");
+ }
+ if (!((InputFile) cacheInstance).isVisibilitySpecified()
+ || cacheInstance.getVisibility() != visibility
+ || cacheInstance.getLicense() != license) {
+ targets.put(filename, new InputFile(
+ pkg, cacheInstance.getLabel(), cacheInstance.getLocation(), visibility, license));
+ }
+ }
+
+ /**
+ * Creates a label for a target inside this package.
+ *
+ * @throws SyntaxException if the {@code targetName} is invalid
+ */
+ Label createLabel(String targetName) throws SyntaxException {
+ return Label.create(pkg.getPackageIdentifier(), targetName);
+ }
+
+ /**
+ * Adds a package group to the package.
+ */
+ void addPackageGroup(String name, Collection<String> packages, Collection<Label> includes,
+ EventHandler eventHandler, Location location)
+ throws NameConflictException, Label.SyntaxException {
+ PackageGroup group =
+ new PackageGroup(createLabel(name), pkg, packages, includes, eventHandler, location);
+ Target existing = targets.get(group.getName());
+ if (existing != null) {
+ throw nameConflict(group, existing);
+ }
+
+ targets.put(group.getName(), group);
+
+ if (group.containsErrors()) {
+ setContainsErrors();
+ }
+ }
+
+ /**
+ * Checks if any labels in the given list appear multiple times and reports an appropriate
+ * error message if so. Returns true if no duplicates were found, false otherwise.
+ *
+ * TODO(bazel-team): apply this to all build functions (maybe automatically?), possibly
+ * integrate with RuleClass.checkForDuplicateLabels.
+ */
+ private static boolean checkForDuplicateLabels(Collection<Label> labels, String owner,
+ String attrName, Location location, EventHandler eventHandler) {
+ Set<Label> dupes = CollectionUtils.duplicatedElementsOf(labels);
+ for (Label dupe : dupes) {
+ eventHandler.handle(Event.error(location, String.format(
+ "label '%s' is duplicated in the '%s' list of '%s'", dupe, attrName, owner)));
+ }
+ return dupes.isEmpty();
+ }
+
+ /**
+ * Adds an environment group to the package.
+ */
+ void addEnvironmentGroup(String name, List<Label> environments, List<Label> defaults,
+ EventHandler eventHandler, Location location)
+ throws NameConflictException, SyntaxException {
+
+ if (!checkForDuplicateLabels(environments, name, "environments", location, eventHandler)
+ || !checkForDuplicateLabels(defaults, name, "defaults", location, eventHandler)) {
+ setContainsErrors();
+ return;
+ }
+
+ EnvironmentGroup group = new EnvironmentGroup(createLabel(name), pkg, environments,
+ defaults, location);
+ Target existing = targets.get(group.getName());
+ if (existing != null) {
+ throw nameConflict(group, existing);
+ }
+
+ targets.put(group.getName(), group);
+ Collection<Event> membershipErrors = group.validateMembership();
+ if (!membershipErrors.isEmpty()) {
+ for (Event error : membershipErrors) {
+ eventHandler.handle(error);
+ }
+ setContainsErrors();
+ return;
+ }
+
+ // For each declared environment, make sure it doesn't also belong to some other group.
+ for (Label environment : group.getEnvironments()) {
+ EnvironmentGroup otherGroup = environmentGroups.get(environment);
+ if (otherGroup != null) {
+ eventHandler.handle(Event.error(location, "environment " + environment + " belongs to"
+ + " both " + group.getLabel() + " and " + otherGroup.getLabel()));
+ setContainsErrors();
+ } else {
+ environmentGroups.put(environment, group);
+ }
+ }
+ }
+
+ void addRule(Rule rule) throws NameConflictException {
+ checkForConflicts(rule);
+ // Now, modify the package:
+ for (OutputFile outputFile : rule.getOutputFiles()) {
+ targets.put(outputFile.getName(), outputFile);
+ PathFragment outputFileFragment = new PathFragment(outputFile.getName());
+ for (int i = 1; i < outputFileFragment.segmentCount(); i++) {
+ String prefix = outputFileFragment.subFragment(0, i).toString();
+ if (!outputFilePrefixes.containsKey(prefix)) {
+ outputFilePrefixes.put(prefix, outputFile);
+ }
+ }
+ }
+ targets.put(rule.getName(), rule);
+ if (rule.containsErrors()) {
+ this.setContainsErrors();
+ }
+ }
+
+ private B beforeBuild() {
+ Preconditions.checkNotNull(pkg);
+ Preconditions.checkNotNull(filename);
+ Preconditions.checkNotNull(buildFileLabel);
+ Preconditions.checkNotNull(makeEnv);
+ // Freeze subincludes.
+ subincludes = (subincludes == null)
+ ? Collections.<Label, Path>emptyMap()
+ : Collections.unmodifiableMap(subincludes);
+
+ // We create the original BUILD InputFile when the package filename is set; however, the
+ // visibility may be overridden with an exports_files directive, so we need to obtain the
+ // current instance here.
+ buildFile = (InputFile) Preconditions.checkNotNull(targets.get(buildFileLabel.getName()));
+
+ List<Rule> rules = Lists.newArrayList(getTargets(Rule.class));
+
+ // All labels mentioned in a rule that refer to an unknown target in the
+ // current package are assumed to be InputFiles, so let's create them:
+ for (final Rule rule : rules) {
+ AggregatingAttributeMapper.of(rule).visitLabels(new AcceptsLabelAttribute() {
+ @Override
+ public void acceptLabelAttribute(Label label, Attribute attribute) {
+ createInputFileMaybe(label, rule.getAttributeLocation(attribute.getName()));
+ }
+ });
+ }
+
+ // "test_suite" rules have the idiosyncratic semantics of implicitly
+ // depending on all tests in the package, iff tests=[] and suites=[].
+ // Note, we implement this here when the Package is fully constructed,
+ // since clearly this information isn't available at Rule construction
+ // time, as forward references are permitted.
+ List<Label> allTests = new ArrayList<>();
+ for (Rule rule : rules) {
+ if (TargetUtils.isTestRule(rule) && !TargetUtils.hasManualTag(rule)
+ && !TargetUtils.isObsolete(rule)) {
+ allTests.add(rule.getLabel());
+ }
+ }
+ for (Rule rule : rules) {
+ AttributeMap attributes = NonconfigurableAttributeMapper.of(rule);
+ if (rule.getRuleClass().equals("test_suite")
+ && attributes.get("tests", Type.LABEL_LIST).isEmpty()
+ && attributes.get("suites", Type.LABEL_LIST).isEmpty()) {
+ rule.setAttributeValueByName("$implicit_tests", allTests);
+ }
+ }
+ return self();
+ }
+
+ /** Intended to be used only by {@link PackageFunction}. */
+ public B buildPartial() {
+ if (alreadyBuilt) {
+ return self();
+ }
+ return beforeBuild();
+ }
+
+ /** Intended to be used only by {@link PackageFunction}. */
+ public P finishBuild() {
+ if (alreadyBuilt) {
+ return pkg;
+ }
+ // Freeze targets and distributions.
+ targets = ImmutableMap.copyOf(targets);
+ defaultDistributionSet =
+ Collections.unmodifiableSet(defaultDistributionSet);
+
+ // Now all targets have been loaded, so we can check all declared environments in an
+ // environment group exist.
+ for (EnvironmentGroup envGroup : ImmutableSet.copyOf(environmentGroups.values())) {
+ Collection<Event> errors = envGroup.checkEnvironmentsExist(targets);
+ if (!errors.isEmpty()) {
+ addEvents(errors);
+ setContainsErrors();
+ }
+ }
+
+ // Build the package.
+ pkg.finishInit(this);
+ alreadyBuilt = true;
+ return pkg;
+ }
+
+ public P build() {
+ if (alreadyBuilt) {
+ return pkg;
+ }
+ beforeBuild();
+ return finishBuild();
+ }
+
+ /**
+ * If "label" refers to a non-existent target in the current package, create
+ * an InputFile target.
+ */
+ void createInputFileMaybe(Label label, Location location) {
+ if (label != null && label.getPackageFragment().equals(pkg.getNameFragment())) {
+ if (!targets.containsKey(label.getName())) {
+ addInputFile(label, location);
+ }
+ }
+ }
+
+ private InputFile addInputFile(Label label, Location location) {
+ InputFile inputFile = new InputFile(pkg, label, location);
+ Target prev = targets.put(label.getName(), inputFile);
+ Preconditions.checkState(prev == null);
+ return inputFile;
+ }
+
+ /**
+ * Precondition check for addRule. We must maintain these invariants of the
+ * package:
+ * - Each name refers to at most one target.
+ * - No rule with errors is inserted into the package.
+ * - The generating rule of every output file in the package must itself be
+ * in the package.
+ */
+ private void checkForConflicts(Rule rule) throws NameConflictException {
+ String name = rule.getName();
+ Target existing = targets.get(name);
+ if (existing != null) {
+ throw nameConflict(rule, existing);
+ }
+ Map<String, OutputFile> outputFiles = new HashMap<>();
+
+ for (OutputFile outputFile : rule.getOutputFiles()) {
+ String outputFileName = outputFile.getName();
+ if (outputFiles.put(outputFileName, outputFile) != null) { // dups within a single rule:
+ throw duplicateOutputFile(outputFile, outputFile);
+ }
+ existing = targets.get(outputFileName);
+ if (existing != null) {
+ throw duplicateOutputFile(outputFile, existing);
+ }
+
+ // Check if this output file is the prefix of an already existing one
+ if (outputFilePrefixes.containsKey(outputFileName)) {
+ throw conflictingOutputFile(outputFile, outputFilePrefixes.get(outputFileName));
+ }
+
+ // Check if a prefix of this output file matches an already existing one
+ PathFragment outputFileFragment = new PathFragment(outputFileName);
+ for (int i = 1; i < outputFileFragment.segmentCount(); i++) {
+ String prefix = outputFileFragment.subFragment(0, i).toString();
+ if (outputFiles.containsKey(prefix)) {
+ throw conflictingOutputFile(outputFile, outputFiles.get(prefix));
+ }
+ if (targets.containsKey(prefix)
+ && targets.get(prefix) instanceof OutputFile) {
+ throw conflictingOutputFile(outputFile, (OutputFile) targets.get(prefix));
+ }
+
+ if (!outputFilePrefixes.containsKey(prefix)) {
+ outputFilePrefixes.put(prefix, outputFile);
+ }
+ }
+ }
+
+ checkForInputOutputConflicts(rule, outputFiles.keySet());
+ }
+
+ /**
+ * A utility method that checks for conflicts between
+ * input file names and output file names for a rule from a build
+ * file.
+ * @param rule the rule whose inputs and outputs are
+ * to be checked for conflicts.
+ * @param outputFiles a set containing the names of output
+ * files to be generated by the rule.
+ * @throws NameConflictException if a conflict is found.
+ */
+ private void checkForInputOutputConflicts(Rule rule, Set<String> outputFiles)
+ throws NameConflictException {
+ PathFragment packageFragment = rule.getLabel().getPackageFragment();
+ for (Label inputLabel : rule.getLabels()) {
+ if (packageFragment.equals(inputLabel.getPackageFragment())
+ && outputFiles.contains(inputLabel.getName())) {
+ throw inputOutputNameConflict(rule, inputLabel.getName());
+ }
+ }
+ }
+
+ /** An output file conflicts with another output file or the BUILD file. */
+ private NameConflictException duplicateOutputFile(OutputFile duplicate, Target existing) {
+ return new NameConflictException(duplicate.getTargetKind() + " '" + duplicate.getName()
+ + "' in rule '" + duplicate.getGeneratingRule().getName() + "' "
+ + conflictsWith(existing));
+ }
+
+ /** The package contains two targets with the same name. */
+ private NameConflictException nameConflict(Target duplicate, Target existing) {
+ return new NameConflictException(duplicate.getTargetKind() + " '" + duplicate.getName()
+ + "' in package '" + duplicate.getLabel().getPackageName() + "' "
+ + conflictsWith(existing));
+ }
+
+ /** A a rule has a input/output name conflict. */
+ private NameConflictException inputOutputNameConflict(Rule rule, String conflictingName) {
+ return new NameConflictException("rule '" + rule.getName() + "' has file '"
+ + conflictingName + "' as both an input and an output");
+ }
+
+ private static NameConflictException conflictingOutputFile(
+ OutputFile added, OutputFile existing) {
+ if (added.getGeneratingRule() == existing.getGeneratingRule()) {
+ return new NameConflictException(String.format(
+ "rule '%s' has conflicting output files '%s' and '%s'", added.getGeneratingRule()
+ .getName(), added.getName(), existing.getName()));
+ } else {
+ return new NameConflictException(String.format(
+ "output file '%s' of rule '%s' conflicts with output file '%s' of rule '%s'", added
+ .getName(), added.getGeneratingRule().getName(), existing.getName(), existing
+ .getGeneratingRule().getName()));
+ }
+ }
+
+ /**
+ * Utility function for generating exception messages.
+ */
+ private static String conflictsWith(Target target) {
+ String message = "conflicts with existing ";
+ if (target instanceof OutputFile) {
+ return message + "generated file from rule '"
+ + ((OutputFile) target).getGeneratingRule().getName()
+ + "'";
+ } else {
+ return message + target.getTargetKind();
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java b/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java
new file mode 100644
index 0000000..5eca0f4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java
@@ -0,0 +1,536 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.packages.License.LicenseParsingException;
+import com.google.devtools.build.lib.packages.Package.AbstractBuilder.GeneratedLabelConflict;
+import com.google.devtools.build.lib.packages.Package.NameConflictException;
+import com.google.devtools.build.lib.packages.RuleClass.ParsedAttributeValue;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.StringDictUnaryEntry;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.syntax.GlobCriteria;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Functionality to deserialize loaded packages.
+ */
+public class PackageDeserializer {
+
+ // Workaround for Java serialization not allowing to pass in a context manually.
+ // volatile is needed to ensure that the objects are published safely.
+ // TODO(bazel-team): Subclass ObjectOutputStream to pass through environment variables.
+ public static volatile RuleClassProvider defaultRuleClassProvider;
+ public static volatile FileSystem defaultDeserializerFileSystem;
+
+ private class Context {
+ private final Package.Builder packageBuilder;
+ private final Path buildFilePath;
+
+ public Context(Path buildFilePath, Package.Builder packageBuilder) {
+ this.buildFilePath = buildFilePath;
+ this.packageBuilder = packageBuilder;
+ }
+
+ Location deserializeLocation(Build.Location location) {
+ return new ExplicitLocation(buildFilePath, location);
+ }
+
+ ParsedAttributeValue deserializeAttribute(Type<?> expectedType,
+ Build.Attribute attrPb)
+ throws PackageDeserializationException {
+ Object value = deserializeAttributeValue(expectedType, attrPb);
+ return new ParsedAttributeValue(
+ attrPb.hasExplicitlySpecified() ? attrPb.getExplicitlySpecified() : false,
+ value,
+ deserializeLocation(attrPb.getParseableLocation()));
+ }
+
+ void deserializeInputFile(Build.SourceFile sourceFile)
+ throws PackageDeserializationException {
+ InputFile inputFile;
+ try {
+ inputFile = packageBuilder.createInputFile(
+ deserializeLabel(sourceFile.getName()).getName(),
+ deserializeLocation(sourceFile.getParseableLocation()));
+ } catch (GeneratedLabelConflict e) {
+ throw new PackageDeserializationException(e);
+ }
+
+ if (!sourceFile.getVisibilityLabelList().isEmpty() || sourceFile.hasLicense()) {
+ packageBuilder.setVisibilityAndLicense(inputFile,
+ PackageFactory.getVisibility(deserializeLabels(sourceFile.getVisibilityLabelList())),
+ deserializeLicense(sourceFile.getLicense()));
+ }
+ }
+
+ void deserializePackageGroup(Build.PackageGroup packageGroupPb)
+ throws PackageDeserializationException {
+ List<String> specifications = new ArrayList<>();
+ for (String containedPackage : packageGroupPb.getContainedPackageList()) {
+ specifications.add("//" + containedPackage);
+ }
+
+ try {
+ packageBuilder.addPackageGroup(
+ deserializeLabel(packageGroupPb.getName()).getName(),
+ specifications,
+ deserializeLabels(packageGroupPb.getIncludedPackageGroupList()),
+ NullEventHandler.INSTANCE, // TODO(bazel-team): Handle errors properly
+ deserializeLocation(packageGroupPb.getParseableLocation()));
+ } catch (Label.SyntaxException | Package.NameConflictException e) {
+ throw new PackageDeserializationException(e);
+ }
+ }
+
+ void deserializeRule(Build.Rule rulePb)
+ throws PackageDeserializationException {
+ RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(rulePb.getRuleClass());
+ if (ruleClass == null) {
+ throw new PackageDeserializationException(
+ String.format("Invalid rule class '%s'", ruleClass));
+ }
+
+ Map<String, ParsedAttributeValue> attributeValues = new HashMap<>();
+ for (Build.Attribute attrPb : rulePb.getAttributeList()) {
+ Type<?> type = ruleClass.getAttributeByName(attrPb.getName()).getType();
+ attributeValues.put(attrPb.getName(), deserializeAttribute(type, attrPb));
+ }
+
+ Label ruleLabel = deserializeLabel(rulePb.getName());
+ Location ruleLocation = deserializeLocation(rulePb.getParseableLocation());
+ try {
+ Rule rule = ruleClass.createRuleWithParsedAttributeValues(
+ ruleLabel, packageBuilder, ruleLocation, attributeValues,
+ NullEventHandler.INSTANCE);
+ packageBuilder.addRule(rule);
+
+ Preconditions.checkState(!rule.containsErrors());
+ } catch (NameConflictException | SyntaxException e) {
+ throw new PackageDeserializationException(e);
+ }
+ }
+ }
+
+ private final FileSystem fileSystem;
+ private final RuleClassProvider ruleClassProvider;
+
+ @Immutable
+ private static final class ExplicitLocation extends Location {
+ private final PathFragment path;
+ private final int startLine;
+ private final int startColumn;
+ private final int endLine;
+ private final int endColumn;
+
+ private ExplicitLocation(Path path, Build.Location location) {
+ super(
+ location.hasStartOffset() && location.hasEndOffset() ? location.getStartOffset() : 0,
+ location.hasStartOffset() && location.hasEndOffset() ? location.getEndOffset() : 0);
+ this.path = path.asFragment();
+ if (location.hasStartLine() && location.hasStartColumn() &&
+ location.hasEndLine() && location.hasEndColumn()) {
+ this.startLine = location.getStartLine();
+ this.startColumn = location.getStartColumn();
+ this.endLine = location.getEndLine();
+ this.endColumn = location.getEndColumn();
+ } else {
+ this.startLine = 0;
+ this.startColumn = 0;
+ this.endLine = 0;
+ this.endColumn = 0;
+ }
+ }
+
+ @Override
+ public PathFragment getPath() {
+ return path;
+ }
+
+ @Override
+ public LineAndColumn getStartLineAndColumn() {
+ return new LineAndColumn(startLine, startColumn);
+ }
+
+ @Override
+ public LineAndColumn getEndLineAndColumn() {
+ return new LineAndColumn(endLine, endColumn);
+ }
+ }
+
+ public PackageDeserializer(FileSystem fileSystem, RuleClassProvider ruleClassProvider) {
+ if (fileSystem == null) {
+ fileSystem = defaultDeserializerFileSystem;
+ }
+ this.fileSystem = Preconditions.checkNotNull(fileSystem);
+ if (ruleClassProvider == null) {
+ ruleClassProvider = defaultRuleClassProvider;
+ }
+ this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider);
+ }
+
+ /**
+ * Exception thrown when something goes wrong during package deserialization.
+ */
+ public static class PackageDeserializationException extends Exception {
+ private PackageDeserializationException(String message) {
+ super(message);
+ }
+
+ private PackageDeserializationException(String message, Exception reason) {
+ super(message, reason);
+ }
+
+ private PackageDeserializationException(Exception reason) {
+ super(reason);
+ }
+ }
+
+ private static Label deserializeLabel(String labelName) throws PackageDeserializationException {
+ try {
+ return Label.parseRepositoryLabel(labelName);
+ } catch (Label.SyntaxException e) {
+ throw new PackageDeserializationException("Invalid label: " + e.getMessage(), e);
+ }
+ }
+
+ private static List<Label> deserializeLabels(List<String> labelNames)
+ throws PackageDeserializationException {
+ ImmutableList.Builder<Label> result = ImmutableList.builder();
+ for (String labelName : labelNames) {
+ result.add(deserializeLabel(labelName));
+ }
+
+ return result.build();
+ }
+
+ private static License deserializeLicense(Build.License licensePb)
+ throws PackageDeserializationException {
+ List<String> licenseStrings = new ArrayList<>();
+ licenseStrings.addAll(licensePb.getLicenseTypeList());
+ for (String exception : licensePb.getExceptionList()) {
+ licenseStrings.add("exception=" + exception);
+ }
+
+ try {
+ return License.parseLicense(licenseStrings);
+ } catch (LicenseParsingException e) {
+ throw new PackageDeserializationException(e);
+ }
+ }
+
+ private static Set<DistributionType> deserializeDistribs(List<String> distributions)
+ throws PackageDeserializationException {
+ try {
+ return License.parseDistributions(distributions);
+ } catch (LicenseParsingException e) {
+ throw new PackageDeserializationException(e);
+ }
+ }
+
+ private static TriState deserializeTriStateValue(String value)
+ throws PackageDeserializationException {
+ if (value.equals("yes")) {
+ return TriState.YES;
+ } else if (value.equals("no")) {
+ return TriState.NO;
+ } else if (value.equals("auto")) {
+ return TriState.AUTO;
+ } else {
+ throw new PackageDeserializationException(
+ String.format("Invalid tristate value: '%s'", value));
+ }
+ }
+
+ private static List<FilesetEntry> deserializeFilesetEntries(
+ List<Build.FilesetEntry> filesetPbs)
+ throws PackageDeserializationException {
+ ImmutableList.Builder<FilesetEntry> result = ImmutableList.builder();
+ for (Build.FilesetEntry filesetPb : filesetPbs) {
+ Label srcLabel = deserializeLabel(filesetPb.getSource());
+ List<Label> files =
+ filesetPb.getFilesPresent() ? deserializeLabels(filesetPb.getFileList()) : null;
+ List<String> excludes =
+ filesetPb.getExcludeList().isEmpty() ?
+ null : ImmutableList.copyOf(filesetPb.getExcludeList());
+ String destDir = filesetPb.getDestinationDirectory();
+ FilesetEntry.SymlinkBehavior symlinkBehavior =
+ pbToSymlinkBehavior(filesetPb.getSymlinkBehavior());
+ String stripPrefix = filesetPb.hasStripPrefix() ? filesetPb.getStripPrefix() : null;
+
+ result.add(
+ new FilesetEntry(srcLabel, files, excludes, destDir, symlinkBehavior, stripPrefix));
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Deserialize a package from its representation as a protocol message. The inverse of
+ * {@link PackageSerializer#serializePackage}.
+ */
+ private void deserializeInternal(Build.Package packagePb, StoredEventHandler eventHandler,
+ Package.Builder builder) throws PackageDeserializationException {
+ Path buildFile = fileSystem.getPath(packagePb.getBuildFilePath());
+ Preconditions.checkNotNull(buildFile);
+ Context context = new Context(buildFile, builder);
+ builder.setFilename(buildFile);
+
+ if (packagePb.hasDefaultVisibilitySet() && packagePb.getDefaultVisibilitySet()) {
+ builder.setDefaultVisibility(
+ PackageFactory.getVisibility(
+ deserializeLabels(packagePb.getDefaultVisibilityLabelList())));
+ }
+
+ // It's important to do this after setting the default visibility, since that implicitly sets
+ // this bit to true
+ builder.setDefaultVisibilitySet(packagePb.getDefaultVisibilitySet());
+ if (packagePb.hasDefaultObsolete()) {
+ builder.setDefaultObsolete(packagePb.getDefaultObsolete());
+ }
+ if (packagePb.hasDefaultTestonly()) {
+ builder.setDefaultTestonly(packagePb.getDefaultTestonly());
+ }
+ if (packagePb.hasDefaultDeprecation()) {
+ builder.setDefaultDeprecation(packagePb.getDefaultDeprecation());
+ }
+
+ builder.setDefaultCopts(packagePb.getDefaultCoptList());
+ if (packagePb.hasDefaultHdrsCheck()) {
+ builder.setDefaultHdrsCheck(packagePb.getDefaultHdrsCheck());
+ }
+ if (packagePb.hasDefaultLicense()) {
+ builder.setDefaultLicense(deserializeLicense(packagePb.getDefaultLicense()));
+ }
+ builder.setDefaultDistribs(deserializeDistribs(packagePb.getDefaultDistribList()));
+
+ for (String subinclude : packagePb.getSubincludeLabelList()) {
+ Label label = deserializeLabel(subinclude);
+ builder.addSubinclude(label, null);
+ }
+
+ ImmutableList.Builder<Label> skylarkFileDependencies = ImmutableList.builder();
+ for (String skylarkFile : packagePb.getSkylarkLabelList()) {
+ skylarkFileDependencies.add(deserializeLabel(skylarkFile));
+ }
+ builder.setSkylarkFileDependencies(skylarkFileDependencies.build());
+
+ MakeEnvironment.Builder makeEnvBuilder = new MakeEnvironment.Builder();
+ for (Build.MakeVar makeVar : packagePb.getMakeVariableList()) {
+ for (Build.MakeVarBinding binding : makeVar.getBindingList()) {
+ makeEnvBuilder.update(
+ makeVar.getName(), binding.getValue(), binding.getPlatformSetRegexp());
+ }
+ }
+ builder.setMakeEnv(makeEnvBuilder);
+
+ for (Build.SourceFile sourceFile : packagePb.getSourceFileList()) {
+ context.deserializeInputFile(sourceFile);
+ }
+
+ for (Build.PackageGroup packageGroupPb :
+ packagePb.getPackageGroupList()) {
+ context.deserializePackageGroup(packageGroupPb);
+ }
+
+ for (Build.Rule rulePb : packagePb.getRuleList()) {
+ context.deserializeRule(rulePb);
+ }
+
+ for (Build.Event event : packagePb.getEventList()) {
+ deserializeEvent(context, eventHandler, event);
+ }
+
+ if (packagePb.hasContainsErrors() && packagePb.getContainsErrors()) {
+ builder.setContainsErrors();
+ }
+ if (packagePb.hasContainsTemporaryErrors() && packagePb.getContainsTemporaryErrors()) {
+ builder.setContainsTemporaryErrors();
+ }
+ }
+
+ /**
+ * Deserialize a protocol message to a package. The inverse of
+ * {@link PackageSerializer#serializePackage}.
+ */
+ public Package deserialize(Build.Package packagePb)
+ throws PackageDeserializationException {
+ Package.Builder builder;
+ try {
+ builder = new Package.Builder(
+ new PackageIdentifier(packagePb.getRepository(), new PathFragment(packagePb.getName())));
+ } catch (SyntaxException e) {
+ throw new PackageDeserializationException(e);
+ }
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ deserializeInternal(packagePb, eventHandler, builder);
+ builder.addEvents(eventHandler.getEvents());
+ return builder.build();
+ }
+
+ private static void deserializeEvent(
+ Context context, StoredEventHandler eventHandler, Build.Event event) {
+ Location location = null;
+ if (event.hasLocation()) {
+ location = context.deserializeLocation(event.getLocation());
+ }
+
+ String message = event.getMessage();
+ switch (event.getKind()) {
+ case ERROR: eventHandler.handle(Event.error(location, message)); break;
+ case WARNING: eventHandler.handle(Event.warn(location, message)); break;
+ case INFO: eventHandler.handle(Event.info(location, message)); break;
+ case PROGRESS: eventHandler.handle(Event.progress(location, message)); break;
+ default: break; // Ignore
+ }
+ }
+
+ private static List<?> deserializeGlobs(List<?> matches,
+ Build.Attribute attrPb) {
+ if (attrPb.getGlobCriteriaCount() == 0) {
+ return matches;
+ }
+
+ Builder<GlobCriteria> criteriaBuilder = ImmutableList.builder();
+ for (Build.GlobCriteria criteriaPb : attrPb.getGlobCriteriaList()) {
+ if (criteriaPb.hasGlob() && criteriaPb.getGlob()) {
+ criteriaBuilder.add(GlobCriteria.fromGlobCall(
+ ImmutableList.copyOf(criteriaPb.getIncludeList()),
+ ImmutableList.copyOf(criteriaPb.getExcludeList())));
+ } else {
+ criteriaBuilder.add(
+ GlobCriteria.fromList(ImmutableList.copyOf(criteriaPb.getIncludeList())));
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"}) GlobList<?> result =
+ new GlobList(criteriaBuilder.build(), matches);
+ return result;
+ }
+
+ // TODO(bazel-team): Verify that these put sane values in the attribute
+ private static Object deserializeAttributeValue(Type<?> expectedType,
+ Build.Attribute attrPb)
+ throws PackageDeserializationException {
+ switch (attrPb.getType()) {
+ case INTEGER:
+ return new Integer(attrPb.getIntValue());
+
+ case STRING:
+ if (expectedType == Type.NODEP_LABEL) {
+ return deserializeLabel(attrPb.getStringValue());
+ } else {
+ return attrPb.getStringValue();
+ }
+
+ case LABEL:
+ case OUTPUT:
+ return deserializeLabel(attrPb.getStringValue());
+
+ case STRING_LIST:
+ if (expectedType == Type.NODEP_LABEL_LIST) {
+ return deserializeGlobs(deserializeLabels(attrPb.getStringListValueList()), attrPb);
+ } else {
+ return deserializeGlobs(ImmutableList.copyOf(attrPb.getStringListValueList()), attrPb);
+ }
+
+ case LABEL_LIST:
+ case OUTPUT_LIST:
+ return deserializeGlobs(deserializeLabels(attrPb.getStringListValueList()), attrPb);
+
+ case DISTRIBUTION_SET:
+ return deserializeDistribs(attrPb.getStringListValueList());
+
+ case LICENSE:
+ return deserializeLicense(attrPb.getLicense());
+
+ case STRING_DICT: {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ for (Build.StringDictEntry entry : attrPb.getStringDictValueList()) {
+ builder.put(entry.getKey(), entry.getValue());
+ }
+ return builder.build();
+ }
+
+ case STRING_DICT_UNARY: {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ for (StringDictUnaryEntry entry : attrPb.getStringDictUnaryValueList()) {
+ builder.put(entry.getKey(), entry.getValue());
+ }
+ return builder.build();
+ }
+
+ case FILESET_ENTRY_LIST:
+ return deserializeFilesetEntries(attrPb.getFilesetListValueList());
+
+ case LABEL_LIST_DICT: {
+ ImmutableMap.Builder<String, List<Label>> builder = ImmutableMap.builder();
+ for (Build.LabelListDictEntry entry : attrPb.getLabelListDictValueList()) {
+ builder.put(entry.getKey(), deserializeLabels(entry.getValueList()));
+ }
+ return builder.build();
+ }
+
+ case STRING_LIST_DICT: {
+ ImmutableMap.Builder<String, List<String>> builder = ImmutableMap.builder();
+ for (Build.StringListDictEntry entry : attrPb.getStringListDictValueList()) {
+ builder.put(entry.getKey(), ImmutableList.copyOf(entry.getValueList()));
+ }
+ return builder.build();
+ }
+
+ case BOOLEAN:
+ return attrPb.getBooleanValue();
+
+ case TRISTATE:
+ return deserializeTriStateValue(attrPb.getStringValue());
+
+ default:
+ throw new PackageDeserializationException("Invalid discriminator: " + attrPb.getType());
+ }
+ }
+
+ private static FilesetEntry.SymlinkBehavior pbToSymlinkBehavior(
+ Build.FilesetEntry.SymlinkBehavior symlinkBehavior) {
+ switch (symlinkBehavior) {
+ case COPY:
+ return FilesetEntry.SymlinkBehavior.COPY;
+ case DEREFERENCE:
+ return FilesetEntry.SymlinkBehavior.DEREFERENCE;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
new file mode 100644
index 0000000..abf63f9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
@@ -0,0 +1,1272 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.GlobCache.BadGlobException;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.AbstractFunction;
+import com.google.devtools.build.lib.syntax.AssignmentStatement;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Environment.NoSuchVariableException;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Expression;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Ident;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.MixedModeFunction;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.Statement;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * The package factory is responsible for constructing Package instances
+ * from a BUILD file's abstract syntax tree (AST).
+ *
+ * <p>A PackageFactory is a heavy-weight object; create them sparingly.
+ * Typically only one is needed per client application.
+ */
+public final class PackageFactory {
+ /**
+ * An argument to the {@code package()} function.
+ */
+ public abstract static class PackageArgument<T> {
+ private final String name;
+ private final Type<T> type;
+
+ protected PackageArgument(String name, Type<T> type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ private void convertAndProcess(
+ Package.LegacyBuilder pkgBuilder, Location location, Object value)
+ throws EvalException, ConversionException {
+ T typedValue = type.convert(value, "'package' argument", pkgBuilder.getBuildFileLabel());
+ process(pkgBuilder, location, typedValue);
+ }
+
+ /**
+ * Process an argument.
+ *
+ * @param pkgBuilder the package builder to be mutated
+ * @param location the location of the {@code package} function for error reporting
+ * @param value the value of the argument. Typically passed to {@link Type#convert}
+ */
+ protected abstract void process(
+ Package.LegacyBuilder pkgBuilder, Location location, T value)
+ throws EvalException;
+ }
+
+ /** Interface for evaluating globs during package loading. */
+ public static interface Globber {
+ /** An opaque token for fetching the result of a glob computation. */
+ abstract static class Token {}
+
+ /**
+ * Asynchronously starts the given glob computation and returns a token for fetching the
+ * result.
+ */
+ Token runAsync(List<String> includes, List<String> excludes, boolean excludeDirs)
+ throws BadGlobException;
+
+ /** Fetches the result of a previously started glob computation. */
+ List<String> fetch(Token token) throws IOException, InterruptedException;
+
+ /** Should be called when the globber is about to be discarded due to an interrupt. */
+ void onInterrupt();
+
+ /** Should be called when the globber is no longer needed. */
+ void onCompletion();
+
+ /** Returns all the glob computations requested before {@link #onCompletion} was called. */
+ Set<Pair<String, Boolean>> getGlobPatterns();
+ }
+
+ /**
+ * An extension to the global namespace of the BUILD language.
+ */
+ public interface EnvironmentExtension {
+ /**
+ * Update the global environment with the identifiers this extension contributes.
+ */
+ void update(Environment environment, MakeEnvironment.Builder pkgMakeEnv,
+ Label buildFileLabel);
+
+ Iterable<PackageArgument<?>> getPackageArguments();
+ }
+
+ private static final int EXCLUDE_DIR_DEFAULT = 1;
+
+ private static class DefaultVisibility extends PackageArgument<List<Label>> {
+ private DefaultVisibility() {
+ super("default_visibility", Type.LABEL_LIST);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ List<Label> value) {
+ pkgBuilder.setDefaultVisibility(getVisibility(value));
+ }
+ }
+
+ private static class DefaultObsolete extends PackageArgument<Boolean> {
+ private DefaultObsolete() {
+ super("default_obsolete", Type.BOOLEAN);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ Boolean value) {
+ pkgBuilder.setDefaultObsolete(value);
+ }
+ }
+
+ private static class DefaultTestOnly extends PackageArgument<Boolean> {
+ private DefaultTestOnly() {
+ super("default_testonly", Type.BOOLEAN);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ Boolean value) {
+ pkgBuilder.setDefaultTestonly(value);
+ }
+ }
+
+ private static class DefaultDeprecation extends PackageArgument<String> {
+ private DefaultDeprecation() {
+ super("default_deprecation", Type.STRING);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ String value) {
+ pkgBuilder.setDefaultDeprecation(value);
+ }
+ }
+
+ private static class Features extends PackageArgument<List<String>> {
+ private Features() {
+ super("features", Type.STRING_LIST);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ List<String> value) {
+ pkgBuilder.addFeatures(value);
+ }
+ }
+
+ private static class DefaultLicenses extends PackageArgument<License> {
+ private DefaultLicenses() {
+ super("licenses", Type.LICENSE);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ License value) {
+ pkgBuilder.setDefaultLicense(value);
+ }
+ }
+
+ private static class DefaultDistribs extends PackageArgument<Set<DistributionType>> {
+ private DefaultDistribs() {
+ super("distribs", Type.DISTRIBUTIONS);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ Set<DistributionType> value) {
+ pkgBuilder.setDefaultDistribs(value);
+ }
+ }
+
+ /**
+ * Declares the package() attribute specifying the default value for
+ * {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} when not explicitly specified.
+ */
+ private static class DefaultCompatibleWith extends PackageArgument<List<Label>> {
+ private DefaultCompatibleWith() {
+ super(Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE, Type.LABEL_LIST);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ List<Label> value) {
+ pkgBuilder.setDefaultCompatibleWith(value, Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE,
+ location);
+ }
+ }
+
+ /**
+ * Declares the package() attribute specifying the default value for
+ * {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} when not explicitly specified.
+ */
+ private static class DefaultRestrictedTo extends PackageArgument<List<Label>> {
+ private DefaultRestrictedTo() {
+ super(Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, Type.LABEL_LIST);
+ }
+
+ @Override
+ protected void process(Package.LegacyBuilder pkgBuilder, Location location,
+ List<Label> value) {
+ pkgBuilder.setDefaultRestrictedTo(value, Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, location);
+ }
+ }
+
+ public static final String PKG_CONTEXT = "$pkg_context";
+
+ /** {@link Globber} that uses the legacy GlobCache. */
+ public static class LegacyGlobber implements Globber {
+
+ private final GlobCache globCache;
+
+ public LegacyGlobber(GlobCache globCache) {
+ this.globCache = globCache;
+ }
+
+ private class Token extends Globber.Token {
+ public final List<String> includes;
+ public final List<String> excludes;
+ public final boolean excludeDirs;
+
+ public Token(List<String> includes, List<String> excludes, boolean excludeDirs) {
+ this.includes = includes;
+ this.excludes = excludes;
+ this.excludeDirs = excludeDirs;
+ }
+ }
+
+ @Override
+ public Set<Pair<String, Boolean>> getGlobPatterns() {
+ return globCache.getKeySet();
+ }
+
+ @Override
+ public Token runAsync(List<String> includes, List<String> excludes, boolean excludeDirs)
+ throws BadGlobException {
+ for (String pattern : Iterables.concat(includes, excludes)) {
+ globCache.getGlobAsync(pattern, excludeDirs);
+ }
+ return new Token(includes, excludes, excludeDirs);
+ }
+
+ @Override
+ public List<String> fetch(Globber.Token token) throws IOException, InterruptedException {
+ Token legacyToken = (Token) token;
+ try {
+ return globCache.glob(legacyToken.includes, legacyToken.excludes,
+ legacyToken.excludeDirs);
+ } catch (BadGlobException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public void onInterrupt() {
+ globCache.cancelBackgroundTasks();
+ }
+
+ @Override
+ public void onCompletion() {
+ globCache.finishBackgroundTasks();
+ }
+ }
+
+ private static final Logger LOG = Logger.getLogger(PackageFactory.class.getName());
+
+ private final RuleFactory ruleFactory;
+ private final RuleClassProvider ruleClassProvider;
+ private final Environment globalEnv;
+
+ private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls;
+ private Preprocessor.Factory preprocessorFactory = Preprocessor.Factory.NullFactory.INSTANCE;
+
+ private final ThreadPoolExecutor threadPool;
+ private Map<String, String> platformSetRegexps;
+
+ private final ImmutableList<EnvironmentExtension> environmentExtensions;
+ private final ImmutableMap<String, PackageArgument<?>> packageArguments;
+
+ /**
+ * Constructs a {@code PackageFactory} instance with the given rule factory.
+ */
+ public PackageFactory(RuleClassProvider ruleClassProvider) {
+ this(ruleClassProvider, null, ImmutableList.<EnvironmentExtension>of());
+ }
+
+ @VisibleForTesting
+ public PackageFactory(RuleClassProvider ruleClassProvider,
+ EnvironmentExtension environmentExtensions) {
+ this(ruleClassProvider, null, ImmutableList.of(environmentExtensions));
+ }
+
+ /**
+ * Constructs a {@code PackageFactory} instance with a specific glob path translator
+ * and rule factory.
+ */
+ @VisibleForTesting
+ public PackageFactory(RuleClassProvider ruleClassProvider,
+ Map<String, String> platformSetRegexps,
+ Iterable<EnvironmentExtension> environmentExtensions) {
+ this.platformSetRegexps = platformSetRegexps;
+ this.ruleFactory = new RuleFactory(ruleClassProvider);
+ this.ruleClassProvider = ruleClassProvider;
+ globalEnv = newGlobalEnvironment();
+ threadPool = new ThreadPoolExecutor(100, 100, 3L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ new ThreadFactoryBuilder().setNameFormat("PackageFactory %d").build());
+ // Do not consume threads when not in use.
+ threadPool.allowCoreThreadTimeOut(true);
+ this.environmentExtensions = ImmutableList.copyOf(environmentExtensions);
+ this.packageArguments = createPackageArguments();
+ }
+
+ /**
+ * Sets the preprocessor used.
+ */
+ public void setPreprocessorFactory(Preprocessor.Factory preprocessorFactory) {
+ this.preprocessorFactory = preprocessorFactory;
+ }
+
+ /**
+ * Sets the syscalls cache used in globbing.
+ */
+ public void setSyscalls(AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls) {
+ this.syscalls = Preconditions.checkNotNull(syscalls);
+ }
+
+ /**
+ * Returns the static environment initialized once and shared by all packages
+ * created by this factory. No updates occur to this environment once created.
+ */
+ @VisibleForTesting
+ public Environment getEnvironment() {
+ return globalEnv;
+ }
+
+ /**
+ * Returns the immutable, unordered set of names of all the known rule
+ * classes.
+ */
+ public Set<String> getRuleClassNames() {
+ return ruleFactory.getRuleClassNames();
+ }
+
+ /**
+ * Returns the {@link RuleClass} for the specified rule class name.
+ */
+ public RuleClass getRuleClass(String ruleClassName) {
+ return ruleFactory.getRuleClass(ruleClassName);
+ }
+
+ /**
+ * Returns the {@link RuleClassProvider} of this {@link PackageFactory}.
+ */
+ public RuleClassProvider getRuleClassProvider() {
+ return ruleClassProvider;
+ }
+
+ /**
+ * Creates the list of arguments for the 'package' function.
+ */
+ private ImmutableMap<String, PackageArgument<?>> createPackageArguments() {
+ ImmutableList.Builder<PackageArgument<?>> arguments =
+ ImmutableList.<PackageArgument<?>>builder()
+ .add(new DefaultDeprecation())
+ .add(new DefaultDistribs())
+ .add(new DefaultLicenses())
+ .add(new DefaultObsolete())
+ .add(new DefaultTestOnly())
+ .add(new DefaultVisibility())
+ .add(new Features())
+ .add(new DefaultCompatibleWith())
+ .add(new DefaultRestrictedTo());
+
+ for (EnvironmentExtension extension : environmentExtensions) {
+ arguments.addAll(extension.getPackageArguments());
+ }
+
+ ImmutableMap.Builder<String, PackageArgument<?>> packageArguments = ImmutableMap.builder();
+ for (PackageArgument<?> argument : arguments.build()) {
+ packageArguments.put(argument.getName(), argument);
+ }
+ return packageArguments.build();
+ }
+
+ /****************************************************************************
+ * Environment function factories.
+ */
+
+ /**
+ * Returns a function-value implementing "glob" in the specified package
+ * context.
+ *
+ * @param async if true, start globs in the background but don't block on their completion.
+ * Only use this for heuristic preloading.
+ */
+ private static Function newGlobFunction(
+ final PackageContext originalContext, final boolean async) {
+ List<String> params = ImmutableList.of("include", "exclude", "exclude_directories");
+ return new MixedModeFunction("glob", params, 1, false) {
+ @Override
+ public Object call(Object[] namedArguments, FuncallExpression ast, Environment env)
+ throws EvalException, ConversionException, InterruptedException {
+
+ // Skylark build extensions need to get the PackageContext from the Environment;
+ // async glob functions cannot do the same because the Environment is not thread safe.
+ PackageContext context;
+ if (originalContext == null) {
+ Preconditions.checkArgument(!async);
+ try {
+ context = (PackageContext) env.lookup(PKG_CONTEXT);
+ } catch (NoSuchVariableException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ } else {
+ context = originalContext;
+ }
+
+ List<String> includes = Type.STRING_LIST.convert(namedArguments[0], "'glob' argument");
+ List<String> excludes = namedArguments[1] == null
+ ? Collections.<String>emptyList()
+ : Type.STRING_LIST.convert(namedArguments[1], "'glob' argument");
+ int excludeDirs = namedArguments[2] == null
+ ? EXCLUDE_DIR_DEFAULT
+ : Type.INTEGER.convert(namedArguments[2], "'glob' argument");
+
+ if (async) {
+ try {
+ context.globber.runAsync(includes, excludes, excludeDirs != 0);
+ } catch (GlobCache.BadGlobException e) {
+ // Ignore: errors will appear during the actual evaluation of the package.
+ }
+ return GlobList.captureResults(includes, excludes, ImmutableList.<String>of());
+ } else {
+ return handleGlob(includes, excludes, excludeDirs != 0, context, ast);
+ }
+ }
+ };
+ }
+
+ /**
+ * Adds a glob to the package, reporting any errors it finds.
+ *
+ * @param includes the list of includes which must be non-null
+ * @param excludes the list of excludes which must be non-null
+ * @param context the package context
+ * @param ast the AST
+ * @return the list of matches
+ * @throws EvalException if globbing failed
+ */
+ private static GlobList<String> handleGlob(List<String> includes, List<String> excludes,
+ boolean excludeDirs, PackageContext context, FuncallExpression ast)
+ throws EvalException, InterruptedException {
+ try {
+ Globber.Token globToken = context.globber.runAsync(includes, excludes, excludeDirs);
+ List<String> matches = context.globber.fetch(globToken);
+ return GlobList.captureResults(includes, excludes, matches);
+ } catch (IOException expected) {
+ context.eventHandler.handle(Event.error(ast.getLocation(),
+ "error globbing [" + Joiner.on(", ").join(includes) + "]: " + expected.getMessage()));
+ context.pkgBuilder.setContainsTemporaryErrors();
+ return GlobList.captureResults(includes, excludes, ImmutableList.<String>of());
+ } catch (GlobCache.BadGlobException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+
+ /**
+ * Returns a function value implementing the "mocksubinclude" function,
+ * emitted by the PythonPreprocessor. We annotate the
+ * package with additional dependencies. (A 'real' subinclude will never be
+ * seen by the parser, because the presence of "subinclude" triggers
+ * preprocessing.)
+ */
+ private static Function newMockSubincludeFunction(final PackageContext context) {
+ return new MixedModeFunction("mocksubinclude", ImmutableList.of("label", "path"), 2, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast)
+ throws ConversionException {
+ Label label = Type.LABEL.convert(args[0], "'mocksubinclude' argument",
+ context.pkgBuilder.getBuildFileLabel());
+ String pathString = Type.STRING.convert(args[1], "'mocksubinclude' argument");
+ Path path = pathString.isEmpty()
+ ? null
+ : context.pkgBuilder.getFilename().getRelative(pathString);
+ // A subinclude within a package counts as a file declaration.
+ if (label.getPackageIdentifier().equals(context.pkgBuilder.getPackageIdentifier())) {
+ Location location = ast.getLocation();
+ if (location == null) {
+ location = Location.fromFile(context.pkgBuilder.getFilename());
+ }
+ context.pkgBuilder.createInputFileMaybe(label, location);
+ }
+
+ context.pkgBuilder.addSubinclude(label, path);
+ return Environment.NONE;
+ }
+ };
+ }
+
+ /**
+ * Fake function: subinclude calls are ignored
+ * They will disappear after the Python preprocessing.
+ */
+ private static Function newSubincludeFunction() {
+ return new MixedModeFunction("subinclude", ImmutableList.of("file"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) {
+ return Environment.NONE;
+ }
+ };
+ }
+
+ /**
+ * Returns a function value implementing "environment_group" in the specified package context.
+ * Syntax is as follows:
+ *
+ * <pre>{@code
+ * environment_group(
+ * name = "sample_group",
+ * environments = [":env1", ":env2", ...],
+ * defaults = [":env1", ...]
+ * )
+ * }</pre>
+ *
+ * <p>Where ":env1", "env2", ... are all environment rules declared in the same package. All
+ * parameters are mandatory.
+ */
+ private static Function newEnvironmentGroupFunction(final PackageContext context) {
+ List<String> params = ImmutableList.of("name", "environments", "defaults");
+ return new MixedModeFunction("environment_group", params, params.size(), true) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast)
+ throws EvalException, ConversionException {
+ Preconditions.checkState(namedArgs[0] != null);
+ String name = Type.STRING.convert(namedArgs[0], "'environment_group' argument");
+ Preconditions.checkState(namedArgs[1] != null);
+ List<Label> environments = Type.LABEL_LIST.convert(
+ namedArgs[1], "'environment_group argument'", context.pkgBuilder.getBuildFileLabel());
+ Preconditions.checkState(namedArgs[2] != null);
+ List<Label> defaults = Type.LABEL_LIST.convert(
+ namedArgs[2], "'environment_group argument'", context.pkgBuilder.getBuildFileLabel());
+
+ try {
+ context.pkgBuilder.addEnvironmentGroup(name, environments, defaults,
+ context.eventHandler, ast.getLocation());
+ return Environment.NONE;
+ } catch (Label.SyntaxException e) {
+ throw new EvalException(ast.getLocation(),
+ "environment group has invalid name: " + name + ": " + e.getMessage());
+ } catch (Package.NameConflictException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+ };
+ }
+
+ /**
+ * Returns a function-value implementing "exports_files" in the specified
+ * package context.
+ */
+ private static Function newExportsFilesFunction(final PackageContext context) {
+ final Package.LegacyBuilder pkgBuilder = context.pkgBuilder;
+ List<String> params = ImmutableList.of("srcs", "visibility", "licenses");
+ return new MixedModeFunction("exports_files", params, 1, false) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast)
+ throws EvalException, ConversionException {
+
+ List<String> files = Type.STRING_LIST.convert(namedArgs[0], "'exports_files' operand");
+
+ RuleVisibility visibility = namedArgs[1] == null
+ ? ConstantRuleVisibility.PUBLIC
+ : getVisibility(Type.LABEL_LIST.convert(
+ namedArgs[1],
+ "'exports_files' operand",
+ pkgBuilder.getBuildFileLabel()));
+ License license = namedArgs[2] == null
+ ? null
+ : Type.LICENSE.convert(namedArgs[2], "'exports_files' operand");
+
+ for (String file : files) {
+ String errorMessage = LabelValidator.validateTargetName(file);
+ if (errorMessage != null) {
+ throw new EvalException(ast.getLocation(), errorMessage);
+ }
+ try {
+ InputFile inputFile = pkgBuilder.createInputFile(file, ast.getLocation());
+ if (inputFile.isVisibilitySpecified()
+ && inputFile.getVisibility() != visibility) {
+ throw new EvalException(ast.getLocation(),
+ String.format("visibility for exported file '%s' declared twice",
+ inputFile.getName()));
+ }
+ if (license != null && inputFile.isLicenseSpecified()) {
+ throw new EvalException(ast.getLocation(),
+ String.format("licenses for exported file '%s' declared twice",
+ inputFile.getName()));
+ }
+ if (license == null && pkgBuilder.getDefaultLicense() == License.NO_LICENSE
+ && pkgBuilder.getBuildFileLabel().toString().startsWith("//third_party/")) {
+ throw new EvalException(ast.getLocation(),
+ "third-party file '" + inputFile.getName() + "' lacks a license declaration "
+ + "with one of the following types: notice, reciprocal, permissive, "
+ + "restricted, unencumbered, by_exception_only");
+ }
+
+ pkgBuilder.setVisibilityAndLicense(inputFile, visibility, license);
+ } catch (Package.Builder.GeneratedLabelConflict e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+ return Environment.NONE;
+ }
+ };
+ }
+
+ /**
+ * Returns a function-value implementing "licenses" in the specified package
+ * context.
+ * TODO(bazel-team): Remove in favor of package.licenses.
+ */
+ private static Function newLicensesFunction(final PackageContext context) {
+ return new MixedModeFunction("licenses", ImmutableList.of("object"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) {
+ try {
+ License license = Type.LICENSE.convert(args[0], "'licenses' operand");
+ context.pkgBuilder.setDefaultLicense(license);
+ } catch (ConversionException e) {
+ context.eventHandler.handle(Event.error(ast.getLocation(), e.getMessage()));
+ context.pkgBuilder.setContainsErrors();
+ }
+ return Environment.NONE;
+ }
+ };
+ }
+
+ /**
+ * Returns a function-value implementing "distribs" in the specified package
+ * context.
+ * TODO(bazel-team): Remove in favor of package.distribs.
+ */
+ private static Function newDistribsFunction(final PackageContext context) {
+ return new MixedModeFunction("distribs", ImmutableList.of("object"), 1, false) {
+ @Override
+ public Object call(Object[] args, FuncallExpression ast) {
+ try {
+ Set<DistributionType> distribs = Type.DISTRIBUTIONS.convert(args[0],
+ "'distribs' operand");
+ context.pkgBuilder.setDefaultDistribs(distribs);
+ } catch (ConversionException e) {
+ context.eventHandler.handle(Event.error(ast.getLocation(), e.getMessage()));
+ context.pkgBuilder.setContainsErrors();
+ }
+ return Environment.NONE;
+ }
+ };
+ }
+
+ private static Function newPackageGroupFunction(final PackageContext context) {
+ List<String> params = ImmutableList.of("name", "packages", "includes");
+ return new MixedModeFunction("package_group", params, 1, true) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast)
+ throws EvalException, ConversionException {
+ Preconditions.checkState(namedArgs[0] != null);
+ String name = Type.STRING.convert(namedArgs[0], "'package_group' argument");
+ List<String> packages = namedArgs[1] == null
+ ? Collections.<String>emptyList()
+ : Type.STRING_LIST.convert(namedArgs[1], "'package_group' argument");
+ List<Label> includes = namedArgs[2] == null
+ ? Collections.<Label>emptyList()
+ : Type.LABEL_LIST.convert(namedArgs[2], "'package_group argument'",
+ context.pkgBuilder.getBuildFileLabel());
+
+ try {
+ context.pkgBuilder.addPackageGroup(name, packages, includes, context.eventHandler,
+ ast.getLocation());
+ return Environment.NONE;
+ } catch (Label.SyntaxException e) {
+ throw new EvalException(ast.getLocation(),
+ "package group has invalid name: " + name + ": " + e.getMessage());
+ } catch (Package.NameConflictException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+ };
+ }
+
+ public static RuleVisibility getVisibility(List<Label> original) {
+ RuleVisibility result;
+
+ result = ConstantRuleVisibility.tryParse(original);
+ if (result != null) {
+ return result;
+ }
+
+ result = PackageGroupsRuleVisibility.tryParse(original);
+ return result;
+ }
+
+ /**
+ * Returns a function-value implementing "package" in the specified package
+ * context.
+ */
+ private static Function newPackageFunction(
+ final Map<String, PackageArgument<?>> packageArguments) {
+ return new MixedModeFunction("package", packageArguments.keySet(), 0, true) {
+ @Override
+ public Object call(Object[] namedArguments, FuncallExpression ast, Environment env)
+ throws EvalException, ConversionException {
+
+ Package.LegacyBuilder pkgBuilder = getContext(env, ast).pkgBuilder;
+
+ // Validate parameter list
+ if (pkgBuilder.isPackageFunctionUsed()) {
+ throw new EvalException(ast.getLocation(),
+ "'package' can only be used once per BUILD file");
+ }
+ pkgBuilder.setPackageFunctionUsed();
+
+ // Parse params
+ boolean foundParameter = false;
+
+ int argNumber = 0;
+ for (Map.Entry<String, PackageArgument<?>> entry : packageArguments.entrySet()) {
+ Object arg = namedArguments[argNumber];
+ argNumber += 1;
+ if (arg == null) {
+ continue;
+ }
+
+ foundParameter = true;
+ entry.getValue().convertAndProcess(pkgBuilder, ast.getLocation(), arg);
+ }
+
+ if (!foundParameter) {
+ throw new EvalException(ast.getLocation(),
+ "at least one argument must be given to the 'package' function");
+ }
+
+ return Environment.NONE;
+ }
+ };
+ }
+
+ // Helper function for createRuleFunction.
+ private static void addRule(RuleFactory ruleFactory,
+ String ruleClassName,
+ PackageContext context,
+ Map<String, Object> kwargs,
+ FuncallExpression ast)
+ throws RuleFactory.InvalidRuleException, Package.NameConflictException {
+ RuleClass ruleClass = getBuiltInRuleClass(ruleClassName, ruleFactory);
+ RuleFactory.createAndAddRule(context, ruleClass, kwargs, ast);
+ }
+
+ private static RuleClass getBuiltInRuleClass(String ruleClassName, RuleFactory ruleFactory) {
+ if (ruleFactory.getRuleClassNames().contains(ruleClassName)) {
+ return ruleFactory.getRuleClass(ruleClassName);
+ }
+ throw new IllegalArgumentException("no such rule class: " + ruleClassName);
+ }
+
+ /**
+ * Get the PackageContext by looking up in the environment.
+ */
+ private static PackageContext getContext(Environment env, FuncallExpression ast)
+ throws EvalException {
+ try {
+ return (PackageContext) env.lookup(PKG_CONTEXT);
+ } catch (NoSuchVariableException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+
+ /**
+ * Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the
+ * specified package context.
+ */
+ private static Function newRuleFunction(final RuleFactory ruleFactory,
+ final String ruleClass) {
+ return new AbstractFunction(ruleClass) {
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ Environment env)
+ throws EvalException {
+ if (!args.isEmpty()) {
+ throw new EvalException(ast.getLocation(),
+ "build rules do not accept positional parameters");
+ }
+
+ try {
+ addRule(ruleFactory, ruleClass, getContext(env, ast), kwargs, ast);
+ } catch (RuleFactory.InvalidRuleException | Package.NameConflictException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ return Environment.NONE;
+ }
+ };
+ }
+
+ /**
+ * Returns a new environment populated with common entries that can be shared
+ * across packages and that don't require the context.
+ */
+ private static Environment newGlobalEnvironment() {
+ Environment env = new Environment();
+ MethodLibrary.setupMethodEnvironment(env);
+ return env;
+ }
+
+ /****************************************************************************
+ * Package creation.
+ */
+
+ /**
+ * Loads, scans parses and evaluates the build file at "buildFile", and
+ * creates and returns a Package builder instance capable of building a package identified by
+ * "packageId".
+ *
+ * <p>This method returns a builder to allow the caller to do additional work, if necessary.
+ *
+ * <p>This method assumes "packageId" is a valid package name according to the
+ * {@link LabelValidator#validatePackageName} heuristic.
+ *
+ * <p>See {@link #evaluateBuildFile} for information on AST retention.
+ *
+ * <p>Executes {@code globber.onCompletion()} on completion and executes
+ * {@code globber.onInterrupt()} on an {@link InterruptedException}.
+ */
+ private Package.LegacyBuilder createPackage(PackageIdentifier packageId, Path buildFile,
+ List<Statement> preludeStatements, ParserInputSource inputSource,
+ Map<PathFragment, SkylarkEnvironment> imports, ImmutableList<Label> skylarkFileDependencies,
+ CachingPackageLocator locator, RuleVisibility defaultVisibility, Globber globber)
+ throws InterruptedException {
+ StoredEventHandler localReporter = new StoredEventHandler();
+ Preprocessor.Result preprocessingResult = preprocess(packageId, buildFile, inputSource, globber,
+ localReporter);
+ return createPackageFromPreprocessingResult(packageId, buildFile, preprocessingResult,
+ localReporter.getEvents(), preludeStatements, imports, skylarkFileDependencies, locator,
+ defaultVisibility, globber);
+ }
+
+ /**
+ * Same as {@link #createPackage}, but using a {@link Preprocessor.Result} from
+ * {@link #preprocess}.
+ *
+ * <p>Executes {@code globber.onCompletion()} on completion and executes
+ * {@code globber.onInterrupt()} on an {@link InterruptedException}.
+ */
+ // Used outside of bazel!
+ public Package.LegacyBuilder createPackageFromPreprocessingResult(PackageIdentifier packageId,
+ Path buildFile,
+ Preprocessor.Result preprocessingResult,
+ Iterable<Event> preprocessingEvents,
+ List<Statement> preludeStatements,
+ Map<PathFragment, SkylarkEnvironment> imports,
+ ImmutableList<Label> skylarkFileDependencies,
+ CachingPackageLocator locator,
+ RuleVisibility defaultVisibility,
+ Globber globber) throws InterruptedException {
+ StoredEventHandler localReporter = new StoredEventHandler();
+ // Run the lexer and parser with a local reporter, so that errors from other threads do not
+ // show up below. Merge the local and global reporters afterwards.
+ // Logged messages are used as a testability hook tracing the parsing progress
+ LOG.fine("Starting to parse " + packageId);
+ BuildFileAST buildFileAST = BuildFileAST.parseBuildFile(
+ preprocessingResult.result, preludeStatements, localReporter, locator, false);
+ LOG.fine("Finished parsing of " + packageId);
+
+ MakeEnvironment.Builder makeEnv = new MakeEnvironment.Builder();
+ if (platformSetRegexps != null) {
+ makeEnv.setPlatformSetRegexps(platformSetRegexps);
+ }
+ try {
+ // At this point the package is guaranteed to exist. It may have parse or
+ // evaluation errors, resulting in a diminished number of rules.
+ prefetchGlobs(packageId, buildFileAST, preprocessingResult.preprocessed,
+ buildFile, globber, defaultVisibility, makeEnv);
+ return evaluateBuildFile(
+ packageId, buildFileAST, buildFile, globber,
+ Iterables.concat(preprocessingEvents, localReporter.getEvents()),
+ defaultVisibility, preprocessingResult.containsErrors,
+ preprocessingResult.containsTransientErrors, makeEnv, imports, skylarkFileDependencies);
+ } catch (InterruptedException e) {
+ globber.onInterrupt();
+ throw e;
+ } finally {
+ globber.onCompletion();
+ }
+ }
+
+ /**
+ * Same as {@link #createPackage}, but does the required validation of "packageName" first,
+ * throwing a {@link NoSuchPackageException} if the name is invalid.
+ */
+ @VisibleForTesting
+ public Package createPackageForTesting(PackageIdentifier packageId,
+ Path buildFile,
+ CachingPackageLocator locator,
+ EventHandler eventHandler) throws NoSuchPackageException, InterruptedException {
+ String error = LabelValidator.validatePackageName(
+ packageId.getPackageFragment().getPathString());
+ if (error != null) {
+ throw new BuildFileNotFoundException(packageId.toString(),
+ "illegal package name: '" + packageId.toString() + "' (" + error + ")");
+ }
+ ParserInputSource inputSource = maybeGetParserInputSource(buildFile, eventHandler);
+ if (inputSource == null) {
+ throw new BuildFileContainsErrorsException(packageId.toString(), "IOException occured");
+ }
+ Package result = createPackage(packageId, buildFile,
+ ImmutableList.<Statement>of(), inputSource,
+ ImmutableMap.<PathFragment, SkylarkEnvironment>of(),
+ ImmutableList.<Label>of(),
+ locator, ConstantRuleVisibility.PUBLIC,
+ createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator)).build();
+ Event.replayEventsOn(eventHandler, result.getEvents());
+ return result;
+ }
+
+ /** Preprocesses the given BUILD file. */
+ // Used outside of bazel!
+ public Preprocessor.Result preprocess(
+ PackageIdentifier packageId,
+ Path buildFile,
+ CachingPackageLocator locator,
+ EventHandler eventHandler) throws InterruptedException {
+ ParserInputSource inputSource = maybeGetParserInputSource(buildFile, eventHandler);
+ if (inputSource == null) {
+ return Preprocessor.Result.transientError(buildFile);
+ }
+ Globber globber = createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator);
+ try {
+ return preprocess(packageId, buildFile, inputSource, globber, eventHandler);
+ } finally {
+ globber.onCompletion();
+ }
+ }
+
+ /**
+ * Preprocesses the given BUILD file, executing {@code globber.onInterrupt()} on an
+ * {@link InterruptedException}.
+ */
+ // Used outside of bazel!
+ public Preprocessor.Result preprocess(
+ PackageIdentifier packageId,
+ Path buildFile,
+ ParserInputSource inputSource,
+ Globber globber,
+ EventHandler eventHandler) throws InterruptedException {
+ Preprocessor preprocessor = preprocessorFactory.getPreprocessor();
+ if (preprocessor == null) {
+ return Preprocessor.Result.noPreprocessing(inputSource);
+ }
+ try {
+ return preprocessor.preprocess(inputSource, packageId.toString(), globber, eventHandler,
+ globalEnv, ruleFactory.getRuleClassNames());
+ } catch (IOException e) {
+ eventHandler.handle(Event.error(Location.fromFile(buildFile),
+ "preprocessing failed: " + e.getMessage()));
+ return Preprocessor.Result.transientError(buildFile);
+ } catch (InterruptedException e) {
+ globber.onInterrupt();
+ throw e;
+ }
+ }
+
+ // Used outside of bazel!
+ public LegacyGlobber createLegacyGlobber(Path packageDirectory, PackageIdentifier packageId,
+ CachingPackageLocator locator) {
+ return new LegacyGlobber(new GlobCache(packageDirectory, packageId, locator, syscalls,
+ threadPool));
+ }
+
+ @Nullable
+ private ParserInputSource maybeGetParserInputSource(Path buildFile, EventHandler eventHandler) {
+ try {
+ return ParserInputSource.create(buildFile);
+ } catch (IOException e) {
+ eventHandler.handle(Event.error(Location.fromFile(buildFile), e.getMessage()));
+ return null;
+ }
+ }
+
+ /**
+ * This tuple holds the current package builder, current lexer, etc, for the
+ * duration of the evaluation of one BUILD file. (We use a PackageContext
+ * object in preference to storing these values in mutable fields of the
+ * PackageFactory.)
+ *
+ * <p>PLEASE NOTE: references to PackageContext objects are held by many
+ * Function closures, but should become unreachable once the Environment is
+ * discarded at the end of evaluation. Please be aware of your memory
+ * footprint when making changes here!
+ */
+ public static class PackageContext {
+
+ final Package.LegacyBuilder pkgBuilder;
+ final Globber globber;
+ final EventHandler eventHandler;
+
+ @VisibleForTesting
+ public PackageContext(Package.LegacyBuilder pkgBuilder, Globber globber,
+ EventHandler eventHandler) {
+ this.pkgBuilder = pkgBuilder;
+ this.eventHandler = eventHandler;
+ this.globber = globber;
+ }
+ }
+
+ /**
+ * Returns the list of native rule functions created using the {@link RuleClassProvider}
+ * of this {@link PackageFactory}.
+ */
+ public ImmutableList<Function> collectNativeRuleFunctions() {
+ ImmutableList.Builder<Function> builder = ImmutableList.builder();
+ for (String ruleClass : ruleFactory.getRuleClassNames()) {
+ builder.add(newRuleFunction(ruleFactory, ruleClass));
+ }
+ builder.add(newGlobFunction(null, false));
+ builder.add(newPackageFunction(packageArguments));
+ return builder.build();
+ }
+
+ private void buildPkgEnv(Environment pkgEnv, String packageName,
+ MakeEnvironment.Builder pkgMakeEnv, PackageContext context, RuleFactory ruleFactory) {
+ pkgEnv.update("distribs", newDistribsFunction(context));
+ pkgEnv.update("glob", newGlobFunction(context, /*async=*/false));
+ pkgEnv.update("mocksubinclude", newMockSubincludeFunction(context));
+ pkgEnv.update("licenses", newLicensesFunction(context));
+ pkgEnv.update("exports_files", newExportsFilesFunction(context));
+ pkgEnv.update("package_group", newPackageGroupFunction(context));
+ pkgEnv.update("package", newPackageFunction(packageArguments));
+ pkgEnv.update("subinclude", newSubincludeFunction());
+ pkgEnv.update("environment_group", newEnvironmentGroupFunction(context));
+
+ pkgEnv.update("PACKAGE_NAME", packageName);
+
+ for (String ruleClass : ruleFactory.getRuleClassNames()) {
+ Function ruleFunction = newRuleFunction(ruleFactory, ruleClass);
+ pkgEnv.update(ruleClass, ruleFunction);
+ }
+
+ for (EnvironmentExtension extension : environmentExtensions) {
+ extension.update(pkgEnv, pkgMakeEnv, context.pkgBuilder.getBuildFileLabel());
+ }
+ }
+
+ /**
+ * Constructs a Package instance, evaluates the BUILD-file AST inside the
+ * build environment, and populates the package with Rule instances as it
+ * goes. As with most programming languages, evaluation stops when an
+ * exception is encountered: no further rules after the point of failure will
+ * be constructed. We assume that rules constructed before the point of
+ * failure are valid; this assumption is not entirely correct, since a
+ * "vardef" after a rule declaration can affect the behavior of that rule.
+ *
+ * <p>Rule attribute checking is performed during evaluation. Each attribute
+ * must conform to the type specified for that <i>(rule class, attribute
+ * name)</i> pair. Errors reported at this stage include: missing value for
+ * mandatory attribute, value of wrong type. Such error cause Rule
+ * construction to be aborted, so the resulting package will have missing
+ * members.
+ *
+ * @see PackageFactory#PackageFactory
+ */
+ @VisibleForTesting // used by PackageFactoryApparatus
+ public Package.LegacyBuilder evaluateBuildFile(PackageIdentifier packageId,
+ BuildFileAST buildFileAST, Path buildFilePath, Globber globber,
+ Iterable<Event> pastEvents, RuleVisibility defaultVisibility, boolean containsError,
+ boolean containsTransientError, MakeEnvironment.Builder pkgMakeEnv,
+ Map<PathFragment, SkylarkEnvironment> imports,
+ ImmutableList<Label> skylarkFileDependencies) throws InterruptedException {
+ // Important: Environment should be unreachable by the end of this method!
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ Environment pkgEnv = new Environment(globalEnv, eventHandler);
+
+ Package.LegacyBuilder pkgBuilder =
+ new Package.LegacyBuilder(packageId)
+ .setGlobber(globber)
+ .setFilename(buildFilePath)
+ .setMakeEnv(pkgMakeEnv)
+ .setDefaultVisibility(defaultVisibility)
+ // "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to
+ // set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag.
+ .setDefaultVisibilitySet(false)
+ .setSkylarkFileDependencies(skylarkFileDependencies);
+
+ Event.replayEventsOn(eventHandler, pastEvents);
+
+ // Stuff that closes over the package context:
+ PackageContext context = new PackageContext(pkgBuilder, globber, eventHandler);
+ buildPkgEnv(pkgEnv, packageId.toString(), pkgMakeEnv, context, ruleFactory);
+
+ if (containsError) {
+ pkgBuilder.setContainsErrors();
+ }
+
+ if (containsTransientError) {
+ pkgBuilder.setContainsTemporaryErrors();
+ }
+
+ if (!validatePackageIdentifier(packageId, buildFileAST.getLocation(), eventHandler)) {
+ pkgBuilder.setContainsErrors();
+ }
+
+ pkgEnv.setImportedExtensions(imports);
+ pkgEnv.updateAndPropagate(PKG_CONTEXT, context);
+ pkgEnv.updateAndPropagate(Environment.PKG_NAME, packageId.toString());
+
+ if (!validateAssignmentStatements(pkgEnv, buildFileAST, eventHandler)) {
+ pkgBuilder.setContainsErrors();
+ }
+
+ if (buildFileAST.containsErrors()) {
+ pkgBuilder.setContainsErrors();
+ }
+
+ // TODO(bazel-team): (2009) the invariant "if errors are reported, mark the package
+ // as containing errors" is strewn all over this class. Refactor to use an
+ // event sensor--and see if we can simplify the calling code in
+ // createPackage().
+ if (!buildFileAST.exec(pkgEnv, eventHandler)) {
+ pkgBuilder.setContainsErrors();
+ }
+
+ pkgBuilder.addEvents(eventHandler.getEvents());
+ return pkgBuilder;
+ }
+
+ /**
+ * Visit all targets and expand the globs in parallel.
+ */
+ private void prefetchGlobs(PackageIdentifier packageId, BuildFileAST buildFileAST,
+ boolean wasPreprocessed, Path buildFilePath, Globber globber,
+ RuleVisibility defaultVisibility, MakeEnvironment.Builder pkgMakeEnv)
+ throws InterruptedException {
+ if (wasPreprocessed) {
+ // No point in prefetching globs here: preprocessing implies eager evaluation
+ // of all globs.
+ return;
+ }
+ // Important: Environment should be unreachable by the end of this method!
+ Environment pkgEnv = new Environment();
+
+ Package.LegacyBuilder pkgBuilder =
+ new Package.LegacyBuilder(packageId)
+ .setFilename(buildFilePath)
+ .setMakeEnv(pkgMakeEnv)
+ .setDefaultVisibility(defaultVisibility)
+ // "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to
+ // set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag.
+ .setDefaultVisibilitySet(false);
+
+ // Stuff that closes over the package context:
+ PackageContext context = new PackageContext(pkgBuilder, globber, NullEventHandler.INSTANCE);
+ buildPkgEnv(pkgEnv, packageId.toString(), pkgMakeEnv, context, ruleFactory);
+ pkgEnv.update("glob", newGlobFunction(context, /*async=*/true));
+ // The Fileset function is heavyweight in that it can run glob(). Avoid this during the
+ // preloading phase.
+ pkgEnv.remove("FilesetEntry");
+
+ buildFileAST.exec(pkgEnv, NullEventHandler.INSTANCE);
+ }
+
+
+ /**
+ * Tests a build AST to ensure that it contains no assignment statements that
+ * redefine built-in build rules.
+ *
+ * @param pkgEnv a package environment initialized with all of the built-in
+ * build rules
+ * @param ast the build file AST to be tested
+ * @param eventHandler a eventHandler where any errors should be logged
+ * @return true if the build file contains no redefinitions of built-in
+ * functions
+ */
+ private static boolean validateAssignmentStatements(Environment pkgEnv,
+ BuildFileAST ast,
+ EventHandler eventHandler) {
+ for (Statement stmt : ast.getStatements()) {
+ if (stmt instanceof AssignmentStatement) {
+ Expression lvalue = ((AssignmentStatement) stmt).getLValue();
+ if (!(lvalue instanceof Ident)) {
+ continue;
+ }
+ String target = ((Ident) lvalue).getName();
+ if (pkgEnv.lookup(target, null) != null) {
+ eventHandler.handle(Event.error(stmt.getLocation(), "Reassignment of builtin build "
+ + "function '" + target + "' not permitted"));
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ // Reports an error and returns false iff package identifier was illegal.
+ private static boolean validatePackageIdentifier(PackageIdentifier packageId, Location location,
+ EventHandler eventHandler) {
+ String error = LabelValidator.validatePackageName(packageId.getPackageFragment().toString());
+ if (error != null) {
+ eventHandler.handle(Event.error(location, error));
+ return false; // Invalid package name 'foo'
+ }
+ return true;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageGroup.java b/src/main/java/com/google/devtools/build/lib/packages/PackageGroup.java
new file mode 100644
index 0000000..17ba29c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageGroup.java
@@ -0,0 +1,152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This class represents a package group. It has a name and a set of packages
+ * and can be asked if a specific package is included in it. The package set is
+ * represented as a list of PathFragments.
+ */
+public class PackageGroup implements Target {
+ private boolean containsErrors;
+ private final Label label;
+ private final Location location;
+ private final Package containingPackage;
+ private final List<PackageSpecification> packageSpecifications;
+ private final List<Label> includes;
+
+ public PackageGroup(Label label, Package pkg, Collection<String> packages,
+ Collection<Label> includes, EventHandler eventHandler, Location location) {
+ this.label = label;
+ this.location = location;
+ this.containingPackage = pkg;
+ this.includes = ImmutableList.copyOf(includes);
+
+ ImmutableList.Builder<PackageSpecification> packagesBuilder = ImmutableList.builder();
+ for (String containedPackage : packages) {
+ PackageSpecification specification = null;
+ try {
+ specification = PackageSpecification.fromString(containedPackage);
+ } catch (PackageSpecification.InvalidPackageSpecificationException e) {
+ containsErrors = true;
+ eventHandler.handle(Event.error(location, e.getMessage()));
+ }
+
+ if (specification != null) {
+ packagesBuilder.add(specification);
+ }
+ }
+ this.packageSpecifications = packagesBuilder.build();
+ }
+
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ public Iterable<PackageSpecification> getPackageSpecifications() {
+ return packageSpecifications;
+ }
+
+ public boolean contains(Package pkg) {
+ for (PackageSpecification specification : packageSpecifications) {
+ if (specification.containsPackage(pkg.getNameFragment())) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ public List<Label> getIncludes() {
+ return includes;
+ }
+
+ public List<String> getContainedPackages() {
+ List<String> result = Lists.newArrayListWithCapacity(packageSpecifications.size());
+ for (PackageSpecification specification : packageSpecifications) {
+ result.add(specification.toString());
+ }
+ return result;
+ }
+
+ @Override
+ public Rule getAssociatedRule() {
+ return null;
+ }
+
+ @Override
+ public Set<DistributionType> getDistributions() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override public String getName() {
+ return label.getName();
+ }
+
+ @Override
+ public License getLicense() {
+ return License.NO_LICENSE;
+ }
+
+ @Override
+ public Package getPackage() {
+ return containingPackage;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return targetKind();
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return targetKind() + " " + getLabel();
+ }
+
+ @Override
+ public RuleVisibility getVisibility() {
+ // Package groups are always public to avoid a PackageGroupConfiguredTarget
+ // needing itself for the visibility check. It may work, but I did not
+ // think it over completely.
+ return ConstantRuleVisibility.PUBLIC;
+ }
+
+ public static String targetKind() {
+ return "package group";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageGroupsRuleVisibility.java b/src/main/java/com/google/devtools/build/lib/packages/PackageGroupsRuleVisibility.java
new file mode 100644
index 0000000..70ffa11
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageGroupsRuleVisibility.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A rule visibility that allows visibility to a list of package groups.
+ */
+@Immutable @ThreadSafe
+public class PackageGroupsRuleVisibility implements RuleVisibility {
+ public static final String PACKAGE_LABEL = "__pkg__";
+ public static final String SUBTREE_LABEL = "__subpackages__";
+ private final List<Label> packageGroups;
+ private final List<PackageSpecification> directPackages;
+ private final List<Label> declaredLabels;
+
+ public PackageGroupsRuleVisibility(List<Label> labels) {
+ declaredLabels = ImmutableList.copyOf(labels);
+ ImmutableList.Builder<PackageSpecification> directPackageBuilder = ImmutableList.builder();
+ ImmutableList.Builder<Label> packageGroupBuilder = ImmutableList.builder();
+
+ for (Label label : labels) {
+ PackageSpecification specification = PackageSpecification.fromLabel(label);
+ if (specification != null) {
+ directPackageBuilder.add(specification);
+ } else {
+ packageGroupBuilder.add(label);
+ }
+ }
+
+ packageGroups = packageGroupBuilder.build();
+ directPackages = directPackageBuilder.build();
+ }
+
+ public Collection<Label> getPackageGroups() {
+ return packageGroups;
+ }
+
+ public Collection<PackageSpecification> getDirectPackages() {
+ return directPackages;
+ }
+
+ @Override
+ public List<Label> getDependencyLabels() {
+ return packageGroups;
+ }
+
+ @Override
+ public List<Label> getDeclaredLabels() {
+ return declaredLabels;
+ }
+
+ /**
+ * Tries to parse a list of labels into a {@link PackageGroupsRuleVisibility}.
+ *
+ * @param labels the list of labels to parse
+ * @return The resulting visibility object. A list of labels can always be
+ * parsed into a PackageGroupsRuleVisibility.
+ */
+ public static PackageGroupsRuleVisibility tryParse(List<Label> labels) {
+ return new PackageGroupsRuleVisibility(labels);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageIdentifier.java b/src/main/java/com/google/devtools/build/lib/packages/PackageIdentifier.java
new file mode 100644
index 0000000..7b16e3c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageIdentifier.java
@@ -0,0 +1,271 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.Canonicalizer;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Uniquely identifies a package, given a repository name and a package's path fragment.
+ *
+ * <p>The repository the build is happening in is the <i>default workspace</i>, and is identified
+ * by the workspace name "". Other repositories can be named in the WORKSPACE file. These
+ * workspaces are prefixed by {@literal @}.</p>
+ */
+@Immutable
+public final class PackageIdentifier implements Comparable<PackageIdentifier>, Serializable {
+
+ /**
+ * A human-readable name for the repository.
+ */
+ public static final class RepositoryName {
+ private final String name;
+
+ /**
+ * Makes sure that name is a valid repository name and creates a new RepositoryName using it.
+ * @throws SyntaxException if the name is invalid.
+ */
+ public static RepositoryName create(String name) throws SyntaxException {
+ String errorMessage = validate(name);
+ if (errorMessage != null) {
+ errorMessage = "invalid repository name '"
+ + StringUtilities.sanitizeControlChars(name) + "': " + errorMessage;
+ throw new SyntaxException(errorMessage);
+ }
+ return new RepositoryName(StringCanonicalizer.intern(name));
+ }
+
+ private RepositoryName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Performs validity checking. Returns null on success, an error message otherwise.
+ */
+ private static String validate(String name) {
+ if (name.isEmpty()) {
+ return null;
+ }
+
+ if (!name.startsWith("@")) {
+ return "workspace name must start with '@'";
+ }
+
+ // "@" isn't a valid workspace name.
+ if (name.length() == 1) {
+ return "empty workspace name";
+ }
+
+ // Check for any character outside of [/0-9A-Z_a-z-]. Try to evaluate the
+ // conditional quickly (by looking in decreasing order of character class
+ // likelihood).
+ for (int i = name.length() - 1; i >= 1; --i) {
+ char c = name.charAt(i);
+ if ((c < 'a' || c > 'z') && c != '_' && c != '-'
+ && (c < '0' || c > '9') && (c < 'A' || c > 'Z')) {
+ return "workspace names may contain only A-Z, a-z, 0-9, '-' and '_'";
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the repository name without the leading "{@literal @}". For the default repository,
+ * returns "".
+ */
+ public String strippedName() {
+ if (name.isEmpty()) {
+ return name;
+ }
+ return name.substring(1);
+ }
+
+ /**
+ * Returns if this is the default repository, that is, {@link #name} is "".
+ */
+ public boolean isDefault() {
+ return name.isEmpty();
+ }
+
+ /**
+ * Returns the repository name, with leading "{@literal @}" (or "" for the default repository).
+ */
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (this == object) {
+ return true;
+ }
+ if (object instanceof RepositoryName) {
+ return name.equals(((RepositoryName) object).name);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+ }
+
+ public static final String DEFAULT_REPOSITORY = "";
+
+ /**
+ * Helper for serializing PackageIdentifiers.
+ *
+ * <p>PackageIdentifier's field should be final, but then it couldn't be deserialized. This
+ * allows the fields to be deserialized and copied into a new PackageIdentifier.</p>
+ */
+ private static final class SerializationProxy implements Serializable {
+ PackageIdentifier packageId;
+
+ public SerializationProxy(PackageIdentifier packageId) {
+ this.packageId = packageId;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeObject(packageId.repository.toString());
+ out.writeObject(packageId.pkgName);
+ }
+
+ private void readObject(ObjectInputStream in)
+ throws IOException, ClassNotFoundException {
+ try {
+ packageId = new PackageIdentifier((String) in.readObject(), (PathFragment) in.readObject());
+ } catch (SyntaxException e) {
+ throw new IOException("Error serializing package identifier: " + e.getMessage());
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private void readObjectNoData() throws ObjectStreamException {
+ }
+
+ private Object readResolve() {
+ return packageId;
+ }
+ }
+
+ // Temporary factory for identifiers without explicit repositories.
+ // TODO(bazel-team): remove all usages of this.
+ public static PackageIdentifier createInDefaultRepo(String name) {
+ return createInDefaultRepo(new PathFragment(name));
+ }
+
+ public static PackageIdentifier createInDefaultRepo(PathFragment name) {
+ try {
+ return new PackageIdentifier(DEFAULT_REPOSITORY, name);
+ } catch (SyntaxException e) {
+ throw new IllegalArgumentException("could not create package identifier for " + name
+ + ": " + e.getMessage());
+ }
+ }
+
+ /**
+ * The identifier for this repository. This is either "" or prefixed with an "@",
+ * e.g., "@myrepo".
+ */
+ private final RepositoryName repository;
+
+ /** The name of the package. Canonical (i.e. x.equals(y) <=> x==y). */
+ private final PathFragment pkgName;
+
+ public PackageIdentifier(String repository, PathFragment pkgName) throws SyntaxException {
+ this(RepositoryName.create(repository), pkgName);
+ }
+
+ public PackageIdentifier(RepositoryName repository, PathFragment pkgName) {
+ Preconditions.checkNotNull(repository);
+ Preconditions.checkNotNull(pkgName);
+ this.repository = repository;
+ this.pkgName = Canonicalizer.fragments().intern(pkgName);
+ }
+
+ private Object writeReplace() throws ObjectStreamException {
+ return new SerializationProxy(this);
+ }
+
+ private void readObject(ObjectInputStream in)
+ throws IOException, ClassNotFoundException {
+ throw new IOException("Serialization is allowed only by proxy");
+ }
+
+ @SuppressWarnings("unused")
+ private void readObjectNoData() throws ObjectStreamException {
+ }
+
+ public RepositoryName getRepository() {
+ return repository;
+ }
+
+ public PathFragment getPackageFragment() {
+ return pkgName;
+ }
+
+ /**
+ * Returns the name of this package.
+ *
+ * <p>There are certain places that expect the path fragment as the package name ('foo/bar') as a
+ * package identifier. This isn't specific enough for packages in other repositories, so their
+ * stringified version is '@baz//foo/bar'.</p>
+ */
+ @Override
+ public String toString() {
+ return (repository.isDefault() ? "" : repository + "//") + pkgName;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (this == object) {
+ return true;
+ }
+ if (object instanceof PackageIdentifier) {
+ PackageIdentifier that = (PackageIdentifier) object;
+ return repository.equals(that.repository) && pkgName.equals(that.pkgName);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(repository, pkgName);
+ }
+
+ @Override
+ public int compareTo(PackageIdentifier that) {
+ return ComparisonChain.start()
+ .compare(repository.toString(), that.repository.toString())
+ .compare(pkgName, that.pkgName)
+ .result();
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageLoadedEvent.java b/src/main/java/com/google/devtools/build/lib/packages/PackageLoadedEvent.java
new file mode 100644
index 0000000..3dbf3c2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageLoadedEvent.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+/**
+ * An event that is fired after a package is loaded.
+ */
+public final class PackageLoadedEvent {
+ private final String packageName;
+ private final long timeInMillis;
+ private final boolean reloading;
+ private final boolean successful;
+
+ public PackageLoadedEvent(String packageName, long timeInMillis, boolean reloading,
+ boolean successful) {
+ this.packageName = packageName;
+ this.timeInMillis = timeInMillis;
+ this.reloading = reloading;
+ this.successful = successful;
+ }
+
+ /**
+ * Returns the package name.
+ */
+ public String getPackageName() {
+ return packageName;
+ }
+
+ /**
+ * Returns time which was spent to load a package.
+ */
+ public long getTimeInMillis() {
+ return timeInMillis;
+ }
+
+ /**
+ * Returns true if package had been loaded before.
+ */
+ public boolean isReloading() {
+ return reloading;
+ }
+
+ /**
+ * Returns true if package was loaded successfully.
+ */
+ public boolean isSuccessful() {
+ return successful;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageNotInCacheException.java b/src/main/java/com/google/devtools/build/lib/packages/PackageNotInCacheException.java
new file mode 100644
index 0000000..f191a1c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageNotInCacheException.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Exception indicating that a package is not in the cache, although it should
+ * have been loaded.
+ */
+public class PackageNotInCacheException extends NoSuchPackageException {
+ public PackageNotInCacheException(String packageName) {
+ super(packageName, "package '" + packageName + "' not in cache");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java b/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java
new file mode 100644
index 0000000..8971cf2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java
@@ -0,0 +1,462 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS;
+import static com.google.devtools.build.lib.packages.Type.FILESET_ENTRY_LIST;
+import static com.google.devtools.build.lib.packages.Type.INTEGER;
+import static com.google.devtools.build.lib.packages.Type.INTEGER_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT_UNARY;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.packages.MakeEnvironment.Binding;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.syntax.GlobCriteria;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Functionality to serialize loaded packages.
+ */
+public class PackageSerializer {
+ private static Build.SourceFile serializeInputFile(InputFile inputFile) {
+ Build.SourceFile.Builder result = Build.SourceFile.newBuilder();
+ result.setName(inputFile.getLabel().toString());
+ if (inputFile.isVisibilitySpecified()) {
+ for (Label visibilityLabel : inputFile.getVisibility().getDeclaredLabels()) {
+ result.addVisibilityLabel(visibilityLabel.toString());
+ }
+ }
+ if (inputFile.isLicenseSpecified()) {
+ result.setLicense(serializeLicense(inputFile.getLicense()));
+ }
+
+ result.setParseableLocation(serializeLocation(inputFile.getLocation()));
+ return result.build();
+ }
+
+ private static Build.Location serializeLocation(Location location) {
+ Build.Location.Builder result = Build.Location.newBuilder();
+
+ result.setStartOffset(location.getStartOffset());
+ if (location.getStartLineAndColumn() != null) {
+ result.setStartLine(location.getStartLineAndColumn().getLine());
+ result.setStartColumn(location.getStartLineAndColumn().getColumn());
+ }
+
+ result.setEndOffset(location.getEndOffset());
+ if (location.getEndLineAndColumn() != null) {
+ result.setEndLine(location.getEndLineAndColumn().getLine());
+ result.setEndColumn(location.getEndLineAndColumn().getColumn());
+ }
+
+ return result.build();
+ }
+
+ private static Build.PackageGroup serializePackageGroup(PackageGroup packageGroup) {
+ Build.PackageGroup.Builder result = Build.PackageGroup.newBuilder();
+
+ result.setName(packageGroup.getLabel().toString());
+ result.setParseableLocation(serializeLocation(packageGroup.getLocation()));
+
+ for (PackageSpecification packageSpecification : packageGroup.getPackageSpecifications()) {
+ result.addContainedPackage(packageSpecification.toString());
+ }
+
+ for (Label include : packageGroup.getIncludes()) {
+ result.addIncludedPackageGroup(include.toString());
+ }
+
+ return result.build();
+ }
+
+ private static Build.Rule serializeRule(Rule rule) {
+ Build.Rule.Builder result = Build.Rule.newBuilder();
+ result.setName(rule.getLabel().toString());
+ result.setRuleClass(rule.getRuleClass());
+ result.setParseableLocation(serializeLocation(rule.getLocation()));
+ for (Attribute attribute : rule.getAttributes()) {
+ Object value = attribute.getName().equals("visibility")
+ ? rule.getVisibility().getDeclaredLabels()
+ // TODO(bazel-team): support configurable attributes. AggregatingAttributeMapper
+ // may make more sense here. Computed defaults may add complications.
+ : RawAttributeMapper.of(rule).get(attribute.getName(), attribute.getType());
+ if (value != null) {
+ PackageSerializer.addAttributeToProto(result, attribute, value,
+ rule.getAttributeLocation(attribute.getName()),
+ rule.isAttributeValueExplicitlySpecified(attribute),
+ true);
+ }
+ }
+
+ return result.build();
+ }
+
+ private static List<Build.MakeVar> serializeMakeEnvironment(MakeEnvironment makeEnv) {
+ List<Build.MakeVar> result = new ArrayList<>();
+
+ for (Map.Entry<String, ImmutableList<Binding>> var : makeEnv.getBindings().entrySet()) {
+ Build.MakeVar.Builder varPb = Build.MakeVar.newBuilder();
+ varPb.setName(var.getKey());
+ for (Binding binding : var.getValue()) {
+ Build.MakeVarBinding.Builder bindingPb = Build.MakeVarBinding.newBuilder();
+ bindingPb.setValue(binding.getValue());
+ bindingPb.setPlatformSetRegexp(binding.getPlatformSetRegexp());
+ varPb.addBinding(bindingPb);
+ }
+
+ result.add(varPb.build());
+ }
+
+ return result;
+ }
+
+ private static Build.License serializeLicense(License license) {
+ Build.License.Builder result = Build.License.newBuilder();
+
+ for (License.LicenseType licenseType : license.getLicenseTypes()) {
+ result.addLicenseType(licenseType.toString());
+ }
+
+ for (Label exception : license.getExceptions()) {
+ result.addException(exception.toString());
+ }
+ return result.build();
+ }
+
+ private static Build.Event serializeEvent(Event event) {
+ Build.Event.Builder result = Build.Event.newBuilder();
+ result.setMessage(event.getMessage());
+ if (event.getLocation() != null) {
+ result.setLocation(serializeLocation(event.getLocation()));
+ }
+
+ Build.Event.EventKind kind;
+
+ switch (event.getKind()) {
+ case ERROR:
+ kind = Build.Event.EventKind.ERROR;
+ break;
+ case WARNING:
+ kind = Build.Event.EventKind.WARNING;
+ break;
+ case INFO:
+ kind = Build.Event.EventKind.INFO;
+ break;
+ case PROGRESS:
+ kind = Build.Event.EventKind.PROGRESS;
+ break;
+ default: throw new IllegalStateException();
+ }
+
+ result.setKind(kind);
+ return result.build();
+ }
+
+ private static void serializePackageInternal(Package pkg, Build.Package.Builder builder) {
+ builder.setName(pkg.getName());
+ builder.setRepository(pkg.getPackageIdentifier().getRepository().toString());
+ builder.setBuildFilePath(pkg.getFilename().getPathString());
+ // The extra bit is needed to handle the corner case when the default visibility is [], i.e.
+ // zero labels.
+ builder.setDefaultVisibilitySet(pkg.isDefaultVisibilitySet());
+ if (pkg.isDefaultVisibilitySet()) {
+ for (Label visibilityLabel : pkg.getDefaultVisibility().getDeclaredLabels()) {
+ builder.addDefaultVisibilityLabel(visibilityLabel.toString());
+ }
+ }
+
+ builder.setDefaultObsolete(pkg.getDefaultObsolete());
+ builder.setDefaultTestonly(pkg.getDefaultTestOnly());
+ if (pkg.getDefaultDeprecation() != null) {
+ builder.setDefaultDeprecation(pkg.getDefaultDeprecation());
+ }
+
+ for (String defaultCopt : pkg.getDefaultCopts()) {
+ builder.addDefaultCopt(defaultCopt);
+ }
+
+ if (pkg.isDefaultHdrsCheckSet()) {
+ builder.setDefaultHdrsCheck(pkg.getDefaultHdrsCheck());
+ }
+
+ builder.setDefaultLicense(serializeLicense(pkg.getDefaultLicense()));
+
+ for (DistributionType distributionType : pkg.getDefaultDistribs()) {
+ builder.addDefaultDistrib(distributionType.toString());
+ }
+
+ for (String feature : pkg.getFeatures()) {
+ builder.addDefaultSetting(feature);
+ }
+
+ for (Label subincludeLabel : pkg.getSubincludeLabels()) {
+ builder.addSubincludeLabel(subincludeLabel.toString());
+ }
+
+ for (Label skylarkLabel : pkg.getSkylarkFileDependencies()) {
+ builder.addSkylarkLabel(skylarkLabel.toString());
+ }
+
+ for (Build.MakeVar makeVar :
+ serializeMakeEnvironment(pkg.getMakeEnvironment())) {
+ builder.addMakeVariable(makeVar);
+ }
+
+ for (Target target : pkg.getTargets()) {
+ if (target instanceof InputFile) {
+ builder.addSourceFile(serializeInputFile((InputFile) target));
+ } else if (target instanceof OutputFile) {
+ // Output files are ignored; they are recorded in rules.
+ } else if (target instanceof PackageGroup) {
+ builder.addPackageGroup(serializePackageGroup((PackageGroup) target));
+ } else if (target instanceof Rule) {
+ builder.addRule(serializeRule((Rule) target));
+ }
+ }
+
+ for (Event event : pkg.getEvents()) {
+ builder.addEvent(serializeEvent(event));
+ }
+
+ builder.setContainsErrors(pkg.containsErrors());
+ builder.setContainsTemporaryErrors(pkg.containsTemporaryErrors());
+ }
+
+ /**
+ * Serialize a package to a protocol message. The inverse of
+ * {@link PackageDeserializer#deserialize}.
+ */
+ public static Build.Package serializePackage(Package pkg) {
+ Build.Package.Builder builder = Build.Package.newBuilder();
+ serializePackageInternal(pkg, builder);
+ return builder.build();
+ }
+
+ /**
+ * Adds the serialized version of the specified attribute to the specified message.
+ *
+ * @param result the message to amend
+ * @param attr the attribute to add
+ * @param value the value of the attribute
+ * @param location the location of the attribute in the source file
+ * @param explicitlySpecified whether the attribute was explicitly specified or not
+ * @param includeGlobs add glob expression for attributes that contain them
+ */
+ // TODO(bazel-team): This is a copy of the code in ProtoOutputFormatter.
+ @SuppressWarnings("unchecked")
+ public static void addAttributeToProto(
+ Build.Rule.Builder result, Attribute attr, Object value, Location location,
+ Boolean explicitlySpecified, boolean includeGlobs) {
+ // Get the attribute type. We need to convert and add appropriately
+ com.google.devtools.build.lib.packages.Type<?> type = attr.getType();
+
+ Build.Attribute.Builder attrPb = Build.Attribute.newBuilder();
+
+ // Set the type, name and source
+ Build.Attribute.Discriminator attributeProtoType = ProtoUtils.getDiscriminatorFromType(type);
+ attrPb.setName(attr.getName());
+ attrPb.setType(attributeProtoType);
+
+ if (location != null) {
+ attrPb.setParseableLocation(serializeLocation(location));
+ }
+
+ if (explicitlySpecified != null) {
+ attrPb.setExplicitlySpecified(explicitlySpecified);
+ }
+
+ /*
+ * Set the appropriate type and value. Since string and string list store
+ * values for multiple types, use the toString() method on the objects
+ * instead of casting them. Note that Boolean and TriState attributes have
+ * both an integer and string representation.
+ */
+ if (type == INTEGER) {
+ attrPb.setIntValue((Integer) value);
+ } else if (type == STRING || type == LABEL || type == NODEP_LABEL || type == OUTPUT) {
+ attrPb.setStringValue(value.toString());
+ } else if (type == STRING_LIST || type == LABEL_LIST || type == NODEP_LABEL_LIST
+ || type == OUTPUT_LIST || type == DISTRIBUTIONS) {
+ Collection<?> values = (Collection<?>) value;
+ for (Object entry : values) {
+ attrPb.addStringListValue(entry.toString());
+ }
+ } else if (type == INTEGER_LIST) {
+ Collection<Integer> values = (Collection<Integer>) value;
+ for (Integer entry : values) {
+ attrPb.addIntListValue(entry);
+ }
+ } else if (type == BOOLEAN) {
+ if ((Boolean) value) {
+ attrPb.setStringValue("true");
+ attrPb.setBooleanValue(true);
+ } else {
+ attrPb.setStringValue("false");
+ attrPb.setBooleanValue(false);
+ }
+ // This maintains partial backward compatibility for external users of the
+ // protobuf that were expecting an integer field and not a true boolean.
+ attrPb.setIntValue((Boolean) value ? 1 : 0);
+ } else if (type == TRISTATE) {
+ switch ((TriState) value) {
+ case AUTO:
+ attrPb.setIntValue(-1);
+ attrPb.setStringValue("auto");
+ attrPb.setTristateValue(Build.Attribute.Tristate.AUTO);
+ break;
+ case NO:
+ attrPb.setIntValue(0);
+ attrPb.setStringValue("no");
+ attrPb.setTristateValue(Build.Attribute.Tristate.NO);
+ break;
+ case YES:
+ attrPb.setIntValue(1);
+ attrPb.setStringValue("yes");
+ attrPb.setTristateValue(Build.Attribute.Tristate.YES);
+ break;
+ }
+ } else if (type == LICENSE) {
+ License license = (License) value;
+ Build.License.Builder licensePb = Build.License.newBuilder();
+ for (License.LicenseType licenseType : license.getLicenseTypes()) {
+ licensePb.addLicenseType(licenseType.toString());
+ }
+ for (Label exception : license.getExceptions()) {
+ licensePb.addException(exception.toString());
+ }
+ attrPb.setLicense(licensePb);
+ } else if (type == STRING_DICT) {
+ Map<String, String> dict = (Map<String, String>) value;
+ for (Map.Entry<String, String> dictEntry : dict.entrySet()) {
+ Build.StringDictEntry entry = Build.StringDictEntry.newBuilder()
+ .setKey(dictEntry.getKey())
+ .setValue(dictEntry.getValue())
+ .build();
+ attrPb.addStringDictValue(entry);
+ }
+ } else if (type == STRING_DICT_UNARY) {
+ Map<String, String> dict = (Map<String, String>) value;
+ for (Map.Entry<String, String> dictEntry : dict.entrySet()) {
+ Build.StringDictUnaryEntry entry = Build.StringDictUnaryEntry.newBuilder()
+ .setKey(dictEntry.getKey())
+ .setValue(dictEntry.getValue())
+ .build();
+ attrPb.addStringDictUnaryValue(entry);
+ }
+ } else if (type == STRING_LIST_DICT) {
+ Map<String, List<String>> dict = (Map<String, List<String>>) value;
+ for (Map.Entry<String, List<String>> dictEntry : dict.entrySet()) {
+ Build.StringListDictEntry.Builder entry = Build.StringListDictEntry.newBuilder()
+ .setKey(dictEntry.getKey());
+ for (Object dictEntryValue : dictEntry.getValue()) {
+ entry.addValue(dictEntryValue.toString());
+ }
+ attrPb.addStringListDictValue(entry);
+ }
+ } else if (type == LABEL_LIST_DICT) {
+ Map<String, List<Label>> dict = (Map<String, List<Label>>) value;
+ for (Map.Entry<String, List<Label>> dictEntry : dict.entrySet()) {
+ Build.LabelListDictEntry.Builder entry = Build.LabelListDictEntry.newBuilder()
+ .setKey(dictEntry.getKey());
+ for (Object dictEntryValue : dictEntry.getValue()) {
+ entry.addValue(dictEntryValue.toString());
+ }
+ attrPb.addLabelListDictValue(entry);
+ }
+ } else if (type == FILESET_ENTRY_LIST) {
+ List<FilesetEntry> filesetEntries = (List<FilesetEntry>) value;
+ for (FilesetEntry filesetEntry : filesetEntries) {
+ Build.FilesetEntry.Builder filesetEntryPb = Build.FilesetEntry.newBuilder()
+ .setSource(filesetEntry.getSrcLabel().toString())
+ .setDestinationDirectory(filesetEntry.getDestDir().getPathString())
+ .setSymlinkBehavior(symlinkBehaviorToPb(filesetEntry.getSymlinkBehavior()))
+ .setStripPrefix(filesetEntry.getStripPrefix())
+ .setFilesPresent(filesetEntry.getFiles() != null);
+
+ if (filesetEntry.getFiles() != null) {
+ for (Label file : filesetEntry.getFiles()) {
+ filesetEntryPb.addFile(file.toString());
+ }
+ }
+
+ if (filesetEntry.getExcludes() != null) {
+ for (String exclude : filesetEntry.getExcludes()) {
+ filesetEntryPb.addExclude(exclude);
+ }
+ }
+
+ attrPb.addFilesetListValue(filesetEntryPb);
+ }
+ } else {
+ throw new IllegalStateException("Unknown type: " + type);
+ }
+
+ if (includeGlobs && value instanceof GlobList<?>) {
+ GlobList<?> globList = (GlobList<?>) value;
+
+ for (GlobCriteria criteria : globList.getCriteria()) {
+ Build.GlobCriteria.Builder criteriaPb = Build.GlobCriteria.newBuilder();
+ criteriaPb.setGlob(criteria.isGlob());
+ for (String include : criteria.getIncludePatterns()) {
+ criteriaPb.addInclude(include);
+ }
+ for (String exclude : criteria.getExcludePatterns()) {
+ criteriaPb.addExclude(exclude);
+ }
+
+ attrPb.addGlobCriteria(criteriaPb);
+ }
+ }
+ result.addAttribute(attrPb);
+ }
+
+ // This is needed because I do not want to use the SymlinkBehavior from the
+ // protocol buffer all over the place, so there are two classes that do
+ // essentially the same thing.
+ private static Build.FilesetEntry.SymlinkBehavior symlinkBehaviorToPb(
+ FilesetEntry.SymlinkBehavior symlinkBehavior) {
+ switch (symlinkBehavior) {
+ case COPY:
+ return Build.FilesetEntry.SymlinkBehavior.COPY;
+ case DEREFERENCE:
+ return Build.FilesetEntry.SymlinkBehavior.DEREFERENCE;
+ default:
+ throw new AssertionError("Unhandled FilesetEntry.SymlinkBehavior");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageSpecification.java b/src/main/java/com/google/devtools/build/lib/packages/PackageSpecification.java
new file mode 100644
index 0000000..20524aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageSpecification.java
@@ -0,0 +1,150 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A class that represents some packages that are included in the visibility list of a rule.
+ */
+public abstract class PackageSpecification {
+ private static final String PACKAGE_LABEL = "__pkg__";
+ private static final String SUBTREE_LABEL = "__subpackages__";
+ private static final String ALL_BENEATH_SUFFIX = "/...";
+ public static final PackageSpecification EVERYTHING =
+ new AllPackagesBeneath(new PathFragment(""));
+
+ public abstract boolean containsPackage(PathFragment packageName);
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+
+ if (!(that instanceof PackageSpecification)) {
+ return false;
+ }
+
+ return this.toString().equals(that.toString());
+ }
+
+ /**
+ * Parses a string as a visibility specification.
+ * Throws {@link InvalidPackageSpecificationException} if the label cannot be parsed.
+ *
+ * <p>Note that these strings are different from what {@link #fromLabel} understands.
+ */
+ public static PackageSpecification fromString(final String spec)
+ throws InvalidPackageSpecificationException {
+ String result = spec;
+ boolean allBeneath = false;
+
+ if (result.startsWith("//")) {
+ result = spec.substring(2);
+ } else {
+ throw new InvalidPackageSpecificationException("invalid package label: " + spec);
+ }
+
+ if (result.indexOf(':') >= 0) {
+ throw new InvalidPackageSpecificationException("invalid package label: " + spec);
+ }
+
+ if (result.equals("...")) {
+ // Special case: //... will not end in /...
+ return EVERYTHING;
+ }
+
+ if (result.endsWith(ALL_BENEATH_SUFFIX)) {
+ allBeneath = true;
+ result = result.substring(0, result.length() - ALL_BENEATH_SUFFIX.length());
+ }
+
+ String errorMessage = LabelValidator.validatePackageName(result);
+ if (errorMessage == null) {
+ return allBeneath ?
+ new AllPackagesBeneath(new PathFragment(result)) :
+ new SinglePackage(new PathFragment(result));
+ } else {
+ throw new InvalidPackageSpecificationException(errorMessage);
+ }
+ }
+
+ /**
+ * Parses a label as a visibility specification. returns null if the label cannot be parsed.
+ *
+ * <p>Note that these strings are different from what {@link #fromString} understands.
+ */
+ public static PackageSpecification fromLabel(Label label) {
+ if (label.getName().equals(PACKAGE_LABEL)) {
+ return new SinglePackage(label.getPackageFragment());
+ } else if (label.getName().equals(SUBTREE_LABEL)) {
+ return new AllPackagesBeneath(label.getPackageFragment());
+ } else {
+ return null;
+ }
+ }
+
+ private static class SinglePackage extends PackageSpecification {
+ private PathFragment singlePackageName;
+
+ public SinglePackage(PathFragment packageName) {
+ this.singlePackageName = packageName;
+ }
+
+ @Override
+ public boolean containsPackage(PathFragment packageName) {
+ return this.singlePackageName.equals(packageName);
+ }
+
+ @Override
+ public String toString() {
+ return singlePackageName.toString();
+ }
+ }
+
+ private static class AllPackagesBeneath extends PackageSpecification {
+ private PathFragment prefix;
+
+ public AllPackagesBeneath(PathFragment prefix) {
+ this.prefix = prefix;
+ }
+
+ @Override
+ public boolean containsPackage(PathFragment packageName) {
+ return packageName.startsWith(prefix);
+ }
+
+ @Override
+ public String toString() {
+ return prefix.equals(new PathFragment("")) ? "..." : prefix.toString() + "/...";
+ }
+ }
+
+ /**
+ * Exception class to be thrown when a specification cannot be parsed.
+ */
+ public static class InvalidPackageSpecificationException extends Exception {
+ public InvalidPackageSpecificationException(String message) {
+ super(message);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PredicateWithMessage.java b/src/main/java/com/google/devtools/build/lib/packages/PredicateWithMessage.java
new file mode 100644
index 0000000..4316f02
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PredicateWithMessage.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Predicate;
+
+/**
+ * A predicate which supports error messages.
+ * @param <T> - the predicate is applied on T objects
+ */
+public interface PredicateWithMessage<T> extends Predicate<T> {
+
+ /**
+ * The error message to display when predicate checks param. Only makes sense to
+ * call this method is apply(param) returns false.
+ */
+ public String getErrorReason(T param);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PredicatesWithMessage.java b/src/main/java/com/google/devtools/build/lib/packages/PredicatesWithMessage.java
new file mode 100644
index 0000000..a7f04bf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/PredicatesWithMessage.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+/**
+ * A helper class for PredicateWithMessage with default predicates.
+ */
+public abstract class PredicatesWithMessage implements PredicateWithMessage<Object> {
+
+ private static final PredicateWithMessage<?> ALWAYS_TRUE = new PredicateWithMessage<Object>() {
+ @Override
+ public boolean apply(Object input) {
+ return true;
+ }
+
+ @Override
+ public String getErrorReason(Object param) {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ @SuppressWarnings("unchecked")
+ public static <T> PredicateWithMessage<T> alwaysTrue() {
+ return (PredicateWithMessage<T>) ALWAYS_TRUE;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Preprocessor.java b/src/main/java/com/google/devtools/build/lib/packages/Preprocessor.java
new file mode 100644
index 0000000..8eda1e9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/Preprocessor.java
@@ -0,0 +1,155 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.PackageFactory.Globber;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/** A Preprocessor is an interface to implement generic text-based preprocessing of BUILD files. */
+public interface Preprocessor {
+ /** Factory for {@link Preprocessor} instances. */
+ interface Factory {
+ /** Supplier for {@link Factory} instances. */
+ interface Supplier {
+ /**
+ * Returns a Preprocessor factory to use for getting Preprocessor instances.
+ *
+ * <p>The CachingPackageLocator is provided so the constructed preprocessors can look up
+ * other BUILD files.
+ */
+ Factory getFactory(CachingPackageLocator loc);
+
+ /** Supplier that always returns {@code NullFactory.INSTANCE}. */
+ static class NullSupplier implements Supplier {
+
+ public static final NullSupplier INSTANCE = new NullSupplier();
+
+ private NullSupplier() {
+ }
+
+ @Override
+ public Factory getFactory(CachingPackageLocator loc) {
+ return NullFactory.INSTANCE;
+ }
+ }
+ }
+
+ /**
+ * Returns whether this {@link Factory} is still suitable for providing {@link Preprocessor}s.
+ * If not, all previous preprocessing results should be assumed to be invalid and a new
+ * {@link Factory} should be created via {@link Supplier#getFactory}.
+ */
+ boolean isStillValid();
+
+ /**
+ * Returns a Preprocessor instance capable of preprocessing a BUILD file independently (e.g. it
+ * ought to be fine to call {@link #getPreprocessor} for each BUILD file).
+ */
+ @Nullable
+ Preprocessor getPreprocessor();
+
+ /** Factory that always returns {@code null} {@link Preprocessor}s. */
+ static class NullFactory implements Factory {
+ public static final NullFactory INSTANCE = new NullFactory();
+
+ private NullFactory() {
+ }
+
+ @Override
+ public boolean isStillValid() {
+ return true;
+ }
+
+ @Override
+ public Preprocessor getPreprocessor() {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * A (result, success) tuple indicating the outcome of preprocessing.
+ */
+ static class Result {
+ public final ParserInputSource result;
+ public final boolean preprocessed;
+ public final boolean containsErrors;
+ public final boolean containsTransientErrors;
+
+ private Result(ParserInputSource result,
+ boolean preprocessed, boolean containsPersistentErrors, boolean containsTransientErrors) {
+ this.result = result;
+ this.preprocessed = preprocessed;
+ this.containsErrors = containsPersistentErrors || containsTransientErrors;
+ this.containsTransientErrors = containsTransientErrors;
+ }
+
+ /** Convenience factory for a {@link Result} wrapping non-preprocessed BUILD file contents. */
+ public static Result noPreprocessing(ParserInputSource buildFileSource) {
+ return new Result(buildFileSource, /*preprocessed=*/false, /*containsErrors=*/false,
+ /*containsTransientErrors=*/false);
+ }
+
+ /**
+ * Factory for a successful preprocessing result, meaning that the BUILD file was able to be
+ * read and has valid syntax and was preprocessed. But note that there may have been be errors
+ * during preprocessing.
+ */
+ public static Result success(ParserInputSource result, boolean containsErrors) {
+ return new Result(result, /*preprocessed=*/true, /*containsPersistentErrors=*/containsErrors,
+ /*containsTransientErrors=*/false);
+ }
+
+ public static Result invalidSyntax(Path buildFile) {
+ return new Result(ParserInputSource.create("", buildFile), /*preprocessed=*/true,
+ /*containsPersistentErrors=*/true, /*containsTransientErrors=*/false);
+ }
+
+ public static Result transientError(Path buildFile) {
+ return new Result(ParserInputSource.create("", buildFile), /*preprocessed=*/false,
+ /*containsPersistentErrors=*/false, /*containsTransientErrors=*/true);
+ }
+ }
+
+ /**
+ * Returns a Result resulting from applying Python preprocessing to the contents of "in". If
+ * errors happen, they must be reported both as an event on eventHandler and in the function
+ * return value.
+ *
+ * @param in the BUILD file to be preprocessed.
+ * @param packageName the BUILD file's package.
+ * @param globber a globber for evaluating globs.
+ * @param eventHandler a eventHandler on which to report warnings/errors.
+ * @param globalEnv the GLOBALS Python environment.
+ * @param ruleNames the set of names of all rules in the build language.
+ * @throws IOException if there was an I/O problem during preprocessing.
+ * @return a pair of the ParserInputSource and a map of subincludes seen during the evaluation
+ */
+ Result preprocess(
+ ParserInputSource in,
+ String packageName,
+ Globber globber,
+ EventHandler eventHandler,
+ Environment globalEnv,
+ Set<String> ruleNames)
+ throws IOException, InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ProtoUtils.java b/src/main/java/com/google/devtools/build/lib/packages/ProtoUtils.java
new file mode 100644
index 0000000..7b6eaf1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/ProtoUtils.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS;
+import static com.google.devtools.build.lib.packages.Type.FILESET_ENTRY_LIST;
+import static com.google.devtools.build.lib.packages.Type.INTEGER;
+import static com.google.devtools.build.lib.packages.Type.INTEGER_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT_UNARY;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.Attribute.Discriminator;
+
+import java.util.Map;
+
+/**
+ * Shared code used in proto buffer output for rules and rule classes.
+ */
+public class ProtoUtils {
+ /**
+ * This map contains all attribute types that are recognized by the protocol
+ * output formatter.
+ */
+ private static final Map<Type<?>, Discriminator> TYPE_MAP
+ = new ImmutableMap.Builder<Type<?>, Discriminator>()
+ .put(INTEGER, Discriminator.INTEGER)
+ .put(DISTRIBUTIONS, Discriminator.DISTRIBUTION_SET)
+ .put(LABEL, Discriminator.LABEL)
+ // NODEP_LABEL attributes are not really strings. This is implemented
+ // this way for the sake of backward compatibility.
+ .put(NODEP_LABEL, Discriminator.STRING)
+ .put(LABEL_LIST, Discriminator.LABEL_LIST)
+ .put(NODEP_LABEL_LIST, Discriminator.STRING_LIST)
+ .put(STRING, Discriminator.STRING)
+ .put(STRING_LIST, Discriminator.STRING_LIST)
+ .put(OUTPUT, Discriminator.OUTPUT)
+ .put(OUTPUT_LIST, Discriminator.OUTPUT_LIST)
+ .put(LICENSE, Discriminator.LICENSE)
+ .put(STRING_DICT, Discriminator.STRING_DICT)
+ .put(FILESET_ENTRY_LIST, Discriminator.FILESET_ENTRY_LIST)
+ .put(LABEL_LIST_DICT, Discriminator.LABEL_LIST_DICT)
+ .put(STRING_LIST_DICT, Discriminator.STRING_LIST_DICT)
+ .put(BOOLEAN, Discriminator.BOOLEAN)
+ .put(TRISTATE, Discriminator.TRISTATE)
+ .put(INTEGER_LIST, Discriminator.INTEGER_LIST)
+ .put(STRING_DICT_UNARY, Discriminator.STRING_DICT_UNARY)
+ .build();
+
+ /**
+ * Returns the appropriate Attribute.Discriminator value from an internal attribute type.
+ */
+ public static Discriminator getDiscriminatorFromType(Type<?> type) {
+ Preconditions.checkArgument(TYPE_MAP.containsKey(type));
+ return TYPE_MAP.get(type);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RawAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/RawAttributeMapper.java
new file mode 100644
index 0000000..a174193
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RawAttributeMapper.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * {@link AttributeMap} implementation that returns raw attribute information as contained
+ * within a {@link Rule}. In particular, configurable attributes of the form
+ * { config1: "value1", config2: "value2" } are passed through without being resolved to a
+ * final value.
+ */
+public class RawAttributeMapper extends AbstractAttributeMapper {
+ RawAttributeMapper(Package pkg, RuleClass ruleClass, Label ruleLabel,
+ AttributeContainer attributes) {
+ super(pkg, ruleClass, ruleLabel, attributes);
+ }
+
+ public static RawAttributeMapper of(Rule rule) {
+ return new RawAttributeMapper(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(),
+ rule.getAttributeContainer());
+ }
+
+ @Override
+ protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) {
+ T value = get(attributeName, type);
+ return value == null ? ImmutableList.<T>of() : ImmutableList.of(value);
+ }
+
+ /**
+ * Returns true if the given attribute is configurable for this rule instance, false
+ * otherwise.
+ */
+ public <T> boolean isConfigurable(String attributeName, Type<T> type) {
+ return getSelector(attributeName, type) != null;
+ }
+
+ /**
+ * If the attribute is configurable for this rule instance, returns its configuration
+ * keys. Else returns an empty list.
+ */
+ public <T> Iterable<Label> getConfigurabilityKeys(String attributeName, Type<T> type) {
+ Type.Selector<T> selector = getSelector(attributeName, type);
+ return selector == null ? ImmutableList.<Label>of() : selector.getEntries().keySet();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RelativePackageNameResolver.java b/src/main/java/com/google/devtools/build/lib/packages/RelativePackageNameResolver.java
new file mode 100644
index 0000000..ade196b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RelativePackageNameResolver.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Resolves relative package names to absolute ones. Handles the absolute
+ * package path marker ("//") and uplevel references ("..").
+ */
+public class RelativePackageNameResolver {
+ private final PathFragment offset;
+ private final boolean discardBuild;
+
+ /**
+ * @param offset the base package path used to resolve relative paths
+ * @param discardBuild if true, discards the last package path segment if
+ * it is called "BUILD"
+ */
+ public RelativePackageNameResolver(PathFragment offset, boolean discardBuild) {
+ Preconditions.checkArgument(!offset.containsUplevelReferences(),
+ "offset should not contain uplevel references");
+
+ this.offset = offset;
+ this.discardBuild = discardBuild;
+ }
+
+ /**
+ * Resolves the given package name with respect to the offset given in the
+ * constructor.
+ *
+ * @param pkg the relative package name to be resolved
+ * @return the absolute package name
+ * @throws InvalidPackageNameException if the package name cannot be resolved
+ * (only syntactic checks are done -- it is not checked if the package
+ * really exists or not)
+ */
+ public String resolve(String pkg) throws InvalidPackageNameException {
+ boolean isAbsolute;
+ String relativePkg;
+
+ if (pkg.startsWith("//")) {
+ isAbsolute = true;
+ relativePkg = pkg.substring(2);
+ } else if (pkg.startsWith("/")) {
+ throw new InvalidPackageNameException(pkg,
+ "package name cannot start with a single slash");
+ } else {
+ isAbsolute = false;
+ relativePkg = pkg;
+ }
+
+ PathFragment relative = new PathFragment(relativePkg);
+
+ if (discardBuild && relative.getBaseName().equals("BUILD")) {
+ relative = relative.getParentDirectory();
+ }
+
+ PathFragment result = isAbsolute ? relative : offset.getRelative(relative);
+ result = result.normalize();
+ if (result.containsUplevelReferences()) {
+ throw new InvalidPackageNameException(pkg,
+ "package name contains too many '..' segments");
+ }
+
+ return result.getPathString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Rule.java b/src/main/java/com/google/devtools/build/lib/packages/Rule.java
new file mode 100644
index 0000000..cff91f6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/Rule.java
@@ -0,0 +1,666 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.util.BinaryPredicate;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * An instance of a build rule in the build language. A rule has a name, a
+ * package to which it belongs, a class such as <code>cc_library</code>, and
+ * set of typed attributes. The set of attribute names and types is a property
+ * of the rule's class. The use of the term "class" here has nothing to do
+ * with Java classes. All rules are implemented by the same Java classes, Rule
+ * and RuleClass.
+ *
+ * <p>Here is a typical rule as it appears in a BUILD file:
+ * <pre>
+ * cc_library(name = 'foo',
+ * defines = ['-Dkey=value'],
+ * srcs = ['foo.cc'],
+ * deps = ['bar'])
+ * </pre>
+ */
+public final class Rule implements Target {
+ /** Dependency predicate that includes all dependencies */
+ public static final BinaryPredicate<Rule, Attribute> ALL_DEPS =
+ new BinaryPredicate<Rule, Attribute>() {
+ @Override
+ public boolean apply(Rule x, Attribute y) {
+ return true;
+ }
+ };
+
+ /** Dependency predicate that excludes host dependencies */
+ public static final BinaryPredicate<Rule, Attribute> NO_HOST_DEPS =
+ new BinaryPredicate<Rule, Attribute>() {
+ @Override
+ public boolean apply(Rule rule, Attribute attribute) {
+ // isHostConfiguration() is only defined for labels and label lists.
+ if (attribute.getType() != Type.LABEL && attribute.getType() != Type.LABEL_LIST) {
+ return true;
+ }
+
+ return attribute.getConfigurationTransition() != ConfigurationTransition.HOST;
+ }
+ };
+
+ /** Dependency predicate that excludes implicit dependencies */
+ public static final BinaryPredicate<Rule, Attribute> NO_IMPLICIT_DEPS =
+ new BinaryPredicate<Rule, Attribute>() {
+ @Override
+ public boolean apply(Rule rule, Attribute attribute) {
+ return rule.isAttributeValueExplicitlySpecified(attribute);
+ }
+ };
+
+ /**
+ * Dependency predicate that excludes those edges that are not present in the
+ * configured target graph.
+ */
+ public static final BinaryPredicate<Rule, Attribute> NO_NODEP_ATTRIBUTES =
+ new BinaryPredicate<Rule, Attribute>() {
+ @Override
+ public boolean apply(Rule rule, Attribute attribute) {
+ return attribute.getType() != Type.NODEP_LABEL &&
+ attribute.getType() != Type.NODEP_LABEL_LIST;
+ }
+ };
+
+ /** Label predicate that allows every label. */
+ public static final Predicate<Label> ALL_LABELS = Predicates.alwaysTrue();
+
+ /**
+ * Checks to see if the attribute has the isDirectCompileTimeInput property.
+ */
+ public static final BinaryPredicate<Rule, Attribute> DIRECT_COMPILE_TIME_INPUT =
+ new BinaryPredicate<Rule, Attribute>() {
+ @Override
+ public boolean apply(Rule rule, Attribute attribute) {
+ return attribute.isDirectCompileTimeInput();
+ }
+ };
+
+ /**
+ * Returns a predicate that computes the logical and of the two given predicates.
+ */
+ public static <X, Y> BinaryPredicate<X, Y> and(
+ final BinaryPredicate<X, Y> a, final BinaryPredicate<X, Y> b) {
+ return new BinaryPredicate<X, Y>() {
+ @Override
+ public boolean apply(X x, Y y) {
+ return a.apply(x, y) && b.apply(x, y);
+ }
+ };
+ }
+
+ private final Label label;
+
+ private final Package pkg;
+
+ private final RuleClass ruleClass;
+
+ private final AttributeContainer attributes;
+ private final RawAttributeMapper attributeMap;
+
+ private RuleVisibility visibility;
+
+ private boolean containsErrors;
+
+ private final Location location;
+
+ private final FuncallExpression ast; // may be null
+
+ // Initialized in the call to populateOutputFiles.
+ private List<OutputFile> outputFiles;
+ private ListMultimap<String, OutputFile> outputFileMap;
+
+ Rule(Package pkg, Label label, RuleClass ruleClass, FuncallExpression ast, Location location) {
+ this.pkg = Preconditions.checkNotNull(pkg);
+ this.label = label;
+ this.ruleClass = Preconditions.checkNotNull(ruleClass);
+ this.location = Preconditions.checkNotNull(location);
+ this.attributes = new AttributeContainer(ruleClass);
+ this.attributeMap = new RawAttributeMapper(pkg, ruleClass, label, attributes);
+ this.containsErrors = false;
+ this.ast = ast;
+ }
+
+ void setVisibility(RuleVisibility visibility) {
+ this.visibility = visibility;
+ }
+
+ void setAttributeValue(Attribute attribute, Object value, boolean explicit) {
+ attributes.setAttributeValue(attribute, value, explicit);
+ }
+
+ void setAttributeValueByName(String attrName, Object value) {
+ attributes.setAttributeValueByName(attrName, value);
+ }
+
+ void setAttributeLocation(int attrIndex, Location location) {
+ attributes.setAttributeLocation(attrIndex, location);
+ }
+
+ void setAttributeLocation(Attribute attribute, Location location) {
+ attributes.setAttributeLocation(attribute, location);
+ }
+
+ void setContainsErrors() {
+ this.containsErrors = true;
+ }
+
+ @Override
+ public Label getLabel() {
+ return attributeMap.getLabel();
+ }
+
+ @Override
+ public String getName() {
+ return attributeMap.getName();
+ }
+
+ @Override
+ public Package getPackage() {
+ return pkg;
+ }
+
+ public RuleClass getRuleClassObject() {
+ return ruleClass;
+ }
+
+ @Override
+ public String getTargetKind() {
+ return ruleClass.getTargetKind();
+ }
+
+ /**
+ * Returns the class of this rule. (e.g. "cc_library")
+ */
+ public String getRuleClass() {
+ return ruleClass.getName();
+ }
+
+ /**
+ * Returns the build features that apply to this rule.
+ */
+ public ImmutableSet<String> getFeatures() {
+ return pkg.getFeatures();
+ }
+
+ /**
+ * Returns true iff the outputs of this rule should be created beneath the
+ * bin directory, false if beneath genfiles. For most rule
+ * classes, this is a constant, but for genrule, it is a property of the
+ * individual rule instance, derived from the 'output_to_bindir' attribute.
+ */
+ public boolean hasBinaryOutput() {
+ return ruleClass.getName().equals("genrule") // this is unfortunate...
+ ? NonconfigurableAttributeMapper.of(this).get("output_to_bindir", Type.BOOLEAN)
+ : ruleClass.hasBinaryOutput();
+ }
+
+ /**
+ * Returns the AST for this rule. Returns null if the package factory chose
+ * not to retain the AST when evaluateBuildFile was called for this rule's
+ * package.
+ */
+ public FuncallExpression getSyntaxTree() {
+ return ast;
+ }
+
+ /**
+ * Returns true iff there were errors while constructing this rule, such as
+ * attributes with missing values or values of the wrong type.
+ */
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ /**
+ * Returns an (unmodifiable, unordered) collection containing all the
+ * Attribute definitions for this kind of rule. (Note, this doesn't include
+ * the <i>values</i> of the attributes, merely the schema. Call
+ * get[Type]Attr() methods to access the actual values.)
+ */
+ public Collection<Attribute> getAttributes() {
+ return ruleClass.getAttributes();
+ }
+
+ /**
+ * Returns true if this rule has any attributes that are configurable.
+ *
+ * <p>Note this is *not* the same as having attribute *types* that are configurable. For example,
+ * "deps" is configurable, in that one can write a rule that sets "deps" to a configuration
+ * dictionary. But if *this* rule's instance of "deps" doesn't do that, its instance
+ * of "deps" is not considered configurable.
+ *
+ * <p>In other words, this method signals which rules might have their attribute values
+ * influenced by the configuration.
+ */
+ public boolean hasConfigurableAttributes() {
+ for (Attribute attribute : getAttributes()) {
+ if (attributeMap.isConfigurable(attribute.getName(), attribute.getType())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the attribute definition whose name is {@code attrName}, or null
+ * if not found. (Use get[X]Attr for the actual value.)
+ *
+ * @deprecated use {@link AbstractAttributeMapper#getAttributeDefinition} instead
+ */
+ @Deprecated
+ public Attribute getAttributeDefinition(String attrName) {
+ return attributeMap.getAttributeDefinition(attrName);
+ }
+
+ /**
+ * Returns an (unmodifiable, ordered) collection containing all the declared output files of this
+ * rule.
+ *
+ * <p>All implicit output files (declared in the {@link RuleClass}) are
+ * listed first, followed by any explicit files (declared via the 'outs' attribute). Additionally
+ * both implicit and explicit outputs will retain the relative order in which they were declared.
+ *
+ * <p>This ordering is useful because it is propagated through to the list of targets returned by
+ * getOuts() and allows targets to access their implicit outputs easily via
+ * {@code getOuts().get(N)} (providing that N is less than the number of implicit outputs).
+ *
+ * <p>The fact that the relative order of the explicit outputs is also retained is less obviously
+ * useful but is still well defined.
+ */
+ public Collection<OutputFile> getOutputFiles() {
+ return outputFiles;
+ }
+
+ /**
+ * Returns an (unmodifiable, ordered) map containing the list of output files for every
+ * output type attribute.
+ */
+ public ListMultimap<String, OutputFile> getOutputFileMap() {
+ return outputFileMap;
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public Rule getAssociatedRule() {
+ return this;
+ }
+
+ /**
+ * Returns this rule's raw attribute info, suitable for being fed into an
+ * {@link AttributeMap} for user-level attribute access. Don't use this method
+ * for direct attribute access.
+ */
+ public AttributeContainer getAttributeContainer() {
+ return attributes;
+ }
+
+ /********************************************************************
+ * Attribute accessor functions.
+ *
+ * The below provide access to attribute definitions and other generic
+ * metadata.
+ *
+ * For access to attribute *values* (e.g. "What's the value of attribute
+ * X for Rule Y?"), go through {@link RuleContext#attributes}. If no
+ * RuleContext is available, create a localized {@link AbstractAttributeMapper}
+ * instance instead.
+ ********************************************************************/
+
+ /**
+ * Returns the default value for the attribute {@code attrName}, which may be
+ * of any type, but must exist (an exception is thrown otherwise).
+ */
+ public Object getAttrDefaultValue(String attrName) {
+ Object defaultValue = ruleClass.getAttributeByName(attrName).getDefaultValue(this);
+ // Computed defaults not expected here.
+ Preconditions.checkState(!(defaultValue instanceof Attribute.ComputedDefault));
+ return defaultValue;
+ }
+
+ /**
+ * Returns true iff the rule class has an attribute with the given name and type.
+ */
+ public boolean isAttrDefined(String attrName, Type<?> type) {
+ return ruleClass.hasAttr(attrName, type);
+ }
+
+ /**
+ * Returns true iff the value of the specified attribute is explicitly set in
+ * the BUILD file (as opposed to its default value). This also returns true if
+ * the value from the BUILD file is the same as the default value.
+ */
+ public boolean isAttributeValueExplicitlySpecified(Attribute attribute) {
+ return attributes.isAttributeValueExplicitlySpecified(attribute);
+ }
+
+ /**
+ * Returns true iff the value of the specified attribute is explicitly set in the BUILD file (as
+ * opposed to its default value). This also returns true if the value from the BUILD file is the
+ * same as the default value. In addition, this method return false if the rule has no attribute
+ * with the given name.
+ */
+ public boolean isAttributeValueExplicitlySpecified(String attrName) {
+ return attributeMap.isAttributeValueExplicitlySpecified(attrName);
+ }
+
+ /**
+ * Returns the location of the attribute definition for this rule, if known;
+ * or the location of the whole rule otherwise. "attrName" need not be a
+ * valid attribute name for this rule.
+ */
+ public Location getAttributeLocation(String attrName) {
+ Location attrLocation = null;
+ if (!attrName.equals("name")) {
+ attrLocation = attributes.getAttributeLocation(attrName);
+ }
+ return attrLocation != null ? attrLocation : getLocation();
+ }
+
+ /**
+ * Returns a new List instance containing all direct dependencies (all types).
+ */
+ public Collection<Label> getLabels() {
+ return getLabels(Rule.ALL_DEPS);
+ }
+
+ /**
+ * Returns a new Collection containing all Labels that match a given Predicate,
+ * not including outputs.
+ *
+ * @param predicate A binary predicate that determines if a label should be
+ * included in the result. The predicate is evaluated with this rule and
+ * the attribute that contains the label. The label will be contained in the
+ * result iff (the predicate returned {@code true} and the labels are not outputs)
+ */
+ public Collection<Label> getLabels(final BinaryPredicate<Rule, Attribute> predicate) {
+ final Set<Label> labels = new HashSet<>();
+ // TODO(bazel-team): move this to AttributeMap, too. Just like visitLabels, which labels should
+ // be visited may depend on the calling context. We shouldn't implicitly decide this for
+ // the caller.
+ AggregatingAttributeMapper.of(this).visitLabels(new AttributeMap.AcceptsLabelAttribute() {
+ @Override
+ public void acceptLabelAttribute(Label label, Attribute attribute) {
+ if (predicate.apply(Rule.this, attribute)) {
+ labels.add(label);
+ }
+ }
+ });
+ return labels;
+ }
+
+ /**
+ * Check if this rule is valid according to the validityPredicate of its RuleClass.
+ */
+ void checkValidityPredicate(EventHandler eventHandler) {
+ PredicateWithMessage<Rule> predicate = getRuleClassObject().getValidityPredicate();
+ if (!predicate.apply(this)) {
+ reportError(predicate.getErrorReason(this), eventHandler);
+ }
+ }
+
+ /**
+ * Collects the output files (both implicit and explicit). All the implicit output files are added
+ * first, followed by any explicit files. Additionally both implicit and explicit output files
+ * will retain the relative order in which they were declared.
+ */
+ void populateOutputFiles(EventHandler eventHandler,
+ Package.AbstractBuilder<?, ?> pkgBuilder) throws SyntaxException {
+ Preconditions.checkState(outputFiles == null);
+ // Order is important here: implicit before explicit
+ outputFiles = Lists.newArrayList();
+ outputFileMap = LinkedListMultimap.create();
+ populateImplicitOutputFiles(eventHandler, pkgBuilder);
+ populateExplicitOutputFiles(eventHandler);
+ outputFiles = ImmutableList.copyOf(outputFiles);
+ outputFileMap = ImmutableListMultimap.copyOf(outputFileMap);
+ }
+
+ // Explicit output files are user-specified attributes of type OUTPUT.
+ private void populateExplicitOutputFiles(EventHandler eventHandler) throws SyntaxException {
+ NonconfigurableAttributeMapper nonConfigurableAttributes =
+ NonconfigurableAttributeMapper.of(this);
+ for (Attribute attribute : ruleClass.getAttributes()) {
+ String name = attribute.getName();
+ Type<?> type = attribute.getType();
+ if (type == Type.OUTPUT) {
+ Label outputLabel = nonConfigurableAttributes.get(name, Type.OUTPUT);
+ if (outputLabel != null) {
+ addLabelOutput(attribute, outputLabel, eventHandler);
+ }
+ } else if (type == Type.OUTPUT_LIST) {
+ for (Label label : nonConfigurableAttributes.get(name, Type.OUTPUT_LIST)) {
+ addLabelOutput(attribute, label, eventHandler);
+ }
+ }
+ }
+ }
+
+ /**
+ * Implicit output files come from rule-specific patterns, and are a function
+ * of the rule's "name", "srcs", and other attributes.
+ */
+ private void populateImplicitOutputFiles(EventHandler eventHandler,
+ Package.AbstractBuilder<?, ?> pkgBuilder) {
+ try {
+ for (String out : ruleClass.getImplicitOutputsFunction().getImplicitOutputs(attributeMap)) {
+ try {
+ addOutputFile(pkgBuilder.createLabel(out), eventHandler);
+ } catch (SyntaxException e) {
+ reportError("illegal output file name '" + out + "' in rule "
+ + getLabel(), eventHandler);
+ }
+ }
+ } catch (EvalException e) {
+ reportError(e.print(), eventHandler);
+ }
+ }
+
+ private void addLabelOutput(Attribute attribute, Label label, EventHandler eventHandler)
+ throws SyntaxException {
+ if (!label.getPackageIdentifier().equals(pkg.getPackageIdentifier())) {
+ throw new IllegalStateException("Label for attribute " + attribute
+ + " should refer to '" + pkg.getName()
+ + "' but instead refers to '" + label.getPackageFragment()
+ + "' (label '" + label.getName() + "')");
+ }
+ if (label.getName().equals(".")) {
+ throw new SyntaxException("output file name can't be equal '.'");
+ }
+ OutputFile outputFile = addOutputFile(label, eventHandler);
+ outputFileMap.put(attribute.getName(), outputFile);
+ }
+
+ private OutputFile addOutputFile(Label label, EventHandler eventHandler) {
+ if (label.getName().equals(getName())) {
+ // TODO(bazel-team): for now (23 Apr 2008) this is just a warning. After
+ // June 1st we should make it an error.
+ reportWarning("target '" + getName() + "' is both a rule and a file; please choose "
+ + "another name for the rule", eventHandler);
+ }
+ OutputFile outputFile = new OutputFile(pkg, label, this);
+ outputFiles.add(outputFile);
+ return outputFile;
+ }
+
+ void reportError(String message, EventHandler eventHandler) {
+ eventHandler.handle(Event.error(location, message));
+ this.containsErrors = true;
+ }
+
+ void reportWarning(String message, EventHandler eventHandler) {
+ eventHandler.handle(Event.warn(location, message));
+ }
+
+ @Override
+ public int hashCode() {
+ return label.hashCode();
+ }
+
+ /**
+ * Returns a string of the form "cc_binary rule //foo:foo"
+ *
+ * @return a string of the form "cc_binary rule //foo:foo"
+ */
+ @Override
+ public String toString() {
+ return getRuleClass() + " rule " + getLabel();
+ }
+
+ /**
+ * Returns the effective visibility of this Rule. Visibility is computed from
+ * these sources in this order of preference:
+ * - 'visibility' attribute
+ * - 'default_visibility;' attribute of package() declaration
+ * - public.
+ */
+ @Override
+ public RuleVisibility getVisibility() {
+ if (visibility != null) {
+ return visibility;
+ }
+
+ if (getRuleClassObject().isPublicByDefault()) {
+ return ConstantRuleVisibility.PUBLIC;
+ }
+
+ return pkg.getDefaultVisibility();
+ }
+
+ public boolean isVisibilitySpecified() {
+ return visibility != null;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Set<DistributionType> getDistributions() {
+ if (isAttrDefined("distribs", Type.DISTRIBUTIONS)
+ && isAttributeValueExplicitlySpecified("distribs")) {
+ return NonconfigurableAttributeMapper.of(this).get("distribs", Type.DISTRIBUTIONS);
+ } else {
+ return getPackage().getDefaultDistribs();
+ }
+ }
+
+ @Override
+ public License getLicense() {
+ if (isAttrDefined("licenses", Type.LICENSE)
+ && isAttributeValueExplicitlySpecified("licenses")) {
+ return NonconfigurableAttributeMapper.of(this).get("licenses", Type.LICENSE);
+ } else {
+ return getPackage().getDefaultLicense();
+ }
+ }
+
+ /**
+ * Returns the license of the output of the binary created by this rule, or
+ * null if it is not specified.
+ */
+ public License getToolOutputLicense(AttributeMap attributes) {
+ if (isAttrDefined("output_licenses", Type.LICENSE)
+ && attributes.isAttributeValueExplicitlySpecified("output_licenses")) {
+ return attributes.get("output_licenses", Type.LICENSE);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the globs that were expanded to create an attribute value, or
+ * null if unknown or not applicable.
+ */
+ public static GlobList<?> getGlobInfo(Object attributeValue) {
+ if (attributeValue instanceof GlobList<?>) {
+ return (GlobList<?>) attributeValue;
+ } else {
+ return null;
+ }
+ }
+
+ private void checkForNullLabel(Label labelToCheck, String where) {
+ if (labelToCheck == null) {
+ throw new IllegalStateException(String.format(
+ "null label in rule %s, %s", getLabel().toString(), where));
+ }
+ }
+
+ // Consistency check: check if this label contains any weird labels (i.e.
+ // null-valued, with a packageFragment that is null...). The bug that prompted
+ // the introduction of this code is #2210848 (NullPointerException in
+ // Package.checkForConflicts() ).
+ void checkForNullLabels() {
+ AggregatingAttributeMapper.of(this).visitLabels(
+ new AttributeMap.AcceptsLabelAttribute() {
+ @Override
+ public void acceptLabelAttribute(Label labelToCheck, Attribute attribute) {
+ checkForNullLabel(labelToCheck, "attribute " + attribute.getName());
+ }
+ });
+ for (OutputFile outputFile : getOutputFiles()) {
+ checkForNullLabel(outputFile.getLabel(), "output file");
+ }
+ }
+
+ /**
+ * Returns the Set of all tags exhibited by this target. May be empty.
+ */
+ public Set<String> getRuleTags() {
+ Set<String> ruleTags = new LinkedHashSet<>();
+ for (Attribute attribute : getRuleClassObject().getAttributes()) {
+ if (attribute.isTaggable()) {
+ Type<?> attrType = attribute.getType();
+ String name = attribute.getName();
+ // This enforces the expectation that taggable attributes are non-configurable.
+ Object value = NonconfigurableAttributeMapper.of(this).get(name, attrType);
+ Set<String> tags = attrType.toTagSet(value, name);
+ ruleTags.addAll(tags);
+ }
+ }
+ return ruleTags;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java
new file mode 100644
index 0000000..f495502
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java
@@ -0,0 +1,1511 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Ordering;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Argument;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Ident;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.UserDefinedFunction;
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * Instances of RuleClass encapsulate the set of attributes of a given "class" of rule, such as
+ * <code>cc_binary</code>.
+ *
+ * <p>This is an instance of the "meta-class" pattern for Rules: we achieve using <i>values</i>
+ * what subclasses achieve using <i>types</i>. (The "Design Patterns" book doesn't include this
+ * pattern, so think of it as something like a cross between a Flyweight and a State pattern. Like
+ * Flyweight, we avoid repeatedly storing data that belongs to many instances. Like State, we
+ * delegate from Rule to RuleClass for the specific behavior of that rule (though unlike state, a
+ * Rule object never changes its RuleClass). This avoids the need to declare one Java class per
+ * class of Rule, yet achieves the same behavior.)
+ *
+ * <p>The use of a metaclass also allows us to compute a mapping from Attributes to small integers
+ * and share this between all rules of the same metaclass. This means we can save the attribute
+ * dictionary for each rule instance using an array, which is much more compact than a hashtable.
+ *
+ * <p>Rule classes whose names start with "$" are considered "abstract"; since they are not valid
+ * identifiers, they cannot be named in the build language. However, they are useful for grouping
+ * related attributes which are inherited.
+ *
+ * <p>The exact values in this class are important. In particular:
+ * <ul>
+ * <li>Changing an attribute from MANDATORY to OPTIONAL creates the potential for null-pointer
+ * exceptions in code that expects a value.
+ * <li>Attributes whose names are preceded by a "$" or a ":" are "hidden", and cannot be redefined
+ * in a BUILD file. They are a useful way of adding a special dependency. By convention,
+ * attributes starting with "$" are implicit dependencies, and those starting with a ":" are
+ * late-bound implicit dependencies, i.e. dependencies that can only be resolved when the
+ * configuration is known.
+ * <li>Attributes should not be introduced into the hierarchy higher then necessary.
+ * <li>The 'deps' and 'data' attributes are treated specially by the code that builds the runfiles
+ * tree. All targets appearing in these attributes appears beneath the ".runfiles" tree; in
+ * addition, "deps" may have rule-specific semantics.
+ * </ul>
+ */
+@Immutable
+public final class RuleClass {
+ /**
+ * A constraint for the package name of the Rule instances.
+ */
+ public static class PackageNameConstraint implements PredicateWithMessage<Rule> {
+
+ public static final int ANY_SEGMENT = 0;
+
+ private final int pathSegment;
+
+ private final Set<String> values;
+
+ /**
+ * The pathSegment-th segment of the package must be one of the specified values.
+ * The path segment indexing starts from 1.
+ */
+ public PackageNameConstraint(int pathSegment, String... values) {
+ this.values = ImmutableSet.copyOf(values);
+ this.pathSegment = pathSegment;
+ }
+
+ @Override
+ public boolean apply(Rule input) {
+ PathFragment path = input.getLabel().getPackageFragment();
+ if (pathSegment == ANY_SEGMENT) {
+ return path.getFirstSegment(values) != PathFragment.INVALID_SEGMENT;
+ } else {
+ return path.segmentCount() >= pathSegment
+ && values.contains(path.getSegment(pathSegment - 1));
+ }
+ }
+
+ @Override
+ public String getErrorReason(Rule param) {
+ if (pathSegment == ANY_SEGMENT) {
+ return param.getRuleClass() + " rules have to be under a " +
+ StringUtil.joinEnglishList(values, "or", "'") + " directory";
+ } else if (pathSegment == 1) {
+ return param.getRuleClass() + " rules are only allowed in "
+ + StringUtil.joinEnglishList(StringUtil.append(values, "//", ""), "or");
+ } else {
+ return param.getRuleClass() + " rules are only allowed in packages which " +
+ StringUtil.ordinal(pathSegment) + " is " + StringUtil.joinEnglishList(values, "or");
+ }
+ }
+
+ @VisibleForTesting
+ public int getPathSegment() {
+ return pathSegment;
+ }
+
+ @VisibleForTesting
+ public Collection<String> getValues() {
+ return values;
+ }
+ }
+
+ /**
+ * Using this callback function, rules can override their own configuration during the
+ * analysis phase.
+ */
+ public interface Configurator<TConfig, TRule> {
+ TConfig apply(TRule rule, TConfig configuration);
+ }
+
+ /**
+ * A factory or builder class for rule implementations.
+ */
+ public interface ConfiguredTargetFactory<TConfiguredTarget, TContext> {
+ /**
+ * Returns a fully initialized configured target instance using the given context.
+ */
+ TConfiguredTarget create(TContext ruleContext) throws InterruptedException;
+ }
+
+ /**
+ * Default rule configurator, it doesn't change the assigned configuration.
+ */
+ public static final RuleClass.Configurator<Object, Object> NO_CHANGE =
+ new RuleClass.Configurator<Object, Object>() {
+ @Override
+ public Object apply(Object rule, Object configuration) {
+ return configuration;
+ }
+ };
+
+ /**
+ * For Bazel's constraint system: the attribute that declares the set of environments a rule
+ * supports, overriding the defaults for their respective groups.
+ */
+ public static final String RESTRICTED_ENVIRONMENT_ATTR = "restricted_to";
+
+ /**
+ * For Bazel's constraint system: the attribute that declares the set of environments a rule
+ * supports, appending them to the defaults for their respective groups.
+ */
+ public static final String COMPATIBLE_ENVIRONMENT_ATTR = "compatible_with";
+
+ /**
+ * For Bazel's constraint system: the implicit attribute used to store rule class restriction
+ * defaults as specified by {@link Builder#restrictedTo}.
+ */
+ public static final String DEFAULT_RESTRICTED_ENVIRONMENT_ATTR =
+ "$" + RESTRICTED_ENVIRONMENT_ATTR;
+
+ /**
+ * For Bazel's constraint system: the implicit attribute used to store rule class compatibility
+ * defaults as specified by {@link Builder#compatibleWith}.
+ */
+ public static final String DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR =
+ "$" + COMPATIBLE_ENVIRONMENT_ATTR;
+
+ /**
+ * Checks if an attribute is part of the constraint system.
+ */
+ public static boolean isConstraintAttribute(String attr) {
+ return RESTRICTED_ENVIRONMENT_ATTR.equals(attr)
+ || COMPATIBLE_ENVIRONMENT_ATTR.equals(attr)
+ || DEFAULT_RESTRICTED_ENVIRONMENT_ATTR.equals(attr)
+ || DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR.equals(attr);
+ }
+
+ /**
+ * A support class to make it easier to create {@code RuleClass} instances.
+ * This class follows the 'fluent builder' pattern.
+ *
+ * <p>The {@link #addAttribute} method will throw an exception if an attribute
+ * of that name already exists. Use {@link #overrideAttribute} in that case.
+ */
+ public static final class Builder {
+ private static final Pattern RULE_NAME_PATTERN = Pattern.compile("[A-Za-z][A-Za-z0-9_]*");
+
+ /**
+ * The type of the rule class, which determines valid names and required
+ * attributes.
+ */
+ public enum RuleClassType {
+ /**
+ * Abstract rules are intended for rule classes that are just used to
+ * factor out common attributes, and for rule classes that are used only
+ * internally. These rules cannot be instantiated by a BUILD file.
+ *
+ * <p>The rule name must contain a '$' and {@link
+ * TargetUtils#isTestRuleName} must return false for the name.
+ */
+ ABSTRACT {
+ @Override
+ public void checkName(String name) {
+ Preconditions.checkArgument(
+ (name.contains("$") && !TargetUtils.isTestRuleName(name)) || name.equals(""));
+ }
+
+ @Override
+ public void checkAttributes(Map<String, Attribute> attributes) {
+ // No required attributes.
+ }
+ },
+
+ /**
+ * Invisible rule classes should contain a dollar sign so that they cannot be instantiated
+ * by the user. They are different from abstract rules in that they can be instantiated
+ * at will.
+ */
+ INVISIBLE {
+ @Override
+ public void checkName(String name) {
+ Preconditions.checkArgument(name.contains("$"));
+ }
+
+ @Override
+ public void checkAttributes(Map<String, Attribute> attributes) {
+ // No required attributes.
+ }
+ },
+
+ /**
+ * Normal rules are instantiable by BUILD files. Their names must therefore
+ * obey the rules for identifiers in the BUILD language. In addition,
+ * {@link TargetUtils#isTestRuleName} must return false for the name.
+ */
+ NORMAL {
+ @Override
+ public void checkName(String name) {
+ Preconditions.checkArgument(!TargetUtils.isTestRuleName(name)
+ && RULE_NAME_PATTERN.matcher(name).matches(), "Invalid rule name: " + name);
+ }
+
+ @Override
+ public void checkAttributes(Map<String, Attribute> attributes) {
+ for (Attribute attribute : REQUIRED_ATTRIBUTES_FOR_NORMAL_RULES) {
+ Attribute presentAttribute = attributes.get(attribute.getName());
+ Preconditions.checkState(presentAttribute != null,
+ "Missing mandatory '%s' attribute in normal rule class.", attribute.getName());
+ Preconditions.checkState(presentAttribute.getType().equals(attribute.getType()),
+ "Mandatory attribute '%s' in normal rule class has incorrect type (expcected" +
+ " %s).", attribute.getName(), attribute.getType());
+ }
+ }
+ },
+
+ /**
+ * Workspace rules can only be instantiated from a WORKSPACE file. Their names obey the
+ * rule for identifiers.
+ */
+ WORKSPACE {
+ @Override
+ public void checkName(String name) {
+ Preconditions.checkArgument(RULE_NAME_PATTERN.matcher(name).matches());
+ }
+
+ @Override
+ public void checkAttributes(Map<String, Attribute> attributes) {
+ // No required attributes.
+ }
+ },
+
+ /**
+ * Test rules are instantiable by BUILD files and are handled specially
+ * when run with the 'test' command. Their names must obey the rules
+ * for identifiers in the BUILD language and {@link
+ * TargetUtils#isTestRuleName} must return true for the name.
+ *
+ * <p>In addition, test rules must contain certain attributes. See {@link
+ * Builder#REQUIRED_ATTRIBUTES_FOR_TESTS}.
+ */
+ TEST {
+ @Override
+ public void checkName(String name) {
+ Preconditions.checkArgument(TargetUtils.isTestRuleName(name)
+ && RULE_NAME_PATTERN.matcher(name).matches());
+ }
+
+ @Override
+ public void checkAttributes(Map<String, Attribute> attributes) {
+ for (Attribute attribute : REQUIRED_ATTRIBUTES_FOR_TESTS) {
+ Attribute presentAttribute = attributes.get(attribute.getName());
+ Preconditions.checkState(presentAttribute != null,
+ "Missing mandatory '%s' attribute in test rule class.", attribute.getName());
+ Preconditions.checkState(presentAttribute.getType().equals(attribute.getType()),
+ "Mandatory attribute '%s' in test rule class has incorrect type (expcected %s).",
+ attribute.getName(), attribute.getType());
+ }
+ }
+ };
+
+ /**
+ * Checks whether the given name is valid for the current rule class type.
+ *
+ * @throws IllegalArgumentException if the name is not valid
+ */
+ public abstract void checkName(String name);
+
+ /**
+ * Checks whether the given set of attributes contains all the required
+ * attributes for the current rule class type.
+ *
+ * @throws IllegalArgumentException if a required attribute is missing
+ */
+ public abstract void checkAttributes(Map<String, Attribute> attributes);
+ }
+
+ /**
+ * A predicate that filters rule classes based on their names.
+ */
+ public static class RuleClassNamePredicate implements Predicate<RuleClass> {
+
+ private final Set<String> ruleClasses;
+
+ public RuleClassNamePredicate(Iterable<String> ruleClasses) {
+ this.ruleClasses = ImmutableSet.copyOf(ruleClasses);
+ }
+
+ public RuleClassNamePredicate(String... ruleClasses) {
+ this.ruleClasses = ImmutableSet.copyOf(ruleClasses);
+ }
+
+ public RuleClassNamePredicate() {
+ this(ImmutableSet.<String>of());
+ }
+
+ @Override
+ public boolean apply(RuleClass ruleClass) {
+ return ruleClasses.contains(ruleClass.getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return ruleClasses.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return (o instanceof RuleClassNamePredicate) &&
+ ruleClasses.equals(((RuleClassNamePredicate) o).ruleClasses);
+ }
+
+ @Override
+ public String toString() {
+ return ruleClasses.isEmpty() ? "nothing" : StringUtil.joinEnglishList(ruleClasses);
+ }
+ }
+
+ /**
+ * List of required attributes for normal rules, name and type.
+ */
+ public static final List<Attribute> REQUIRED_ATTRIBUTES_FOR_NORMAL_RULES = ImmutableList.of(
+ attr("tags", Type.STRING_LIST).build()
+ );
+
+ /**
+ * List of required attributes for test rules, name and type.
+ */
+ public static final List<Attribute> REQUIRED_ATTRIBUTES_FOR_TESTS = ImmutableList.of(
+ attr("tags", Type.STRING_LIST).build(),
+ attr("size", Type.STRING).build(),
+ attr("timeout", Type.STRING).build(),
+ attr("flaky", Type.BOOLEAN).build(),
+ attr("shard_count", Type.INTEGER).build(),
+ attr("local", Type.BOOLEAN).build()
+ );
+
+ private String name;
+ private final RuleClassType type;
+ private final boolean skylark;
+ private boolean documented;
+ private boolean publicByDefault = false;
+ private boolean binaryOutput = true;
+ private boolean workspaceOnly = false;
+ private boolean outputsDefaultExecutable = false;
+ private ImplicitOutputsFunction implicitOutputsFunction = ImplicitOutputsFunction.NONE;
+ private Configurator<?, ?> configurator = NO_CHANGE;
+ private ConfiguredTargetFactory<?, ?> configuredTargetFactory = null;
+ private PredicateWithMessage<Rule> validityPredicate =
+ PredicatesWithMessage.<Rule>alwaysTrue();
+ private Predicate<String> preferredDependencyPredicate = Predicates.alwaysFalse();
+ private List<Class<?>> advertisedProviders = new ArrayList<>();
+ private UserDefinedFunction configuredTargetFunction = null;
+ private SkylarkEnvironment ruleDefinitionEnvironment = null;
+ private Set<Class<?>> configurationFragments = new LinkedHashSet<>();
+ private boolean failIfMissingConfigurationFragment;
+
+ private final Map<String, Attribute> attributes = new LinkedHashMap<>();
+
+ /**
+ * Constructs a new {@code RuleClassBuilder} using all attributes from all
+ * parent rule classes. An attribute cannot exist in more than one parent.
+ *
+ * <p>The rule type affects the the allowed names and the required
+ * attributes (see {@link RuleClassType}).
+ *
+ * @throws IllegalArgumentException if an attribute with the same name exists
+ * in more than one parent
+ */
+ public Builder(String name, RuleClassType type, boolean skylark, RuleClass... parents) {
+ this.name = name;
+ this.skylark = skylark;
+ this.type = type;
+ this.documented = type != RuleClassType.ABSTRACT;
+ for (RuleClass parent : parents) {
+ if (parent.getValidityPredicate() != PredicatesWithMessage.<Rule>alwaysTrue()) {
+ setValidityPredicate(parent.getValidityPredicate());
+ }
+ if (parent.preferredDependencyPredicate != Predicates.<String>alwaysFalse()) {
+ setPreferredDependencyPredicate(parent.preferredDependencyPredicate);
+ }
+ configurationFragments.addAll(parent.requiredConfigurationFragments);
+ failIfMissingConfigurationFragment |= parent.failIfMissingConfigurationFragment;
+
+ for (Attribute attribute : parent.getAttributes()) {
+ String attrName = attribute.getName();
+ Preconditions.checkArgument(
+ !attributes.containsKey(attrName) || attributes.get(attrName) == attribute,
+ String.format("Attribute %s is inherited multiple times in %s ruleclass",
+ attrName, name));
+ attributes.put(attrName, attribute);
+ }
+ }
+ // TODO(bazel-team): move this testonly attribute setting to somewhere else
+ // preferably to some base RuleClass implementation.
+ if (this.type.equals(RuleClassType.TEST)) {
+ Attribute.Builder<Boolean> testOnlyAttr = attr("testonly", BOOLEAN).value(true)
+ .nonconfigurable("policy decision: this shouldn't depend on the configuration");
+ if (attributes.containsKey("testonly")) {
+ override(testOnlyAttr);
+ } else {
+ add(testOnlyAttr);
+ }
+ }
+ }
+
+ /**
+ * Checks that required attributes for test rules are present, creates the
+ * {@link RuleClass} object and returns it.
+ *
+ * @throws IllegalStateException if any of the required attributes is missing
+ */
+ public RuleClass build() {
+ return build(name);
+ }
+
+ /**
+ * Same as {@link #build} except with setting the name parameter.
+ */
+ public RuleClass build(String name) {
+ Preconditions.checkArgument(this.name.isEmpty() || this.name.equals(name));
+ type.checkName(name);
+ type.checkAttributes(attributes);
+ boolean skylarkExecutable =
+ skylark && (type == RuleClassType.NORMAL || type == RuleClassType.TEST);
+ Preconditions.checkState(
+ (type == RuleClassType.ABSTRACT)
+ == (configuredTargetFactory == null && configuredTargetFunction == null));
+ Preconditions.checkState(skylarkExecutable == (configuredTargetFunction != null));
+ Preconditions.checkState(skylarkExecutable == (ruleDefinitionEnvironment != null));
+ return new RuleClass(name, skylarkExecutable, documented, publicByDefault, binaryOutput,
+ workspaceOnly, outputsDefaultExecutable, implicitOutputsFunction, configurator,
+ configuredTargetFactory, validityPredicate, preferredDependencyPredicate,
+ ImmutableSet.copyOf(advertisedProviders), configuredTargetFunction,
+ ruleDefinitionEnvironment, configurationFragments, failIfMissingConfigurationFragment,
+ attributes.values().toArray(new Attribute[0]));
+ }
+
+ /**
+ * Declares that the implementation of this rule class requires the given configuration
+ * fragments to be present in the configuration. The value is inherited by subclasses.
+ *
+ * <p>For backwards compatibility, if the set is empty, all fragments may be accessed. But note
+ * that this is only enforced in the {@link com.google.devtools.build.lib.analysis.RuleContext}
+ * class.
+ */
+ public Builder requiresConfigurationFragments(Class<?>... configurationFragment) {
+ Collections.addAll(configurationFragments, configurationFragment);
+ return this;
+ }
+
+ public Builder failIfMissingConfigurationFragment() {
+ this.failIfMissingConfigurationFragment = true;
+ return this;
+ }
+
+ public Builder setUndocumented() {
+ documented = false;
+ return this;
+ }
+
+ public Builder publicByDefault() {
+ publicByDefault = true;
+ return this;
+ }
+
+ public Builder setWorkspaceOnly() {
+ workspaceOnly = true;
+ return this;
+ }
+
+ /**
+ * Determines the outputs of this rule to be created beneath the {@code
+ * genfiles} directory. By default, files are created beneath the {@code bin}
+ * directory.
+ *
+ * <p>This property is not inherited and this method should not be called by
+ * builder of {@link RuleClassType#ABSTRACT} rule class.
+ *
+ * @throws IllegalStateException if called for abstract rule class builder
+ */
+ public Builder setOutputToGenfiles() {
+ Preconditions.checkState(type != RuleClassType.ABSTRACT,
+ "Setting not inherited property (output to genrules) of abstract rule class '%s'", name);
+ this.binaryOutput = false;
+ return this;
+ }
+
+ /**
+ * Sets the implicit outputs function of the rule class. The default implicit
+ * outputs function is {@link ImplicitOutputsFunction#NONE}.
+ *
+ * <p>This property is not inherited and this method should not be called by
+ * builder of {@link RuleClassType#ABSTRACT} rule class.
+ *
+ * @throws IllegalStateException if called for abstract rule class builder
+ */
+ public Builder setImplicitOutputsFunction(
+ ImplicitOutputsFunction implicitOutputsFunction) {
+ Preconditions.checkState(type != RuleClassType.ABSTRACT,
+ "Setting not inherited property (implicit output function) of abstract rule class '%s'",
+ name);
+ this.implicitOutputsFunction = implicitOutputsFunction;
+ return this;
+ }
+
+ public Builder cfg(Configurator<?, ?> configurator) {
+ Preconditions.checkState(type != RuleClassType.ABSTRACT,
+ "Setting not inherited property (cfg) of abstract rule class '%s'", name);
+ this.configurator = configurator;
+ return this;
+ }
+
+ public Builder factory(ConfiguredTargetFactory<?, ?> factory) {
+ this.configuredTargetFactory = factory;
+ return this;
+ }
+
+ public Builder setValidityPredicate(PredicateWithMessage<Rule> predicate) {
+ this.validityPredicate = predicate;
+ return this;
+ }
+
+ public Builder setPreferredDependencyPredicate(Predicate<String> predicate) {
+ this.preferredDependencyPredicate = predicate;
+ return this;
+ }
+
+ /**
+ * State that the rule class being built possibly supplies the specified provider to its direct
+ * dependencies.
+ *
+ * <p>When computing the set of aspects required for a rule, only the providers listed here are
+ * considered. The presence of a provider here does not mean that the rule <b>must</b> implement
+ * said provider, merely that it <b>can</b>. After the configured target is constructed from
+ * this rule, aspects will be filtered according to the set of actual providers.
+ *
+ * <p>This is here so that we can do the loading phase overestimation required for
+ * "blaze query", which does not have the configured targets available.
+ *
+ * <p>It's okay for the rule class eventually not to supply it (possibly based on analysis phase
+ * logic), but if a provider is not advertised but is supplied, aspects that require the it will
+ * not be evaluated for the rule.
+ */
+ public Builder advertiseProvider(Class<?>... providers) {
+ Collections.addAll(advertisedProviders, providers);
+ return this;
+ }
+
+ private void addAttribute(Attribute attribute) {
+ Preconditions.checkState(!attributes.containsKey(attribute.getName()),
+ "An attribute with the name '%s' already exists.", attribute.getName());
+ attributes.put(attribute.getName(), attribute);
+ }
+
+ private void overrideAttribute(Attribute attribute) {
+ String attrName = attribute.getName();
+ Preconditions.checkState(attributes.containsKey(attrName),
+ "No such attribute '%s' to override in ruleclass '%s'.", attrName, name);
+ Type<?> origType = attributes.get(attrName).getType();
+ Type<?> newType = attribute.getType();
+ Preconditions.checkState(origType.equals(newType),
+ "The type of the new attribute '%s' is different from the original one '%s'.",
+ newType, origType);
+ attributes.put(attrName, attribute);
+ }
+
+ /**
+ * Builds attribute from the attribute builder and adds it to this rule
+ * class.
+ *
+ * @param attr attribute builder
+ */
+ public <TYPE> Builder add(Attribute.Builder<TYPE> attr) {
+ addAttribute(attr.build());
+ return this;
+ }
+
+ /**
+ * Builds attribute from the attribute builder and overrides the attribute
+ * with the same name.
+ *
+ * @throws IllegalArgumentException if the attribute does not override one of the same name
+ */
+ public <TYPE> Builder override(Attribute.Builder<TYPE> attr) {
+ overrideAttribute(attr.build());
+ return this;
+ }
+
+ /**
+ * Adds or overrides the attribute in the rule class. Meant for Skylark usage.
+ */
+ public void addOrOverrideAttribute(Attribute attribute) {
+ if (attributes.containsKey(attribute.getName())) {
+ overrideAttribute(attribute);
+ } else {
+ addAttribute(attribute);
+ }
+ }
+
+ /**
+ * Sets the rule implementation function. Meant for Skylark usage.
+ */
+ public Builder setConfiguredTargetFunction(UserDefinedFunction func) {
+ this.configuredTargetFunction = func;
+ return this;
+ }
+
+ /**
+ * Sets the rule definition environment. Meant for Skylark usage.
+ */
+ public Builder setRuleDefinitionEnvironment(SkylarkEnvironment env) {
+ this.ruleDefinitionEnvironment = env;
+ return this;
+ }
+
+ /**
+ * Removes an attribute with the same name from this rule class.
+ *
+ * @throws IllegalArgumentException if the attribute with this name does
+ * not exist
+ */
+ public <TYPE> Builder removeAttribute(String name) {
+ Preconditions.checkState(attributes.containsKey(name), "No such attribute '%s' to remove.",
+ name);
+ attributes.remove(name);
+ return this;
+ }
+
+ /**
+ * This rule class outputs a default executable for every rule with the same name as
+ * the rules's. Only works for Skylark.
+ */
+ public <TYPE> Builder setOutputsDefaultExecutable() {
+ this.outputsDefaultExecutable = true;
+ return this;
+ }
+
+ /**
+ * Declares that instances of this rule are compatible with the specified environments,
+ * in addition to the defaults declared by their environment groups. This can be overridden
+ * by rule-specific declarations. See
+ * {@link com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics} for details.
+ */
+ public <TYPE> Builder compatibleWith(Label... environments) {
+ add(attr(DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR, LABEL_LIST).cfg(HOST)
+ .value(ImmutableList.copyOf(environments)));
+ return this;
+ }
+
+ /**
+ * Declares that instances of this rule are restricted to the specified environments, i.e.
+ * these override the defaults declared by their environment groups. This can be overridden
+ * by rule-specific declarations. See
+ * {@link com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics} for details.
+ *
+ * <p>The input list cannot be empty.
+ */
+ public <TYPE> Builder restrictedTo(Label firstEnvironment, Label... otherEnvironments) {
+ ImmutableList<Label> environments = ImmutableList.<Label>builder().add(firstEnvironment)
+ .add(otherEnvironments).build();
+ add(attr(DEFAULT_RESTRICTED_ENVIRONMENT_ATTR, LABEL_LIST).cfg(HOST).value(environments));
+ return this;
+
+ }
+
+ /**
+ * Returns an Attribute.Builder object which contains a replica of the
+ * same attribute in the parent rule if exists.
+ *
+ * @param name the name of the attribute
+ */
+ public Attribute.Builder<?> copy(String name) {
+ Preconditions.checkArgument(attributes.containsKey(name),
+ "Attribute %s does not exist in parent rule class.", name);
+ return attributes.get(name).cloneBuilder();
+ }
+ }
+
+ private final String name; // e.g. "cc_library"
+
+ /**
+ * The kind of target represented by this RuleClass (e.g. "cc_library rule").
+ * Note: Even though there is partial duplication with the {@link RuleClass#name} field,
+ * we want to store this as a separate field instead of generating it on demand in order to
+ * avoid string duplication.
+ */
+ private final String targetKind;
+
+ private final boolean skylarkExecutable;
+ private final boolean documented;
+ private final boolean publicByDefault;
+ private final boolean binaryOutput;
+ private final boolean workspaceOnly;
+ private final boolean outputsDefaultExecutable;
+
+ /**
+ * A (unordered) mapping from attribute names to small integers indexing into
+ * the {@code attributes} array.
+ */
+ private final Map<String, Integer> attributeIndex = new HashMap<>();
+
+ /**
+ * All attributes of this rule class (including inherited ones) ordered by
+ * attributeIndex value.
+ */
+ private final Attribute[] attributes;
+
+ /**
+ * The set of implicit outputs generated by a rule, expressed as a function
+ * of that rule.
+ */
+ private final ImplicitOutputsFunction implicitOutputsFunction;
+
+ /**
+ * The set of implicit outputs generated by a rule, expressed as a function
+ * of that rule.
+ */
+ private final Configurator<?, ?> configurator;
+
+ /**
+ * The factory that creates configured targets from this rule.
+ */
+ private final ConfiguredTargetFactory<?, ?> configuredTargetFactory;
+
+ /**
+ * The constraint the package name of the rule instance must fulfill
+ */
+ private final PredicateWithMessage<Rule> validityPredicate;
+
+ /**
+ * See {@link #isPreferredDependency}.
+ */
+ private final Predicate<String> preferredDependencyPredicate;
+
+ /**
+ * The list of transitive info providers this class advertises to aspects.
+ */
+ private final ImmutableSet<Class<?>> advertisedProviders;
+
+ /**
+ * The Skylark rule implementation of this RuleClass. Null for non Skylark executable RuleClasses.
+ */
+ @Nullable private final UserDefinedFunction configuredTargetFunction;
+
+ /**
+ * The Skylark rule definition environment of this RuleClass.
+ * Null for non Skylark executable RuleClasses.
+ */
+ @Nullable private final SkylarkEnvironment ruleDefinitionEnvironment;
+
+ /**
+ * The set of required configuration fragments; this should list all fragments that can be
+ * accessed by the rule implementation. If empty, all fragments are allowed to be accessed for
+ * backwards compatibility.
+ */
+ private final ImmutableSet<Class<?>> requiredConfigurationFragments;
+
+ /**
+ * Whether to fail during analysis if a configuration fragment is missing. The default behavior is
+ * to create fail actions for all declared outputs, i.e., to fail during execution, if any of the
+ * outputs is actually attempted to be built.
+ */
+ private final boolean failIfMissingConfigurationFragment;
+
+ /**
+ * Constructs an instance of RuleClass whose name is 'name', attributes
+ * are 'attributes'. The {@code srcsAllowedFiles} determines which types of
+ * files are allowed as parameters to the "srcs" attribute; rules are always
+ * allowed. For the "deps" attribute, there are four cases:
+ * <ul>
+ * <li>if the parameter is a file, it is allowed if its file type is given
+ * in {@code depsAllowedFiles},
+ * <li>if the parameter is a rule and the rule class is accepted by
+ * {@code depsAllowedRules}, then it is allowed,
+ * <li>if the parameter is a rule and the rule class is not accepted by
+ * {@code depsAllowedRules}, but accepted by
+ * {@code depsAllowedRulesWithWarning}, then it is allowed, but
+ * triggers a warning;
+ * <li>all other parameters trigger an error.
+ * </ul>
+ *
+ * <p>The {@code depsAllowedRules} predicate should have a {@code toString}
+ * method which returns a plain English enumeration of the allowed rule class
+ * names, if it does not allow all rule classes.
+ * @param workspaceOnly
+ */
+ @VisibleForTesting
+ RuleClass(String name,
+ boolean skylarkExecutable, boolean documented, boolean publicByDefault,
+ boolean binaryOutput, boolean workspaceOnly, boolean outputsDefaultExecutable,
+ ImplicitOutputsFunction implicitOutputsFunction,
+ Configurator<?, ?> configurator,
+ ConfiguredTargetFactory<?, ?> configuredTargetFactory,
+ PredicateWithMessage<Rule> validityPredicate, Predicate<String> preferredDependencyPredicate,
+ ImmutableSet<Class<?>> advertisedProviders,
+ @Nullable UserDefinedFunction configuredTargetFunction,
+ @Nullable SkylarkEnvironment ruleDefinitionEnvironment,
+ Set<Class<?>> allowedConfigurationFragments, boolean failIfMissingConfigurationFragment,
+ Attribute... attributes) {
+ this.name = name;
+ this.targetKind = name + " rule";
+ this.skylarkExecutable = skylarkExecutable;
+ this.documented = documented;
+ this.publicByDefault = publicByDefault;
+ this.binaryOutput = binaryOutput;
+ this.implicitOutputsFunction = implicitOutputsFunction;
+ this.configurator = Preconditions.checkNotNull(configurator);
+ this.configuredTargetFactory = configuredTargetFactory;
+ this.validityPredicate = validityPredicate;
+ this.preferredDependencyPredicate = preferredDependencyPredicate;
+ this.advertisedProviders = advertisedProviders;
+ this.configuredTargetFunction = configuredTargetFunction;
+ this.ruleDefinitionEnvironment = ruleDefinitionEnvironment;
+ // Do not make a defensive copy as builder does that already
+ this.attributes = attributes;
+ this.workspaceOnly = workspaceOnly;
+ this.outputsDefaultExecutable = outputsDefaultExecutable;
+ this.requiredConfigurationFragments = ImmutableSet.copyOf(allowedConfigurationFragments);
+ this.failIfMissingConfigurationFragment = failIfMissingConfigurationFragment;
+
+ // create the index:
+ int index = 0;
+ for (Attribute attribute : attributes) {
+ attributeIndex.put(attribute.getName(), index++);
+ }
+ }
+
+ /**
+ * Returns the function which determines the set of implicit outputs
+ * generated by a given rule.
+ *
+ * <p>An implicit output is an OutputFile that automatically comes into
+ * existence when a rule of this class is declared, and whose name is derived
+ * from the name of the rule.
+ *
+ * <p>Implicit outputs are a widely-relied upon. All ".so",
+ * and "_deploy.jar" targets referenced in BUILD files are examples.
+ */
+ @VisibleForTesting
+ public ImplicitOutputsFunction getImplicitOutputsFunction() {
+ return implicitOutputsFunction;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <C, R> Configurator<C, R> getConfigurator() {
+ return (Configurator<C, R>) configurator;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <CT, RC> ConfiguredTargetFactory<CT, RC> getConfiguredTargetFactory() {
+ return (ConfiguredTargetFactory<CT, RC>) configuredTargetFactory;
+ }
+
+ /**
+ * Returns the class of rule that this RuleClass represents (e.g. "cc_library").
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns the target kind of this class of rule (e.g. "cc_library rule").
+ */
+ String getTargetKind() {
+ return targetKind;
+ }
+
+ public boolean getWorkspaceOnly() {
+ return workspaceOnly;
+ }
+
+ /**
+ * Returns true iff the attribute 'attrName' is defined for this rule class,
+ * and has type 'type'.
+ */
+ public boolean hasAttr(String attrName, Type<?> type) {
+ Integer index = getAttributeIndex(attrName);
+ return index != null && getAttribute(index).getType() == type;
+ }
+
+ /**
+ * Returns the index of the specified attribute name. Use of indices allows
+ * space-efficient storage of attribute values in rules, since hashtables are
+ * not required. (The index mapping is specific to each RuleClass and an
+ * attribute may have a different index in the parent RuleClass.)
+ *
+ * <p>Returns null if the named attribute is not defined for this class of Rule.
+ */
+ Integer getAttributeIndex(String attrName) {
+ return attributeIndex.get(attrName);
+ }
+
+ /**
+ * Returns the attribute whose index is 'attrIndex'. Fails if attrIndex is
+ * not in range.
+ */
+ Attribute getAttribute(int attrIndex) {
+ return attributes[attrIndex];
+ }
+
+ /**
+ * Returns the attribute whose name is 'attrName'; fails if not found.
+ */
+ public Attribute getAttributeByName(String attrName) {
+ return attributes[getAttributeIndex(attrName)];
+ }
+
+ /**
+ * Returns the attribute whose name is {@code attrName}, or null if not
+ * found.
+ */
+ Attribute getAttributeByNameMaybe(String attrName) {
+ Integer i = getAttributeIndex(attrName);
+ return i == null ? null : attributes[i];
+ }
+
+ /**
+ * Returns the number of attributes defined for this rule class.
+ */
+ public int getAttributeCount() {
+ return attributeIndex.size();
+ }
+
+ /**
+ * Returns an (immutable) list of all Attributes defined for this class of
+ * rule, ordered by increasing index.
+ */
+ public List<Attribute> getAttributes() {
+ return ImmutableList.copyOf(attributes);
+ }
+
+ public PredicateWithMessage<Rule> getValidityPredicate() {
+ return validityPredicate;
+ }
+
+ /**
+ * Returns the set of advertised transitive info providers.
+ *
+ * <p>When computing the set of aspects required for a rule, only the providers listed here are
+ * considered. The presence of a provider here does not mean that the rule <b>must</b> implement
+ * said provider, merely that it <b>can</b>. After the configured target is constructed from this
+ * rule, aspects will be filtered according to the set of actual providers.
+ *
+ * <p>This is here so that we can do the loading phase overestimation required for "blaze query",
+ * which does not have the configured targets available.
+ *
+ * <p>This should in theory only contain subclasses of
+ * {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider}, but our current dependency
+ * structure does not allow a reference to that class here.
+ */
+ public ImmutableSet<Class<?>> getAdvertisedProviders() {
+ return advertisedProviders;
+ }
+
+ /**
+ * For --compile_one_dependency: if multiple rules consume the specified target,
+ * should we choose this one over the "unpreferred" options?
+ */
+ public boolean isPreferredDependency(String filename) {
+ return preferredDependencyPredicate.apply(filename);
+ }
+
+ /**
+ * The set of required configuration fragments; this contains all fragments that can be
+ * accessed by the rule implementation. If empty, all fragments are allowed to be accessed for
+ * backwards compatibility.
+ */
+ public Set<Class<?>> getRequiredConfigurationFragments() {
+ return requiredConfigurationFragments;
+ }
+
+ /**
+ * Checks if the configuration fragment may be accessed (i.e., if it's declared). If no fragments
+ * are declared, this allows access to all fragments for backwards compatibility.
+ */
+ public boolean isLegalConfigurationFragment(Class<?> configurationFragment) {
+ // For now, we allow all rules that don't declare allowed fragments to access any fragment.
+ // TODO(bazel-team): Declare fragment dependencies for all rules and remove this.
+ if (requiredConfigurationFragments.isEmpty()) {
+ return true;
+ }
+ return requiredConfigurationFragments.contains(configurationFragment);
+ }
+
+ /**
+ * Whether to fail analysis if any of the required configuration fragments are missing.
+ */
+ public boolean failIfMissingConfigurationFragment() {
+ return failIfMissingConfigurationFragment;
+ }
+
+ /**
+ * Helper function for {@link RuleFactory#createRule}.
+ */
+ Rule createRuleWithLabel(Package.AbstractBuilder<?, ?> pkgBuilder, Label ruleLabel,
+ Map<String, Object> attributeValues, EventHandler eventHandler, FuncallExpression ast,
+ Location location) throws SyntaxException {
+ Rule rule = pkgBuilder.newRuleWithLabel(ruleLabel, this, null, location);
+ createRuleCommon(rule, pkgBuilder, attributeValues, eventHandler, ast);
+ return rule;
+ }
+
+ private void createRuleCommon(Rule rule, Package.AbstractBuilder<?, ?> pkgBuilder,
+ Map<String, Object> attributeValues, EventHandler eventHandler, FuncallExpression ast)
+ throws SyntaxException {
+ populateRuleAttributeValues(
+ rule, pkgBuilder, attributeValues, eventHandler, ast);
+ rule.populateOutputFiles(eventHandler, pkgBuilder);
+ rule.checkForNullLabels();
+ rule.checkValidityPredicate(eventHandler);
+ }
+
+ static class ParsedAttributeValue {
+ private final boolean explicitlySpecified;
+ private final Object value;
+ private final Location location;
+
+ ParsedAttributeValue(boolean explicitlySpecified, Object value, Location location) {
+ this.explicitlySpecified = explicitlySpecified;
+ this.value = value;
+ this.location = location;
+ }
+
+ public boolean getExplicitlySpecified() {
+ return explicitlySpecified;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+ }
+
+ /**
+ * Creates a rule with the attribute values that are already parsed.
+ *
+ * <p><b>WARNING:</b> This assumes that the attribute values here have the right type and
+ * bypasses some sanity checks. If they are of the wrong type, everything will come down burning.
+ */
+ @SuppressWarnings("unchecked")
+ Rule createRuleWithParsedAttributeValues(Label label,
+ Package.AbstractBuilder<?, ?> pkgBuilder, Location ruleLocation,
+ Map<String, ParsedAttributeValue> attributeValues, EventHandler eventHandler)
+ throws SyntaxException{
+ Rule rule = pkgBuilder.newRuleWithLabel(label, this, null, ruleLocation);
+ rule.checkValidityPredicate(eventHandler);
+
+ for (Attribute attribute : rule.getRuleClassObject().getAttributes()) {
+ ParsedAttributeValue value = attributeValues.get(attribute.getName());
+ if (attribute.isMandatory()) {
+ Preconditions.checkState(value != null);
+ }
+
+ if (value == null) {
+ continue;
+ }
+
+ checkAllowedValues(rule, attribute, value.getValue(), eventHandler);
+ rule.setAttributeValue(attribute, value.getValue(), value.getExplicitlySpecified());
+ rule.setAttributeLocation(attribute, value.getLocation());
+
+ if (attribute.getName().equals("visibility")) {
+ // TODO(bazel-team): Verify that this cast works
+ rule.setVisibility(PackageFactory.getVisibility((List<Label>) value.getValue()));
+ }
+ }
+
+ rule.populateOutputFiles(eventHandler, pkgBuilder);
+ Preconditions.checkState(!rule.containsErrors());
+ return rule;
+ }
+
+ /**
+ * Populates the attributes table of new rule "rule" from the
+ * "attributeValues" mapping from attribute names to values in the build
+ * language. Errors are reported on "reporter". "ast" is used to associate
+ * location information with each rule attribute.
+ */
+ private void populateRuleAttributeValues(Rule rule,
+ Package.AbstractBuilder<?, ?> pkgBuilder,
+ Map<String, Object> attributeValues,
+ EventHandler eventHandler,
+ FuncallExpression ast) {
+ BitSet definedAttrs = new BitSet(); // set of attr indices
+
+ for (Map.Entry<String, Object> entry : attributeValues.entrySet()) {
+ String attributeName = entry.getKey();
+ Object attributeValue = entry.getValue();
+ if (attributeValue == Environment.NONE) { // Ignore all None values.
+ continue;
+ }
+ Integer attrIndex = setRuleAttributeValue(rule, eventHandler, attributeName, attributeValue);
+ if (attrIndex != null) {
+ definedAttrs.set(attrIndex);
+ checkAttrValNonEmpty(rule, eventHandler, attributeValue, attrIndex);
+ }
+ }
+
+ // Save the location of each non-default attribute definition:
+ if (ast != null) {
+ for (Argument arg : ast.getArguments()) {
+ Ident keyword = arg.getName();
+ if (keyword != null) {
+ String name = keyword.getName();
+ Integer attrIndex = getAttributeIndex(name);
+ if (attrIndex != null) {
+ rule.setAttributeLocation(attrIndex, arg.getValue().getLocation());
+ }
+ }
+ }
+ }
+
+ List<Attribute> attrsWithComputedDefaults = new ArrayList<>();
+
+ // Set defaults; ensure that every mandatory attribute has a value. Use
+ // the default if none is specified.
+ int numAttributes = getAttributeCount();
+ for (int attrIndex = 0; attrIndex < numAttributes; ++attrIndex) {
+ if (!definedAttrs.get(attrIndex)) {
+ Attribute attr = getAttribute(attrIndex);
+ if (attr.isMandatory()) {
+ rule.reportError(rule.getLabel() + ": missing value for mandatory "
+ + "attribute '" + attr.getName() + "' in '"
+ + name + "' rule", eventHandler);
+ }
+
+ if (attr.hasComputedDefault()) {
+ attrsWithComputedDefaults.add(attr);
+ } else {
+ Object defaultValue = getAttributeNoncomputedDefaultValue(attr, pkgBuilder);
+ checkAttrValNonEmpty(rule, eventHandler, defaultValue, attrIndex);
+ checkAllowedValues(rule, attr, defaultValue, eventHandler);
+ rule.setAttributeValue(attr, defaultValue, /*explicit=*/false);
+ }
+ }
+ }
+
+ // Evaluate and set any computed defaults now that all non-computed
+ // TODO(bazel-team): remove this special casing. Thanks to configurable attributes refactoring,
+ // computed defaults don't get bound to their final values at this point, so we no longer
+ // have to wait until regular attributes have been initialized.
+ for (Attribute attr : attrsWithComputedDefaults) {
+ rule.setAttributeValue(attr, attr.getDefaultValue(rule), /*explicit=*/false);
+ }
+
+ // Now that all attributes are bound to values, collect and store configurable attribute keys.
+ populateConfigDependenciesAttribute(rule);
+ checkForDuplicateLabels(rule, eventHandler);
+ checkThirdPartyRuleHasLicense(rule, pkgBuilder, eventHandler);
+ checkForValidSizeAndTimeoutValues(rule, eventHandler);
+ }
+
+ /**
+ * Collects all labels used as keys for configurable attributes and places them into
+ * the special implicit attribute that tracks them.
+ */
+ private static void populateConfigDependenciesAttribute(Rule rule) {
+ RawAttributeMapper attributes = RawAttributeMapper.of(rule);
+ Attribute configDepsAttribute = attributes.getAttributeDefinition("$config_dependencies");
+ if (configDepsAttribute == null) {
+ // Not currently compatible with Skylark rules.
+ return;
+ }
+
+ Set<Label> configLabels = new LinkedHashSet<>();
+ for (Attribute attr : rule.getAttributes()) {
+ Type.Selector<?> selector = attributes.getSelector(attr.getName(), attr.getType());
+ if (selector != null) {
+ for (Label label : selector.getEntries().keySet()) {
+ if (!Type.Selector.isReservedLabel(label)) {
+ configLabels.add(label);
+ }
+ }
+ }
+ }
+
+ rule.setAttributeValue(configDepsAttribute, ImmutableList.copyOf(configLabels),
+ /*explicit=*/false);
+ }
+
+ private void checkAttrValNonEmpty(
+ Rule rule, EventHandler eventHandler, Object attributeValue, Integer attrIndex) {
+ if (attributeValue instanceof List<?>) {
+ Attribute attr = getAttribute(attrIndex);
+ if (attr.isNonEmpty() && ((List<?>) attributeValue).isEmpty()) {
+ rule.reportError(rule.getLabel() + ": non empty " + "attribute '" + attr.getName()
+ + "' in '" + name + "' rule '" + rule.getLabel() + "' has to have at least one value",
+ eventHandler);
+ }
+ }
+ }
+
+ /**
+ * Report an error for each label that appears more than once in a LABEL_LIST attribute
+ * of the given rule.
+ *
+ * @param rule The rule.
+ * @param eventHandler The eventHandler to use to report the duplicated deps.
+ */
+ private static void checkForDuplicateLabels(Rule rule, EventHandler eventHandler) {
+ for (Attribute attribute : rule.getAttributes()) {
+ if (attribute.getType() == Type.LABEL_LIST) {
+ checkForDuplicateLabels(rule, attribute, eventHandler);
+ }
+ }
+ }
+
+ /**
+ * Reports an error against the specified rule if it's beneath third_party
+ * but does not have a declared license.
+ */
+ private static void checkThirdPartyRuleHasLicense(Rule rule,
+ Package.AbstractBuilder<?, ?> pkgBuilder, EventHandler eventHandler) {
+ if (rule.getLabel().getPackageName().startsWith("third_party/")) {
+ License license = rule.getLicense();
+ if (license == null) {
+ license = pkgBuilder.getDefaultLicense();
+ }
+ if (license == License.NO_LICENSE) {
+ rule.reportError("third-party rule '" + rule.getLabel() + "' lacks a license declaration "
+ + "with one of the following types: notice, reciprocal, permissive, "
+ + "restricted, unencumbered, by_exception_only",
+ eventHandler);
+ }
+ }
+ }
+
+ /**
+ * Report an error for each label that appears more than once in the given attribute
+ * of the given rule.
+ *
+ * @param rule The rule.
+ * @param attribute The attribute to check. Must exist in rule and be of type LABEL_LIST.
+ * @param eventHandler The eventHandler to use to report the duplicated deps.
+ */
+ private static void checkForDuplicateLabels(Rule rule, Attribute attribute,
+ EventHandler eventHandler) {
+ final String attrName = attribute.getName();
+ // This attribute may be selectable, so iterate over each selection possibility in turn.
+ // TODO(bazel-team): merge '*' condition into all lists when implemented.
+ AggregatingAttributeMapper attributeMap = AggregatingAttributeMapper.of(rule);
+ for (List<Label> labels : attributeMap.visitAttribute(attrName, Type.LABEL_LIST)) {
+ if (!labels.isEmpty()) {
+ Set<Label> duplicates = CollectionUtils.duplicatedElementsOf(labels);
+ for (Label label : duplicates) {
+ rule.reportError(
+ String.format("Label '%s' is duplicated in the '%s' attribute of rule '%s'",
+ label, attrName, rule.getName()), eventHandler);
+ }
+ }
+ }
+ }
+
+ /**
+ * Report an error if the rule has a timeout or size attribute that is not a
+ * legal value. These attributes appear on all tests.
+ *
+ * @param rule the rule to check
+ * @param eventHandler the eventHandler to use to report the duplicated deps
+ */
+ private static void checkForValidSizeAndTimeoutValues(Rule rule, EventHandler eventHandler) {
+ if (rule.getRuleClassObject().hasAttr("size", Type.STRING)) {
+ String size = NonconfigurableAttributeMapper.of(rule).get("size", Type.STRING);
+ if (TestSize.getTestSize(size) == null) {
+ rule.reportError(
+ String.format("In rule '%s', size '%s' is not a valid size.", rule.getName(), size),
+ eventHandler);
+ }
+ }
+ if (rule.getRuleClassObject().hasAttr("timeout", Type.STRING)) {
+ String timeout = NonconfigurableAttributeMapper.of(rule).get("timeout", Type.STRING);
+ if (TestTimeout.getTestTimeout(timeout) == null) {
+ rule.reportError(
+ String.format(
+ "In rule '%s', timeout '%s' is not a valid timeout.", rule.getName(), timeout),
+ eventHandler);
+ }
+ }
+ }
+
+ /**
+ * Returns the default value for the specified rule attribute.
+ *
+ * For most rule attributes, the default value is either explicitly specified
+ * in the attribute, or implicitly based on the type of the attribute, except
+ * for some special cases (e.g. "licenses", "distribs") where it comes from
+ * some other source, such as state in the package.
+ *
+ * Precondition: {@code !attr.hasComputedDefault()}. (Computed defaults are
+ * evaluated in second pass.)
+ */
+ private static Object getAttributeNoncomputedDefaultValue(Attribute attr,
+ Package.AbstractBuilder<?, ?> pkgBuilder) {
+ if (attr.getName().equals("licenses")) {
+ return pkgBuilder.getDefaultLicense();
+ }
+ if (attr.getName().equals("distribs")) {
+ return pkgBuilder.getDefaultDistribs();
+ }
+ return attr.getDefaultValue(null);
+ }
+
+ /**
+ * Sets the value of attribute "attrName" in rule "rule", by converting the
+ * build-language value "attrVal" to the appropriate type for the attribute.
+ * Returns the attribute index iff successful, null otherwise.
+ *
+ * <p>In case of failure, error messages are reported on "handler", and "rule"
+ * is marked as containing errors.
+ */
+ @SuppressWarnings("unchecked")
+ private Integer setRuleAttributeValue(Rule rule,
+ EventHandler eventHandler,
+ String attrName,
+ Object attrVal) {
+ if (attrName.equals("name")) {
+ return null; // "name" is handled specially
+ }
+
+ Integer attrIndex = getAttributeIndex(attrName);
+ if (attrIndex == null) {
+ rule.reportError(rule.getLabel() + ": no such attribute '" + attrName +
+ "' in '" + name + "' rule", eventHandler);
+ return null;
+ }
+
+ Attribute attr = getAttribute(attrIndex);
+ Object converted;
+ try {
+ String what = "attribute '" + attrName + "' in '" + name + "' rule";
+ converted = attr.getType().selectableConvert(attrVal, what, rule.getLabel());
+
+ if ((converted instanceof Type.Selector<?>) && !attr.isConfigurable()) {
+ rule.reportError(rule.getLabel() + ": attribute \"" + attr.getName()
+ + "\" is not configurable", eventHandler);
+ return null;
+ }
+
+ if ((converted instanceof List<?>) && !(converted instanceof GlobList<?>)) {
+ if (attr.isOrderIndependent()) {
+ converted = Ordering.natural().sortedCopy((List<? extends Comparable<?>>) converted);
+ }
+ converted = ImmutableList.copyOf((List<?>) converted);
+ }
+ } catch (Type.ConversionException e) {
+ rule.reportError(rule.getLabel() + ": " + e.getMessage(), eventHandler);
+ return null;
+ }
+
+ if (attrName.equals("visibility")) {
+ List<Label> attrList = (List<Label>) converted;
+ if (!attrList.isEmpty() &&
+ ConstantRuleVisibility.LEGACY_PUBLIC_LABEL.equals(attrList.get(0))) {
+ rule.reportError(rule.getLabel() + ": //visibility:legacy_public only allowed in package "
+ + "declaration", eventHandler);
+ }
+ rule.setVisibility(PackageFactory.getVisibility(attrList));
+ }
+
+ checkAllowedValues(rule, attr, converted, eventHandler);
+ rule.setAttributeValue(attr, converted, /*explicit=*/true);
+ return attrIndex;
+ }
+
+ private void checkAllowedValues(Rule rule, Attribute attribute, Object value,
+ EventHandler eventHandler) {
+ if (attribute.checkAllowedValues()) {
+ PredicateWithMessage<Object> allowedValues = attribute.getAllowedValues();
+ if (!allowedValues.apply(value)) {
+ rule.reportError(String.format(rule.getLabel() + ": invalid value in '%s' attribute: %s",
+ attribute.getName(),
+ allowedValues.getErrorReason(value)), eventHandler);
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public boolean isDocumented() {
+ return documented;
+ }
+
+ public boolean isPublicByDefault() {
+ return publicByDefault;
+ }
+
+ /**
+ * Returns true iff the outputs of this rule should be created beneath the
+ * <i>bin</i> directory, false if beneath <i>genfiles</i>. For most rule
+ * classes, this is a constant, but for genrule, it is a property of the
+ * individual rule instance, derived from the 'output_to_bindir' attribute;
+ * see Rule.hasBinaryOutput().
+ */
+ boolean hasBinaryOutput() {
+ return binaryOutput;
+ }
+
+ /**
+ * Returns this RuleClass's custom Skylark rule implementation.
+ */
+ @Nullable public UserDefinedFunction getConfiguredTargetFunction() {
+ return configuredTargetFunction;
+ }
+
+ /**
+ * Returns this RuleClass's rule definition environment.
+ */
+ @Nullable public SkylarkEnvironment getRuleDefinitionEnvironment() {
+ return ruleDefinitionEnvironment;
+ }
+
+ /**
+ * Returns true if this RuleClass is an executable Skylark RuleClass (i.e. it is
+ * Skylark and Normal or Test RuleClass).
+ */
+ public boolean isSkylarkExecutable() {
+ return skylarkExecutable;
+ }
+
+ /**
+ * Returns true if this rule class outputs a default executable for every rule.
+ */
+ public boolean outputsDefaultExecutable() {
+ return outputsDefaultExecutable;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/packages/RuleClassProvider.java
new file mode 100644
index 0000000..90fdfca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RuleClassProvider.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.ValidationEnvironment;
+
+import java.util.Map;
+
+/**
+ * The collection of the supported build rules. Provides an Environment for Skylark rule creation.
+ */
+public interface RuleClassProvider {
+ /**
+ * Returns a map from rule names to rule class objects.
+ */
+ Map<String, RuleClass> getRuleClassMap();
+
+ /**
+ * Returns a new Skylark Environment instance for rule creation. Implementations need to be
+ * thread safe.
+ */
+ SkylarkEnvironment createSkylarkRuleClassEnvironment(
+ EventHandler eventHandler, String astFileContentHashCode);
+
+ /**
+ * Returns a validation environment for static analysis of skylark files.
+ * The environment has to contain all built-in functions and objects.
+ */
+ ValidationEnvironment getSkylarkValidationEnvironment();
+
+ /**
+ * Returns the Skylark module to register the native rules with.
+ */
+ Object getNativeModule();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleErrorConsumer.java b/src/main/java/com/google/devtools/build/lib/packages/RuleErrorConsumer.java
new file mode 100644
index 0000000..84d00c0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RuleErrorConsumer.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * A thin interface exposing only the warning and error reporting functionality
+ * of a rule.
+ *
+ * <p>When a class or a method needs only this functionality but not the whole
+ * {@code RuleConfiguredTarget}, it can use this thin interface instead.
+ *
+ * <p>This interface should only be implemented by {@code RuleConfiguredTarget}
+ * and its subclasses.
+ */
+public interface RuleErrorConsumer {
+ /**
+ * Consume a non-attribute-specific warning in a rule.
+ */
+ void ruleWarning(String message);
+
+ /**
+ * Consume a non-attribute-specific error in a rule.
+ */
+ void ruleError(String message);
+
+ /**
+ * Consume an attribute-specific warning in a rule.
+ */
+ void attributeWarning(String attrName, String message);
+
+ /**
+ * Consume an attribute-specific error in a rule.
+ */
+ void attributeError(String attrName, String message);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java b/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java
new file mode 100644
index 0000000..c79bbaa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Package.NameConflictException;
+import com.google.devtools.build.lib.packages.PackageFactory.PackageContext;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Given a rule class and a set of attributes, returns a Rule instance. Also
+ * performs a number of checks and associates the rule and the owning package
+ * with each other.
+ *
+ * <p>Note: the code that actually populates the RuleClass map has been moved
+ * to {@link RuleClassProvider}.
+ */
+public class RuleFactory {
+
+ /**
+ * Maps rule class name to the metaclass instance for that rule.
+ */
+ private final ImmutableMap<String, RuleClass> ruleClassMap;
+
+ /**
+ * Constructs a RuleFactory instance.
+ */
+ public RuleFactory(RuleClassProvider provider) {
+ this.ruleClassMap = ImmutableMap.copyOf(provider.getRuleClassMap());
+ }
+
+ /**
+ * Returns the (immutable, unordered) set of names of all the known rule classes.
+ */
+ public Set<String> getRuleClassNames() {
+ return ruleClassMap.keySet();
+ }
+
+ /**
+ * Returns the RuleClass for the specified rule class name.
+ */
+ public RuleClass getRuleClass(String ruleClassName) {
+ return ruleClassMap.get(ruleClassName);
+ }
+
+ /**
+ * Creates and returns a rule instance.
+ *
+ * <p>It is the caller's responsibility to add the rule to the package (the
+ * caller may choose not to do so if, for example, the rule has errors).
+ *
+ * @param pkgBuilder the under-construction package to which the rule belongs
+ * @param ruleClass the class of the rule; this must not be null
+ * @param attributeValues a map of attribute names to attribute values. Each
+ * attribute must be defined for this class of rule, and have a value
+ * of the appropriate type. There must be a map entry for each
+ * non-optional attribute of this class of rule.
+ * @param eventHandler a eventHandler on which errors and warnings are reported during
+ * rule creation
+ * @param ast the abstract syntax tree of the rule expression (optional)
+ * @param location the location at which this rule was declared
+ * @throws InvalidRuleException if the rule could not be constructed for any
+ * reason (e.g. no <code>name</code> attribute is defined)
+ * @throws NameConflictException
+ */
+ static Rule createAndAddRule(Package.AbstractBuilder<?, ?> pkgBuilder,
+ RuleClass ruleClass,
+ Map<String, Object> attributeValues,
+ EventHandler eventHandler,
+ FuncallExpression ast,
+ Location location) throws InvalidRuleException, NameConflictException {
+ Preconditions.checkNotNull(ruleClass);
+ String ruleClassName = ruleClass.getName();
+ Object nameObject = attributeValues.get("name");
+ if (!(nameObject instanceof String)) {
+ throw new InvalidRuleException(ruleClassName + " rule has no 'name' attribute");
+ }
+ String name = (String) nameObject;
+ Label label;
+ try {
+ // Test that this would form a valid label name -- in particular, this
+ // catches cases where Makefile variables $(foo) appear in "name".
+ label = pkgBuilder.createLabel(name);
+ } catch (Label.SyntaxException e) {
+ throw new InvalidRuleException("illegal rule name: " + name + ": " + e.getMessage());
+ }
+ boolean inWorkspaceFile = location.getPath() != null
+ && location.getPath().endsWith(new PathFragment("WORKSPACE"));
+ if (ruleClass.getWorkspaceOnly() && !inWorkspaceFile) {
+ throw new RuleFactory.InvalidRuleException(ruleClass + " must be in the WORKSPACE file "
+ + "(used by " + label + ")");
+ } else if (!ruleClass.getWorkspaceOnly() && inWorkspaceFile) {
+ throw new RuleFactory.InvalidRuleException(ruleClass + " cannot be in the WORKSPACE file "
+ + "(used by " + label + ")");
+ }
+
+ try {
+ Rule rule = ruleClass.createRuleWithLabel(pkgBuilder, label, attributeValues,
+ eventHandler, ast, location);
+ pkgBuilder.addRule(rule);
+ return rule;
+ } catch (SyntaxException e) {
+ throw new RuleFactory.InvalidRuleException(ruleClass + " " + e.getMessage());
+ }
+ }
+
+ public static Rule createAndAddRule(PackageContext context,
+ RuleClass ruleClass,
+ Map<String, Object> attributeValues,
+ FuncallExpression ast) throws InvalidRuleException, NameConflictException {
+ return createAndAddRule(context.pkgBuilder, ruleClass, attributeValues, context.eventHandler,
+ ast, ast.getLocation());
+ }
+
+ /**
+ * InvalidRuleException is thrown by createRule() if the Rule could not be
+ * constructed. It contains an error message.
+ */
+ public static class InvalidRuleException extends Exception {
+ private InvalidRuleException(String message) {
+ super(message);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleVisibility.java b/src/main/java/com/google/devtools/build/lib/packages/RuleVisibility.java
new file mode 100644
index 0000000..ef1a126
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/RuleVisibility.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.List;
+
+/**
+ * A RuleVisibility specifies which other rules can depend on a specified rule.
+ * Note that the actual method that performs this check is declared in
+ * RuleConfiguredTargetVisibility.
+ *
+ * <p>The conversion to ConfiguredTargetVisibility is handled in an ugly
+ * if-ladder, because I want to avoid this package depending on build.lib.view.
+ *
+ * All implementations of this interface are immutable.
+ */
+public interface RuleVisibility {
+ /**
+ * Returns the list of labels that need to be loaded so that the visibility
+ * decision can be made during analysis time. E.g. for package group
+ * visibility, this is the list of package groups referenced. Does not include
+ * labels that have special meanings in the visibility declaration, e.g.
+ * "//visibility:*" or "//*:__pkg__".
+ */
+ List<Label> getDependencyLabels();
+
+ /**
+ * Returns the list of labels used during the declaration of this visibility.
+ * These do not necessarily represent loadable labels: for example, for public
+ * or private visibilities, the special labels "//visibility:*" will be
+ * returned, and so will be the special "//*:__pkg__" labels indicating a
+ * single package.
+ */
+ List<Label> getDeclaredLabels();
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/packages/SkylarkFileType.java b/src/main/java/com/google/devtools/build/lib/packages/SkylarkFileType.java
new file mode 100644
index 0000000..f6098cf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/SkylarkFileType.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileType.HasFilename;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+/**
+ * A wrapper class for FileType and FileTypeSet functionality in Skylark.
+ */
+@SkylarkModule(name = "FileType", doc = "File type for file filtering.")
+public class SkylarkFileType {
+
+ private final FileType fileType;
+
+ private SkylarkFileType(FileType fileType) {
+ this.fileType = fileType;
+ }
+
+ public static SkylarkFileType of(Iterable<String> extensions) {
+ return new SkylarkFileType(FileType.of(extensions));
+ }
+
+ public FileTypeSet getFileTypeSet() {
+ return FileTypeSet.of(fileType);
+ }
+
+ @SkylarkCallable(doc = "")
+ public ImmutableList<HasFilename> filter(Iterable<HasFilename> files) {
+ return ImmutableList.copyOf(FileType.filter(files, fileType));
+ }
+
+ @SkylarkCallable(doc = "")
+ public boolean matches(String fileName) {
+ return fileType.apply(fileName);
+ }
+
+ @VisibleForTesting
+ public Object getExtensions() {
+ return fileType.getExtensions();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Target.java b/src/main/java/com/google/devtools/build/lib/packages/Target.java
new file mode 100644
index 0000000..ec5bc86
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/Target.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+
+import java.util.Set;
+
+/**
+ * A node in the build dependency graph, identified by a Label.
+ */
+@SkylarkModule(name = "target", doc = "A BUILD target.")
+public interface Target {
+
+ /**
+ * Returns the label of this target. (e.g. "//foo:bar")
+ */
+ @SkylarkCallable(name = "label", doc = "")
+ Label getLabel();
+
+ /**
+ * Returns the name of this rule (relative to its owning package).
+ */
+ @SkylarkCallable(name = "name", doc = "")
+ String getName();
+
+ /**
+ * Returns the Package to which this rule belongs.
+ */
+ Package getPackage();
+
+ /**
+ * Returns a string describing this kind of target: e.g. "cc_library rule",
+ * "source file", "generated file".
+ */
+ String getTargetKind();
+
+ /**
+ * Returns the rule associated with this target, if any.
+ *
+ * If this is a Rule, returns itself; it this is an OutputFile, returns its
+ * generating rule; if this is an input file, returns null.
+ */
+ Rule getAssociatedRule();
+
+ /**
+ * Returns the license associated with this target.
+ */
+ License getLicense();
+
+ /**
+ * Returns the place where the target was defined.
+ */
+ Location getLocation();
+
+ /**
+ * Returns the set of distribution types associated with this target.
+ */
+ Set<DistributionType> getDistributions();
+
+ /**
+ * Returns the visibility of this target.
+ */
+ RuleVisibility getVisibility();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java b/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java
new file mode 100644
index 0000000..3710eeb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java
@@ -0,0 +1,265 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Utility functions over Targets that don't really belong in the base {@link
+ * Target} interface.
+ */
+public final class TargetUtils {
+
+ // *_test / test_suite attribute that used to specify constraint keywords.
+ private static final String CONSTRAINTS_ATTR = "tags";
+
+ private TargetUtils() {} // Uninstantiable.
+
+ public static boolean isTestRuleName(String name) {
+ return name.endsWith("_test");
+ }
+
+ public static boolean isTestSuiteRuleName(String name) {
+ return name.equals("test_suite");
+ }
+
+ /**
+ * Returns true iff {@code target} is a {@code *_test} rule; excludes {@code
+ * test_suite}.
+ */
+ public static boolean isTestRule(Target target) {
+ return (target instanceof Rule) && isTestRuleName(((Rule) target).getRuleClass());
+ }
+
+ /**
+ * Returns true iff {@code target} is a {@code test_suite} rule.
+ */
+ public static boolean isTestSuiteRule(Target target) {
+ return target instanceof Rule &&
+ isTestSuiteRuleName(((Rule) target).getRuleClass());
+ }
+
+ /**
+ * Returns true iff {@code target} is a {@code *_test} or {@code test_suite}.
+ */
+ public static boolean isTestOrTestSuiteRule(Target target) {
+ return isTestRule (target) || isTestSuiteRule(target);
+ }
+
+ /**
+ * Returns true if {@code target} has "manual" in the tags attribute and thus should be ignored by
+ * command-line wildcards or by test_suite $implicit_tests attribute.
+ */
+ public static boolean hasManualTag(Target target) {
+ return (target instanceof Rule) && hasConstraint((Rule) target, "manual");
+ }
+
+ /**
+ * Returns true if test marked as "exclusive" by the appropriate keyword
+ * in the tags attribute.
+ *
+ * Method assumes that passed target is a test rule, so usually it should be
+ * used only after isTestRule() or isTestOrTestSuiteRule(). Behavior is
+ * undefined otherwise.
+ */
+ public static boolean isExclusiveTestRule(Rule rule) {
+ return hasConstraint(rule, "exclusive");
+ }
+
+ /**
+ * Returns true if test marked as "local" by the appropriate keyword
+ * in the tags attribute.
+ *
+ * Method assumes that passed target is a test rule, so usually it should be
+ * used only after isTestRule() or isTestOrTestSuiteRule(). Behavior is
+ * undefined otherwise.
+ */
+ public static boolean isLocalTestRule(Rule rule) {
+ return hasConstraint(rule, "local")
+ || NonconfigurableAttributeMapper.of(rule).get("local", Type.BOOLEAN);
+ }
+
+ /**
+ * Returns true if the rule is a test or test suite and is local or exclusive.
+ * Wraps the above calls into one generic check safely applicable to any rule.
+ */
+ public static boolean isTestRuleAndRunsLocally(Rule rule) {
+ return isTestOrTestSuiteRule(rule) &&
+ (isLocalTestRule(rule) || isExclusiveTestRule(rule));
+ }
+
+ /**
+ * Returns true if test marked as "external" by the appropriate keyword
+ * in the tags attribute.
+ *
+ * Method assumes that passed target is a test rule, so usually it should be
+ * used only after isTestRule() or isTestOrTestSuiteRule(). Behavior is
+ * undefined otherwise.
+ */
+ public static boolean isExternalTestRule(Rule rule) {
+ return hasConstraint(rule, "external");
+ }
+
+ /**
+ * Returns true, iff the given target is a rule and it has the attribute
+ * <code>obsolete<code/> set to one.
+ */
+ public static boolean isObsolete(Target target) {
+ if (!(target instanceof Rule)) {
+ return false;
+ }
+ Rule rule = (Rule) target;
+ return (rule.isAttrDefined("obsolete", Type.BOOLEAN))
+ && NonconfigurableAttributeMapper.of(rule).get("obsolete", Type.BOOLEAN);
+ }
+
+ /**
+ * If the given target is a rule, returns its <code>deprecation<code/> value, or null if unset.
+ */
+ @Nullable
+ public static String getDeprecation(Target target) {
+ if (!(target instanceof Rule)) {
+ return null;
+ }
+ Rule rule = (Rule) target;
+ return (rule.isAttrDefined("deprecation", Type.STRING))
+ ? NonconfigurableAttributeMapper.of(rule).get("deprecation", Type.STRING)
+ : null;
+ }
+
+ /**
+ * Checks whether specified constraint keyword is present in the
+ * tags attribute of the test or test suite rule.
+ *
+ * Method assumes that provided rule is a test or a test suite. Behavior is
+ * undefined otherwise.
+ */
+ private static boolean hasConstraint(Rule rule, String keyword) {
+ return NonconfigurableAttributeMapper.of(rule).get(CONSTRAINTS_ATTR, Type.STRING_LIST)
+ .contains(keyword);
+ }
+
+ /**
+ * Returns the execution info. These include execution requirement
+ * tags ('requires-*' as well as "local") as keys with empty values.
+ */
+ public static Map<String, String> getExecutionInfo(Rule rule) {
+ // tags may contain duplicate values.
+ Map<String, String> map = new HashMap<>();
+ for (String tag :
+ NonconfigurableAttributeMapper.of(rule).get(CONSTRAINTS_ATTR, Type.STRING_LIST)) {
+ if (tag.startsWith("requires-") || tag.equals("local")) {
+ map.put(tag, "");
+ }
+ }
+ return ImmutableMap.copyOf(map);
+ }
+
+ /**
+ * Returns the language part of the rule name (e.g. "foo" for foo_test or foo_binary).
+ *
+ * <p>In practice this is the part before the "_", if any, otherwise the entire rule class name.
+ *
+ * <p>Precondition: isTestRule(target) || isRunnableNonTestRule(target).
+ */
+ public static String getRuleLanguage(Target target) {
+ return getRuleLanguage(((Rule) target).getRuleClass());
+ }
+
+ /**
+ * Returns the language part of the rule name (e.g. "foo" for foo_test or foo_binary).
+ *
+ * <p>In practice this is the part before the "_", if any, otherwise the entire rule class name.
+ */
+ public static String getRuleLanguage(String ruleClass) {
+ int index = ruleClass.lastIndexOf("_");
+ // Chop off "_binary" or "_test".
+ return index != -1 ? ruleClass.substring(0, index) : ruleClass;
+ }
+
+ private static boolean isExplicitDependency(Rule rule, Label label) {
+ if (rule.getVisibility().getDependencyLabels().contains(label)) {
+ return true;
+ }
+
+ ExplicitEdgeVisitor visitor = new ExplicitEdgeVisitor(rule, label);
+ AggregatingAttributeMapper.of(rule).visitLabels(visitor);
+ return visitor.isExplicit();
+ }
+
+ private static class ExplicitEdgeVisitor implements AttributeMap.AcceptsLabelAttribute {
+ private final Label expectedLabel;
+ private final Rule rule;
+ private boolean isExplicit = false;
+
+ public ExplicitEdgeVisitor(Rule rule, Label expected) {
+ this.rule = rule;
+ this.expectedLabel = expected;
+ }
+
+ @Override
+ public void acceptLabelAttribute(Label label, Attribute attr) {
+ if (isExplicit || !rule.isAttributeValueExplicitlySpecified(attr)) {
+ // Nothing to do here.
+ } else if (expectedLabel.equals(label)) {
+ isExplicit = true;
+ }
+ }
+
+ public boolean isExplicit() {
+ return isExplicit;
+ }
+ }
+
+ /**
+ * Return {@link Location} for {@link Target} target, if it should not be null.
+ */
+ public static Location getLocationMaybe(Target target) {
+ return (target instanceof Rule) || (target instanceof InputFile) ? target.getLocation() : null;
+ }
+
+ /**
+ * Return nicely formatted error message that {@link Label} label that was pointed to by
+ * {@link Target} target did not exist, due to {@link NoSuchThingException} e.
+ */
+ public static String formatMissingEdge(@Nullable Target target, Label label,
+ NoSuchThingException e) {
+ // instanceof returns false if target is null (which is exploited here)
+ if (target instanceof Rule) {
+ Rule rule = (Rule) target;
+ return !isExplicitDependency(rule, label)
+ ? ("every rule of type " + rule.getRuleClass() + " implicitly depends upon the target '"
+ + label + "', but this target could not be found. "
+ + "If this is an integration test, maybe you forgot to add a mock for your new tool?")
+ : e.getMessage() + " and referenced by '" + target.getLabel() + "'";
+ } else if (target instanceof InputFile) {
+ return e.getMessage() + " (this is usually caused by a missing package group in the"
+ + " package-level visibility declaration)";
+ } else {
+ if (target != null) {
+ return "in target '" + target.getLabel() + "', no such label '" + label + "': "
+ + e.getMessage();
+ }
+ return e.getMessage();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TestSize.java b/src/main/java/com/google/devtools/build/lib/packages/TestSize.java
new file mode 100644
index 0000000..425a343
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/TestSize.java
@@ -0,0 +1,123 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.Set;
+
+/**
+ * Possible test sizes.
+ *
+ * Test size may affect the way how test is executed - e.g., it will determine
+ * default timeout value and estimated local resource usage.
+ */
+public enum TestSize {
+
+ // Small tests use small amount of memory, but CPU intensive.
+ SMALL(TestTimeout.SHORT, 2),
+ // Medium tests tend to use larger amount of memory.
+ MEDIUM(TestTimeout.MODERATE, 10),
+ // All other tests estimated to use fairly large amount of memory.
+ LARGE(TestTimeout.LONG, 20),
+ ENORMOUS(TestTimeout.ETERNAL, 30);
+
+ private final TestTimeout timeout;
+ private final int defaultShards;
+
+ private TestSize(TestTimeout defaultTimeout, int defaultShards) {
+ this.timeout = defaultTimeout;
+ this.defaultShards = defaultShards;
+ }
+
+ /**
+ * Returns default timeout in seconds.
+ */
+ public TestTimeout getDefaultTimeout() {
+ return timeout;
+ }
+
+ /**
+ * Returns default number of shards.
+ */
+ public int getDefaultShards() { return defaultShards; }
+
+ /**
+ * Returns test size of the given test target, or null if the size attribute is unrecognized.
+ */
+ public static TestSize getTestSize(Rule testTarget) {
+ String attr = NonconfigurableAttributeMapper.of(testTarget).get("size", Type.STRING);
+ return getTestSize(attr);
+ }
+
+ /**
+ * Returns {@link TestSize} matching the given timeout or null if the
+ * given timeout doesn't match any {@link TestSize}.
+ *
+ * @param timeout The timeout associated with the desired TestSize.
+ */
+ public static TestSize getTestSize(TestTimeout timeout) {
+ for (TestSize size : TestSize.values()) {
+ if (size.timeout == timeout) {
+ return size;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Normal practice is to always use size tags as lower case strings.
+ */
+ @Override
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+
+ /**
+ * Returns the enum associated with a test's size or null if the tag is
+ * not lower case or an unknown size.
+ */
+ public static TestSize getTestSize(String attr) {
+ if (!attr.equals(attr.toLowerCase())) {
+ return null;
+ }
+ try {
+ return TestSize.valueOf(attr.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Converter for the --test_size_filters option.
+ */
+ public static class TestSizeFilterConverter extends EnumFilterConverter<TestSize> {
+ public TestSizeFilterConverter() {
+ super(TestSize.class, "test size");
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>This override is necessary to prevent OptionsData
+ * from throwing a "must be assignable from the converter return type" exception.
+ * OptionsData doesn't recognize the generic type and actual type are the same.
+ */
+ @Override
+ public final Set<TestSize> convert(String input) throws OptionsParsingException {
+ return super.convert(input);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TestTargetUtils.java b/src/main/java/com/google/devtools/build/lib/packages/TestTargetUtils.java
new file mode 100644
index 0000000..dbd4dae
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/TestTargetUtils.java
@@ -0,0 +1,404 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.pkgcache.TargetProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Utility functions over test Targets that don't really belong in the base {@link Target}
+ * interface.
+ */
+public final class TestTargetUtils {
+ /**
+ * Returns a predicate to be used for test size filtering, i.e., that only accepts tests of the
+ * given size.
+ */
+ public static Predicate<Target> testSizeFilter(final Set<TestSize> allowedSizes) {
+ return new Predicate<Target>() {
+ @Override
+ public boolean apply(Target target) {
+ if (!(target instanceof Rule)) {
+ return false;
+ }
+ return allowedSizes.contains(TestSize.getTestSize((Rule) target));
+ }
+ };
+ }
+
+ /**
+ * Returns a predicate to be used for test timeout filtering, i.e., that only accepts tests of
+ * the given timeout.
+ **/
+ public static Predicate<Target> testTimeoutFilter(final Set<TestTimeout> allowedTimeouts) {
+ return new Predicate<Target>() {
+ @Override
+ public boolean apply(Target target) {
+ if (!(target instanceof Rule)) {
+ return false;
+ }
+ return allowedTimeouts.contains(TestTimeout.getTestTimeout((Rule) target));
+ }
+ };
+ }
+
+ /**
+ * Returns a predicate to be used for test language filtering, i.e., that only accepts tests of
+ * the specified languages. The reporter and the list of rule names are only used to warn about
+ * unknown languages.
+ */
+ public static Predicate<Target> testLangFilter(List<String> langFilterList,
+ EventHandler reporter, Set<String> allRuleNames) {
+ final Set<String> requiredLangs = new HashSet<>();
+ final Set<String> excludedLangs = new HashSet<>();
+
+ for (String lang : langFilterList) {
+ if (lang.startsWith("-")) {
+ lang = lang.substring(1);
+ excludedLangs.add(lang);
+ } else {
+ requiredLangs.add(lang);
+ }
+ if (!allRuleNames.contains(lang + "_test")) {
+ reporter.handle(
+ Event.warn("Unknown language '" + lang + "' in --test_lang_filters option"));
+ }
+ }
+
+ return new Predicate<Target>() {
+ @Override
+ public boolean apply(Target rule) {
+ String ruleLang = TargetUtils.getRuleLanguage(rule);
+ return (requiredLangs.isEmpty() || requiredLangs.contains(ruleLang))
+ && !excludedLangs.contains(ruleLang);
+ }
+ };
+ }
+
+ /**
+ * Returns whether a test with the specified tags matches a filter (as specified by the set
+ * of its positive and its negative filters).
+ */
+ public static boolean testMatchesFilters(Collection<String> testTags,
+ Collection<String> requiredTags, Collection<String> excludedTags,
+ boolean mustMatchAllPositive) {
+
+ for (String tag : excludedTags) {
+ if (testTags.contains(tag)) {
+ return false;
+ }
+ }
+
+ // Check required tags, if there are any.
+ if (!requiredTags.isEmpty()) {
+ if (mustMatchAllPositive) {
+ // Require all tags to be present.
+ for (String tag : requiredTags) {
+ if (!testTags.contains(tag)) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ // Require at least one positive tag.
+ for (String tag : requiredTags) {
+ if (testTags.contains(tag)) {
+ return true;
+ }
+ }
+ }
+
+ return false; // No positive tag found.
+ }
+
+ return true; // No tags are required.
+ }
+
+ /**
+ * Returns a predicate to be used for test tag filtering, i.e., that only accepts tests that match
+ * all of the required tags and none of the excluded tags.
+ */
+ // TODO(bazel-team): This also applies to non-test rules, so should probably be moved to
+ // TargetUtils.
+ public static Predicate<Target> tagFilter(List<String> tagFilterList) {
+ Pair<Collection<String>, Collection<String>> tagLists = sortTagsBySense(tagFilterList);
+ final Collection<String> requiredTags = tagLists.first;
+ final Collection<String> excludedTags = tagLists.second;
+ return new Predicate<Target>() {
+ @Override
+ public boolean apply(Target input) {
+ if (!(input instanceof Rule)) {
+ return false;
+ }
+ // Note that test_tags are those originating from the XX_test rule,
+ // whereas the requiredTags and excludedTags originate from the command
+ // line or test_suite rule.
+ return testMatchesFilters(((Rule) input).getRuleTags(),
+ requiredTags, excludedTags, false);
+ }
+ };
+ }
+
+ /**
+ * Separates a list of text "tags" into a Pair of Collections, where
+ * the first element are the required or positive tags and the second element
+ * are the excluded or negative tags.
+ * This should work on tag list provided from the command line
+ * --test_tags_filters flag or on tag filters explicitly declared in the
+ * suite.
+ *
+ * @param tagList A collection of text targets to separate.
+ */
+ public static Pair<Collection<String>, Collection<String>> sortTagsBySense(
+ Iterable<String> tagList) {
+ Collection<String> requiredTags = new HashSet<>();
+ Collection<String> excludedTags = new HashSet<>();
+
+ for (String tag : tagList) {
+ if (tag.startsWith("-")) {
+ excludedTags.add(tag.substring(1));
+ } else if (tag.startsWith("+")) {
+ requiredTags.add(tag.substring(1));
+ } else if (tag.equals("manual")) {
+ // Ignore manual attribute because it is an exception: it is not a filter
+ // but a property of test_suite
+ continue;
+ } else {
+ requiredTags.add(tag);
+ }
+ }
+ return Pair.of(requiredTags, excludedTags);
+ }
+
+ /**
+ * Returns the (new, mutable) set of test rules, expanding all 'test_suite' rules into the
+ * individual tests they group together and preserving other test target instances.
+ *
+ * Method assumes that passed collection contains only *_test and test_suite rules. While, at this
+ * point it will successfully preserve non-test rules as well, there is no guarantee that this
+ * behavior will be kept in the future.
+ *
+ * @param targetProvider a target provider
+ * @param eventHandler a failure eventHandler to report loading failures to
+ * @param targets Collection of the *_test and test_suite configured targets
+ * @return a duplicate-free iterable of the tests under the specified targets
+ */
+ public static ResolvedTargets<Target> expandTestSuites(TargetProvider targetProvider,
+ EventHandler eventHandler, Iterable<? extends Target> targets, boolean strict,
+ boolean keepGoing)
+ throws TargetParsingException {
+ Closure closure = new Closure(targetProvider, eventHandler, strict, keepGoing);
+ ResolvedTargets.Builder<Target> result = ResolvedTargets.builder();
+ for (Target target : targets) {
+ if (TargetUtils.isTestRule(target)) {
+ result.add(target);
+ } else if (TargetUtils.isTestSuiteRule(target)) {
+ result.addAll(closure.getTestsInSuite((Rule) target));
+ } else {
+ result.add(target);
+ }
+ }
+ if (closure.hasError) {
+ result.setError();
+ }
+ return result.build();
+ }
+
+ // TODO(bazel-team): This is a copy of TestsExpression.Closure with some minor changes; this
+ // should be unified.
+ private static final class Closure {
+ private final TargetProvider targetProvider;
+
+ private final EventHandler eventHandler;
+
+ private final boolean keepGoing;
+
+ private final boolean strict;
+
+ private final Map<Target, Set<Target>> testsInSuite = new HashMap<>();
+
+ private boolean hasError;
+
+ public Closure(TargetProvider targetProvider, EventHandler eventHandler, boolean strict,
+ boolean keepGoing) {
+ this.targetProvider = targetProvider;
+ this.eventHandler = eventHandler;
+ this.strict = strict;
+ this.keepGoing = keepGoing;
+ }
+
+ /**
+ * Computes and returns the set of test rules in a particular suite. Uses
+ * dynamic programming---a memoized version of {@link #computeTestsInSuite}.
+ */
+ private Set<Target> getTestsInSuite(Rule testSuite) throws TargetParsingException {
+ Set<Target> tests = testsInSuite.get(testSuite);
+ if (tests == null) {
+ tests = Sets.newHashSet();
+ testsInSuite.put(testSuite, tests); // break cycles by inserting empty set early.
+ computeTestsInSuite(testSuite, tests);
+ }
+ return tests;
+ }
+
+ /**
+ * Populates 'result' with all the tests associated with the specified
+ * 'testSuite'. Throws an exception if any target is missing.
+ *
+ * CAUTION! Keep this logic consistent with {@code TestsSuiteConfiguredTarget}!
+ */
+ private void computeTestsInSuite(Rule testSuite, Set<Target> result)
+ throws TargetParsingException {
+ List<Target> testsAndSuites = new ArrayList<>();
+ // Note that testsAndSuites can contain input file targets; the test_suite rule does not
+ // restrict the set of targets that can appear in tests or suites.
+ testsAndSuites.addAll(getPrerequisites(testSuite, "tests"));
+ testsAndSuites.addAll(getPrerequisites(testSuite, "suites"));
+
+ // 1. Add all tests
+ for (Target test : testsAndSuites) {
+ if (TargetUtils.isTestRule(test)) {
+ result.add(test);
+ } else if (strict && !TargetUtils.isTestSuiteRule(test)) {
+ // If strict mode is enabled, then give an error for any non-test, non-test-suite targets.
+ eventHandler.handle(Event.error(testSuite.getLocation(),
+ "in test_suite rule '" + testSuite.getLabel()
+ + "': expecting a test or a test_suite rule but '" + test.getLabel()
+ + "' is not one."));
+ hasError = true;
+ if (!keepGoing) {
+ throw new TargetParsingException("Test suite expansion failed.");
+ }
+ }
+ }
+
+ // 2. Add implicit dependencies on tests in same package, if any.
+ for (Target target : getPrerequisites(testSuite, "$implicit_tests")) {
+ // The Package construction of $implicit_tests ensures that this check never fails, but we
+ // add it here anyway for compatibility with future code.
+ if (TargetUtils.isTestRule(target)) {
+ result.add(target);
+ }
+ }
+
+ // 3. Filter based on tags, size, env.
+ filterTests(testSuite, result);
+
+ // 4. Expand all suites recursively.
+ for (Target suite : testsAndSuites) {
+ if (TargetUtils.isTestSuiteRule(suite)) {
+ result.addAll(getTestsInSuite((Rule) suite));
+ }
+ }
+ }
+
+ /**
+ * Returns the set of rules named by the attribute 'attrName' of test_suite rule 'testSuite'.
+ * The attribute must be a list of labels. If a target cannot be resolved, then an error is
+ * reported to the environment (which may throw an exception if {@code keep_going} is disabled).
+ */
+ private Collection<Target> getPrerequisites(Rule testSuite, String attrName)
+ throws TargetParsingException {
+ try {
+ List<Target> targets = new ArrayList<>();
+ // TODO(bazel-team): This serializes package loading in some cases. We might want to make
+ // this multi-threaded.
+ for (Label label :
+ NonconfigurableAttributeMapper.of(testSuite).get(attrName, Type.LABEL_LIST)) {
+ targets.add(targetProvider.getTarget(eventHandler, label));
+ }
+ return targets;
+ } catch (NoSuchThingException e) {
+ if (keepGoing) {
+ hasError = true;
+ eventHandler.handle(Event.error(e.getMessage()));
+ return ImmutableList.of();
+ }
+ throw new TargetParsingException(e.getMessage(), e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new TargetParsingException("interrupted", e);
+ }
+ }
+
+ /**
+ * Filters 'tests' (by mutation) according to the 'tags' attribute, specifically those that
+ * match ALL of the tags in tagsAttribute.
+ *
+ * @precondition {@code env.getAccessor().isTestSuite(testSuite)}
+ * @precondition {@code env.getAccessor().isTestRule(test)} for all test in tests
+ */
+ private void filterTests(Rule testSuite, Set<Target> tests) {
+ List<String> tagsAttribute =
+ NonconfigurableAttributeMapper.of(testSuite).get("tags", Type.STRING_LIST);
+ // Split the tags list into positive and negative tags
+ Pair<Collection<String>, Collection<String>> tagLists = sortTagsBySense(tagsAttribute);
+ Collection<String> positiveTags = tagLists.first;
+ Collection<String> negativeTags = tagLists.second;
+
+ Iterator<Target> it = tests.iterator();
+ while (it.hasNext()) {
+ Rule test = (Rule) it.next();
+ AttributeMap nonConfigurableAttributes = NonconfigurableAttributeMapper.of(test);
+ List<String> testTags =
+ new ArrayList<>(nonConfigurableAttributes.get("tags", Type.STRING_LIST));
+ testTags.add(nonConfigurableAttributes.get("size", Type.STRING));
+ if (!includeTest(testTags, positiveTags, negativeTags)) {
+ it.remove();
+ }
+ }
+ }
+
+ /**
+ * Decides whether to include a test in a test_suite or not.
+ * @param testTags Collection of all tags exhibited by a given test.
+ * @param positiveTags Tags declared by the suite. A Test must match ALL of these.
+ * @param negativeTags Tags declared by the suite. A Test must match NONE of these.
+ * @return false is the test is to be removed.
+ */
+ private static boolean includeTest(Collection<String> testTags,
+ Collection<String> positiveTags, Collection<String> negativeTags) {
+ // Add this test if it matches ALL of the positive tags and NONE of the
+ // negative tags in the tags attribute.
+ for (String tag : negativeTags) {
+ if (testTags.contains(tag)) {
+ return false;
+ }
+ }
+ for (String tag : positiveTags) {
+ if (!testTags.contains(tag)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TestTimeout.java b/src/main/java/com/google/devtools/build/lib/packages/TestTimeout.java
new file mode 100644
index 0000000..cfa8047
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/TestTimeout.java
@@ -0,0 +1,198 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Maps;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Symbolic labels of test timeout. Borrows heavily from {@link TestSize}.
+ */
+public enum TestTimeout {
+
+ // These symbolic labels are used in the build files.
+ SHORT(0, 60, 60),
+ MODERATE(30, 300, 300),
+ LONG(300, 900, 900),
+ ETERNAL(900, 365 * 24 * 60 /* One year */, 3600);
+
+ /**
+ * Default --test_timeout flag, used when collecting code coverage.
+ */
+ public static String COVERAGE_CMD_TIMEOUT = "--test_timeout=300,600,1200,3600";
+
+ private final Integer rangeMin;
+ private final Integer rangeMax;
+ private final Integer timeout;
+
+ private TestTimeout(Integer rangeMin, Integer rangeMax, Integer timeout) {
+ this.rangeMin = rangeMin;
+ this.rangeMax = rangeMax;
+ this.timeout = timeout;
+ }
+
+ /**
+ * Returns the enum associated with a test's timeout or null if the tag is
+ * not lower case or an unknown size.
+ */
+ public static TestTimeout getTestTimeout(String attr) {
+ if (!attr.equals(attr.toLowerCase())) {
+ return null;
+ }
+ try {
+ return TestTimeout.valueOf(attr.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+
+ /**
+ * We print to upper case to make the test timeout warnings more readable.
+ */
+ public String prettyPrint() {
+ return super.toString().toUpperCase();
+ }
+
+ public Integer getTimeout() {
+ return timeout;
+ }
+ /**
+ * Returns true iff the given time in seconds is exactly in the range of valid
+ * execution times for this TestSize.
+ */
+ public boolean isInRangeExact(Integer timeInSeconds) {
+ return timeInSeconds >= rangeMin && timeInSeconds < rangeMax;
+ }
+
+ /**
+ * Returns true iff the given time in seconds is approximately (+/- 75%) in the range of valid
+ * execution times for this TestSize.
+ */
+ public boolean isInRangeFuzzy(Integer timeInSeconds) {
+ return timeInSeconds >= rangeMin - (rangeMin * .75)
+ && (this == ETERNAL || timeInSeconds <= rangeMax + (rangeMax * .75));
+ }
+
+ /**
+ * Returns suggested test size for the given time in seconds.
+ */
+ public static TestTimeout getSuggestedTestTimeout(Integer timeInSeconds) {
+ for (TestTimeout testTimeout : values()) {
+ if (testTimeout.isInRangeExact(timeInSeconds)) {
+ return testTimeout;
+ }
+ }
+ return ETERNAL;
+ }
+
+ /**
+ * Returns test timeout of the given test target using explicitly specified timeout
+ * or default through to the size label's associated default.
+ */
+ public static TestTimeout getTestTimeout(Rule testTarget) {
+ String attr = NonconfigurableAttributeMapper.of(testTarget).get("timeout", Type.STRING);
+ if (!attr.equals(attr.toLowerCase())) {
+ return null; // attribute values must be lowercase
+ }
+ try {
+ return TestTimeout.valueOf(attr.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Converter for the --test_timeout option.
+ */
+ public static class TestTimeoutConverter implements Converter<Map<TestTimeout, Integer>> {
+ public TestTimeoutConverter() {}
+
+ @Override
+ public Map<TestTimeout, Integer> convert(String input) throws OptionsParsingException {
+ List<Integer> values = new ArrayList<>();
+ for (String token : Splitter.on(',').limit(6).split(input)) {
+ // Handle the case of "2," which is accepted as legal... Because Splitter.split is lazy,
+ // there's no way of knowing if an empty string is a trailing or an intermediate one,
+ // so we can't fully emulate String.split(String, 0).
+ if (!token.isEmpty() || values.size() > 1) {
+ try {
+ values.add(Integer.valueOf(token));
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("'" + input + "' is not an int");
+ }
+ }
+ }
+ EnumMap<TestTimeout, Integer> timeouts = Maps.newEnumMap(TestTimeout.class);
+ if (values.size() == 1) {
+ timeouts.put(SHORT, values.get(0));
+ timeouts.put(MODERATE, values.get(0));
+ timeouts.put(LONG, values.get(0));
+ timeouts.put(ETERNAL, values.get(0));
+ } else if (values.size() == 4) {
+ timeouts.put(SHORT, values.get(0));
+ timeouts.put(MODERATE, values.get(1));
+ timeouts.put(LONG, values.get(2));
+ timeouts.put(ETERNAL, values.get(3));
+ } else {
+ throw new OptionsParsingException("Invalid number of comma-separated entries");
+ }
+ for (TestTimeout label : values()) {
+ if (!timeouts.containsKey(label) || timeouts.get(label) <= 0) {
+ timeouts.put(label, label.getTimeout());
+ }
+ }
+ return timeouts;
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a single integer or comma-separated list of 4 integers";
+ }
+ }
+
+ /**
+ * Converter for the --test_timeout_filters option.
+ */
+ public static class TestTimeoutFilterConverter extends EnumFilterConverter<TestTimeout> {
+ public TestTimeoutFilterConverter() {
+ super(TestTimeout.class, "test timeout");
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>This override is necessary to prevent OptionsData
+ * from throwing a "must be assignable from the converter return type" exception.
+ * OptionsData doesn't recognize the generic type and actual type are the same.
+ */
+ @Override
+ public final Set<TestTimeout> convert(String input) throws OptionsParsingException {
+ return super.convert(input);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TriState.java b/src/main/java/com/google/devtools/build/lib/packages/TriState.java
new file mode 100644
index 0000000..0cea28a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/TriState.java
@@ -0,0 +1,22 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+/**
+ * Enum used to represent tri-state parameters in rule attributes (yes/no/auto).
+ */
+public enum TriState {
+ YES, NO, AUTO
+}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Type.java b/src/main/java/com/google/devtools/build/lib/packages/Type.java
new file mode 100644
index 0000000..9172a1f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/packages/Type.java
@@ -0,0 +1,1025 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.packages.License.DistributionType;
+import com.google.devtools.build.lib.packages.License.LicenseParsingException;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SelectorValue;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.logging.Level;
+
+import javax.annotation.Nullable;
+
+/**
+ * <p>Root of Type symbol hierarchy for values in the build language.</p>
+ *
+ * <p>Type symbols are primarily used for their <code>convert</code> method,
+ * which is a kind of cast operator enabling conversion from untyped (Object)
+ * references to values in the build language, to typed references.</p>
+ *
+ * <p>For example, this code type-converts a value <code>x</code> returned by
+ * the evaluator, to a list of strings:</p>
+ *
+ * <pre>
+ * Object x = expr.eval(env);
+ * List<String> s = Type.STRING_LIST.convert(x);
+ * </pre>
+ */
+public abstract class Type<T> {
+
+ private Type() {}
+
+ /**
+ * Converts untyped Object x resulting from the evaluation of an expression in the build language,
+ * into a typed object of type T.
+ *
+ * <p>x must be *directly* convertible to this type. This therefore disqualifies "selector
+ * expressions" of the form "{ config1: 'value1_of_orig_type', config2: 'value2_of_orig_type; }"
+ * (which support configurable attributes). To handle those expressions, see
+ * {@link #selectableConvert}.
+ *
+ * @param x the build-interpreter value to convert.
+ * @param what a string description of what x is for; should be included in
+ * any exception thrown. Grammatically, must describe a syntactic
+ * construct, e.g. "attribute 'srcs' of rule foo".
+ * @param currentRule the label of the current BUILD rule; must be non-null if resolution of
+ * package-relative label strings is required
+ * @throws ConversionException if there was a problem performing the type conversion
+ */
+ public abstract T convert(Object x, String what, @Nullable Label currentRule)
+ throws ConversionException;
+ // TODO(bazel-team): Check external calls (e.g. in PackageFactory), verify they always want
+ // this over selectableConvert.
+
+ /**
+ * Equivalent to <code>convert(x, null)</code>. Useful for converting values to types that do not
+ * involve the type <code>LABEL</code> and hence do not require the label of the current package.
+ */
+ public final T convert(Object x, String what) throws ConversionException {
+ return convert(x, what, null);
+ }
+
+ /**
+ * Variation of {@link #convert} that supports selector expressions for configurable attributes
+ * (i.e. "{ config1: 'value1_of_orig_type', config2: 'value2_of_orig_type; }"). If x is a
+ * selector expression, returns a {@link Selector} instance that contains key-mapped entries
+ * of the native type. Else, returns the native type directly.
+ *
+ * <p>The caller is responsible for casting the returned value appropriately.
+ */
+ public Object selectableConvert(Object x, String what, @Nullable Label currentRule)
+ throws ConversionException {
+ if (x instanceof SelectorValue) {
+ return new Selector<T>(((SelectorValue) x).getDictionary(), what, currentRule, this);
+ }
+ return convert(x, what, currentRule);
+ }
+
+ public abstract T cast(Object value);
+
+ @Override
+ public abstract String toString();
+
+ /**
+ * Returns the default value for this type; may return null iff no default is defined for this
+ * type.
+ */
+ public abstract T getDefaultValue();
+
+ /**
+ * If this type contains labels (e.g. it *is* a label or it's a collection of labels),
+ * returns a list of those labels for a value of that type. If this type doesn't
+ * contain labels, returns an empty list.
+ *
+ * <p>This is used to support reliable label visitation in
+ * {@link AbstractAttributeMapper#visitLabels}. To preserve that reliability, every
+ * type should faithfully define its own instance of this method. In other words,
+ * be careful about defining default instances in base types that get auto-inherited
+ * by their children. Keep all definitions as explicit as possible.
+ */
+ public abstract Iterable<Label> getLabels(Object value);
+
+ /**
+ * {@link #getLabels} return value for types that don't contain labels.
+ */
+ private static final Iterable<Label> NO_LABELS_HERE = ImmutableList.of();
+
+ /**
+ * Converts an initialized Type object into a tag set representation.
+ * This operation is only valid for certain sub-Types which are guaranteed
+ * to be properly initialized.
+ *
+ * @param value the actual value
+ * @throws UnsupportedOperationException if the concrete type does not support
+ * tag conversion or if a convertible type has no initialized value.
+ */
+ public Set<String> toTagSet(Object value, String name) {
+ String msg = "Attribute " + name + " does not support tag conversion.";
+ throw new UnsupportedOperationException(msg);
+ }
+
+ /**
+ * The type of an integer.
+ */
+ public static final Type<Integer> INTEGER = new IntegerType();
+
+ /**
+ * The type of a string.
+ */
+ public static final Type<String> STRING = new StringType();
+
+ /**
+ * The type of a boolean.
+ */
+ public static final Type<Boolean> BOOLEAN = new BooleanType();
+
+ /**
+ * The type of a TriState with values: true (x>0), false (x==0), auto (x<0).
+ */
+ public static final Type<TriState> TRISTATE = new TriStateType();
+
+ /**
+ * The type of a label. Labels are not actually a first-class datatype in
+ * the build language, but they are so frequently used in the definitions of
+ * attributes that it's worth treating them specially (and providing support
+ * for resolution of relative-labels in the <code>convert()</code> method).
+ */
+ public static final Type<Label> LABEL = new LabelType();
+
+ /**
+ * This is a label type that does not cause dependencies. It is needed because
+ * certain rules want to verify the type of a target referenced by one of their attributes, but
+ * if there was a dependency edge there, it would be a circular dependency.
+ */
+ public static final Type<Label> NODEP_LABEL = new LabelType();
+
+ /**
+ * The type of a license. Like Label, licenses aren't first-class, but
+ * they're important enough to justify early syntax error detection.
+ */
+ public static final Type<License> LICENSE = new LicenseType();
+
+ /**
+ * The type of a single distribution. Only used internally, as a type
+ * symbol, not a converter.
+ */
+ public static final Type<DistributionType> DISTRIBUTION = new Type<DistributionType>() {
+ @Override
+ public DistributionType cast(Object value) {
+ return (DistributionType) value;
+ }
+
+ @Override
+ public DistributionType convert(Object x, String what, Label currentRule) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DistributionType getDefaultValue() {
+ return null;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "distribution";
+ }
+ };
+
+ /**
+ * The type of a set of distributions. Distributions are not a first-class type,
+ * but they do warrant early syntax checking.
+ */
+ public static final Type<Set<DistributionType>> DISTRIBUTIONS = new Distributions();
+
+ /**
+ * The type of an output file, treated as a {@link #LABEL}.
+ */
+ public static final Type<Label> OUTPUT = new OutputType();
+
+ /**
+ * The type of a FilesetEntry attribute inside a Fileset.
+ */
+ public static final Type<FilesetEntry> FILESET_ENTRY = new FilesetEntryType();
+
+ /**
+ * The type of a list of not-yet-typed objects.
+ */
+ public static final ObjectListType OBJECT_LIST = new ObjectListType();
+
+ /**
+ * The type of a list of {@linkplain #STRING strings}.
+ */
+ public static final ListType<String> STRING_LIST = ListType.create(STRING);
+
+ /**
+ * The type of a list of {@linkplain #INTEGER strings}.
+ */
+ public static final ListType<Integer> INTEGER_LIST = ListType.create(INTEGER);
+
+ /**
+ * The type of a dictionary of {@linkplain #STRING strings}.
+ */
+ public static final DictType<String, String> STRING_DICT = DictType.create(STRING, STRING);
+
+ /**
+ * The type of a list of {@linkplain #OUTPUT outputs}.
+ */
+ public static final ListType<Label> OUTPUT_LIST = ListType.create(OUTPUT);
+
+ /**
+ * The type of a list of {@linkplain #LABEL labels}.
+ */
+ public static final ListType<Label> LABEL_LIST = ListType.create(LABEL);
+
+ /**
+ * The type of a list of {@linkplain #NODEP_LABEL labels} that do not cause
+ * dependencies.
+ */
+ public static final ListType<Label> NODEP_LABEL_LIST = ListType.create(NODEP_LABEL);
+
+ /**
+ * The type of a dictionary of {@linkplain #STRING_LIST label lists}.
+ */
+ public static final DictType<String, List<String>> STRING_LIST_DICT =
+ DictType.create(STRING, STRING_LIST);
+
+ /**
+ * The type of a dictionary of {@linkplain #STRING strings}, where each entry
+ * maps to a single string value.
+ */
+ public static final DictType<String, String> STRING_DICT_UNARY = DictType.create(STRING, STRING);
+
+ /**
+ * The type of a dictionary of {@linkplain #LABEL_LIST label lists}.
+ */
+ public static final DictType<String, List<Label>> LABEL_LIST_DICT =
+ DictType.create(STRING, LABEL_LIST);
+
+ /**
+ * The type of a list of {@linkplain #FILESET_ENTRY FilesetEntries}.
+ */
+ public static final ListType<FilesetEntry> FILESET_ENTRY_LIST = ListType.create(FILESET_ENTRY);
+
+ /**
+ * For ListType objects, returns the type of the elements of the list; for
+ * all other types, returns null. (This non-obvious implementation strategy
+ * is necessitated by the wildcard capture rules of the Java type system,
+ * which disallow conversion from Type{List{ELEM}} to Type{List{?}}.)
+ */
+ public Type<?> getListElementType() {
+ return null;
+ }
+
+ /**
+ * ConversionException is thrown when a type-conversion fails; it contains
+ * an explanatory error message.
+ */
+ public static class ConversionException extends Exception {
+ private static String message(Type<?> type, Object value, String what) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("expected value of type '").append(type).append("'");
+ if (what != null) {
+ builder.append(" for ").append(what);
+ }
+ builder.append(", but got '");
+ EvalUtils.printValue(value, builder);
+ builder.append("' (").append(EvalUtils.getDatatypeName(value)).append(")");
+ return builder.toString();
+ }
+
+ private ConversionException(Type<?> type, Object value, String what) {
+ super(message(type, value, what));
+ }
+
+ private ConversionException(String message) {
+ super(message);
+ }
+ }
+
+ /********************************************************************
+ * *
+ * Subclasses *
+ * *
+ ********************************************************************/
+
+ private static class ObjectType extends Type<Object> {
+ @Override
+ public Object cast(Object value) {
+ return value;
+ }
+
+ @Override
+ public String getDefaultValue() {
+ throw new UnsupportedOperationException(
+ "ObjectType has no default value");
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "object";
+ }
+
+ @Override
+ public Object convert(Object x, String what, Label currentRule) {
+ return x;
+ }
+ }
+
+ private static class IntegerType extends Type<Integer> {
+ @Override
+ public Integer cast(Object value) {
+ return (Integer) value;
+ }
+
+ @Override
+ public Integer getDefaultValue() {
+ return 0;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "int";
+ }
+
+ @Override
+ public Integer convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (!(x instanceof Integer)) {
+ throw new ConversionException(this, x, what);
+ }
+ return (Integer) x;
+ }
+ }
+
+ private static class BooleanType extends Type<Boolean> {
+ @Override
+ public Boolean cast(Object value) {
+ return (Boolean) value;
+ }
+
+ @Override
+ public Boolean getDefaultValue() {
+ return false;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "boolean";
+ }
+
+ // Conversion to boolean must also tolerate integers of 0 and 1 only.
+ @Override
+ public Boolean convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (x instanceof Boolean) {
+ return (Boolean) x;
+ }
+ Integer xAsInteger = INTEGER.convert(x, what, currentRule);
+ if (xAsInteger == 0) {
+ return false;
+ } else if (xAsInteger == 1) {
+ return true;
+ }
+ throw new ConversionException("boolean is not one of [0, 1]");
+ }
+
+ /**
+ * Booleans attributes are converted to tags based on their names.
+ */
+ @Override
+ public Set<String> toTagSet(Object value, String name) {
+ if (value == null) {
+ String msg = "Illegal tag conversion from null on Attribute " + name + ".";
+ throw new IllegalStateException(msg);
+ }
+ String tag = (Boolean) value ? name : "no" + name;
+ return new ImmutableSet.Builder<String>()
+ .add(tag)
+ .build();
+ }
+ }
+
+ /**
+ * Tristate values are needed for cases where user intent matters.
+ *
+ * <p>Tristate values are not explicitly interchangeable with booleans and are
+ * handled explicitly as TriStates. Prefer Booleans with default values where
+ * possible. The main use case for TriState values is when a Rule's behavior
+ * must interact with a Flag value in a complicated way.</p>
+ */
+ private static class TriStateType extends Type<TriState> {
+ @Override
+ public TriState cast(Object value) {
+ return (TriState) value;
+ }
+
+ @Override
+ public TriState getDefaultValue() {
+ return TriState.AUTO;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "tristate";
+ }
+
+ // Like BooleanType, this must handle integers as well.
+ @Override
+ public TriState convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (x instanceof TriState) {
+ return (TriState) x;
+ }
+ if (x instanceof Boolean) {
+ return ((Boolean) x) ? TriState.YES : TriState.NO;
+ }
+ Integer xAsInteger = INTEGER.convert(x, what, currentRule);
+ if (xAsInteger == -1) {
+ return TriState.AUTO;
+ } else if (xAsInteger == 1) {
+ return TriState.YES;
+ } else if (xAsInteger == 0) {
+ return TriState.NO;
+ }
+ throw new ConversionException(this, x, "TriState values is not one of [-1, 0, 1]");
+ }
+ }
+
+ private static class StringType extends Type<String> {
+ @Override
+ public String cast(Object value) {
+ return (String) value;
+ }
+
+ @Override
+ public String getDefaultValue() {
+ return "";
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "string";
+ }
+
+ @Override
+ public String convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (!(x instanceof String)) {
+ throw new ConversionException(this, x, what);
+ }
+ return StringCanonicalizer.intern((String) x);
+ }
+
+ /**
+ * A String is representable as a set containing its value.
+ */
+ @Override
+ public Set<String> toTagSet(Object value, String name) {
+ if (value == null) {
+ String msg = "Illegal tag conversion from null on Attribute " + name + ".";
+ throw new IllegalStateException(msg);
+ }
+ return new ImmutableSet.Builder<String>()
+ .add((String) value)
+ .build();
+ }
+ }
+
+ private static class FilesetEntryType extends Type<FilesetEntry> {
+ @Override
+ public FilesetEntry cast(Object value) {
+ return (FilesetEntry) value;
+ }
+
+ @Override
+ public FilesetEntry convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (!(x instanceof FilesetEntry)) {
+ throw new ConversionException(this, x, what);
+ }
+ return (FilesetEntry) x;
+ }
+
+ @Override
+ public String toString() {
+ return "FilesetEntry";
+ }
+
+ @Override
+ public FilesetEntry getDefaultValue() {
+ return null;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return cast(value).getLabels();
+ }
+ }
+
+ private static class LabelType extends Type<Label> {
+ @Override
+ public Label cast(Object value) {
+ return (Label) value;
+ }
+
+ @Override
+ public Label getDefaultValue() {
+ return null; // Labels have no default value
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return ImmutableList.of(cast(value));
+ }
+
+ @Override
+ public String toString() {
+ return "label";
+ }
+
+ @Override
+ public Label convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (x instanceof Label) {
+ return (Label) x;
+ }
+ try {
+ return currentRule.getRelative(
+ STRING.convert(x, what, currentRule));
+ } catch (Label.SyntaxException e) {
+ throw new ConversionException("invalid label '" + x + "' in "
+ + what + ": "+ e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Like Label, LicenseType is a derived type, which is declared specially
+ * in order to allow syntax validation. It represents the licenses, as
+ * described in {@ref License}.
+ */
+ public static class LicenseType extends Type<License> {
+ @Override
+ public License cast(Object value) {
+ return (License) value;
+ }
+
+ @Override
+ public License convert(Object x, String what, Label currentRule) throws ConversionException {
+ try {
+ List<String> licenseStrings = STRING_LIST.convert(x, what);
+ return License.parseLicense(licenseStrings);
+ } catch (LicenseParsingException e) {
+ throw new ConversionException(e.getMessage());
+ }
+ }
+
+ @Override
+ public License getDefaultValue() {
+ return License.NO_LICENSE;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "license";
+ }
+ }
+
+ /**
+ * Like Label, Distributions is a derived type, which is declared specially
+ * in order to allow syntax validation. It represents the declared distributions
+ * of a target, as described in {@ref License}.
+ */
+ private static class Distributions extends Type<Set<DistributionType>> {
+ @SuppressWarnings("unchecked")
+ @Override
+ public Set<DistributionType> cast(Object value) {
+ return (Set<DistributionType>) value;
+ }
+
+ @Override
+ public Set<DistributionType> convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ try {
+ List<String> distribStrings = STRING_LIST.convert(x, what);
+ return License.parseDistributions(distribStrings);
+ } catch (LicenseParsingException e) {
+ throw new ConversionException(e.getMessage());
+ }
+ }
+
+ @Override
+ public Set<DistributionType> getDefaultValue() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object what) {
+ return NO_LABELS_HERE;
+ }
+
+ @Override
+ public String toString() {
+ return "distributions";
+ }
+
+ @Override
+ public Type<DistributionType> getListElementType() {
+ return DISTRIBUTION;
+ }
+ }
+
+ private static class OutputType extends Type<Label> {
+ @Override
+ public Label cast(Object value) {
+ return (Label) value;
+ }
+
+ @Override
+ public Label getDefaultValue() {
+ return null;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ return ImmutableList.of(cast(value));
+ }
+
+ @Override
+ public String toString() {
+ return "output";
+ }
+
+ @Override
+ public Label convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+
+ String value;
+ try {
+ value = STRING.convert(x, what, currentRule);
+ } catch (ConversionException e) {
+ throw new ConversionException(this, x, what);
+ }
+ try {
+ // Enforce value is relative to the currentRule.
+ Label result = currentRule.getRelative(value);
+ if (!result.getPackageName().equals(currentRule.getPackageName())) {
+ throw new ConversionException("label '" + value + "' is not in the current package");
+ }
+ return result;
+ } catch (Label.SyntaxException e) {
+ throw new ConversionException(
+ "illegal output file name '" + value + "' in rule " + currentRule + ": "
+ + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * A type to support dictionary attributes.
+ */
+ public static class DictType<KEY, VALUE> extends Type<Map<KEY, VALUE>> {
+
+ private final Type<KEY> keyType;
+ private final Type<VALUE> valueType;
+
+ private final Map<KEY, VALUE> empty = ImmutableMap.of();
+
+ private static <KEY, VALUE> DictType<KEY, VALUE> create(
+ Type<KEY> keyType, Type<VALUE> valueType) {
+ return new DictType<>(keyType, valueType);
+ }
+
+ private DictType(Type<KEY> keyType, Type<VALUE> valueType) {
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ public Type<KEY> getKeyType() {
+ return keyType;
+ }
+
+ public Type<VALUE> getValueType() {
+ return valueType;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<KEY, VALUE> cast(Object value) {
+ return (Map<KEY, VALUE>) value;
+ }
+
+ @Override
+ public String toString() {
+ return "dict(" + keyType + ", " + valueType + ")";
+ }
+
+ @Override
+ public Map<KEY, VALUE> convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (!(x instanceof Map<?, ?>)) {
+ throw new ConversionException(String.format(
+ "Expected a map for dictionary but got a %s", x.getClass().getName()));
+ }
+ ImmutableMap.Builder<KEY, VALUE> result = ImmutableMap.builder();
+ Map<?, ?> o = (Map<?, ?>) x;
+ for (Entry<?, ?> elem : o.entrySet()) {
+ result.put(
+ keyType.convert(elem.getKey(), "dict key element", currentRule),
+ valueType.convert(elem.getValue(), "dict value element", currentRule));
+ }
+ return result.build();
+ }
+
+ @Override
+ public Map<KEY, VALUE> getDefaultValue() {
+ return empty;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ ImmutableList.Builder<Label> labels = ImmutableList.builder();
+ for (Map.Entry<KEY, VALUE> entry : cast(value).entrySet()) {
+ labels.addAll(keyType.getLabels(entry.getKey()));
+ labels.addAll(valueType.getLabels(entry.getValue()));
+ }
+ return labels.build();
+ }
+ }
+
+ public static class ListType<ELEM> extends Type<List<ELEM>> {
+
+ private final Type<ELEM> elemType;
+
+ private final List<ELEM> empty = ImmutableList.of();
+
+ private static <ELEM> ListType<ELEM> create(Type<ELEM> elemType) {
+ return new ListType<>(elemType);
+ }
+
+ private ListType(Type<ELEM> elemType) {
+ this.elemType = elemType;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<ELEM> cast(Object value) {
+ return (List<ELEM>) value;
+ }
+
+ @Override
+ public Type<ELEM> getListElementType() {
+ return elemType;
+ }
+
+ @Override
+ public List<ELEM> getDefaultValue() {
+ return empty;
+ }
+
+ @Override
+ public Iterable<Label> getLabels(Object value) {
+ ImmutableList.Builder<Label> labels = ImmutableList.builder();
+ for (ELEM entry : cast(value)) {
+ labels.addAll(elemType.getLabels(entry));
+ }
+ return labels.build();
+ }
+
+ @Override
+ public String toString() {
+ return "list(" + elemType + ")";
+ }
+
+ @Override
+ public List<ELEM> convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (!(x instanceof Iterable<?>)) {
+ throw new ConversionException(this, x, what);
+ }
+ List<ELEM> result = new ArrayList<>();
+ int index = 0;
+ for (Object elem : (Iterable<?>) x) {
+ ELEM converted = elemType.convert(elem, "element " + index + " of " + what, currentRule);
+ if (converted != null) {
+ result.add(converted);
+ } else {
+ // shouldn't happen but it does, rarely
+ String message = "Converting a list with a null element: "
+ + "element " + index + " of " + what + " in " + currentRule;
+ LoggingUtil.logToRemote(Level.WARNING, message,
+ new ConversionException(message));
+ }
+ ++index;
+ }
+ if (x instanceof GlobList<?>) {
+ return new GlobList<>(((GlobList<?>) x).getCriteria(), result);
+ } else {
+ return result;
+ }
+ }
+
+ /**
+ * A list is representable as a tag set as the contents of itself expressed
+ * as Strings. So a List<String> is effectively converted to a Set<String>.
+ */
+ @Override
+ public Set<String> toTagSet(Object items, String name) {
+ if (items == null) {
+ String msg = "Illegal tag conversion from null on Attribute" + name + ".";
+ throw new IllegalStateException(msg);
+ }
+ Set<String> tags = new LinkedHashSet<>();
+ @SuppressWarnings("unchecked")
+ List<ELEM> itemsAsListofElem = (List<ELEM>) items;
+ for (ELEM element : itemsAsListofElem) {
+ tags.add(element.toString());
+ }
+ return tags;
+ }
+ }
+
+ public static class ObjectListType extends ListType<Object> {
+
+ private static final Type<Object> elemType = new ObjectType();
+
+ private ObjectListType() {
+ super(elemType);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List<Object> convert(Object x, String what, Label currentRule)
+ throws ConversionException {
+ if (x instanceof List) {
+ return (List<Object>) x;
+ } else if (x instanceof Iterable) {
+ return ImmutableList.copyOf((Iterable<?>) x);
+ } else {
+ throw new ConversionException(this, x, what);
+ }
+ }
+ }
+
+ /**
+ * The type of a general list.
+ */
+ public static final ListType<Object> LIST = new ListType<>(new ObjectType());
+
+ /**
+ * Returns whether the specified type is a label type or not.
+ */
+ public static boolean isLabelType(Type<?> type) {
+ return type == LABEL || type == LABEL_LIST
+ || type == NODEP_LABEL || type == NODEP_LABEL_LIST
+ || type == LABEL_LIST_DICT || type == FILESET_ENTRY_LIST;
+ }
+
+ /**
+ * Special Type that represents a selector expression for configurable attributes. Holds a
+ * mapping of <Label, T> entries, where keys are configurability patterns and values are
+ * objects of the attribute's native Type.
+ */
+ public static final class Selector<T> {
+
+ private final Type<T> originalType;
+ private final Map<Label, T> map;
+ private final Label defaultConditionLabel;
+ private final boolean hasDefaultCondition;
+
+ /**
+ * Value to use when none of an attribute's selection criteria match.
+ */
+ @VisibleForTesting
+ public static final String DEFAULT_CONDITION_KEY = "//conditions:default";
+
+ @VisibleForTesting
+ Selector(Object x, String what, @Nullable Label currentRule, Type<T> originalType)
+ throws ConversionException {
+ Preconditions.checkState(x instanceof Map<?, ?>);
+
+ try {
+ defaultConditionLabel = Label.parseAbsolute(DEFAULT_CONDITION_KEY);
+ } catch (Label.SyntaxException e) {
+ throw new IllegalStateException(DEFAULT_CONDITION_KEY + " is not a valid label");
+ }
+
+
+ this.originalType = originalType;
+ Map<Label, T> result = Maps.newLinkedHashMap();
+ boolean foundDefaultCondition = false;
+ for (Entry<?, ?> entry : ((Map<?, ?>) x).entrySet()) {
+ Label key = LABEL.convert(entry.getKey(), what, currentRule);
+ if (key.equals(defaultConditionLabel)) {
+ foundDefaultCondition = true;
+ }
+ result.put(key, originalType.convert(entry.getValue(), what, currentRule));
+ }
+ map = ImmutableMap.copyOf(result);
+ hasDefaultCondition = foundDefaultCondition;
+ }
+
+ /**
+ * Returns the selector's (configurability pattern --gt; matching values) map.
+ */
+ public Map<Label, T> getEntries() {
+ return map;
+ }
+
+ /**
+ * Returns the value to use when none of the attribute's selection keys match.
+ */
+ public T getDefault() {
+ return map.get(defaultConditionLabel);
+ }
+
+ /**
+ * Returns whether or not this selector has a default condition.
+ */
+ public boolean hasDefault() {
+ return hasDefaultCondition;
+ }
+
+ /**
+ * Returns the native Type for this attribute (i.e. what this would be if it wasn't a
+ * selector expression).
+ */
+ public Type<T> getOriginalType() {
+ return originalType;
+ }
+
+ /**
+ * Returns true for labels that are "reserved selector key words" and not intended to
+ * map to actual targets.
+ */
+ public static boolean isReservedLabel(Label label) {
+ return label.toString().equals(DEFAULT_CONDITION_KEY);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/CompileOneDependencyTransformer.java b/src/main/java/com/google/devtools/build/lib/pkgcache/CompileOneDependencyTransformer.java
new file mode 100644
index 0000000..ae14f18
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/CompileOneDependencyTransformer.java
@@ -0,0 +1,186 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.FileTarget;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Implementation of --compile_one_dependency.
+ */
+final class CompileOneDependencyTransformer {
+
+ private final PackageManager pkgManager;
+
+ public CompileOneDependencyTransformer(PackageManager pkgManager) {
+ this.pkgManager = pkgManager;
+ }
+
+ /**
+ * For each input file in the original result, returns a rule in the same package which has the
+ * input file as a source.
+ */
+ public ResolvedTargets<Target> transformCompileOneDependency(EventHandler eventHandler,
+ ResolvedTargets<Target> original) throws TargetParsingException {
+ if (original.hasError()) {
+ return original;
+ }
+ ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder();
+ for (Target target : original.getTargets()) {
+ builder.add(transformCompileOneDependency(eventHandler, target));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns a list of rules in the given package sorted by BUILD file order. When
+ * multiple rules depend on a target, we choose the first match in this list (after
+ * filtering for preferred dependencies - see below).
+ *
+ * <p>Rules with configurable attributes are skipped, as this code doesn't know which
+ * configuration will be applied, so it can't reliably determine what their 'srcs'
+ * will look like.
+ */
+ private Iterable<Rule> getOrderedRuleList(Package pkg) {
+ List<Rule> orderedList = Lists.newArrayList();
+ for (Rule rule : pkg.getTargets(Rule.class)) {
+ if (!rule.hasConfigurableAttributes()) {
+ orderedList.add(rule);
+ }
+ }
+
+ Collections.sort(orderedList, new Comparator<Rule>() {
+ @Override
+ public int compare(Rule o1, Rule o2) {
+ return Integer.compare(
+ o1.getLocation().getStartOffset(),
+ o2.getLocation().getStartOffset());
+ }
+ });
+ return orderedList;
+ }
+
+ private Target transformCompileOneDependency(EventHandler eventHandler, Target target)
+ throws TargetParsingException {
+ if (!(target instanceof FileTarget)) {
+ throw new TargetParsingException("--compile_one_dependency target '" +
+ target.getLabel() + "' must be a file");
+ }
+
+ Package pkg;
+ try {
+ pkg = pkgManager.getLoadedPackage(target.getLabel().getPackageIdentifier());
+ } catch (NoSuchPackageException e) {
+ throw new IllegalStateException(e);
+ }
+
+ Iterable<Rule> orderedRuleList = getOrderedRuleList(pkg);
+ // Consuming rule to return if no "preferred" rules have been found.
+ Rule fallbackRule = null;
+
+ for (Rule rule : orderedRuleList) {
+ try {
+ // The call to getSrcTargets here can be removed in favor of the
+ // rule.getLabels() call below once we update "srcs" for all rules.
+ if (SrcTargetUtil.getSrcTargets(eventHandler, rule, pkgManager).contains(target)) {
+ if (rule.getRuleClassObject().isPreferredDependency(target.getName())) {
+ return rule;
+ } else if (fallbackRule == null) {
+ fallbackRule = rule;
+ }
+ }
+ } catch (NoSuchThingException e) {
+ // Nothing to see here. Move along.
+ } catch (InterruptedException e) {
+ throw new TargetParsingException("interrupted");
+ }
+ }
+
+ Rule result = null;
+
+ // For each rule, see if it has directCompileTimeInputAttribute,
+ // and if so check the targets listed in that attribute match the label.
+ for (Rule rule : orderedRuleList) {
+ if (rule.getLabels(Rule.DIRECT_COMPILE_TIME_INPUT).contains(target.getLabel())) {
+ if (rule.getRuleClassObject().isPreferredDependency(target.getName())) {
+ result = rule;
+ } else if (fallbackRule == null) {
+ fallbackRule = rule;
+ }
+ }
+ }
+
+ if (result == null) {
+ result = fallbackRule;
+ }
+
+ if (result == null) {
+ throw new TargetParsingException(
+ "Couldn't find dependency on target '" + target.getLabel() + "'");
+ }
+
+ try {
+ // If the rule has source targets, return it.
+ if (!SrcTargetUtil.getSrcTargets(eventHandler, result, pkgManager).isEmpty()) {
+ return result;
+ }
+ } catch (NoSuchThingException e) {
+ throw new TargetParsingException(
+ "Couldn't find dependency on target '" + target.getLabel() + "'");
+ } catch (InterruptedException e) {
+ throw new TargetParsingException("interrupted");
+ }
+
+ for (Rule rule : orderedRuleList) {
+ RawAttributeMapper attributes = RawAttributeMapper.of(rule);
+ // We don't know what configuration we're using at this point, so we can't be sure
+ // which deps/srcs apply to this invocation if they're configurable for this rule.
+ // So exclude such rules for consideration.
+ if (attributes.isConfigurable("deps", Type.LABEL_LIST)
+ || attributes.isConfigurable("srcs", Type.LABEL_LIST)) {
+ continue;
+ }
+ RuleClass ruleClass = rule.getRuleClassObject();
+ if (ruleClass.hasAttr("deps", Type.LABEL_LIST) &&
+ ruleClass.hasAttr("srcs", Type.LABEL_LIST)) {
+ for (Label dep : attributes.get("deps", Type.LABEL_LIST)) {
+ if (dep.equals(result.getLabel())) {
+ if (!attributes.get("srcs", Type.LABEL_LIST).isEmpty()) {
+ return rule;
+ }
+ }
+ }
+ }
+ }
+
+ throw new TargetParsingException(
+ "Couldn't find dependency on target '" + target.getLabel() + "'");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicies.java b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicies.java
new file mode 100644
index 0000000..df01326
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicies.java
@@ -0,0 +1,126 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+
+import java.util.Objects;
+
+/**
+ * Utility class for predefined filtering policies.
+ */
+public final class FilteringPolicies {
+
+ private FilteringPolicies() {
+ }
+
+ /**
+ * Base class for singleton filtering policies.
+ */
+ private abstract static class AbstractFilteringPolicy implements FilteringPolicy {
+ @Override
+ public int hashCode() {
+ return getClass().getSimpleName().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+ return getClass().equals(obj.getClass());
+ }
+ }
+
+ private static class NoFilter extends AbstractFilteringPolicy {
+ @Override
+ public boolean shouldRetain(Target target, boolean explicit) {
+ return true;
+ }
+ }
+
+ public static final FilteringPolicy NO_FILTER = new NoFilter();
+
+ private static class FilterManualAndObsolete extends AbstractFilteringPolicy {
+ @Override
+ public boolean shouldRetain(Target target, boolean explicit) {
+ return explicit || !(TargetUtils.hasManualTag(target) || TargetUtils.isObsolete(target));
+ }
+ }
+
+ public static final FilteringPolicy FILTER_MANUAL_AND_OBSOLETE = new FilterManualAndObsolete();
+
+ private static class FilterTests extends AbstractFilteringPolicy {
+ @Override
+ public boolean shouldRetain(Target target, boolean explicit) {
+ return TargetUtils.isTestOrTestSuiteRule(target)
+ && FILTER_MANUAL_AND_OBSOLETE.shouldRetain(target, explicit);
+ }
+ }
+
+ public static final FilteringPolicy FILTER_TESTS = new FilterTests();
+
+ private static class RulesOnly extends AbstractFilteringPolicy {
+ @Override
+ public boolean shouldRetain(Target target, boolean explicit) {
+ return target instanceof Rule;
+ }
+ }
+
+ public static final FilteringPolicy RULES_ONLY = new RulesOnly();
+
+ /**
+ * Returns the result of applying y, if target passes x.
+ */
+ public static FilteringPolicy and(final FilteringPolicy x,
+ final FilteringPolicy y) {
+ return new AndFilteringPolicy(x, y);
+ }
+
+ private static class AndFilteringPolicy implements FilteringPolicy {
+ private final FilteringPolicy firstPolicy;
+ private final FilteringPolicy secondPolicy;
+
+ public AndFilteringPolicy(FilteringPolicy firstPolicy, FilteringPolicy secondPolicy) {
+ this.firstPolicy = Preconditions.checkNotNull(firstPolicy);
+ this.secondPolicy = Preconditions.checkNotNull(secondPolicy);
+ }
+
+ @Override
+ public boolean shouldRetain(Target target, boolean explicit) {
+ return firstPolicy.shouldRetain(target, explicit)
+ && secondPolicy.shouldRetain(target, explicit);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(firstPolicy, secondPolicy);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof AndFilteringPolicy)) {
+ return false;
+ }
+ AndFilteringPolicy other = (AndFilteringPolicy) obj;
+ return other.firstPolicy.equals(firstPolicy) && other.secondPolicy.equals(secondPolicy);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicy.java b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicy.java
new file mode 100644
index 0000000..ac27fb0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicy.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.packages.Target;
+
+import java.io.Serializable;
+
+/**
+ * A filtering policy defines how target patterns are matched. For instance, we may wish to select
+ * only tests, no tests, or remove obsolete targets.
+ */
+public interface FilteringPolicy extends Serializable {
+
+ /**
+ * Returns true if this target should be retained.
+ *
+ * @param explicit true iff the label was specified explicitly, as opposed to being discovered by
+ * a wildcard.
+ */
+ @ThreadSafety.ThreadSafe
+ boolean shouldRetain(Target target, boolean explicit);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackage.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackage.java
new file mode 100644
index 0000000..3c6f70f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackage.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.events.Event;
+
+/**
+ * A loaded package that can verify whether it is still up to date.
+ */
+interface LoadedPackage {
+ /**
+ * Returns the actual loaded {@link Package} object.
+ */
+ Package getPackage();
+
+ /**
+ * Returns true iff the entry is still valid.
+ *
+ * <p>An entry is valid when the package it denotes has not been moved, deleted, or changed. This
+ * requires disk I/O to fetch metadata and re-evaluate globs.
+ */
+ boolean isValid() throws InterruptedException;
+
+ /**
+ * Returns true iff the the contents of the package are guaranteed not to have changed after
+ * between {@link #isValid()} calls and syncs of the associated package loader.
+ */
+ boolean contentsCouldNotHaveChanged();
+
+ /**
+ * Returns the set of events (sorted by the order they were reported) that occurred during the
+ * loading of the package.
+ */
+ Iterable<Event> getEvents();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackageProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackageProvider.java
new file mode 100644
index 0000000..1c51810
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackageProvider.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Read-only API for retrieving packages, i.e., calling this API should not result in packages being
+ * loaded.
+ *
+ * <p><b>Concurrency</b>: Implementations should be thread-safe.
+ */
+// TODO(bazel-team): Skyframe doesn't really implement this - can we remove it?
+public interface LoadedPackageProvider {
+
+ /**
+ * Returns a package if it was recently loaded, i.e., since the most recent cache sync. This
+ * throws an exception if the package was not loaded, even if it exists on disk.
+ */
+ Package getLoadedPackage(PackageIdentifier packageIdentifier) throws NoSuchPackageException;
+
+ /**
+ * Returns a target if it was recently loaded, i.e., since the most recent cache sync. This
+ * throws an exception if the target was not loaded or not validated, even if it exists in the
+ * surrounding package.
+ */
+ Target getLoadedTarget(Label label) throws NoSuchPackageException, NoSuchTargetException;
+
+ /**
+ * Returns true iff the specified target is current, i.e. a request for its label using {@link
+ * #getLoadedTarget} would return the same target instance.
+ */
+ boolean isTargetCurrent(Target target);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailedException.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailedException.java
new file mode 100644
index 0000000..537746b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailedException.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+/**
+ * An exception indicating that there was a problem during the loading phase for one or more
+ * targets in such a way that the build cannot proceed (for example because keep_going is disabled).
+ */
+public class LoadingFailedException extends Exception {
+
+ public LoadingFailedException(String message) {
+ super(message);
+ }
+
+ public LoadingFailedException(String message, Throwable cause) {
+ super(message + ": " + cause.getMessage(), cause);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailureEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailureEvent.java
new file mode 100644
index 0000000..d7e0256
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailureEvent.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * This event is fired during the build, when it becomes known that the loading
+ * of a target cannot be completed because of an error in one of its
+ * dependencies.
+ */
+public class LoadingFailureEvent {
+ private final Label failedTarget;
+ private final Label failureReason;
+
+ public LoadingFailureEvent(Label failedTarget, Label failureReason) {
+ this.failedTarget = failedTarget;
+ this.failureReason = failureReason;
+ }
+
+ public Label getFailedTarget() {
+ return failedTarget;
+ }
+
+ public Label getFailureReason() {
+ return failureReason;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseCompleteEvent.java
new file mode 100644
index 0000000..19c22ec
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseCompleteEvent.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+
+/**
+ * This event is fired after the loading phase is complete.
+ */
+public class LoadingPhaseCompleteEvent {
+ private final Collection<Target> targets;
+ private final Collection<Target> filteredTargets;
+ private final PackageManager.PackageManagerStatistics pkgManagerStats;
+ private final long timeInMs;
+
+ /**
+ * Construct the event.
+ *
+ * @param targets the set of active targets that remain
+ * @param pkgManagerStats statistics about the package cache
+ */
+ public LoadingPhaseCompleteEvent(Collection<Target> targets, Collection<Target> filteredTargets,
+ PackageManager.PackageManagerStatistics pkgManagerStats, long timeInMs) {
+ this.targets = targets;
+ this.filteredTargets = filteredTargets;
+ this.pkgManagerStats = pkgManagerStats;
+ this.timeInMs = timeInMs;
+ }
+
+ /**
+ * @return The set of active targets remaining, which is a subset of the
+ * targets we attempted to load.
+ */
+ public Collection<Target> getTargets() {
+ return targets;
+ }
+
+ /**
+ * @return The set of filtered targets.
+ */
+ public Collection<Target> getFilteredTargets() {
+ return filteredTargets;
+ }
+
+ /**
+ * @return The set of active target labels remaining, which is a subset of the
+ * targets we attempted to load.
+ */
+ public Iterable<Label> getLabels() {
+ return Iterables.transform(targets, TO_LABEL);
+ }
+
+ public long getTimeInMs() {
+ return timeInMs;
+ }
+
+ /**
+ * Returns the PackageCache statistics.
+ */
+ public PackageManager.PackageManagerStatistics getPkgManagerStats() {
+ return pkgManagerStats;
+ }
+
+ private static final Function<Target, Label> TO_LABEL = new Function<Target, Label>() {
+ @Override
+ public Label apply(Target input) {
+ return input.getLabel();
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java
new file mode 100644
index 0000000..4d8eea6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java
@@ -0,0 +1,661 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.events.DelegatingEventHandler;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.TestTargetUtils;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converters.CommaSeparatedOptionListConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * Implements the loading phase; responsible for:
+ * <ul>
+ * <li>target pattern evaluation
+ * <li>test suite expansion
+ * <li>loading the labels needed to construct the build configuration
+ * <li>loading the labels needed for the analysis with the build configuration
+ * <li>loading the transitive closure of the targets and the configuration labels
+ * </ul>
+ *
+ * <p>In order to ensure correctness of incremental loading and of full cache hits, this class is
+ * very restrictive about access to its internal state and to its collaborators. In particular, none
+ * of the collaborators of this class may change in incompatible ways, such as changing the relative
+ * working directory for the target pattern parser, without notifying this class.
+ *
+ * <p>For full caching, this class tracks the exact values of all inputs to the loading phase. To
+ * maximize caching, it is vital that these change as rarely as possible.
+ */
+public class LoadingPhaseRunner {
+
+ /**
+ * Loading phase options.
+ */
+ public static class Options extends OptionsBase {
+
+ @Option(name = "loading_phase_threads",
+ defaultValue = "200",
+ category = "undocumented",
+ help = "Number of parallel threads to use for the loading phase.")
+ public int loadingPhaseThreads;
+
+ @Option(name = "build_tests_only",
+ defaultValue = "false",
+ category = "what",
+ help = "If specified, only *_test and test_suite rules will be built "
+ + "and other targets specified on the command line will be ignored. "
+ + "By default everything that was requested will be built.")
+ public boolean buildTestsOnly;
+
+ @Option(name = "compile_one_dependency",
+ defaultValue = "false",
+ category = "what",
+ help = "Compile a single dependency of the argument files. "
+ + "This is useful for syntax checking source files in IDEs, "
+ + "for example, by rebuilding a single target that depends on "
+ + "the source file to detect errors as early as possible in the "
+ + "edit/build/test cycle. This argument affects the way all "
+ + "non-flag arguments are interpreted; instead of being targets "
+ + "to build they are source filenames. For each source filename "
+ + "an arbitrary target that depends on it will be built.")
+ public boolean compileOneDependency;
+
+ @Option(name = "test_tag_filters",
+ converter = CommaSeparatedOptionListConverter.class,
+ defaultValue = "",
+ category = "what",
+ help = "Specifies a comma-separated list of test tags. Each tag can be optionally " +
+ "preceded with '-' to specify excluded tags. Only those test targets will be " +
+ "found that contain at least one included tag and do not contain any excluded " +
+ "tags. This option affects --build_tests_only behavior and the test command."
+ )
+ public List<String> testTagFilterList;
+
+ @Option(name = "test_size_filters",
+ converter = TestSize.TestSizeFilterConverter.class,
+ defaultValue = "",
+ category = "what",
+ help = "Specifies a comma-separated list of test sizes. Each size can be optionally " +
+ "preceded with '-' to specify excluded sizes. Only those test targets will be " +
+ "found that contain at least one included size and do not contain any excluded " +
+ "sizes. This option affects --build_tests_only behavior and the test command."
+ )
+ public Set<TestSize> testSizeFilterSet;
+
+ @Option(name = "test_timeout_filters",
+ converter = TestTimeout.TestTimeoutFilterConverter.class,
+ defaultValue = "",
+ category = "what",
+ help = "Specifies a comma-separated list of test timeouts. Each timeout can be " +
+ "optionally preceded with '-' to specify excluded timeouts. Only those test " +
+ "targets will be found that contain at least one included timeout and do not " +
+ "contain any excluded timeouts. This option affects --build_tests_only behavior " +
+ "and the test command."
+ )
+ public Set<TestTimeout> testTimeoutFilterSet;
+
+ @Option(name = "test_lang_filters",
+ converter = CommaSeparatedOptionListConverter.class,
+ defaultValue = "",
+ category = "what",
+ help = "Specifies a comma-separated list of test languages. Each language can be " +
+ "optionally preceded with '-' to specify excluded languages. Only those " +
+ "test targets will be found that are written in the specified languages. " +
+ "The name used for each language should be the same as the language prefix in the " +
+ "*_test rule, e.g. one of 'cc', 'java', 'py', etc." +
+ "This option affects --build_tests_only behavior and the test command."
+ )
+ public List<String> testLangFilterList;
+ }
+
+ /**
+ * A callback interface to notify the caller about specific events.
+ * TODO(bazel-team): maybe we should use the EventBus instead?
+ */
+ public interface Callback {
+ /**
+ * Called after the target patterns have been resolved to give the caller a chance to validate
+ * the list before proceeding.
+ */
+ void notifyTargets(Collection<Target> targets) throws LoadingFailedException;
+
+ /**
+ * Called after loading has finished, to notify the caller about the visited packages.
+ *
+ * <p>The set of visited packages is the set of packages in the transitive closure of the
+ * union of the top level targets.
+ */
+ void notifyVisitedPackages(Set<PackageIdentifier> visitedPackages);
+ }
+
+ /**
+ * The result of the loading phase, i.e., whether there were errors, and which targets were
+ * successfully loaded, plus some related metadata.
+ */
+ public static final class LoadingResult {
+ private final boolean hasTargetPatternError;
+ private final boolean hasLoadingError;
+ private final ImmutableSet<Target> targetsToAnalyze;
+ private final ImmutableSet<Target> testsToRun;
+ private final ImmutableMap<PackageIdentifier, Path> packageRoots;
+ // TODO(bazel-team): consider moving this to LoadedPackageProvider
+ private final ImmutableSet<PackageIdentifier> visitedPackages;
+
+ public LoadingResult(boolean hasTargetPatternError, boolean hasLoadingError,
+ Collection<Target> targetsToAnalyze, Collection<Target> testsToRun,
+ ImmutableMap<PackageIdentifier, Path> packageRoots,
+ Set<PackageIdentifier> visitedPackages) {
+ this.hasTargetPatternError = hasTargetPatternError;
+ this.hasLoadingError = hasLoadingError;
+ this.targetsToAnalyze =
+ targetsToAnalyze == null ? null : ImmutableSet.copyOf(targetsToAnalyze);
+ this.testsToRun = testsToRun == null ? null : ImmutableSet.copyOf(testsToRun);
+ this.packageRoots = packageRoots;
+ this.visitedPackages = ImmutableSet.copyOf(visitedPackages);
+ }
+
+ /** Whether there were errors during target pattern evaluation. */
+ public boolean hasTargetPatternError() {
+ return hasTargetPatternError;
+ }
+
+ /** Whether there were errors during the loading phase. */
+ public boolean hasLoadingError() {
+ return hasLoadingError;
+ }
+
+ /** Successfully loaded targets that should be built. */
+ public Collection<Target> getTargets() {
+ return targetsToAnalyze;
+ }
+
+ /** Successfully loaded targets that should be run as tests. Must be a subset of the targets. */
+ public Collection<Target> getTestsToRun() {
+ return testsToRun;
+ }
+
+ /**
+ * The map from package names to the package root where each package was found; this is used to
+ * set up the symlink tree.
+ */
+ public ImmutableMap<PackageIdentifier, Path> getPackageRoots() {
+ return packageRoots;
+ }
+
+ /**
+ * Returns all packages that were visited during this loading phase.
+ *
+ * <p>We use this to decide when to evict ConfiguredTarget nodes from the graph.
+ */
+ @ThreadCompatible
+ private ImmutableSet<PackageIdentifier> getVisitedPackages() {
+ return visitedPackages;
+ }
+ }
+
+ private static final class ParseFailureListenerImpl extends DelegatingEventHandler
+ implements ParseFailureListener {
+ private final EventBus eventBus;
+
+ private ParseFailureListenerImpl(EventHandler delegate, EventBus eventBus) {
+ super(delegate);
+ this.eventBus = eventBus;
+ }
+
+ @Override
+ public void parsingError(String targetPattern, String message) {
+ if (eventBus != null) {
+ eventBus.post(new ParsingFailedEvent(targetPattern, message));
+ }
+ }
+ }
+
+ private static final Logger LOG = Logger.getLogger(LoadingPhaseRunner.class.getName());
+
+ private final PackageManager packageManager;
+ private final TargetPatternEvaluator targetPatternEvaluator;
+ private final Set<String> ruleNames;
+ private final TransitivePackageLoader pkgLoader;
+
+ public LoadingPhaseRunner(PackageManager packageManager,
+ Set<String> ruleNames) {
+ this.packageManager = packageManager;
+ this.targetPatternEvaluator = packageManager.getTargetPatternEvaluator();
+ this.ruleNames = ruleNames;
+ this.pkgLoader = packageManager.newTransitiveLoader();
+ }
+
+ public TargetPatternEvaluator getTargetPatternEvaluator() {
+ return targetPatternEvaluator;
+ }
+
+ public void updatePatternEvaluator(PathFragment relativeWorkingDirectory) {
+ targetPatternEvaluator.updateOffset(relativeWorkingDirectory);
+ }
+
+ /**
+ * This method only exists for the benefit of InfoCommand, which needs to construct
+ * a {@code BuildConfigurationCollection} without running a full loading phase. Don't
+ * add any more clients; instead, we should change info so that it doesn't need the configuration.
+ */
+ public LoadedPackageProvider loadForConfigurations(EventHandler eventHandler,
+ Set<Label> labelsToLoad, boolean keepGoing) throws InterruptedException {
+ // Use a new Label Visitor here to avoid erasing the cache on the existing one.
+ TransitivePackageLoader transitivePackageLoader = packageManager.newTransitiveLoader();
+ boolean loadingSuccessful = transitivePackageLoader.sync(
+ eventHandler, ImmutableSet.<Target>of(),
+ labelsToLoad, keepGoing, /*parallelThreads=*/10,
+ /*maxDepth=*/Integer.MAX_VALUE);
+ return loadingSuccessful ? packageManager : null;
+ }
+
+ /**
+ * Performs target pattern evaluation, test suite expansion (if requested), and loads the
+ * transitive closure of the resulting targets as well as of the targets needed to use the
+ * given build configuration provider.
+ */
+ public LoadingResult execute(EventHandler eventHandler, EventBus eventBus,
+ List<String> targetPatterns, Options options,
+ ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing,
+ boolean determineTests, @Nullable Callback callback)
+ throws TargetParsingException, LoadingFailedException, InterruptedException {
+ LOG.info("Starting pattern evaluation");
+ Stopwatch timer = Stopwatch.createStarted();
+ if (options.buildTestsOnly && options.compileOneDependency) {
+ throw new LoadingFailedException("--compile_one_dependency cannot be used together with "
+ + "the --build_tests_only option or the 'bazel test' command ");
+ }
+
+ EventHandler parseFailureListener = new ParseFailureListenerImpl(eventHandler, eventBus);
+ // Determine targets to build:
+ ResolvedTargets<Target> targets = getTargetsToBuild(parseFailureListener,
+ targetPatterns, options.compileOneDependency, keepGoing);
+
+ ImmutableSet<Target> filteredTargets = targets.getFilteredTargets();
+
+ boolean buildTestsOnly = options.buildTestsOnly;
+ ImmutableSet<Target> testsToRun = null;
+ ImmutableSet<Target> testFilteredTargets = ImmutableSet.of();
+
+ // Now we have a list of targets to build. If the --build_tests_only option was specified or we
+ // want to run tests, we need to determine the list of targets to test. For that, we remove
+ // manual tests and apply the command line filters. Also, if --build_tests_only is specified,
+ // then the list of filtered targets will be set as build list as well.
+ if (determineTests || buildTestsOnly) {
+ // Parse the targets to get the tests.
+ ResolvedTargets<Target> testTargets = determineTests(parseFailureListener,
+ targetPatterns, options, keepGoing);
+ if (testTargets.getTargets().isEmpty() && !testTargets.getFilteredTargets().isEmpty()) {
+ eventHandler.handle(Event.warn("All specified test targets were excluded by filters"));
+ }
+
+ if (buildTestsOnly) {
+ // Replace original targets to build with test targets, so that only targets that are
+ // actually going to be built are loaded in the loading phase. Note that this has a side
+ // effect that any test_suite target requested to be built is replaced by the set of *_test
+ // targets it represents; for example, this affects the status and the summary reports.
+ Set<Target> allFilteredTargets = new HashSet<>();
+ allFilteredTargets.addAll(targets.getTargets());
+ allFilteredTargets.addAll(targets.getFilteredTargets());
+ allFilteredTargets.removeAll(testTargets.getTargets());
+ allFilteredTargets.addAll(testTargets.getFilteredTargets());
+ testFilteredTargets = ImmutableSet.copyOf(allFilteredTargets);
+ filteredTargets = ImmutableSet.of();
+
+ targets = ResolvedTargets.<Target>builder()
+ .merge(testTargets)
+ .mergeError(targets.hasError())
+ .build();
+ if (determineTests) {
+ testsToRun = testTargets.getTargets();
+ }
+ } else /*if (determineTests)*/ {
+ testsToRun = testTargets.getTargets();
+ targets = ResolvedTargets.<Target>builder()
+ .merge(targets)
+ // Avoid merge() here which would remove the filteredTargets from the targets.
+ .addAll(testsToRun)
+ .mergeError(testTargets.hasError())
+ .build();
+ // filteredTargets is correct in this case - it cannot contain tests that got back in
+ // through test_suite expansion, because the test determination would also filter those out.
+ // However, that's not obvious, and it might be better to explicitly recompute it.
+ }
+ if (testsToRun != null) {
+ // Note that testsToRun can still be null here, if buildTestsOnly && !shouldRunTests.
+ Preconditions.checkState(targets.getTargets().containsAll(testsToRun));
+ }
+ }
+
+ eventBus.post(new TargetParsingCompleteEvent(targets.getTargets(),
+ filteredTargets, testFilteredTargets,
+ timer.stop().elapsed(TimeUnit.MILLISECONDS)));
+
+ if (targets.hasError()) {
+ eventHandler.handle(Event.warn("Target pattern parsing failed. Continuing anyway"));
+ }
+
+ if (callback != null) {
+ callback.notifyTargets(targets.getTargets());
+ }
+
+ maybeReportDeprecation(eventHandler, targets.getTargets());
+
+ // Load the transitive closure of all targets.
+ LoadingResult result = doLoadingPhase(eventHandler, eventBus, targets.getTargets(),
+ testsToRun, labelsToLoadUnconditionally, keepGoing, options.loadingPhaseThreads,
+ targets.hasError());
+
+ if (callback != null) {
+ callback.notifyVisitedPackages(result.getVisitedPackages());
+ }
+
+ return result;
+ }
+
+ /**
+ * Visit the transitive closure of the targets, populating the package cache
+ * and ensuring that all labels can be resolved and all rules were free from
+ * errors.
+ *
+ * @param targetsToLoad the list of command-line target patterns specified by the user
+ * @param testsToRun the tests to run as a subset of the targets to load
+ * @param labelsToLoadUnconditionally the labels to load unconditionally (presumably for the build
+ * configuration)
+ * @param keepGoing if true, don't throw ViewCreationFailedException if some
+ * targets could not be loaded, just skip thm.
+ */
+ private LoadingResult doLoadingPhase(EventHandler eventHandler, EventBus eventBus,
+ ImmutableSet<Target> targetsToLoad, Collection<Target> testsToRun,
+ ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing,
+ int loadingPhaseThreads, boolean hasError)
+ throws InterruptedException, LoadingFailedException {
+ eventHandler.handle(Event.progress("Loading..."));
+ Stopwatch timer = Stopwatch.createStarted();
+ LOG.info("Starting loading phase");
+
+ Set<Label> labelsToLoad = ImmutableSet.copyOf(labelsToLoadUnconditionally.values());
+
+ // For each label in {@code targetsToLoad}, ensure that the target to which
+ // it refers exists, and also every target in its transitive closure of label
+ // dependencies. Success guarantees that a call to
+ // {@code getConfiguredTarget} for the same targets will not fail; the
+ // configuration process is intolerant of missing packages/targets. Before
+ // calling getConfiguredTarget(), clients must ensure that all necessary
+ // packages/targets have been visited since the last sync/clear.
+ boolean loadingSuccessful = pkgLoader.sync(eventHandler, targetsToLoad, labelsToLoad,
+ keepGoing, loadingPhaseThreads, Integer.MAX_VALUE);
+
+ ImmutableSet<Target> targetsToAnalyze;
+ if (loadingSuccessful) {
+ // Success: all loaded targets will be analyzed.
+ targetsToAnalyze = targetsToLoad;
+ } else if (keepGoing) {
+ // Keep going: filter out the error-free targets and only continue with those.
+ targetsToAnalyze = filterErrorFreeTargets(eventBus, targetsToLoad,
+ pkgLoader, labelsToLoadUnconditionally);
+
+ // Tell the user about the subset of successful targets.
+ int requested = targetsToLoad.size();
+ int loaded = targetsToAnalyze.size();
+ if (0 < loaded && loaded < requested) {
+ String message = String.format("Loading succeeded for only %d of %d targets", loaded,
+ requested);
+ eventHandler.handle(Event.info(message));
+ LOG.info(message);
+ }
+ } else {
+ throw new LoadingFailedException("Loading failed; build aborted");
+ }
+
+ Set<Target> filteredTargets = targetsToAnalyze;
+ try {
+ // We use strict test_suite expansion here to match the analysis-time checks.
+ ResolvedTargets<Target> expandedResult = TestTargetUtils.expandTestSuites(
+ packageManager, eventHandler, targetsToAnalyze, /*strict=*/true, /*keepGoing=*/true);
+ targetsToAnalyze = expandedResult.getTargets();
+ filteredTargets = Sets.difference(filteredTargets, targetsToAnalyze);
+ if (expandedResult.hasError()) {
+ if (!keepGoing) {
+ throw new LoadingFailedException("Could not expand test suite target");
+ }
+ loadingSuccessful = false;
+ }
+ } catch (TargetParsingException e) {
+ // This shouldn't happen, because we've already loaded the targets successfully.
+ throw (AssertionError) (new AssertionError("Unexpected target failure").initCause(e));
+ }
+
+ // Perform some operations on the set of packages containing the collected targets.
+ ImmutableMap<PackageIdentifier, Path> packageRoots = collectPackageRoots(
+ pkgLoader.getErrorFreeVisitedPackages());
+
+ Set<PackageIdentifier> visitedPackageNames = pkgLoader.getVisitedPackageNames();
+
+ // Clear some targets from the cache to free memory.
+ packageManager.partiallyClear();
+
+ eventBus.post(new LoadingPhaseCompleteEvent(
+ targetsToAnalyze, filteredTargets, packageManager.getStatistics(),
+ timer.stop().elapsed(TimeUnit.MILLISECONDS)));
+ LOG.info("Loading phase finished");
+
+ // testsToRun can contain targets that aren't analyzed, but the BuildView ignores those.
+ return new LoadingResult(hasError, !loadingSuccessful, targetsToAnalyze, testsToRun,
+ packageRoots, visitedPackageNames);
+ }
+
+ private Collection<Target> getTargetsForLabels(Collection<Label> labels) {
+ Set<Target> result = new HashSet<>();
+
+ for (Label label : labels) {
+ try {
+ result.add(packageManager.getLoadedTarget(label));
+ } catch (NoSuchPackageException e) {
+ Package pkg = Preconditions.checkNotNull(e.getPackage());
+ try {
+ result.add(pkg.getTarget(label.getName()));
+ } catch (NoSuchTargetException ex) {
+ throw new IllegalStateException(ex);
+ }
+ } catch (NoSuchThingException e) {
+ throw new IllegalStateException(e); // The target should have been loaded
+ }
+ }
+
+ return result;
+ }
+
+ private ImmutableSet<Target> filterErrorFreeTargets(
+ EventBus eventBus, Collection<Target> targetsToLoad,
+ TransitivePackageLoader pkgLoader,
+ ListMultimap<String, Label> labelsToLoadUnconditionally) throws LoadingFailedException {
+ // Error out if any of the labels needed for the configuration could not be loaded.
+ Collection<Label> labelsToLoad = new ArrayList<>(labelsToLoadUnconditionally.values());
+ for (Target target : targetsToLoad) {
+ labelsToLoad.add(target.getLabel());
+ }
+ Multimap<Label, Label> rootCauses = pkgLoader.getRootCauses(labelsToLoad);
+ for (Map.Entry<String, Label> entry : labelsToLoadUnconditionally.entries()) {
+ if (rootCauses.containsKey(entry.getValue())) {
+ throw new LoadingFailedException("Failed to load required " + entry.getKey()
+ + " target: '" + entry.getValue() + "'");
+ }
+ }
+
+ // Post root causes for command-line targets that could not be loaded.
+ for (Map.Entry<Label, Label> entry : rootCauses.entries()) {
+ eventBus.post(new LoadingFailureEvent(entry.getKey(), entry.getValue()));
+ }
+
+ return ImmutableSet.copyOf(Sets.difference(ImmutableSet.copyOf(targetsToLoad),
+ ImmutableSet.copyOf(getTargetsForLabels(rootCauses.keySet()))));
+ }
+
+ /**
+ * Returns a map of collected package names to root paths.
+ */
+ private static ImmutableMap<PackageIdentifier, Path> collectPackageRoots(
+ Collection<Package> packages) {
+ // Make a map of the package names to their root paths.
+ ImmutableMap.Builder<PackageIdentifier, Path> packageRoots = ImmutableMap.builder();
+ for (Package pkg : packages) {
+ packageRoots.put(pkg.getPackageIdentifier(), pkg.getSourceRoot());
+ }
+ return packageRoots.build();
+ }
+
+ /**
+ * Interpret the command-line arguments.
+ *
+ * @param targetPatterns the list of command-line target patterns specified by the user
+ * @param compileOneDependency if true, enables alternative interpretation of targetPatterns; see
+ * {@link Options#compileOneDependency}
+ * @throws TargetParsingException if parsing failed and !keepGoing
+ */
+ private ResolvedTargets<Target> getTargetsToBuild(EventHandler eventHandler,
+ List<String> targetPatterns, boolean compileOneDependency,
+ boolean keepGoing) throws TargetParsingException, InterruptedException {
+ ResolvedTargets<Target> result =
+ targetPatternEvaluator.parseTargetPatternList(eventHandler, targetPatterns,
+ FilteringPolicies.FILTER_MANUAL_AND_OBSOLETE, keepGoing);
+ if (compileOneDependency) {
+ return new CompileOneDependencyTransformer(packageManager)
+ .transformCompileOneDependency(eventHandler, result);
+ }
+ return result;
+ }
+
+ /**
+ * Interpret test target labels from the command-line arguments and return the corresponding set
+ * of targets, handling the filter flags, and expanding test suites.
+ *
+ * @param eventHandler the error event eventHandler
+ * @param targetPatterns the list of command-line target patterns specified by the user
+ * @param options the loading phase options
+ * @param keepGoing value of the --keep_going flag
+ */
+ private ResolvedTargets<Target> determineTests(EventHandler eventHandler,
+ List<String> targetPatterns, Options options, boolean keepGoing)
+ throws TargetParsingException, InterruptedException {
+ // Parse the targets to get the tests.
+ ResolvedTargets.Builder<Target> testTargetsBuilder = ResolvedTargets.builder();
+ for (String targetPattern : targetPatterns) {
+ if (targetPattern.startsWith("-")) {
+ ResolvedTargets<Target> someNegativeTargets = targetPatternEvaluator.parseTargetPatternList(
+ eventHandler, ImmutableList.of(targetPattern.substring(1)),
+ FilteringPolicies.FILTER_TESTS, keepGoing);
+ ResolvedTargets<Target> moreNegativeTargets = TestTargetUtils.expandTestSuites(
+ packageManager, eventHandler, someNegativeTargets.getTargets(), /*strict=*/false,
+ keepGoing);
+ testTargetsBuilder.filter(Predicates.not(Predicates.in(moreNegativeTargets.getTargets())));
+ testTargetsBuilder.mergeError(moreNegativeTargets.hasError());
+ } else {
+ ResolvedTargets<Target> somePositiveTargets = targetPatternEvaluator.parseTargetPatternList(
+ eventHandler, ImmutableList.of(targetPattern),
+ FilteringPolicies.FILTER_TESTS, keepGoing);
+ ResolvedTargets<Target> morePositiveTargets = TestTargetUtils.expandTestSuites(
+ packageManager, eventHandler, somePositiveTargets.getTargets(), /*strict=*/false,
+ keepGoing);
+ testTargetsBuilder.addAll(morePositiveTargets.getTargets());
+ testTargetsBuilder.mergeError(morePositiveTargets.hasError());
+ }
+ }
+ testTargetsBuilder.filter(getTestFilter(eventHandler, options));
+ return testTargetsBuilder.build();
+ }
+
+ /**
+ * Convert the options into a test filter.
+ */
+ private Predicate<Target> getTestFilter(EventHandler eventHandler, Options options) {
+ Predicate<Target> testFilter = Predicates.alwaysTrue();
+ if (!options.testSizeFilterSet.isEmpty()) {
+ testFilter = Predicates.and(testFilter,
+ TestTargetUtils.testSizeFilter(options.testSizeFilterSet));
+ }
+ if (!options.testTimeoutFilterSet.isEmpty()) {
+ testFilter = Predicates.and(testFilter,
+ TestTargetUtils.testTimeoutFilter(options.testTimeoutFilterSet));
+ }
+ if (!options.testTagFilterList.isEmpty()) {
+ testFilter = Predicates.and(testFilter,
+ TestTargetUtils.tagFilter(options.testTagFilterList));
+ }
+ if (!options.testLangFilterList.isEmpty()) {
+ testFilter = Predicates.and(testFilter,
+ TestTargetUtils.testLangFilter(options.testLangFilterList, eventHandler, ruleNames));
+ }
+ return testFilter;
+ }
+
+ /**
+ * Emit a warning when a deprecated target is mentioned on the command line.
+ *
+ * <p>Note that this does not stop us from emitting "target X depends on deprecated target Y"
+ * style warnings for the same target and it is a good thing; <i>depending</i> on a target and
+ * <i>wanting</i> to build it are different things.
+ */
+ private void maybeReportDeprecation(EventHandler eventHandler, Collection<Target> targets) {
+ for (Rule rule : Iterables.filter(targets, Rule.class)) {
+ if (rule.isAttributeValueExplicitlySpecified("deprecation")) {
+ eventHandler.handle(Event.warn(rule.getLocation(), String.format(
+ "target '%s' is deprecated: %s", rule.getLabel(),
+ NonconfigurableAttributeMapper.of(rule).get("deprecation", Type.STRING))));
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheBackedTargetPatternResolver.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheBackedTargetPatternResolver.java
new file mode 100644
index 0000000..e4dc010
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheBackedTargetPatternResolver.java
@@ -0,0 +1,263 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * An implementation of the {@link TargetPatternResolver} that uses the {@link
+ * RecursivePackageProvider} as the backing implementation.
+ */
+final class PackageCacheBackedTargetPatternResolver implements TargetPatternResolver<Target> {
+
+ private final RecursivePackageProvider packageProvider;
+ private final EventHandler eventHandler;
+ private final boolean keepGoing;
+ private final FilteringPolicy policy;
+ private final ThreadPoolExecutor packageVisitorPool;
+
+ PackageCacheBackedTargetPatternResolver(RecursivePackageProvider packageProvider,
+ EventHandler eventHandler, boolean keepGoing, FilteringPolicy policy,
+ ThreadPoolExecutor packageVisitorPool) {
+ this.packageProvider = packageProvider;
+ this.eventHandler = eventHandler;
+ this.keepGoing = keepGoing;
+ this.policy = policy;
+ this.packageVisitorPool = packageVisitorPool;
+ }
+
+ @Override
+ public void warn(String msg) {
+ eventHandler.handle(Event.warn(msg));
+ }
+
+ @Override
+ public Target getTargetOrNull(String targetName) throws InterruptedException {
+ try {
+ return packageProvider.getTarget(eventHandler, Label.parseAbsolute(targetName));
+ } catch (NoSuchPackageException | NoSuchTargetException | Label.SyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Target> getExplicitTarget(String targetName)
+ throws TargetParsingException, InterruptedException {
+ Label label = TargetPatternResolverUtil.label(targetName);
+ return getExplicitTarget(label, targetName);
+ }
+
+ private ResolvedTargets<Target> getExplicitTarget(Label label, String originalLabel)
+ throws TargetParsingException, InterruptedException {
+ try {
+ Target target = packageProvider.getTarget(eventHandler, label);
+ if (policy.shouldRetain(target, true)) {
+ return ResolvedTargets.of(target);
+ }
+ return ResolvedTargets.<Target>empty();
+ } catch (BuildFileContainsErrorsException e) {
+ // We don't need to report an error here because errors
+ // would have already been reported in this case.
+ return handleParsingError(eventHandler, originalLabel,
+ new TargetParsingException(e.getMessage(), e), keepGoing);
+ } catch (NoSuchThingException e) {
+ return handleParsingError(eventHandler, originalLabel,
+ new TargetParsingException(e.getMessage(), e), keepGoing);
+ }
+ }
+
+ /**
+ * Handles an error differently based on the value of keepGoing.
+ *
+ * @param badPattern The pattern we were unable to parse.
+ * @param e The underlying exception.
+ * @param keepGoing It true, report a warning and return.
+ * If false, throw the exception.
+ * @return the empty set.
+ * @throws TargetParsingException if !keepGoing.
+ */
+ private ResolvedTargets<Target> handleParsingError(EventHandler eventHandler, String badPattern,
+ TargetParsingException e, boolean keepGoing) throws TargetParsingException {
+ if (eventHandler instanceof ParseFailureListener) {
+ ((ParseFailureListener) eventHandler).parsingError(badPattern, e.getMessage());
+ }
+ if (keepGoing) {
+ eventHandler.handle(Event.error("Skipping '" + badPattern + "': " + e.getMessage()));
+ return ResolvedTargets.<Target>failed();
+ } else {
+ throw e;
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName,
+ boolean rulesOnly) throws TargetParsingException, InterruptedException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+ return getTargetsInPackage(originalPattern, packageName, actualPolicy);
+ }
+
+ private ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName,
+ FilteringPolicy policy) throws TargetParsingException, InterruptedException {
+ // Normalise, e.g "foo//bar" -> "foo/bar"; "foo/" -> "foo":
+ packageName = new PathFragment(packageName).toString();
+
+ // it's possible for this check to pass, but for Label.validatePackageNameFull to report an
+ // error because the package name is illegal. That's a little weird, but we can live with
+ // that for now--see test case: testBadPackageNameButGoodEnoughForALabel. (BTW I tried
+ // duplicating that validation logic in Label but it was extremely tricky.)
+ if (LabelValidator.validatePackageName(packageName) != null) {
+ return handleParsingError(eventHandler, originalPattern,
+ new TargetParsingException(
+ "'" + packageName + "' is not a valid package name"), keepGoing);
+ }
+ Package pkg;
+ try {
+ pkg = packageProvider.getPackage(
+ eventHandler, PackageIdentifier.createInDefaultRepo(packageName));
+ } catch (NoSuchPackageException e) {
+ return handleParsingError(eventHandler, originalPattern, new TargetParsingException(
+ TargetPatternResolverUtil.getParsingErrorMessage(
+ e.getMessage(), originalPattern)), keepGoing);
+ }
+
+ if (pkg.containsErrors()) {
+ // Report an error, but continue (and return partial results) if keepGoing is specified.
+ handleParsingError(eventHandler, originalPattern, new TargetParsingException(
+ TargetPatternResolverUtil.getParsingErrorMessage(
+ "package contains errors", originalPattern)), keepGoing);
+ }
+
+ return TargetPatternResolverUtil.resolvePackageTargets(pkg, policy);
+ }
+
+ @Override
+ public ResolvedTargets<Target> findTargetsBeneathDirectory(String originalPattern,
+ String pathPrefix, boolean rulesOnly) throws TargetParsingException, InterruptedException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+ return findTargetsBeneathDirectory(eventHandler, originalPattern, pathPrefix, actualPolicy,
+ keepGoing, pathPrefix.isEmpty());
+ }
+
+ private ResolvedTargets<Target> findTargetsBeneathDirectory(final EventHandler eventHandler,
+ final String originalPattern, String pathPrefix, final FilteringPolicy policy,
+ final boolean keepGoing, boolean useTopLevelExcludes)
+ throws TargetParsingException, InterruptedException {
+ PathFragment directory = new PathFragment(pathPrefix);
+ if (directory.containsUplevelReferences()) {
+ throw new TargetParsingException("up-level references are not permitted: '"
+ + pathPrefix + "'");
+ }
+ if (!pathPrefix.isEmpty() && (LabelValidator.validatePackageName(pathPrefix) != null)) {
+ return handleParsingError(eventHandler, pathPrefix, new TargetParsingException(
+ "'" + pathPrefix + "' is not a valid package name"), keepGoing);
+ }
+
+ final ResolvedTargets.Builder<Target> builder = ResolvedTargets.concurrentBuilder();
+ try {
+ packageProvider.visitPackageNamesRecursively(eventHandler, directory,
+ useTopLevelExcludes, packageVisitorPool,
+ new PathPackageLocator.AcceptsPathFragment() {
+ @Override
+ public void accept(PathFragment packageName) {
+ String pkgName = packageName.getPathString();
+ try {
+ // Get the targets without transforming. We'll do that later below.
+ builder.merge(getTargetsInPackage(originalPattern, pkgName,
+ FilteringPolicies.NO_FILTER));
+ } catch (InterruptedException e) {
+ throw new RuntimeParsingException(new TargetParsingException("interrupted"));
+ } catch (TargetParsingException e) {
+ // We'd like to make visitPackageNamesRecursively() generic
+ // over some checked exception type (TargetParsingException in
+ // this case). To do so, we'd have to make AbstractQueueVisitor
+ // generic over the same exception type. That won't work due to
+ // type erasure. As a workaround, we wrap the exception here,
+ // and unwrap it below.
+ throw new RuntimeParsingException(e);
+ }
+ }
+ });
+ } catch (RuntimeParsingException e) {
+ throw e.unwrap();
+ } catch (UnsupportedOperationException e) {
+ throw new TargetParsingException("recursive target patterns are not permitted: '"
+ + originalPattern + "'");
+ }
+
+ if (builder.isEmpty()) {
+ return handleParsingError(eventHandler, originalPattern,
+ new TargetParsingException("no targets found beneath '" + directory + "'"),
+ keepGoing);
+ }
+
+ // Apply the transform after the check so we only return the
+ // error if the tree really contains no targets.
+ ResolvedTargets<Target> intermediateResult = builder.build();
+ ResolvedTargets.Builder<Target> filteredBuilder = ResolvedTargets.builder();
+ if (intermediateResult.hasError()) {
+ filteredBuilder.setError();
+ }
+ for (Target target : intermediateResult.getTargets()) {
+ if (policy.shouldRetain(target, false)) {
+ filteredBuilder.add(target);
+ }
+ }
+ return filteredBuilder.build();
+ }
+
+ @Override
+ public boolean isPackage(String packageName) {
+ return packageProvider.isPackage(packageName);
+ }
+
+ @Override
+ public String getTargetKind(Target target) {
+ return target.getTargetKind();
+ }
+
+ private static final class RuntimeParsingException extends RuntimeException {
+ private TargetParsingException parsingException;
+
+ public RuntimeParsingException(TargetParsingException cause) {
+ super(cause);
+ this.parsingException = Preconditions.checkNotNull(cause);
+ }
+
+ public TargetParsingException unwrap() {
+ return parsingException;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheOptions.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheOptions.java
new file mode 100644
index 0000000..f9d3efc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheOptions.java
@@ -0,0 +1,142 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.syntax.CommaSeparatedPackageNameListConverter;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.List;
+
+/**
+ * Options for configuring the PackageCache.
+ */
+public class PackageCacheOptions extends OptionsBase {
+ /**
+ * A converter for package path that defaults to {@code Constants.DEFAULT_PACKAGE_PATH} if the
+ * option is not given.
+ *
+ * <p>Required because you cannot specify a non-constant value in annotation attributes.
+ */
+ public static class PackagePathConverter implements Converter<List<String>> {
+ @Override
+ public List<String> convert(String input) throws OptionsParsingException {
+ return input.isEmpty()
+ ? Constants.DEFAULT_PACKAGE_PATH
+ : new Converters.ColonSeparatedOptionListConverter().convert(input);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a string";
+ }
+ }
+
+ /**
+ * Converter for the {@code --default_visibility} option.
+ */
+ public static class DefaultVisibilityConverter implements Converter<RuleVisibility> {
+ @Override
+ public RuleVisibility convert(String input) throws OptionsParsingException {
+ if (input.equals("public")) {
+ return ConstantRuleVisibility.PUBLIC;
+ } else if (input.equals("private")) {
+ return ConstantRuleVisibility.PRIVATE;
+ } else {
+ throw new OptionsParsingException("Not a valid default visibility: '" + input
+ + "' (should be 'public' or 'private'");
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "default visibility";
+ }
+ }
+
+ @Option(name = "package_path",
+ defaultValue = "",
+ category = "package loading",
+ converter = PackagePathConverter.class,
+ help = "A colon-separated list of where to look for packages. "
+ + "Elements beginning with '%workspace%' are relative to the enclosing "
+ + "workspace. If omitted or empty, the default is the output of "
+ + "'blaze info default-package-path'.")
+ public List<String> packagePath;
+
+ @Option(name = "show_package_location",
+ defaultValue = "false",
+ category = "verbosity",
+ deprecationWarning = "This flag is no longer supported and will go away soon.",
+ help = "If enabled, causes Blaze to print the location on the --package_path "
+ + "from which each package was loaded.")
+ public boolean showPackageLocation;
+
+ @Option(name = "show_loading_progress",
+ defaultValue = "true",
+ category = "verbosity",
+ help = "If enabled, causes Blaze to print \"Loading package:\" messages.")
+ public boolean showLoadingProgress;
+
+ @Option(name = "deleted_packages",
+ defaultValue = "",
+ category = "package loading",
+ converter = CommaSeparatedPackageNameListConverter.class,
+ help = "A comma-separated list of names of packages which the "
+ + "build system will consider non-existent, even if they are "
+ + "visible somewhere on the package path."
+ + "\n"
+ + "Use this option when deleting a subpackage 'x/y' of an "
+ + "existing package 'x'. For example, after deleting x/y/BUILD "
+ + "in your client, the build system may complain if it "
+ + "encounters a label '//x:y/z' if that is still provided by another "
+ + "package_path entry. Specifying --deleted_packages x/y avoids this "
+ + "problem.")
+ public List<String> deletedPackages;
+
+ @Option(name = "default_visibility",
+ defaultValue = "private",
+ category = "undocumented",
+ converter = DefaultVisibilityConverter.class,
+ help = "Default visibility for packages that don't set it explicitly ('public' or "
+ + "'private').")
+ public RuleVisibility defaultVisibility;
+
+ @Option(name = "min_pkg_count_for_ct_node_eviction",
+ defaultValue = "3700",
+ // Why is the default value 3700? As of December 2013, a medium target loads about this many
+ // packages, uses ~310MB RAM to only load [1] or ~990MB to load and analyze [2,3]. So we
+ // can likely load and analyze this many packages without worrying about Blaze OOM'ing.
+ //
+ // If the total number of unique packages so far [4] is higher than the value of this flag,
+ // then we evict CT nodes [5] from the Skyframe graph.
+ //
+ // [1] blaze -x build --nobuild --noanalyze //medium:target
+ // [2] blaze -x build --nobuild //medium:target
+ // [3] according to "blaze info used-heap-size"
+ // [4] this means the number of unique packages loaded by builds, including the current one,
+ // since the last CT node eviction [5]
+ // [5] "CT node eviction" means clearing those nodes from the Skyframe graph that correspond
+ // to ConfiguredTargets; this is done using SkyframeExecutor.resetConfiguredTargets
+ category = "undocumented",
+ help = "Threshold for number of loaded packages before skyframe-m1 cache eviction kicks in")
+ public int minLoadedPkgCountForCtNodeEviction;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageManager.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageManager.java
new file mode 100644
index 0000000..4d5e687
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageManager.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+
+import java.io.PrintStream;
+
+/**
+ * A PackageManager keeps state about loaded packages around for quick lookup, and provides
+ * related functionality: Recursive package finding, loaded package checking, etc.
+ */
+public interface PackageManager extends PackageProvider, CachingPackageLocator,
+ LoadedPackageProvider {
+
+ /**
+ * Returns the package cache statistics.
+ */
+ PackageManagerStatistics getStatistics();
+
+ /**
+ * Removes cached data which is not needed anymore after loading is complete, to reduce memory
+ * consumption between builds. Whether or not this method is called does not affect correctness.
+ */
+ void partiallyClear();
+
+ /**
+ * Dump the contents of the package manager in human-readable form.
+ * Used by 'bazel dump' and the BuildTool's unexpected exception handler.
+ */
+ void dump(PrintStream printStream);
+
+ /**
+ * Returns the package locator used by this package manager.
+ *
+ * <p>If you are tempted to call {@code getPackagePath().getPathEntries().get(0)}, be warned that
+ * this is probably not the value you are looking for! Look at the methods of {@code
+ * BazelRuntime} instead.
+ */
+ @ThreadSafety.ThreadSafe
+ PathPackageLocator getPackagePath();
+
+ /**
+ * Collects statistics of the package manager since the last sync.
+ */
+ interface PackageManagerStatistics {
+
+ /**
+ * Returns the number of packages loaded since the last sync. I.e. the cache
+ * misses.
+ */
+ int getPackagesLoaded();
+
+ /**
+ * Returns the number of packages looked up since the last sync.
+ */
+ int getPackagesLookedUp();
+
+ /**
+ * Returns the number of all the packages currently loaded.
+ *
+ * <p>
+ * Note that this method is not affected by sync(), and the packages it
+ * returns are not guaranteed to be up-to-date.
+ */
+ int getCacheSize();
+ }
+
+ /**
+ * Retrieve a target pattern parser that works with this package manager.
+ */
+ TargetPatternEvaluator getTargetPatternEvaluator();
+
+ /**
+ * Construct a new {@link TransitivePackageLoader}.
+ */
+ TransitivePackageLoader newTransitiveLoader();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageProvider.java
new file mode 100644
index 0000000..0573768e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageProvider.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+
+/**
+ * API for retrieving packages. Implementations generally load packages to fulfill requests.
+ *
+ * <p><b>Concurrency</b>: Implementations should be thread safe for {@link #getPackage}.
+ */
+public interface PackageProvider extends TargetProvider {
+
+ /**
+ * Returns the {@link Package} named "packageName". If there is no such package (e.g.
+ * {@code isPackage(packageName)} returns false), throws a {@link NoSuchPackageException}.
+ *
+ * <p>The returned package may contain lexical/grammatical errors, in which
+ * case <code>pkg.containsErrors() == true</code>. Such packages may be
+ * missing some rules. Any rules that are present may soundly be used for
+ * builds, though.
+ *
+ * @param eventHandler the eventHandler on which to report warning and errors; if the package
+ * has been loaded by another thread, this eventHandler won't see any warnings or errors
+ * @param packageName a legal package name.
+ * @throws NoSuchPackageException if the package could not be found.
+ * @throws InterruptedException if the package loading was interrupted.
+ */
+ Package getPackage(EventHandler eventHandler, PackageIdentifier packageName)
+ throws NoSuchPackageException, InterruptedException;
+
+ /**
+ * Returns whether a package with the given name exists. That is, returns whether all the
+ * following hold
+ * <ol>
+ * <li>{@code packageName} is a valid package name</li>
+ * <li>there is a BUILD file for the package</li>
+ * <li>the package is not considered deleted via --deleted_packages</li>
+ * </ol>
+ *
+ * <p> If these don't hold, then attempting to read the package with {@link #getPackage} may fail
+ * or may return a package containing errors.
+ */
+ boolean isPackage(String packageName);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/ParseFailureListener.java b/src/main/java/com/google/devtools/build/lib/pkgcache/ParseFailureListener.java
new file mode 100644
index 0000000..e3cf5ca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/ParseFailureListener.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.events.EventHandler;
+
+/**
+ * Represents a listener which reports parse errors to the underlying
+ * {@link EventHandler} and {@link EventBus} (if non-null).
+ */
+public interface ParseFailureListener {
+
+ /** Reports a parsing failure. */
+ void parsingError(String badPattern, String message);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/ParsingFailedEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/ParsingFailedEvent.java
new file mode 100644
index 0000000..7eb9169
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/ParsingFailedEvent.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+/**
+ * This event is fired when a target or target pattern fails to parse.
+ * In some cases (not all) this happens before targets are created,
+ * and thus in these cases there are no status lines.
+ * Therefore, the parse failure is reported separately.
+ */
+public class ParsingFailedEvent {
+ private final String targetPattern;
+ private final String message;
+
+ /**
+ * Creates a new parsing failed event with the given pattern and message.
+ */
+ public ParsingFailedEvent(String targetPattern, String message) {
+ this.targetPattern = targetPattern;
+ this.message = message;
+ }
+
+ public String getPattern() {
+ return targetPattern;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java
new file mode 100644
index 0000000..a2af5f2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java
@@ -0,0 +1,213 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+/**
+ * A mapping from the name of a package to the location of its BUILD file.
+ * The implementation composes an ordered sequence of directories according to
+ * the package-path rules.
+ *
+ * <p>All methods are thread-safe, and (assuming no change to the underlying
+ * filesystem) idempotent.
+ */
+public class PathPackageLocator {
+
+ public static final Set<String> DEFAULT_TOP_LEVEL_EXCLUDES =
+ ImmutableSet.of("experimental", "obsolete");
+
+ /**
+ * An interface which accepts {@link PathFragment}s.
+ */
+ public interface AcceptsPathFragment {
+
+ /**
+ * Accept a {@link PathFragment}.
+ *
+ * @param fragment The path fragment.
+ */
+ void accept(PathFragment fragment);
+ }
+
+ private static final Logger LOG = Logger.getLogger(PathPackageLocator.class.getName());
+
+ private final ImmutableList<Path> pathEntries;
+
+ /**
+ * Constructs a PathPackageLocator based on the specified list of package root directories.
+ */
+ public PathPackageLocator(List<Path> pathEntries) {
+ this.pathEntries = ImmutableList.copyOf(pathEntries);
+ }
+
+ /**
+ * Constructs a PathPackageLocator based on the specified array of package root directories.
+ */
+ public PathPackageLocator(Path... pathEntries) {
+ this(Arrays.asList(pathEntries));
+ }
+
+ /**
+ * Returns the path to the build file for this package.
+ *
+ * <p>The package's root directory may be computed by calling getParentFile()
+ * on the result of this function.
+ *
+ * <p>Instances of this interface do not attempt to do any caching, nor
+ * implement checks for package-boundary crossing logic; the PackageCache
+ * does that.
+ *
+ * <p>If the same package exists beneath multiple package path entries, the
+ * first path that matches always wins.
+ */
+ public Path getPackageBuildFile(String packageName) throws NoSuchPackageException {
+ Path buildFile = getPackageBuildFileNullable(packageName, UnixGlob.DEFAULT_SYSCALLS_REF);
+ if (buildFile == null) {
+ throw new BuildFileNotFoundException(packageName, "BUILD file not found on package path");
+ }
+ return buildFile;
+ }
+
+ /**
+ * Like #getPackageBuildFile(), but returns null instead of throwing.
+ *
+ * @param packageName the name of the package.
+ * @param cache a filesystem-level cache of stat() calls.
+ */
+ public Path getPackageBuildFileNullable(String packageName,
+ AtomicReference<? extends UnixGlob.FilesystemCalls> cache) {
+ return getFilePath(new PathFragment(packageName).getRelative("BUILD"), cache);
+ }
+
+
+ /**
+ * Returns an immutable ordered list of the directories on the package path.
+ */
+ public ImmutableList<Path> getPathEntries() {
+ return pathEntries;
+ }
+
+ @Override
+ public String toString() {
+ return "PathPackageLocator" + pathEntries;
+ }
+
+ /**
+ * A factory of PathPackageLocators from a list of path elements. Elements
+ * may contain "%workspace%", indicating the workspace.
+ *
+ * @param pathElements Each element must be an absolute path, relative path,
+ * or some string "%workspace%" + relative, where relative is itself a
+ * relative path. The special symbol "%workspace%" means to interpret
+ * the path relative to the nearest enclosing workspace. Relative
+ * paths are interpreted relative to the client's working directory,
+ * which may be below the workspace.
+ * @param eventHandler The eventHandler.
+ * @param workspace The nearest enclosing package root directory.
+ * @param clientWorkingDirectory The client's working directory.
+ * @return a list of {@link Path}s.
+ */
+ public static PathPackageLocator create(List<String> pathElements,
+ EventHandler eventHandler,
+ Path workspace,
+ Path clientWorkingDirectory) {
+ List<Path> resolvedPaths = new ArrayList<>();
+ final String workspaceWildcard = "%workspace%";
+
+ for (String pathElement : pathElements) {
+ // Replace "%workspace%" with the path of the enclosing workspace directory.
+ pathElement = pathElement.replace(workspaceWildcard, workspace.getPathString());
+
+ PathFragment pathElementFragment = new PathFragment(pathElement);
+
+ // If the path string started with "%workspace%" or "/", it is already absolute,
+ // so the following line is a no-op.
+ Path rootPath = clientWorkingDirectory.getRelative(pathElementFragment);
+
+ if (!pathElementFragment.isAbsolute() && !clientWorkingDirectory.equals(workspace)) {
+ eventHandler.handle(
+ Event.warn("The package path element '" + pathElementFragment + "' will be "
+ + "taken relative to your working directory. You may have intended "
+ + "to have the path taken relative to your workspace directory. "
+ + "If so, please use the '" + workspaceWildcard + "' wildcard."));
+ }
+
+ if (rootPath.exists()) {
+ resolvedPaths.add(rootPath);
+ } else {
+ LOG.fine("package path element " + rootPath + " does not exist, ignoring");
+ }
+ }
+ return new PathPackageLocator(resolvedPaths);
+ }
+
+ /**
+ * Returns the path to the WORKSPACE file for this build.
+ *
+ * <p>If there are WORKSPACE files beneath multiple package path entries, the first one always
+ * wins.
+ */
+ public Path getWorkspaceFile() {
+ AtomicReference<? extends UnixGlob.FilesystemCalls> cache = UnixGlob.DEFAULT_SYSCALLS_REF;
+ // TODO(bazel-team): correctness in the presence of changes to the location of the WORKSPACE
+ // file.
+ return getFilePath(new PathFragment("WORKSPACE"), cache);
+ }
+
+ private Path getFilePath(PathFragment suffix,
+ AtomicReference<? extends UnixGlob.FilesystemCalls> cache) {
+ for (Path pathEntry : pathEntries) {
+ Path buildFile = pathEntry.getRelative(suffix);
+ FileStatus stat = cache.get().statNullable(buildFile, Symlinks.FOLLOW);
+ if (stat != null && stat.isFile()) {
+ return buildFile;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public int hashCode() {
+ return pathEntries.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof PathPackageLocator)) {
+ return false;
+ }
+ return this.getPathEntries().equals(((PathPackageLocator) other).getPathEntries());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/RecursivePackageProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/RecursivePackageProvider.java
new file mode 100644
index 0000000..8d7bd1d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/RecursivePackageProvider.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+import javax.annotation.Nullable;
+
+/**
+ * Support for resolving {@code package/...} target patterns.
+ */
+public interface RecursivePackageProvider extends PackageProvider {
+
+ /**
+ * <p>Visits the names of all packages beneath the given directory recursively and concurrently.
+ *
+ * <p>Note: This operation needs to stat directories recursively. It could be very expensive when
+ * there is a big tree under the given directory.
+ *
+ * <p>Over a single iteration, package names are unique.
+ *
+ * <p>This method uses the given thread pool to call the observer method, possibly concurrently
+ * (depending on the thread pool). When this method terminates, however, all such threads will
+ * have completed.
+ *
+ * <p>To abort the traversal, call {@link Thread#interrupt()} on the calling thread.
+ *
+ * <p>This method guarantees that all BUILD files it returns correspond to valid package names
+ * that are not marked as deleted within the current build.
+ *
+ * @param eventHandler an eventHandler which should be used to log any errors that occur while
+ * scanning directories for BUILD files
+ * @param directory a relative, canonical path specifying the directory to search
+ * @param useTopLevelExcludes whether to skip a pre-set list of top level directories
+ * @param visitorPool the thread pool to use to visit packages in parallel
+ * @param observer is called for each path fragment found; thread-safe if the thread pool supports
+ * multiple parallel threads
+ * @throws InterruptedException if the calling thread was interrupted.
+ */
+ void visitPackageNamesRecursively(EventHandler eventHandler, PathFragment directory,
+ boolean useTopLevelExcludes, @Nullable ThreadPoolExecutor visitorPool,
+ PathPackageLocator.AcceptsPathFragment observer) throws InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/SrcTargetUtil.java b/src/main/java/com/google/devtools/build/lib/pkgcache/SrcTargetUtil.java
new file mode 100644
index 0000000..85d967e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/SrcTargetUtil.java
@@ -0,0 +1,148 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.FileTarget;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A helper class for getting source and header files from a given {@link Rule}.
+ */
+public final class SrcTargetUtil {
+ private SrcTargetUtil() {
+ }
+
+ /**
+ * Given a Rule, returns an immutable list of FileTarget for its sources, in the order they appear
+ * in its "srcs", "src" or "srcjar" attribute, and any filegroups or other rules it references.
+ * An empty list is returned if no "srcs" or "src" attribute exists for this rule. The list may
+ * contain OutputFiles if the sources were generated by another rule.
+ *
+ * <p>This method should be considered only a heuristic, and should not be used during the
+ * analysis phase.
+ *
+ * <p>(We could remove the throws clauses if we restrict the results to srcs within the same
+ * package.)
+ *
+ * @throws NoSuchTargetException or NoSuchPackageException when a source label cannot be resolved
+ * to a Target
+ */
+ @ThreadSafety.ThreadSafe
+ public static List<FileTarget> getSrcTargets(EventHandler eventHandler, Rule rule,
+ TargetProvider provider)
+ throws NoSuchTargetException, NoSuchPackageException, InterruptedException {
+ return getTargets(eventHandler, rule, SOURCE_ATTRIBUTES, Sets.newHashSet(rule), provider);
+ }
+
+ // Attributes referring to "sources".
+ private static final ImmutableSet<String> SOURCE_ATTRIBUTES =
+ ImmutableSet.of("srcs", "src", "srcjar");
+
+ // Attributes referring to "headers".
+ private static final ImmutableSet<String> HEADER_ATTRIBUTES =
+ ImmutableSet.of("hdrs");
+
+ // The attribute to search in filegroups.
+ private static final ImmutableSet<String> FILEGROUP_ATTRIBUTES =
+ ImmutableSet.of("srcs");
+
+ /**
+ * Same as {@link #getSrcTargets}, but for both source and headers (i.e. also traversing
+ * the "hdrs" attribute).
+ */
+ @ThreadSafety.ThreadSafe
+ public static List<FileTarget> getSrcAndHdrTargets(EventHandler eventHandler, Rule rule,
+ TargetProvider provider)
+ throws NoSuchTargetException, NoSuchPackageException, InterruptedException {
+ ImmutableSet<String> srcAndHdrAttributes = ImmutableSet.<String>builder()
+ .addAll(SOURCE_ATTRIBUTES)
+ .addAll(HEADER_ATTRIBUTES)
+ .build();
+ return getTargets(eventHandler, rule, srcAndHdrAttributes, Sets.newHashSet(rule), provider);
+ }
+
+ @ThreadSafety.ThreadSafe
+ public static List<FileTarget> getHdrTargets(EventHandler eventHandler, Rule rule,
+ TargetProvider provider)
+ throws NoSuchTargetException, NoSuchPackageException, InterruptedException {
+ ImmutableSet<String> srcAndHdrAttributes = ImmutableSet.<String>builder()
+ .addAll(HEADER_ATTRIBUTES)
+ .build();
+ return getTargets(eventHandler, rule, srcAndHdrAttributes, Sets.newHashSet(rule), provider);
+ }
+
+ /**
+ * @see #getSrcTargets(EventHandler, Rule, TargetProvider)
+ */
+ private static List<FileTarget> getTargets(EventHandler eventHandler,
+ Rule rule,
+ ImmutableSet<String> attributes,
+ Set<Rule> visitedRules,
+ TargetProvider targetProvider)
+ throws NoSuchTargetException, NoSuchPackageException, InterruptedException {
+ Preconditions.checkState(!rule.hasConfigurableAttributes()); // Not currently supported.
+ List<Label> srcLabels = Lists.newArrayList();
+ AttributeMap attributeMap = RawAttributeMapper.of(rule);
+ for (String attrName : attributes) {
+ if (rule.isAttrDefined(attrName, Type.LABEL_LIST)) {
+ srcLabels.addAll(attributeMap.get(attrName, Type.LABEL_LIST));
+ } else if (rule.isAttrDefined(attrName, Type.LABEL)) {
+ Label srcLabel = attributeMap.get(attrName, Type.LABEL);
+ if (srcLabel != null) {
+ srcLabels.add(srcLabel);
+ }
+ }
+ }
+ if (srcLabels.isEmpty()) {
+ return ImmutableList.of();
+ }
+ List<FileTarget> srcTargets = new ArrayList<>();
+ for (Label label : srcLabels) {
+ Target target = targetProvider.getTarget(eventHandler, label);
+ if (target instanceof FileTarget) {
+ srcTargets.add((FileTarget) target);
+ } else {
+ Rule srcRule = target.getAssociatedRule();
+ if (srcRule != null && !visitedRules.contains(srcRule)) {
+ visitedRules.add(srcRule);
+ if ("filegroup".equals(srcRule.getRuleClass())) {
+ srcTargets.addAll(getTargets(eventHandler, srcRule, FILEGROUP_ATTRIBUTES, visitedRules,
+ targetProvider));
+ } else {
+ srcTargets.addAll(srcRule.getOutputFiles());
+ }
+ }
+ }
+ }
+ return ImmutableList.copyOf(srcTargets);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetEdgeObserver.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetEdgeObserver.java
new file mode 100644
index 0000000..acc858f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetEdgeObserver.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import javax.annotation.Nullable;
+
+/**
+ * An observer of the visitation over a target graph.
+ */
+public interface TargetEdgeObserver {
+
+ /**
+ * Called when an edge is discovered.
+ * May be called more than once for the same
+ * (from, to) pair.
+ *
+ * @param from the originating node.
+ * @param attribute The attribute which defines the edge.
+ * Non-null iff (from instanceof Rule).
+ * @param to the target node.
+ */
+ void edge(Target from, Attribute attribute, Target to);
+
+ /**
+ * Called when a Target has a reference to a non-existent target.
+ *
+ * @param target the target. May be null (e.g. in the case of an implicit
+ * dependency on a subincluded file).
+ * @param to a label reference in the rule, which does not correspond
+ * to a valid target.
+ * @param e the corresponding exception thrown
+ */
+ void missingEdge(@Nullable Target target, Label to, NoSuchThingException e);
+
+ /**
+ * Called when a node is discovered. May be called
+ * more than once for the same node.
+ *
+ * @param node the target.
+ */
+ void node(Target node);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetParsingCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetParsingCompleteEvent.java
new file mode 100644
index 0000000..48d8861
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetParsingCompleteEvent.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+
+/**
+ * This event is fired just after target pattern evaluation is completed.
+ */
+public class TargetParsingCompleteEvent {
+
+ private final ImmutableSet<Target> targets;
+ private final ImmutableSet<Target> filteredTargets;
+ private final ImmutableSet<Target> testFilteredTargets;
+ private final long timeInMs;
+
+ /**
+ * Construct the event.
+ * @param targets The targets that were parsed from the
+ * command-line pattern.
+ */
+ public TargetParsingCompleteEvent(Collection<Target> targets,
+ Collection<Target> filteredTargets, Collection<Target> testFilteredTargets,
+ long timeInMs) {
+ this.timeInMs = timeInMs;
+ this.targets = ImmutableSet.copyOf(targets);
+ this.filteredTargets = ImmutableSet.copyOf(filteredTargets);
+ this.testFilteredTargets = ImmutableSet.copyOf(testFilteredTargets);
+ }
+
+ @VisibleForTesting
+ public TargetParsingCompleteEvent(Collection<Target> targets) {
+ this(targets, ImmutableSet.<Target>of(), ImmutableSet.<Target>of(), 0);
+ }
+
+ /**
+ * @return the parsed targets, which will subsequently be loaded
+ */
+ public ImmutableSet<Target> getTargets() {
+ return targets;
+ }
+
+ public Iterable<Label> getLabels() {
+ return Iterables.transform(targets, new Function<Target, Label>() {
+ @Override
+ public Label apply(Target input) {
+ return input.getLabel();
+ }
+ });
+ }
+
+ /**
+ * @return the filtered targets (i.e., using -//foo:bar on the command-line)
+ */
+ public ImmutableSet<Target> getFilteredTargets() {
+ return filteredTargets;
+ }
+
+ /**
+ * @return the test-filtered targets, if --build_test_only is in effect
+ */
+ public ImmutableSet<Target> getTestFilteredTargets() {
+ return testFilteredTargets;
+ }
+
+ public long getTimeInMs() {
+ return timeInMs;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternEvaluator.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternEvaluator.java
new file mode 100644
index 0000000..94e956b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternEvaluator.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A parser for target patterns. Target patterns are a generalisation of
+ * labels to include wildcards for finding all packages recursively
+ * beneath some root, and for finding all targets within a package.
+ *
+ * <p>A list of target patterns implies a union of all the labels of each
+ * pattern. Each item in a list of target patterns may include a prefix
+ * negation operator, indicating that the sets of targets for this pattern
+ * should be subtracted from the set of targets for the preceding patterns (note
+ * this means that order matters). Thus, the following list of target patterns:
+ * <pre>foo/... -foo/bar:all</pre>
+ * means "all targets beneath <tt>foo</tt> except for those targets in
+ * package <tt>foo/bar</tt>.
+ */
+@ThreadSafety.ConditionallyThreadSafe // as long as you don't call updateOffset.
+public interface TargetPatternEvaluator {
+ /**
+ * Attempts to parse an ordered list of target patterns, computing the union
+ * of the set of targets represented by each pattern, unless it is preceded by
+ * "-", in which case the set difference is computed. Implements the
+ * specification described in the class-level comment.
+ */
+ ResolvedTargets<Target> parseTargetPatternList(EventHandler eventHandler,
+ List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing)
+ throws TargetParsingException, InterruptedException;
+
+ /**
+ * Attempts to parse a single target pattern while consulting the package
+ * cache to check for the existence of packages and directories and the build
+ * targets in them. Implements the specification described in the
+ * class-level comment. Returns a {@link ResolvedTargets} object.
+ *
+ * <p>If an error is encountered, a {@link TargetParsingException} is thrown,
+ * unless {@code keepGoing} is set to true. In that case, the returned object
+ * will have its error bit set.
+ */
+ ResolvedTargets<Target> parseTargetPattern(EventHandler eventHandler, String pattern,
+ boolean keepGoing) throws TargetParsingException, InterruptedException;
+
+ /**
+ * Attempts to parse and load the given collection of patterns; the returned map contains the
+ * results for each pattern successfully parsed.
+ *
+ * <p>If an error is encountered, a {@link TargetParsingException} is thrown, unless {@code
+ * keepGoing} is set to true. In that case, the patterns that failed to load have the error flag
+ * set.
+ */
+ Map<String, ResolvedTargets<Target>> preloadTargetPatterns(EventHandler eventHandler,
+ Collection<String> patterns, boolean keepGoing)
+ throws TargetParsingException, InterruptedException;
+
+
+ /**
+ * Update the parser's offset, given the workspace and working directory.
+ *
+ * @param relativeWorkingDirectory the working directory relative to the workspace
+ */
+ @ThreadHostile
+ void updateOffset(PathFragment relativeWorkingDirectory);
+
+ /**
+ * @return the offset of this parser from the root of the workspace.
+ * Non-absolute package-names will be resolved relative
+ * to this offset.
+ */
+ @VisibleForTesting
+ String getOffset();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternResolverUtil.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternResolverUtil.java
new file mode 100644
index 0000000..6bbf55c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternResolverUtil.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.StringUtilities;
+
+/**
+ * Common utility methods for target pattern resolution.
+ */
+public final class TargetPatternResolverUtil {
+ private TargetPatternResolverUtil() {
+ // Utility class.
+ }
+
+ // Parse 'label' as a Label, mapping Label.SyntaxException into
+ // TargetParsingException.
+ public static Label label(String label) throws TargetParsingException {
+ try {
+ return Label.parseAbsolute(label);
+ } catch (Label.SyntaxException e) {
+ throw invalidTarget(label, e.getMessage());
+ }
+ }
+
+ /**
+ * Returns a new exception indicating that a command-line target is invalid.
+ */
+ private static TargetParsingException invalidTarget(String packageName,
+ String additionalMessage) {
+ return new TargetParsingException("invalid target format: '" +
+ StringUtilities.sanitizeControlChars(packageName) + "'; " +
+ StringUtilities.sanitizeControlChars(additionalMessage));
+ }
+
+ public static String getParsingErrorMessage(String message, String originalPattern) {
+ if (originalPattern == null) {
+ return message;
+ } else {
+ return String.format("while parsing '%s': %s", originalPattern, message);
+ }
+ }
+
+ public static ResolvedTargets<Target> resolvePackageTargets(Package pkg,
+ FilteringPolicy policy) {
+ ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder();
+ for (Target target : pkg.getTargets()) {
+ if (policy.shouldRetain(target, false)) {
+ builder.add(target);
+ }
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetProvider.java
new file mode 100644
index 0000000..72dc834
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetProvider.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * API for retrieving targets.
+ *
+ * <p><b>Concurrency</b>: Implementations should be thread safe.
+ */
+public interface TargetProvider {
+ /**
+ * Returns the Target identified by "label", loading, parsing and evaluating the package if it is
+ * not already loaded.
+ *
+ * @throws NoSuchPackageException if the package could not be found
+ * @throws NoSuchTargetException if the package was loaded successfully, but
+ * the specified {@link Target} was not found in it
+ * @throws InterruptedException if the package loading was interrupted
+ */
+ Target getTarget(EventHandler eventHandler, Label label) throws NoSuchPackageException,
+ NoSuchTargetException, InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TransitivePackageLoader.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TransitivePackageLoader.java
new file mode 100644
index 0000000..14990b2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TransitivePackageLoader.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * Visits a set of Targets and Labels transitively.
+ */
+public interface TransitivePackageLoader {
+
+ /**
+ * Visit the specified labels and follow the transitive closure of their
+ * outbound dependencies. If the targets have previously been visited,
+ * may do an up-to-date check which will not trigger any of the observers.
+ *
+ * @param eventHandler the error and warnings eventHandler; must be thread-safe
+ * @param targetsToVisit the targets to visit
+ * @param labelsToVisit the labels to visit in addition to the targets
+ * @param keepGoing if false, stop visitation upon first error.
+ * @param parallelThreads number of threads to use in the visitation.
+ * @param maxDepth the maximum depth to traverse to.
+ */
+ boolean sync(EventHandler eventHandler,
+ Set<Target> targetsToVisit,
+ Set<Label> labelsToVisit,
+ boolean keepGoing,
+ int parallelThreads,
+ int maxDepth) throws InterruptedException;
+
+ /**
+ * Returns a read-only view of the set of targets visited since this visitor
+ * was constructed.
+ *
+ * <p>Not thread-safe; do not call during visitation.
+ */
+ // TODO(bazel-team): This is only used in legacy non-Skyframe code.
+ Set<Label> getVisitedTargets();
+
+ /**
+ * Returns a read-only view of the set of packages visited since this visitor
+ * was constructed.
+ *
+ * <p>Not thread-safe; do not call during visitation.
+ */
+ Set<PackageIdentifier> getVisitedPackageNames();
+
+ /**
+ * Returns a read-only view of the set of the actual packages visited without error since this
+ * visitor was constructed.
+ *
+ * <p>Use {@link #getVisitedPackageNames()} instead when possible.
+ *
+ * <p>Not thread-safe; do not call during visitation.
+ */
+ Set<Package> getErrorFreeVisitedPackages();
+
+ /**
+ * Return a mapping between the specified top-level targets and root causes. Note that targets in
+ * the input that are transitively error free will not be in the output map. "Top-level" targets
+ * are the targetsToVisit and labelsToVisit specified in the last sync.
+ *
+ * <p>May only be called once a keep_going visitation is complete, and prior to
+ * trimErrorTracking().
+ *
+ * @param targetsToLoad the set of targets to be checked. Implementations may choose to only
+ * return root causes for targets in this set that were requested top-level targets.
+ * @return a mapping of targets to root causes
+ */
+ Multimap<Label, Label> getRootCauses(Collection<Label> targetsToLoad);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/Describable.java b/src/main/java/com/google/devtools/build/lib/profiler/Describable.java
new file mode 100644
index 0000000..a5cc08a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/Describable.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler;
+
+/**
+ * Allows class to implement profiler-friendly (and user-friendly)
+ * textual description of the object that would uniquely identify an object in
+ * the profiler data dump.
+ */
+public interface Describable {
+
+ /**
+ * Returns textual description that will uniquely identify an object.
+ */
+ String describe();
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/MemoryProfiler.java b/src/main/java/com/google/devtools/build/lib/profiler/MemoryProfiler.java
new file mode 100644
index 0000000..25ebd53
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/MemoryProfiler.java
@@ -0,0 +1,80 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler;
+
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+
+/**
+ * Blaze memory profiler.
+ *
+ * <p>At each call to {@code profile} performs garbage collection and stores
+ * heap and non-heap memory usage in an external file.
+ *
+ * <p><em>Heap memory</em> is the runtime data area from which memory for all
+ * class instances and arrays is allocated. <em>Non-heap memory</em> includes
+ * the method area and memory required for the internal processing or
+ * optimization of the JVM. It stores per-class structures such as a runtime
+ * constant pool, field and method data, and the code for methods and
+ * constructors. The Java Native Interface (JNI) code or the native library of
+ * an application and the JVM implementation allocate memory from the
+ * <em>native heap</em>.
+ *
+ * <p>The script in /devtools/blaze/scripts/blaze-memchart.sh can be used for post processing.
+ */
+public final class MemoryProfiler {
+
+ private static final MemoryProfiler INSTANCE = new MemoryProfiler();
+
+ public static MemoryProfiler instance() {
+ return INSTANCE;
+ }
+
+ private PrintStream memoryProfile;
+ private ProfilePhase currentPhase;
+
+ public synchronized void start(OutputStream out) {
+ this.memoryProfile = (out == null) ? null : new PrintStream(out);
+ this.currentPhase = ProfilePhase.INIT;
+ }
+
+ public synchronized void stop() {
+ if (memoryProfile != null) {
+ memoryProfile.close();
+ memoryProfile = null;
+ }
+ }
+
+ public synchronized void markPhase(ProfilePhase nextPhase) {
+ if (memoryProfile != null) {
+ String name = currentPhase.description;
+ ManagementFactory.getMemoryMXBean().gc();
+ MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+ memoryProfile.println(name + ":heap:init:" + memoryUsage.getInit());
+ memoryProfile.println(name + ":heap:used:" + memoryUsage.getUsed());
+ memoryProfile.println(name + ":heap:commited:" + memoryUsage.getCommitted());
+ memoryProfile.println(name + ":heap:max:" + memoryUsage.getMax());
+
+ memoryUsage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage();
+ memoryProfile.println(name + ":non-heap:init:" + memoryUsage.getInit());
+ memoryProfile.println(name + ":non-heap:used:" + memoryUsage.getUsed());
+ memoryProfile.println(name + ":non-heap:commited:" + memoryUsage.getCommitted());
+ memoryProfile.println(name + ":non-heap:max:" + memoryUsage.getMax());
+ currentPhase = nextPhase;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfileInfo.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfileInfo.java
new file mode 100644
index 0000000..4ce3a93
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfileInfo.java
@@ -0,0 +1,926 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.profiler;
+
+import static com.google.devtools.build.lib.profiler.ProfilerTask.CRITICAL_PATH;
+import static com.google.devtools.build.lib.profiler.ProfilerTask.CRITICAL_PATH_COMPONENT;
+import static com.google.devtools.build.lib.profiler.ProfilerTask.TASK_COUNT;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.util.VarInt;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+
+/**
+ * Holds parsed profile file information and provides various ways of
+ * accessing it (mostly through different dictionaries or sorted lists).
+ *
+ * Class should not be instantiated directly but through the use of the
+ * ProfileLoader.loadProfile() method.
+ */
+public class ProfileInfo {
+
+ /**
+ * Immutable container for the aggregated stats.
+ */
+ public static final class AggregateAttr {
+ public final int count;
+ public final long totalTime;
+
+ AggregateAttr(int count, long totalTime) {
+ this.count = count;
+ this.totalTime = totalTime;
+ }
+ }
+
+ /**
+ * Immutable compact representation of the Map<ProfilerTask, AggregateAttr>.
+ */
+ static final class CompactStatistics {
+ final byte[] content;
+
+ CompactStatistics(byte[] content) {
+ this.content = content;
+ }
+
+ /**
+ * Create compact task statistic instance using provided array.
+ * Array length must exactly match ProfilerTask value space.
+ * Each statistic is stored in the array according to the ProfilerTask
+ * value ordinal() number. Absent statistics are represented by null.
+ */
+ CompactStatistics(AggregateAttr[] stats) {
+ Preconditions.checkArgument(stats.length == TASK_COUNT);
+ ByteBuffer sink = ByteBuffer.allocate(TASK_COUNT * (1 + 5 + 10));
+ for (int i = 0; i < TASK_COUNT; i++) {
+ if (stats[i] != null && stats[i].count > 0) {
+ sink.put((byte) i);
+ VarInt.putVarInt(stats[i].count, sink);
+ VarInt.putVarLong(stats[i].totalTime, sink);
+ }
+ }
+ content = sink.position() > 0 ? Arrays.copyOfRange(sink.array(), 0, sink.position()) : null;
+ }
+
+ boolean isEmpty() { return content == null; }
+
+ /**
+ * Converts instance back into AggregateAttr[TASK_COUNT]. See
+ * constructor documentation for more information.
+ */
+ AggregateAttr[] toArray() {
+ AggregateAttr[] stats = new AggregateAttr[TASK_COUNT];
+ if (!isEmpty()) {
+ ByteBuffer source = ByteBuffer.wrap(content);
+ while (source.hasRemaining()) {
+ byte id = source.get();
+ int count = VarInt.getVarInt(source);
+ long time = VarInt.getVarLong(source);
+ stats[id] = new AggregateAttr(count, time);
+ }
+ }
+ return stats;
+ }
+
+ /**
+ * Returns AggregateAttr instance for the given ProfilerTask value.
+ */
+ AggregateAttr getAttr(ProfilerTask task) {
+ if (isEmpty()) { return ZERO; }
+ ByteBuffer source = ByteBuffer.wrap(content);
+ byte id = (byte) task.ordinal();
+ while (source.hasRemaining()) {
+ if (id == source.get()) {
+ int count = VarInt.getVarInt(source);
+ long time = VarInt.getVarLong(source);
+ return new AggregateAttr(count, time);
+ } else {
+ VarInt.getVarInt(source);
+ VarInt.getVarLong(source);
+ }
+ }
+ return ZERO;
+ }
+
+ /**
+ * Returns cumulative time stored in this instance across whole
+ * ProfilerTask dimension.
+ */
+ long getTotalTime() {
+ if (isEmpty()) { return 0; }
+ ByteBuffer source = ByteBuffer.wrap(content);
+ long totalTime = 0;
+ while (source.hasRemaining()) {
+ source.get();
+ VarInt.getVarInt(source);
+ totalTime += VarInt.getVarLong(source);
+ }
+ return totalTime;
+ }
+ }
+
+ /**
+ * Container for the profile record information.
+ *
+ * <p> TODO(bazel-team): (2010) Current Task instance heap size is 72 bytes. And there are
+ * millions of them. Consider trimming some attributes.
+ */
+ public final class Task implements Comparable<Task> {
+ public final long threadId;
+ public final int id;
+ public final int parentId;
+ public final long startTime;
+ public final long duration;
+ public final ProfilerTask type;
+ final CompactStatistics stats;
+ // Contains statistic for a task and all subtasks. Populated only for root tasks.
+ CompactStatistics aggregatedStats = null;
+ // Subtasks are stored as an array for performance and memory utilization
+ // reasons (we can easily deal with millions of those objects).
+ public Task[] subtasks = NO_TASKS;
+ final int descIndex;
+ // Reference to the related task (e.g. ACTION_GRAPH->ACTION task relation).
+ private Task relatedTask;
+
+ Task(long threadId, int id, int parentId, long startTime, long duration,
+ ProfilerTask type, int descIndex, CompactStatistics stats) {
+ this.threadId = threadId;
+ this.id = id;
+ this.parentId = parentId;
+ this.startTime = startTime;
+ this.duration = duration;
+ this.type = type;
+ this.descIndex = descIndex;
+ this.stats = stats;
+ relatedTask = null;
+ }
+
+ public String getDescription() {
+ return descriptionList.get(descIndex);
+ }
+
+ public boolean hasStats() {
+ return !stats.isEmpty();
+ }
+
+ public long getInheritedDuration() {
+ return stats.getTotalTime();
+ }
+
+ public AggregateAttr[] getStatAttrArray() {
+ Preconditions.checkNotNull(stats);
+ return stats.toArray();
+ }
+
+ private void combineStats(int[] counts, long[] duration) {
+ int ownIndex = type.ordinal();
+ if (parentId != 0) {
+ // Parent task already accounted for this task total duration. We need to adjust
+ // for the inherited duration.
+ duration[ownIndex] -= getInheritedDuration();
+ }
+ AggregateAttr[] ownStats = stats.toArray();
+ for (int i = 0; i < TASK_COUNT; i++) {
+ AggregateAttr attr = ownStats[i];
+ if (attr != null) {
+ counts[i] += attr.count;
+ duration[i] += attr.totalTime;
+ }
+ }
+ for (Task task : subtasks) {
+ task.combineStats(counts, duration);
+ }
+ }
+
+ /**
+ * Calculates aggregated statistics covering all subtasks (including
+ * nested ones). Must be called only for parent tasks.
+ */
+ void calculateRootStats() {
+ Preconditions.checkState(parentId == 0);
+ int[] counts = new int[TASK_COUNT];
+ long[] duration = new long[TASK_COUNT];
+ combineStats(counts, duration);
+ AggregateAttr[] statArray = ProfileInfo.createEmptyStatArray();
+ for (int i = 0; i < TASK_COUNT; i++) {
+ statArray[i] = new AggregateAttr(counts[i], duration[i]);
+ }
+ this.aggregatedStats = new CompactStatistics(statArray);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return (o instanceof ProfileInfo.Task) && ((Task) o).id == this.id;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.id;
+ }
+
+ @Override
+ public String toString() {
+ return type + "(" + id + "," + getDescription() + ")";
+ }
+
+ /**
+ * Tasks records by default sorted by their id. Since id was obtained using
+ * AtomicInteger, this comparison will correctly sort tasks in time-ascending
+ * order regardless of their origin thread.
+ */
+ @Override
+ public int compareTo(Task task) {
+ return this.id - task.id;
+ }
+ }
+
+ /**
+ * Represents node on critical build path
+ */
+ public static final class CriticalPathEntry {
+ public final Task task;
+ public final long duration;
+ public final long cumulativeDuration;
+ public final CriticalPathEntry next;
+
+ private long criticalTime = 0L;
+
+ public CriticalPathEntry(Task task, long duration, CriticalPathEntry next) {
+ this.task = task;
+ this.duration = duration;
+ this.next = next;
+ this.cumulativeDuration =
+ duration + (next != null ? next.cumulativeDuration : 0);
+ }
+
+ private void setCriticalTime(long duration) {
+ criticalTime = duration;
+ }
+
+ public long getCriticalTime() {
+ return criticalTime;
+ }
+ }
+
+ /**
+ * Helper class to create space-efficient task multimap, used to associate
+ * array of tasks with specific key.
+ */
+ private abstract static class TaskMapCreator<K> implements Comparator<Task> {
+ @Override
+ public abstract int compare(Task a, Task b);
+ public abstract K getKey(Task task);
+
+ public Map<K, Task[]> createTaskMap(List<Task> taskList) {
+ // Created map usually will end up with thousands of entries, so we
+ // preinitialize it to the 10000.
+ Map<K, Task[]> taskMap = Maps.newHashMapWithExpectedSize(10000);
+ if (taskList.size() == 0) { return taskMap; }
+ Task[] taskArray = taskList.toArray(new Task[taskList.size()]);
+ Arrays.sort(taskArray, this);
+ K key = getKey(taskArray[0]);
+ int start = 0;
+ for (int i = 0; i < taskArray.length; i++) {
+ K currentKey = getKey(taskArray[i]);
+ if (!key.equals(currentKey)) {
+ taskMap.put(key, Arrays.copyOfRange(taskArray, start, i));
+ key = currentKey;
+ start = i;
+ }
+ }
+ if (start < taskArray.length) {
+ taskMap.put(key, Arrays.copyOfRange(taskArray, start, taskArray.length));
+ }
+ return taskMap;
+ }
+ }
+
+ /**
+ * An interface to pass back profile loading and aggregation messages.
+ */
+ public interface InfoListener {
+ void info(String text);
+ void warn(String text);
+ }
+
+ private static final Task[] NO_TASKS = new Task[0];
+ private static final AggregateAttr ZERO = new AggregateAttr(0, 0);
+
+ public final String comment;
+ private boolean corruptedOrIncomplete = false;
+
+ // TODO(bazel-team): (2010) In one case, this list took 277MB of heap. Ideally it should be
+ // replaced with a trie.
+ private final List<String> descriptionList;
+ private final Map<Task, Task> parallelBuilderCompletionQueueTasks;
+ public final Map<Long, Task[]> tasksByThread;
+ public final List<Task> allTasksById;
+ public List<Task> rootTasksById; // Not final due to the late initialization.
+ public final List<Task> phaseTasks;
+
+ public final Map<Task, Task[]> actionDependencyMap;
+ // Used to create fake Action tasks if ACTIONG_GRAPH task does not have
+ // corresponding ACTION task. For action dependency calculations we will
+ // create fake ACTION tasks and assign them negative ids.
+ private int fakeActionId = 0;
+
+ private ProfileInfo(String comment) {
+ this.comment = comment;
+
+ descriptionList = Lists.newArrayListWithExpectedSize(10000);
+ tasksByThread = Maps.newHashMap();
+ parallelBuilderCompletionQueueTasks = Maps.newHashMap();
+ allTasksById = Lists.newArrayListWithExpectedSize(50000);
+ phaseTasks = Lists.newArrayList();
+ actionDependencyMap = Maps.newHashMapWithExpectedSize(10000);
+ }
+
+ private void addTask(Task task) {
+ allTasksById.add(task);
+ }
+
+ /**
+ * Returns true if profile datafile was corrupted or incomplete
+ * and false otherwise.
+ */
+ public boolean isCorruptedOrIncomplete() {
+ return corruptedOrIncomplete;
+ }
+
+ /**
+ * Returns number of missing actions which were faked in order to complete
+ * action graph.
+ */
+ public int getMissingActionsCount() {
+ return -fakeActionId;
+ }
+
+ /**
+ * Initializes minimum internal data structures necessary to obtain individual
+ * task statistic. This method is sufficient to initialize data for dumping.
+ */
+ public void calculateStats() {
+ if (allTasksById.size() == 0) {
+ return;
+ }
+
+ Collections.sort(allTasksById);
+
+ Map<Integer, Task[]> subtaskMap = new TaskMapCreator<Integer>() {
+ @Override
+ public int compare(Task a, Task b) {
+ return a.parentId != b.parentId ? a.parentId - b.parentId : a.compareTo(b);
+ }
+ @Override
+ public Integer getKey(Task task) { return task.parentId; }
+ }.createTaskMap(allTasksById);
+ for (Task task : allTasksById) {
+ Task[] subtasks = subtaskMap.get(task.id);
+ if (subtasks != null) {
+ task.subtasks = subtasks;
+ }
+ }
+ rootTasksById = Arrays.asList(subtaskMap.get(0));
+
+ for (Task task : rootTasksById) {
+ task.calculateRootStats();
+ if (task.type == ProfilerTask.PHASE) {
+ if (!phaseTasks.isEmpty()) {
+ phaseTasks.get(phaseTasks.size() - 1).relatedTask = task;
+ }
+ phaseTasks.add(task);
+ }
+ }
+ }
+
+ /**
+ * Analyzes task relationships and dependencies. Used for the detailed profile
+ * analysis.
+ */
+ public void analyzeRelationships() {
+ tasksByThread.putAll(new TaskMapCreator<Long>() {
+ @Override
+ public int compare(Task a, Task b) {
+ return a.threadId != b.threadId ? (a.threadId < b.threadId ? -1 : 1) : a.compareTo(b);
+ }
+ @Override
+ public Long getKey(Task task) { return task.threadId; }
+ }.createTaskMap(rootTasksById));
+
+ buildDependencyMap();
+ }
+
+ /**
+ * Calculates cumulative time attributed to the specific task type.
+ * Expects to be called only for root (parentId = 0) tasks.
+ * calculateStats() must have been called first.
+ */
+ public AggregateAttr getStatsForType(ProfilerTask type, Collection<Task> tasks) {
+ long totalTime = 0;
+ int count = 0;
+ for (Task task : tasks) {
+ if (task.parentId > 0) {
+ throw new IllegalArgumentException("task " + task.id + " is not a root task");
+ }
+ AggregateAttr attr = task.aggregatedStats.getAttr(type);
+ count += attr.count;
+ totalTime += attr.totalTime;
+ if (task.type == type) {
+ count++;
+ totalTime += (task.duration - task.getInheritedDuration());
+ }
+ }
+ return new AggregateAttr(count, totalTime);
+ }
+
+ /**
+ * Returns list of all root tasks related to (in other words, started during)
+ * the specified phase task.
+ */
+ public List<Task> getTasksForPhase(Task phaseTask) {
+ Preconditions.checkArgument(phaseTask.type == ProfilerTask.PHASE,
+ "Unsupported task type %s", phaseTask.type);
+
+ // Algorithm below takes into account fact that rootTasksById list is sorted
+ // by the task id and task id values are monotonically increasing with time
+ // (this property is guaranteed by the profiler). Thus list is effectively
+ // sorted by the startTime. We are trying to select a sublist that includes
+ // all tasks that were started later than the given task but earlier than
+ // its completion time.
+ int startIndex = Collections.binarySearch(rootTasksById, phaseTask);
+ Preconditions.checkState(startIndex >= 0,
+ "Phase task %s is not a root task", phaseTask.id);
+ int endIndex = (phaseTask.relatedTask != null)
+ ? Collections.binarySearch(rootTasksById, phaseTask.relatedTask)
+ : rootTasksById.size();
+ Preconditions.checkState(endIndex >= startIndex,
+ "Failed to find end of the phase marked by the task %s", phaseTask.id);
+ return rootTasksById.subList(startIndex, endIndex);
+ }
+
+ /**
+ * Returns task with "Build artifacts" description - corresponding to the
+ * execution phase. Usually used to location ACTION_GRAPH task tree.
+ */
+ public Task getPhaseTask(ProfilePhase phase) {
+ for (Task task : phaseTasks) {
+ if (task.getDescription().equals(phase.description)) {
+ return task;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns duration of the given phase in ns.
+ */
+ public long getPhaseDuration(Task phaseTask) {
+ Preconditions.checkArgument(phaseTask.type == ProfilerTask.PHASE,
+ "Unsupported task type %s", phaseTask.type);
+
+ long duration;
+ if (phaseTask.relatedTask != null) {
+ duration = phaseTask.relatedTask.startTime - phaseTask.startTime;
+ } else {
+ Task lastTask = rootTasksById.get(rootTasksById.size() - 1);
+ duration = lastTask.startTime + lastTask.duration - phaseTask.startTime;
+ }
+ Preconditions.checkState(duration >= 0);
+ return duration;
+ }
+
+
+ /**
+ * Builds map of dependencies between ACTION tasks based on dependencies
+ * between ACTION_GRAPH tasks
+ */
+ private Task buildActionTaskTree(Task actionGraphTask, List<Task> actionTasksByDescription) {
+ Task actionTask = actionGraphTask.relatedTask;
+ if (actionTask == null) {
+ actionTask = actionTasksByDescription.get(actionGraphTask.descIndex);
+ if (actionTask == null) {
+ // If we cannot find ACTION task that corresponds to the ACTION_GRAPH task,
+ // most likely scenario is that we dealing with either aborted or failed
+ // build. In this case we will find or create fake zero-duration action
+ // task and still reconstruct dependency graph.
+ actionTask = new Task(-1, --fakeActionId, 0, 0, 0,
+ ProfilerTask.ACTION, actionGraphTask.descIndex, new CompactStatistics((byte[]) null));
+ actionTask.calculateRootStats();
+ actionTasksByDescription.set(actionGraphTask.descIndex, actionTask);
+ }
+ actionGraphTask.relatedTask = actionTask;
+ }
+ if (actionGraphTask.subtasks.length != 0) {
+ List<Task> list = Lists.newArrayListWithCapacity(actionGraphTask.subtasks.length);
+ for (Task task : actionGraphTask.subtasks) {
+ if (task.type == ProfilerTask.ACTION_GRAPH) {
+ list.add(buildActionTaskTree(task, actionTasksByDescription));
+ }
+ }
+ if (!list.isEmpty()) {
+ Task[] actionPrerequisites = list.toArray(new Task[list.size()]);
+ Arrays.sort(actionPrerequisites);
+ actionDependencyMap.put(actionTask, actionPrerequisites);
+ }
+ }
+ return actionTask;
+ }
+
+ /**
+ * Builds map of dependencies between ACTION tasks based on dependencies
+ * between ACTION_GRAPH tasks. Root of that dependency tree would be
+ * getBuildPhaseTask().
+ *
+ * <p> Also marks related ACTION and ACTION_SUBMIT tasks.
+ */
+ private void buildDependencyMap() {
+ Task analysisPhaseTask = getPhaseTask(ProfilePhase.ANALYZE);
+ Task executionPhaseTask = getPhaseTask(ProfilePhase.EXECUTE);
+ if ((executionPhaseTask == null) || (analysisPhaseTask == null)) {
+ return;
+ }
+ // Association between ACTION_GRAPH tasks and ACTION tasks can be established through
+ // description id. So we create appropriate xref list.
+ List<Task> actionTasksByDescription = Lists.newArrayList(new Task[descriptionList.size()]);
+ for (Task task : getTasksForPhase(executionPhaseTask)) {
+ if (task.type == ProfilerTask.ACTION) {
+ actionTasksByDescription.set(task.descIndex, task);
+ }
+ }
+ List<Task> list = new ArrayList<>();
+ for (Task task : getTasksForPhase(analysisPhaseTask)) {
+ if (task.type == ProfilerTask.ACTION_GRAPH) {
+ list.add(buildActionTaskTree(task, actionTasksByDescription));
+ }
+ }
+ Task[] actionPrerequisites = list.toArray(new Task[list.size()]);
+ Arrays.sort(actionPrerequisites);
+ actionDependencyMap.put(executionPhaseTask, actionPrerequisites);
+
+ // Scan through all execution phase tasks to identify ACTION_SUBMIT tasks and associate
+ // them with ACTION task counterparts. ACTION_SUBMIT tasks are not necessarily root
+ // tasks so we need to scan ALL tasks.
+ for (Task task : allTasksById.subList(executionPhaseTask.id, allTasksById.size())) {
+ if (task.type == ProfilerTask.ACTION_SUBMIT) {
+ Task actionTask = actionTasksByDescription.get(task.descIndex);
+ if (actionTask != null) {
+ task.relatedTask = actionTask;
+ actionTask.relatedTask = task;
+ }
+ } else if (task.type == ProfilerTask.ACTION_BUILDER) {
+ Task actionTask = actionTasksByDescription.get(task.descIndex);
+ if (actionTask != null) {
+ parallelBuilderCompletionQueueTasks.put(actionTask, task);
+ }
+ }
+ }
+ }
+
+ /**
+ * Calculates critical path for the specific action
+ * excluding specified nested task types (e.g. VFS-related time) and not
+ * accounting for overhead related to the Blaze scheduler.
+ */
+ private CriticalPathEntry computeCriticalPathForAction(
+ Set<ProfilerTask> ignoredTypes, Set<Task> ignoredTasks,
+ Task actionTask, Map<Task, CriticalPathEntry> cache, Deque<Task> stack) {
+
+ // Loop check is expensive for the Deque (and we don't want to use hash sets because adding
+ // and removing elements was shown to be very expensive). To avoid quadratic costs we're
+ // checking for infinite loop only when deque's size equal to the power of 2 and >= 32.
+ if ((stack.size() & 0x1F) == 0 && Integer.bitCount(stack.size()) == 1) {
+ if (stack.contains(actionTask)) {
+ // This situation will appear if build has ended with the
+ // IllegalStateException thrown by the
+ // ParallelBuilder.getNextCompletedAction(), warning user about
+ // possible cycle in the dependency graph. But the exception text
+ // is more friendly and will actually identify the loop.
+ // Do not use Preconditions class below due to the very expensive
+ // toString() calls used in the message.
+ throw new IllegalStateException ("Dependency graph contains loop:\n"
+ + actionTask + " in the\n" + Joiner.on('\n').join(stack));
+ }
+ }
+ stack.addLast(actionTask);
+ CriticalPathEntry entry;
+ try {
+ entry = cache.get(actionTask);
+ long entryDuration = 0;
+ if (entry == null) {
+ Task[] actionPrerequisites = actionDependencyMap.get(actionTask);
+ if (actionPrerequisites != null) {
+ for (Task task : actionPrerequisites) {
+ CriticalPathEntry candidate =
+ computeCriticalPathForAction(ignoredTypes, ignoredTasks, task, cache, stack);
+ if (entry == null || entryDuration < candidate.cumulativeDuration) {
+ entry = candidate;
+ entryDuration = candidate.cumulativeDuration;
+ }
+ }
+ }
+ if (actionTask.type == ProfilerTask.ACTION) {
+ long duration = actionTask.duration;
+ if (ignoredTasks.contains(actionTask)) {
+ duration = 0L;
+ } else {
+ for (ProfilerTask type : ignoredTypes) {
+ duration -= actionTask.aggregatedStats.getAttr(type).totalTime;
+ }
+ }
+
+ entry = new CriticalPathEntry(actionTask, duration, entry);
+ cache.put(actionTask, entry);
+ }
+ }
+ } finally {
+ stack.removeLast();
+ }
+ return entry;
+ }
+
+ /**
+ * Returns the critical path information from the {@code CriticalPathComputer} recorded stats.
+ * This code does not have the "Critical" column (Time difference if we removed this node from
+ * the critical path).
+ */
+ public CriticalPathEntry getCriticalPathNewVersion() {
+ for (Task task : rootTasksById) {
+ if (task.type == CRITICAL_PATH) {
+ CriticalPathEntry entry = null;
+ for (Task shared : task.subtasks) {
+ entry = new CriticalPathEntry(shared, shared.duration, entry);
+ }
+ return entry;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Calculates critical path for the given action graph excluding
+ * specified tasks (usually ones that belong to the "real" critical path).
+ */
+ public CriticalPathEntry getCriticalPath(Set<ProfilerTask> ignoredTypes) {
+ Task actionTask = getPhaseTask(ProfilePhase.EXECUTE);
+ if (actionTask == null) {
+ return null;
+ }
+ Map <Task, CriticalPathEntry> cache = Maps.newHashMapWithExpectedSize(1000);
+ CriticalPathEntry result = computeCriticalPathForAction(ignoredTypes,
+ new HashSet<Task>(), actionTask, cache,
+ new ArrayDeque<Task>());
+ if (result != null) {
+ return result;
+ }
+ return getCriticalPathNewVersion();
+ }
+
+ /**
+ * Calculates critical path time that will be saved by eliminating specific
+ * entry from the critical path
+ */
+ public void analyzeCriticalPath(Set<ProfilerTask> ignoredTypes, CriticalPathEntry path) {
+ // With light critical path we do not need to analyze since it is already preprocessed
+ // by blaze build.
+ if (path != null && path.task.type == CRITICAL_PATH_COMPONENT) {
+ return;
+ }
+ for (CriticalPathEntry entry = path; entry != null; entry = entry.next) {
+ Map <Task, CriticalPathEntry> cache = Maps.newHashMapWithExpectedSize(1000);
+ entry.setCriticalTime(path.cumulativeDuration -
+ computeCriticalPathForAction(ignoredTypes, Sets.newHashSet(entry.task),
+ getPhaseTask(ProfilePhase.EXECUTE), cache, new ArrayDeque<Task>())
+ .cumulativeDuration);
+ }
+ }
+
+ /**
+ * Return the next critical path entry for the task or null if there is none.
+ */
+ public CriticalPathEntry getNextCriticalPathEntryForTask(CriticalPathEntry path, Task task) {
+ for (CriticalPathEntry entry = path; entry != null; entry = entry.next) {
+ if (entry.task.id == task.id) {
+ return entry;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns time action waited in the execution queue (difference between
+ * ACTION task start time and ACTION_SUBMIT task start time).
+ */
+ public long getActionWaitTime(Task actionTask) {
+ // Light critical path does not record wait time.
+ if (actionTask.type == ProfilerTask.CRITICAL_PATH_COMPONENT) {
+ return 0;
+ }
+ Preconditions.checkArgument(actionTask.type == ProfilerTask.ACTION);
+ if (actionTask.relatedTask != null) {
+ Preconditions.checkState(actionTask.relatedTask.type == ProfilerTask.ACTION_SUBMIT);
+ long time = actionTask.startTime - actionTask.relatedTask.startTime;
+ Preconditions.checkState(time >= 0);
+ return time;
+ } else {
+ return 0L; // submission time is not available.
+ }
+ }
+
+ /**
+ * Returns time action waited in the parallel builder completion queue
+ * (difference between ACTION task end time and ACTION_BUILDER start time).
+ */
+ public long getActionQueueTime(Task actionTask) {
+ // Light critical path does not record queue time.
+ if (actionTask.type == ProfilerTask.CRITICAL_PATH_COMPONENT) {
+ return 0;
+ }
+ Preconditions.checkArgument(actionTask.type == ProfilerTask.ACTION);
+ Task related = parallelBuilderCompletionQueueTasks.get(actionTask);
+ if (related != null) {
+ Preconditions.checkState(related.type == ProfilerTask.ACTION_BUILDER);
+ long time = related.startTime - (actionTask.startTime + actionTask.duration);
+ Preconditions.checkState(time >= 0);
+ return time;
+ } else {
+ return 0L; // queue task is not available.
+ }
+ }
+
+ /**
+ * Returns an empty array used to store task statistics. Array index
+ * corresponds to the ProfilerTask ordinal() value associated with the
+ * given statistic. Absent statistics are stored as null.
+ * <p>
+ * In essence, it is a fast equivalent of Map<ProfilerTask, AggregateAttr>.
+ */
+ public static AggregateAttr[] createEmptyStatArray() {
+ return new AggregateAttr[TASK_COUNT];
+ }
+
+ /**
+ * Loads and parses Blaze profile file.
+ *
+ * @param profileFile profile file path
+ *
+ * @return ProfileInfo object with some fields populated (call calculateStats()
+ * and analyzeRelationships() to populate the remaining fields)
+ * @throws UnsupportedEncodingException if the file format is invalid
+ * @throws IOException if the file can't be read
+ */
+ public static ProfileInfo loadProfile(Path profileFile)
+ throws IOException {
+ // It is extremely important to wrap InflaterInputStream using
+ // BufferedInputStream because majority of reads would be done using
+ // readInt()/readLong() methods and InflaterInputStream is very inefficient
+ // in handling small read requests (performance difference with 1MB buffer
+ // used below is almost 10x).
+ DataInputStream in = new DataInputStream(
+ new BufferedInputStream(new InflaterInputStream(
+ profileFile.getInputStream(), new Inflater(false), 65536), 1024 * 1024));
+
+ if (in.readInt() != Profiler.MAGIC) {
+ in.close();
+ throw new UnsupportedEncodingException("Invalid profile datafile format");
+ }
+ if (in.readInt() != Profiler.VERSION) {
+ in.close();
+ throw new UnsupportedEncodingException("Incompatible profile datafile version");
+ }
+ String fileComment = in.readUTF();
+
+ // Read list of used record types
+ int typeCount = in.readInt();
+ boolean hasUnknownTypes = false;
+ Set<String> supportedTasks = new HashSet<>();
+ for (ProfilerTask task : ProfilerTask.values()) {
+ supportedTasks.add(task.toString());
+ }
+ List<ProfilerTask> typeList = new ArrayList<>();
+ for (int i = 0; i < typeCount; i++) {
+ String name = in.readUTF();
+ if (supportedTasks.contains(name)) {
+ typeList.add(ProfilerTask.valueOf(name));
+ } else {
+ hasUnknownTypes = true;
+ typeList.add(ProfilerTask.UNKNOWN);
+ }
+ }
+
+ ProfileInfo info = new ProfileInfo(fileComment);
+
+ // Read record until we encounter end marker (-1).
+ // TODO(bazel-team): Maybe this still should handle corrupted(truncated) files.
+ try {
+ int size;
+ while ((size = in.readInt()) != Profiler.EOF_MARKER) {
+ byte[] backingArray = new byte[size];
+ in.readFully(backingArray);
+ ByteBuffer buffer = ByteBuffer.wrap(backingArray);
+ long threadId = VarInt.getVarLong(buffer);
+ int id = VarInt.getVarInt(buffer);
+ int parentId = VarInt.getVarInt(buffer);
+ long startTime = VarInt.getVarLong(buffer);
+ long duration = VarInt.getVarLong(buffer);
+ int descIndex = VarInt.getVarInt(buffer) - 1;
+ if (descIndex == -1) {
+ String desc = in.readUTF();
+ descIndex = info.descriptionList.size();
+ info.descriptionList.add(desc);
+ }
+ ProfilerTask type = typeList.get(buffer.get());
+ byte[] stats = null;
+ if (buffer.hasRemaining()) {
+ // Copy aggregated stats.
+ int offset = buffer.position();
+ stats = Arrays.copyOfRange(backingArray, offset, size);
+ if (hasUnknownTypes) {
+ while (buffer.hasRemaining()) {
+ byte attrType = buffer.get();
+ if (typeList.get(attrType) == ProfilerTask.UNKNOWN) {
+ // We're dealing with unknown aggregated type - update stats array to
+ // use ProfilerTask.UNKNOWN.ordinal() value.
+ stats[buffer.position() - 1 - offset] = (byte) ProfilerTask.UNKNOWN.ordinal();
+ }
+ VarInt.getVarInt(buffer);
+ VarInt.getVarLong(buffer);
+ }
+ }
+ }
+ ProfileInfo.Task task = info.new Task(threadId, id, parentId, startTime, duration, type,
+ descIndex, new CompactStatistics(stats));
+ info.addTask(task);
+ }
+ } catch (IOException e) {
+ info.corruptedOrIncomplete = true;
+ } finally {
+ in.close();
+ }
+
+ return info;
+ }
+
+ /**
+ * Loads and parses Blaze profile file, and reports what it is doing.
+ *
+ * @param profileFile profile file path
+ * @param reporter for progress messages and warnings
+ *
+ * @return ProfileInfo object with most fields populated
+ * (call analyzeRelationships() to populate the remaining fields)
+ * @throws UnsupportedEncodingException if the file format is invalid
+ * @throws IOException if the file can't be read
+ */
+ public static ProfileInfo loadProfileVerbosely(Path profileFile, InfoListener reporter)
+ throws IOException {
+ reporter.info("Loading " + profileFile.getPathString());
+ ProfileInfo profileInfo = ProfileInfo.loadProfile(profileFile);
+ if (profileInfo.isCorruptedOrIncomplete()) {
+ reporter.warn("Profile file is incomplete or corrupted - not all records were parsed");
+ }
+ reporter.info(profileInfo.comment + ", " + profileInfo.allTasksById.size() + " record(s)");
+ return profileInfo;
+ }
+
+ /*
+ * Sorts and aggregates Blaze profile file, and reports what it is doing.
+ */
+ public static void aggregateProfile(ProfileInfo profileInfo, InfoListener reporter) {
+ reporter.info("Aggregating task statistics");
+ profileInfo.calculateStats();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhase.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhase.java
new file mode 100644
index 0000000..fa4b862
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhase.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler;
+
+/**
+ * Build phase markers. Used as a separators between different build phases.
+ */
+public enum ProfilePhase {
+ LAUNCH("launch", "Launch Blaze", 0x3F9FCF9F), // 9C9
+ INIT("init", "Initialize command", 0x3F9F9FCF), // 99C
+ LOAD("loading", "Load packages", 0x3FCFFFCF), // CFC
+ ANALYZE("analysis", "Analyze dependencies", 0x3FCFCFFF), // CCF
+ LICENSE("license checking", "Analyze licenses", 0x3FCFFFFF), // CFF
+ PREPARE("preparation", "Prepare for build", 0x3FFFFFCF), // FFC
+ EXECUTE("execution", "Build artifacts", 0x3FFFCFCF), // FCC
+ FINISH("finish", "Complete build",0x3FFFCFFF); // FCF
+
+ /** Short name for the phase */
+ public final String nick;
+ /** Human readable description for the phase. */
+ public final String description;
+ /** Default color of the task, when rendered in a chart. */
+ public final int color;
+
+ ProfilePhase(String nick, String description, int color) {
+ this.nick = nick;
+ this.description = description;
+ this.color = color;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhaseStatistics.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhaseStatistics.java
new file mode 100644
index 0000000..f3eb525
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhaseStatistics.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler;
+
+/**
+ * Hold pre-formatted statistics of a profiled execution phase.
+ *
+ * TODO(bazel-team): Change String statistics into StatisticsTable[], where StatisticsTable is an
+ * Object with a title (can be null), header[columns] (can be null), data[rows][columns],
+ * alignment[columns] (left/right).
+ * The HtmlChartsVisitor can turn that into HTML tables, the text formatter can calculate the max
+ * for each column and format the text accordingly.
+ */
+public class ProfilePhaseStatistics {
+ private final String title;
+ private final String statistics;
+
+ public ProfilePhaseStatistics (String title, String statistics) {
+ this.title = title;
+ this.statistics = statistics;
+ }
+
+ public String getTitle(){
+ return title;
+ }
+
+ public String getStatistics(){
+ return statistics;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java b/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java
new file mode 100644
index 0000000..d592848
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java
@@ -0,0 +1,871 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.profiler;
+
+import static com.google.devtools.build.lib.profiler.ProfilerTask.TASK_COUNT;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.VarInt;
+
+import java.io.BufferedOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+
+/**
+ * Blaze internal profiler. Provides facility to report various Blaze tasks and
+ * store them (asynchronously) in the file for future analysis.
+ * <p>
+ * Implemented as singleton so any caller should use Profiler.instance() to
+ * obtain reference.
+ * <p>
+ * Internally, profiler uses two data structures - ThreadLocal task stack to track
+ * nested tasks and single ConcurrentLinkedQueue to gather all completed tasks.
+ * <p>
+ * Also, due to the nature of the provided functionality (instrumentation of all
+ * Blaze components), build.lib.profiler package will be used by almost every
+ * other Blaze package, so special attention should be paid to avoid any
+ * dependencies on the rest of the Blaze code, including build.lib.util and
+ * build.lib.vfs. This is important because build.lib.util and build.lib.vfs
+ * contain Profiler invocations and any dependency on those two packages would
+ * create circular relationship.
+ * <p>
+ * All gathered instrumentation data will be stored in the file. Please, note,
+ * that while file format is described here it is considered internal and can
+ * change at any time. For scripting, using blaze analyze-profile --dump=raw
+ * would be more robust and stable solution.
+ * <p>
+ * <pre>
+ * Profiler file consists of the deflated stream with following overall structure:
+ * HEADER
+ * TASK_TYPE_TABLE
+ * TASK_RECORD...
+ * EOF_MARKER
+ *
+ * HEADER:
+ * int32: magic token (Profiler.MAGIC)
+ * int32: version format (Profiler.VERSION)
+ * string: file comment
+ *
+ * TASK_TYPE_TABLE:
+ * int32: number of type names below
+ * string... : type names. Each of the type names is assigned id according to
+ * their position in this table starting from 0.
+ *
+ * TASK_RECORD:
+ * int32 size: size of the encoded task record
+ * byte[size] encoded_task_record:
+ * varint64: thread id - as was returned by Thread.getId()
+ * varint32: task id - starting from 1.
+ * varint32: parent task id for subtasks or 0 for root tasks
+ * varint64: start time in ns, relative to the Profiler.start() invocation
+ * varint64: task duration in ns
+ * byte: task type id (see TASK_TYPE_TABLE)
+ * varint32: description string index incremented by 1 (>0) or 0 this is
+ * a first occurrence of the description string
+ * AGGREGATED_STAT...: remainder of the field (if present) represents
+ * aggregated stats for that task
+ * string: *optional* description string, will appear only if description
+ * string index above was 0. In that case this string will be
+ * assigned next sequential id so every unique description string
+ * will appear in the file only once - after that it will be
+ * referenced by id.
+ *
+ * AGGREGATE_STAT:
+ * byte: stat type
+ * varint32: total number of subtask invocations
+ * varint64: cumulative duration of subtask invocations in ns.
+ *
+ * EOF_MARKER:
+ * int64: -1 - please note that this corresponds to the thread id in the
+ * TASK_RECORD which is always > 0
+ * </pre>
+ *
+ * @see ProfilerTask enum for recognized task types.
+ */
+//@ThreadSafe - commented out to avoid cyclic dependency with lib.util package
+public final class Profiler {
+ static final int MAGIC = 0x11223344;
+
+ // File version number. Note that merely adding new record types in
+ // the ProfilerTask does not require bumping version number as long as original
+ // enum values are not renamed or deleted.
+ static final int VERSION = 0x03;
+
+ // EOF marker. Must be < 0.
+ static final int EOF_MARKER = -1;
+
+ // Profiler will check for gathered data and persist all of it in the
+ // separate thread every SAVE_DELAY ms.
+ private static final int SAVE_DELAY = 2000; // ms
+
+ /**
+ * The profiler (a static singleton instance). Inactive by default.
+ */
+ private static final Profiler instance = new Profiler();
+
+ /**
+ * A task that was very slow.
+ */
+ public final class SlowTask implements Comparable<SlowTask> {
+ final long durationNanos;
+ final Object object;
+ ProfilerTask type;
+
+ private SlowTask(TaskData taskData) {
+ this.durationNanos = taskData.duration;
+ this.object = taskData.object;
+ this.type = taskData.type;
+ }
+
+ @Override
+ public int compareTo(SlowTask other) {
+ long delta = durationNanos - other.durationNanos;
+ if (delta < 0) { // Very clumsy
+ return -1;
+ } else if (delta > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ public long getDurationNanos() {
+ return durationNanos;
+ }
+
+ public String getDescription() {
+ return toDescription(object);
+ }
+
+ public ProfilerTask getType() {
+ return type;
+ }
+ }
+
+ /**
+ * Container for the single task record.
+ * Should never be instantiated directly - use TaskStack.create() instead.
+ *
+ * Class itself is not thread safe, but all access to it from Profiler
+ * methods is.
+ */
+ //@ThreadCompatible - commented out to avoid cyclic dependency with lib.util.
+ private final class TaskData {
+ final long threadId;
+ final long startTime;
+ long duration = 0L;
+ final int id;
+ final int parentId;
+ int[] counts; // number of invocations per ProfilerTask type
+ long[] durations; // time spend in the task per ProfilerTask type
+ final ProfilerTask type;
+ final Object object;
+
+ TaskData(long startTime, TaskData parent,
+ ProfilerTask eventType, Object object) {
+ threadId = Thread.currentThread().getId();
+ counts = null;
+ durations = null;
+ id = taskId.incrementAndGet();
+ parentId = (parent == null ? 0 : parent.id);
+ this.startTime = startTime;
+ this.type = eventType;
+ this.object = Preconditions.checkNotNull(object);
+ }
+
+ /**
+ * Aggregates information about an *immediate* subtask.
+ */
+ public void aggregateChild(ProfilerTask type, long duration) {
+ int index = type.ordinal();
+ if (counts == null) {
+ // one entry for each ProfilerTask type
+ counts = new int[TASK_COUNT];
+ durations = new long[TASK_COUNT];
+ }
+ counts[index]++;
+ durations[index] += duration;
+ }
+
+ @Override
+ public String toString() {
+ return "Thread " + threadId + ", task " + id + ", type " + type + ", " + object;
+ }
+ }
+
+ /**
+ * Tracks nested tasks for each thread.
+ *
+ * java.util.ArrayDeque is the most efficient stack implementation in the
+ * Java Collections Framework (java.util.Stack class is older synchronized
+ * alternative). It is, however, used here strictly for LIFO operations.
+ * However, ArrayDeque is 1.6 only. For 1.5 best approach would be to utilize
+ * ArrayList and emulate stack using it.
+ */
+ //@ThreadSafe - commented out to avoid cyclic dependency with lib.util.
+ private final class TaskStack extends ThreadLocal<List<TaskData>> {
+
+ @Override
+ public List<TaskData> initialValue() {
+ return new ArrayList<>();
+ }
+
+ public TaskData peek() {
+ List<TaskData> list = get();
+ if (list.isEmpty()) {
+ return null;
+ }
+ return list.get(list.size() - 1);
+ }
+
+ public TaskData pop() {
+ List<TaskData> list = get();
+ return list.remove(list.size() - 1);
+ }
+
+ public boolean isEmpty() {
+ return get().isEmpty();
+ }
+
+ public void push(ProfilerTask eventType, Object object) {
+ get().add(create(clock.nanoTime(), eventType, object));
+ }
+
+ public TaskData create(long startTime, ProfilerTask eventType, Object object) {
+ return new TaskData(startTime, peek(), eventType, object);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder(
+ "Current task stack for thread " + Thread.currentThread().getName() + ":\n");
+ List<TaskData> list = get();
+ for (int i = list.size() - 1; i >= 0; i--) {
+ builder.append(list.get(i).toString());
+ builder.append("\n");
+ }
+ return builder.toString();
+ }
+ }
+
+ private static String toDescription(Object object) {
+ return (object instanceof Describable)
+ ? ((Describable) object).describe()
+ : object.toString();
+ }
+
+ /**
+ * Implements datastore for object description indices. Intended to be used
+ * only by the Profiler.save() method.
+ */
+ //@ThreadCompatible - commented out to avoid cyclic dependency with lib.util.
+ private final class ObjectDescriber {
+ private Map<Object, Integer> descMap = new IdentityHashMap<>(2000);
+ private int indexCounter = 0;
+
+ ObjectDescriber() { }
+
+ int getDescriptionIndex(Object object) {
+ Integer index = descMap.get(object);
+ return (index != null) ? index : -1;
+ }
+
+ String getDescription(Object object) {
+ String description = toDescription(object);
+
+ Integer oldIndex = descMap.put(object, indexCounter++);
+ // Do not use Preconditions class below due to the rather expensive
+ // toString() calls used in the message.
+ if (oldIndex != null) {
+ throw new IllegalStateException(" Object '" + description + "' @ "
+ + System.identityHashCode(object) + " already had description index "
+ + oldIndex + " while assigning index " + descMap.get(object));
+ } else if (description.length() > 20000) {
+ // Note size 64k byte limitation in DataOutputStream#writeUTF().
+ description = description.substring(0, 20000);
+ }
+ return description;
+ }
+
+ boolean isUnassigned(int index) {
+ return (index < 0);
+ }
+ }
+
+ /**
+ * Aggregator class that keeps track of the slowest tasks of the specified type.
+ *
+ * <p><code>priorityQueues</p> is sharded so that all threads need not compete for the same
+ * lock if they do the same operation at the same time. Access to the individual queues is
+ * synchronized on the queue objects themselves.
+ */
+ private final class SlowestTaskAggregator {
+ private static final int SHARDS = 16;
+ private final int size;
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private final PriorityQueue<SlowTask>[] priorityQueues = new PriorityQueue[SHARDS];
+
+ SlowestTaskAggregator(int size) {
+ this.size = size;
+
+ for (int i = 0; i < SHARDS; i++) {
+ priorityQueues[i] = new PriorityQueue<SlowTask>(size + 1);
+ }
+ }
+
+ // @ThreadSafe
+ void add(TaskData taskData) {
+ PriorityQueue<SlowTask> queue =
+ priorityQueues[(int) (Thread.currentThread().getId() % SHARDS)];
+ synchronized (queue) {
+ if (queue.size() == size) {
+ // Optimization: check if we are faster than the fastest element. If we are, we would
+ // be the ones to fall off the end of the queue, therefore, we can safely return early.
+ if (queue.peek().getDurationNanos() > taskData.duration) {
+ return;
+ }
+
+ queue.add(new SlowTask(taskData));
+ queue.remove();
+ } else {
+ queue.add(new SlowTask(taskData));
+ }
+ }
+ }
+
+ // @ThreadSafe
+ void clear() {
+ for (int i = 0; i < SHARDS; i++) {
+ PriorityQueue<SlowTask> queue = priorityQueues[i];
+ synchronized (queue) {
+ queue.clear();
+ }
+ }
+ }
+
+ // @ThreadSafe
+ Iterable<SlowTask> getSlowestTasks() {
+ // This is slow, but since it only happens during the end of the invocation, it's OK
+ PriorityQueue<SlowTask> merged = new PriorityQueue<>(size * SHARDS);
+ for (int i = 0; i < SHARDS; i++) {
+ PriorityQueue<SlowTask> queue = priorityQueues[i];
+ synchronized (queue) {
+ merged.addAll(queue);
+ }
+ }
+
+ while (merged.size() > size) {
+ merged.remove();
+ }
+
+ return merged;
+ }
+ }
+
+ /**
+ * Which {@link ProfilerTask}s are profiled.
+ */
+ public enum ProfiledTaskKinds {
+ /**
+ * Do not profile anything.
+ *
+ * <p>Performance is best with this case, but we lose critical path analysis and slowest
+ * operation tracking.
+ */
+ NONE {
+ @Override
+ boolean isProfiling(ProfilerTask type) {
+ return false;
+ }
+ },
+
+ /**
+ * Profile on a few, known-to-be-slow tasks.
+ *
+ * <p>Performance is somewhat decreased in comparison to {@link #NONE}, but we still track the
+ * slowest operations (VFS).
+ */
+ SLOWEST {
+ @Override
+ boolean isProfiling(ProfilerTask type) {
+ return type.collectsSlowestInstances();
+ }
+ },
+
+ /**
+ * Profile all tasks.
+ *
+ * <p>This is in use when {@code --profile} is specified.
+ */
+ ALL {
+ @Override
+ boolean isProfiling(ProfilerTask type) {
+ return true;
+ }
+ };
+
+ /** Whether the Profiler collects data for the given task type. */
+ abstract boolean isProfiling(ProfilerTask type);
+ }
+
+ private Clock clock;
+ private ProfiledTaskKinds profiledTaskKinds;
+ private volatile long profileStartTime = 0L;
+ private volatile boolean recordAllDurations = false;
+ private AtomicInteger taskId = new AtomicInteger();
+
+ private TaskStack taskStack;
+ private Queue<TaskData> taskQueue;
+ private DataOutputStream out;
+ private Timer timer;
+ private IOException saveException;
+ private ObjectDescriber describer;
+ @SuppressWarnings("unchecked")
+ private final SlowestTaskAggregator[] slowestTasks =
+ new SlowestTaskAggregator[ProfilerTask.values().length];
+
+ private Profiler() {
+ for (ProfilerTask task : ProfilerTask.values()) {
+ if (task.slowestInstancesCount != 0) {
+ slowestTasks[task.ordinal()] = new SlowestTaskAggregator(task.slowestInstancesCount);
+ }
+ }
+ }
+
+ public static Profiler instance() {
+ return instance;
+ }
+
+ /**
+ * Returns the nanoTime of the current profiler instance, or an arbitrary
+ * constant if not active.
+ */
+ public static long nanoTimeMaybe() {
+ if (instance.isActive()) {
+ return instance.clock.nanoTime();
+ }
+ return -1;
+ }
+
+ /**
+ * Enable profiling.
+ *
+ * <p>Subsequent calls to beginTask/endTask will be recorded
+ * in the provided output stream. Please note that stream performance is
+ * extremely important and buffered streams should be utilized.
+ *
+ * @param profiledTaskKinds which kinds of {@link ProfilerTask}s to track
+ * @param stream output stream to store profile data. Note: passing unbuffered stream object
+ * reference may result in significant performance penalties
+ * @param comment a comment to insert in the profile data
+ * @param recordAllDurations iff true, record all tasks regardless of their duration; otherwise
+ * some tasks may get aggregated if they finished quick enough
+ * @param clock a {@code BlazeClock.instance()}
+ * @param execStartTimeNanos execution start time in nanos obtained from {@code clock.nanoTime()}
+ */
+ public synchronized void start(ProfiledTaskKinds profiledTaskKinds, OutputStream stream,
+ String comment, boolean recordAllDurations, Clock clock, long execStartTimeNanos)
+ throws IOException {
+ Preconditions.checkState(!isActive(), "Profiler already active");
+ taskStack = new TaskStack();
+ taskQueue = new ConcurrentLinkedQueue<>();
+ describer = new ObjectDescriber();
+
+ this.profiledTaskKinds = profiledTaskKinds;
+ this.clock = clock;
+
+ // sanity check for current limitation on the number of supported types due
+ // to using enum.ordinal() to store them instead of EnumSet for performance reasons.
+ Preconditions.checkState(TASK_COUNT < 256,
+ "The profiler implementation supports only up to 255 different ProfilerTask values.");
+
+ // reset state for the new profiling session
+ taskId.set(0);
+ this.recordAllDurations = recordAllDurations;
+ this.saveException = null;
+ if (stream != null) {
+ this.timer = new Timer("ProfilerTimer", true);
+ // Wrapping deflater stream in the buffered stream proved to reduce CPU consumption caused by
+ // the save() method. Values for buffer sizes were chosen by running small amount of tests
+ // and identifying point of diminishing returns - but I have not really tried to optimize
+ // them.
+ this.out = new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(
+ stream, new Deflater(Deflater.BEST_SPEED, false), 65536), 262144));
+
+ this.out.writeInt(MAGIC); // magic
+ this.out.writeInt(VERSION); // protocol_version
+ this.out.writeUTF(comment);
+ // ProfileTask.values() method sorts enums using their ordinal() value, so
+ // there there is no need to store ordinal() value for each entry.
+ this.out.writeInt(TASK_COUNT);
+ for (ProfilerTask type : ProfilerTask.values()) {
+ this.out.writeUTF(type.toString());
+ }
+
+ // Start save thread
+ timer.schedule(new TimerTask() {
+ @Override public void run() { save(); }
+ }, SAVE_DELAY, SAVE_DELAY);
+ } else {
+ this.out = null;
+ }
+
+ // activate profiler
+ profileStartTime = execStartTimeNanos;
+ }
+
+ public synchronized Iterable<SlowTask> getSlowestTasks() {
+ List<Iterable<SlowTask>> slowestTasksByType = new ArrayList<>();
+
+ for (SlowestTaskAggregator aggregator : slowestTasks) {
+ if (aggregator != null) {
+ slowestTasksByType.add(aggregator.getSlowestTasks());
+ }
+ }
+
+ return Iterables.concat(slowestTasksByType);
+ }
+
+ /**
+ * Disable profiling and complete profile file creation.
+ * Subsequent calls to beginTask/endTask will no longer
+ * be recorded in the profile.
+ */
+ public synchronized void stop() throws IOException {
+ if (saveException != null) {
+ throw saveException;
+ }
+ if (!isActive()) {
+ return;
+ }
+ // Log a final event to update the duration of ProfilePhase.FINISH.
+ logEvent(ProfilerTask.INFO, "Finishing");
+ save();
+ clear();
+
+ for (SlowestTaskAggregator aggregator : slowestTasks) {
+ if (aggregator != null) {
+ aggregator.clear();
+ }
+ }
+
+ if (saveException != null) {
+ throw saveException;
+ }
+ if (out != null) {
+ out.writeInt(EOF_MARKER);
+ out.close();
+ out = null;
+ }
+ }
+
+ /**
+ * Returns true iff profiling is currently enabled.
+ */
+ public boolean isActive() {
+ return profileStartTime != 0L;
+ }
+
+ public boolean isProfiling(ProfilerTask type) {
+ return profiledTaskKinds.isProfiling(type);
+ }
+
+ /**
+ * Saves all gathered information from taskQueue queue to the file.
+ * Method is invoked internally by the Timer-based thread and at the end of
+ * profiling session.
+ */
+ private synchronized void save() {
+ if (out == null) {
+ return;
+ }
+ try {
+ // Allocate the sink once to avoid GC
+ ByteBuffer sink = ByteBuffer.allocate(1024);
+ while (!taskQueue.isEmpty()) {
+ sink.clear();
+ TaskData data = taskQueue.poll();
+
+ VarInt.putVarLong(data.threadId, sink);
+ VarInt.putVarInt(data.id, sink);
+ VarInt.putVarInt(data.parentId, sink);
+ VarInt.putVarLong(data.startTime - profileStartTime, sink);
+ VarInt.putVarLong(data.duration, sink);
+
+ // To save space (and improve performance), convert all description
+ // strings to the canonical object and use IdentityHashMap to assign
+ // unique numbers for each string.
+ int descIndex = describer.getDescriptionIndex(data.object);
+ VarInt.putVarInt(descIndex + 1, sink); // Add 1 to avoid encoding negative values.
+
+ // Save types using their ordinal() value
+ sink.put((byte) data.type.ordinal());
+
+ // Save aggregated data stats.
+ if (data.counts != null) {
+ for (int i = 0; i < TASK_COUNT; i++) {
+ if (data.counts[i] > 0) {
+ sink.put((byte) i); // aggregated type ordinal value
+ VarInt.putVarInt(data.counts[i], sink);
+ VarInt.putVarLong(data.durations[i], sink);
+ }
+ }
+ }
+
+ this.out.writeInt(sink.position());
+ this.out.write(sink.array(), 0, sink.position());
+ if (describer.isUnassigned(descIndex)) {
+ this.out.writeUTF(describer.getDescription(data.object));
+ }
+ }
+ this.out.flush();
+ } catch (IOException e) {
+ saveException = e;
+ clear();
+ try {
+ out.close();
+ } catch (IOException e2) {
+ // ignore it
+ }
+ }
+ }
+
+ private synchronized void clear() {
+ profileStartTime = 0L;
+ if (timer != null) {
+ timer.cancel();
+ timer = null;
+ }
+ taskStack = null;
+ taskQueue = null;
+ describer = null;
+
+ // Note that slowest task aggregator are not cleared here because clearing happens
+ // periodically over the course of a command invocation.
+ }
+
+ /**
+ * Unless --record_full_profiler_data is given we drop small tasks and add their time to the
+ * parents duration.
+ */
+ private boolean wasTaskSlowEnoughToRecord(ProfilerTask type, long duration) {
+ return (recordAllDurations || duration >= type.minDuration);
+ }
+
+ /**
+ * Adds task directly to the main queue bypassing task stack. Used for simple
+ * tasks that are known to not have any subtasks.
+ *
+ * @param startTime task start time (obtained through {@link Profiler#nanoTimeMaybe()})
+ * @param duration task duration
+ * @param type task type
+ * @param object object associated with that task. Can be String object that
+ * describes it.
+ */
+ private void logTask(long startTime, long duration, ProfilerTask type, Object object) {
+ Preconditions.checkNotNull(object);
+ Preconditions.checkState(startTime > 0, "startTime was " + startTime);
+ if (duration < 0) {
+ // See note in Clock#nanoTime, which is used by Profiler#nanoTimeMaybe.
+ duration = 0;
+ }
+
+ TaskData parent = taskStack.peek();
+ if (parent != null) {
+ parent.aggregateChild(type, duration);
+ }
+ if (wasTaskSlowEnoughToRecord(type, duration)) {
+ TaskData data = taskStack.create(startTime, type, object);
+ data.duration = duration;
+ if (out != null) {
+ taskQueue.add(data);
+ }
+
+ SlowestTaskAggregator aggregator = slowestTasks[type.ordinal()];
+
+ if (aggregator != null) {
+ aggregator.add(data);
+ }
+ }
+ }
+
+ /**
+ * Used externally to submit simple task (one that does not have any subtasks).
+ * Depending on the minDuration attribute of the task type, task may be
+ * just aggregated into the parent task and not stored directly.
+ *
+ * @param startTime task start time (obtained through {@link
+ * Profiler#nanoTimeMaybe()})
+ * @param type task type
+ * @param object object associated with that task. Can be String object that
+ * describes it.
+ */
+ public void logSimpleTask(long startTime, ProfilerTask type, Object object) {
+ if (isActive() && isProfiling(type)) {
+ logTask(startTime, clock.nanoTime() - startTime, type, object);
+ }
+ }
+
+ /**
+ * Used externally to submit simple task (one that does not have any
+ * subtasks). Depending on the minDuration attribute of the task type, task
+ * may be just aggregated into the parent task and not stored directly.
+ *
+ * <p>Note that start and stop time must both be acquired from the same clock
+ * instance.
+ *
+ * @param startTime task start time
+ * @param stopTime task stop time
+ * @param type task type
+ * @param object object associated with that task. Can be String object that
+ * describes it.
+ */
+ public void logSimpleTask(long startTime, long stopTime, ProfilerTask type, Object object) {
+ if (isActive() && isProfiling(type)) {
+ logTask(startTime, stopTime - startTime, type, object);
+ }
+ }
+
+ /**
+ * Used externally to submit simple task (one that does not have any
+ * subtasks). Depending on the minDuration attribute of the task type, task
+ * may be just aggregated into the parent task and not stored directly.
+ *
+ * @param startTime task start time (obtained through {@link
+ * Profiler#nanoTimeMaybe()})
+ * @param duration the duration of the task
+ * @param type task type
+ * @param object object associated with that task. Can be String object that
+ * describes it.
+ */
+ public void logSimpleTaskDuration(long startTime, long duration, ProfilerTask type,
+ Object object) {
+ if (isActive() && isProfiling(type)) {
+ logTask(startTime, duration, type, object);
+ }
+ }
+
+ /**
+ * Used to log "events" - tasks with zero duration.
+ */
+ public void logEvent(ProfilerTask type, Object object) {
+ if (isActive() && isProfiling(type)) {
+ logTask(clock.nanoTime(), 0, type, object);
+ }
+ }
+
+ /**
+ * Records the beginning of the task specified by the parameters. This method
+ * should always be followed by completeTask() invocation to mark the end of
+ * task execution (usually ensured by try {} finally {} block). Failure to do
+ * so will result in task stack corruption.
+ *
+ * Use of this method allows to support nested task monitoring. For tasks that
+ * are known to not have any subtasks, logSimpleTask() should be used instead.
+ *
+ * @param type predefined task type - see ProfilerTask for available types.
+ * @param object object associated with that task. Can be String object that
+ * describes it.
+ */
+ public void startTask(ProfilerTask type, Object object) {
+ // ProfilerInfo.allTasksById is supposed to be an id -> Task map, but it is in fact a List,
+ // which means that we cannot drop tasks to which we had already assigned ids. Therefore,
+ // non-leaf tasks must not have a minimum duration. However, we don't quite consistently
+ // enforce this, and Blaze only works because we happen not to add child tasks to those parent
+ // tasks that have a minimum duration.
+ Preconditions.checkNotNull(object);
+ if (isActive() && isProfiling(type)) {
+ taskStack.push(type, object);
+ }
+ }
+
+ /**
+ * Records the end of the task and moves tasks from the thread-local stack to
+ * the main queue. Will validate that given task type matches task at the top
+ * of the stack.
+ *
+ * @param type task type.
+ */
+ public void completeTask(ProfilerTask type) {
+ if (isActive() && isProfiling(type)) {
+ long endTime = clock.nanoTime();
+ TaskData data = taskStack.pop();
+ // Do not use Preconditions class below due to the very expensive
+ // toString() calls used in the message.
+ if (data.type != type) {
+ throw new IllegalStateException("Inconsistent Profiler.completeTask() call for the "
+ + type + " task.\n " + taskStack);
+ }
+ data.duration = endTime - data.startTime;
+ if (data.parentId > 0) {
+ taskStack.peek().aggregateChild(data.type, data.duration);
+ }
+ boolean shouldRecordTask = wasTaskSlowEnoughToRecord(type, data.duration);
+ if (out != null && (shouldRecordTask || data.counts != null)) {
+ taskQueue.add(data);
+ }
+
+ if (shouldRecordTask) {
+ SlowestTaskAggregator aggregator = slowestTasks[type.ordinal()];
+
+ if (aggregator != null) {
+ aggregator.add(data);
+ }
+ }
+ }
+ }
+
+ /**
+ * Convenience method to log phase marker tasks.
+ */
+ public void markPhase(ProfilePhase phase) {
+ MemoryProfiler.instance().markPhase(phase);
+ if (isActive() && isProfiling(ProfilerTask.PHASE)) {
+ Preconditions.checkState(taskStack.isEmpty(), "Phase tasks must not be nested");
+ logEvent(ProfilerTask.PHASE, phase.description);
+ }
+ }
+
+ /**
+ * Convenience method to log spawn tasks.
+ *
+ * TODO(bazel-team): Right now method expects single string of the spawn action
+ * as task description (usually either argv[0] or a name of the main executable
+ * in case of complex shell commands). Maybe it should accept Command object
+ * and create more user friendly description.
+ */
+ public void logSpawn(long startTime, String arg0) {
+ if (isActive() && isProfiling(ProfilerTask.SPAWN)) {
+ logTask(startTime, clock.nanoTime() - startTime, ProfilerTask.SPAWN, arg0);
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
new file mode 100644
index 0000000..d06626c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
@@ -0,0 +1,101 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.profiler;
+
+/**
+ * All possible types of profiler tasks. Each type also defines description and
+ * minimum duration in nanoseconds for it to be recorded as separate event and
+ * not just be aggregated into the parent event.
+ */
+public enum ProfilerTask {
+ /* WARNING:
+ * Add new Tasks at the end (before Unknown) to not break the profiles that people have created!
+ * The profile file format uses the ordinal() of this enumeration to identify the task.
+ */
+ PHASE("build phase marker", -1, 0x336699, 0),
+ ACTION("action processing", -1, 0x666699, 0),
+ ACTION_BUILDER("parallel builder completion queue", -1, 0xCC3399, 0),
+ ACTION_SUBMIT("execution queue submission", -1, 0xCC3399, 0),
+ ACTION_CHECK("action dependency checking", 10000000, 0x999933, 0),
+ ACTION_EXECUTE("action execution", -1, 0x99CCFF, 0),
+ ACTION_LOCK("action resource lock", 10000000, 0xCC9933, 0),
+ ACTION_RELEASE("action resource release", 10000000, 0x006666, 0),
+ ACTION_GRAPH("action graph dependency", -1, 0x3399FF, 0),
+ ACTION_UPDATE("update action information", 10000000, 0x993300, 0),
+ ACTION_COMPLETE("complete action execution", -1, 0xCCCC99, 0),
+ INFO("general information", -1, 0x000066, 0),
+ EXCEPTION("exception", -1, 0xFFCC66, 0),
+ CREATE_PACKAGE("package creation", -1, 0x6699CC, 0),
+ PACKAGE_VALIDITY_CHECK("package validity check", -1, 0x336699, 0),
+ SPAWN("local process spawn", -1, 0x663366, 0),
+ REMOTE_EXECUTION("remote action execution", -1, 0x9999CC, 0),
+ LOCAL_EXECUTION("local action execution", -1, 0xCCCCCC, 0),
+ SCANNER("include scanner", -1, 0x669999, 0),
+ // 30 is a good number because the slowest items are stored in a heap, with temporarily
+ // one more element, and with 31 items, a heap becomes a complete binary tree
+ LOCAL_PARSE("Local parse to prepare for remote execution", 50000000, 0x6699CC, 30),
+ UPLOAD_TIME("Remote execution upload time", 50000000, 0x6699CC, 0),
+ PROCESS_TIME("Remote execution process wall time", 50000000, 0xF999CC, 0),
+ REMOTE_QUEUE("Remote execution queuing time", 50000000, 0xCC6600, 0),
+ REMOTE_SETUP("Remote execution setup", 50000000, 0xA999CC, 0),
+ FETCH("Remote execution file fetching", 50000000, 0xBB99CC, 0),
+ VFS_STAT("VFS stat", 10000000, 0x9999FF, 30),
+ VFS_DIR("VFS readdir", 10000000, 0x0066CC, 30),
+ VFS_LINK("VFS readlink", 10000000, 0x99CCCC, 30),
+ VFS_MD5("VFS md5", 10000000, 0x999999, 30),
+ VFS_XATTR("VFS xattr", 10000000, 0x9999DD, 30),
+ VFS_DELETE("VFS delete", 10000000, 0xFFCC00, 0),
+ VFS_OPEN("VFS open", 10000000, 0x009999, 30),
+ VFS_READ("VFS read", 10000000, 0x99CC33, 30),
+ VFS_WRITE("VFS write", 10000000, 0xFF9900, 30),
+ VFS_GLOB("globbing", -1, 0x999966, 30),
+ VFS_VMFS_STAT("VMFS stat", 10000000, 0x9999FF, 0),
+ VFS_VMFS_DIR("VMFS readdir", 10000000, 0x0066CC, 0),
+ VFS_VMFS_READ("VMFS read", 10000000, 0x99CC33, 0),
+ WAIT("thread wait", 5000000, 0x66CCCC, 0),
+ CONFIGURED_TARGET("configured target creation", -1, 0x663300, 0),
+ TRANSITIVE_CLOSURE("transitive closure creation", -1, 0x996600, 0),
+ TEST("for testing only", -1, 0x000000, 0),
+ SKYFRAME_EVAL("skyframe evaluator", -1, 0xCC9900, 0),
+ SKYFUNCTION("skyfunction", -1, 0xCC6600, 0),
+ CRITICAL_PATH("critical path", -1, 0x666699, 0),
+ CRITICAL_PATH_COMPONENT("critical path component", -1, 0x666699, 0),
+ IDE_BUILD_INFO("ide_build_info", -1, 0xCC6633, 0),
+ UNKNOWN("Unknown event", -1, 0x339966, 0);
+
+ // Size of the ProfilerTask value space.
+ public static final int TASK_COUNT = ProfilerTask.values().length;
+
+ /** Human readable description for the task. */
+ public final String description;
+ /** Threshold for skipping tasks in the profile in nanoseconds, unless --record_full_profiler_data
+ * is used */
+ public final long minDuration;
+ /** Default color of the task, when rendered in a chart. */
+ public final int color;
+ /** How many of the slowest instances to keep. If 0, no slowest instance calculation is done. */
+ public final int slowestInstancesCount;
+
+ ProfilerTask(String description, long minDuration, int color, int slowestInstanceCount) {
+ this.description = description;
+ this.minDuration = minDuration;
+ this.color = color;
+ this.slowestInstancesCount = slowestInstanceCount;
+ }
+
+ /** Whether the Profiler collects the slowest instances of this task. */
+ public boolean collectsSlowestInstances() {
+ return slowestInstancesCount > 0;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/AggregatingChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/AggregatingChartCreator.java
new file mode 100644
index 0000000..469e605
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/AggregatingChartCreator.java
@@ -0,0 +1,161 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.devtools.build.lib.profiler.ProfileInfo;
+import com.google.devtools.build.lib.profiler.ProfileInfo.Task;
+import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Implementation of {@link ChartCreator} that creates Gantt Charts that try to
+ * minimize the number of bars while preserving as much information about the
+ * execution of actions as possible.
+ *
+ * <p>Profiler tasks are categorized into four categories:
+ * <ul>
+ * <li>Actions: Actions executed.
+ * <li>Blaze Internal: This category contains internal blaze tasks, like loading
+ * packages, saving the action cache etc.
+ * <li>Locks: Contains tasks that indicate that a thread is waiting for
+ * resources.
+ * <li>VFS: Contains tasks that access the file system.
+ * </ul>
+ */
+public class AggregatingChartCreator implements ChartCreator {
+
+ /** The tasks in the 'actions' category. */
+ private static final Set<ProfilerTask> ACTION_TASKS = EnumSet.of(ProfilerTask.ACTION,
+ ProfilerTask.ACTION_SUBMIT);
+
+ /** The tasks in the 'blaze internal' category. */
+ private static Set<ProfilerTask> BLAZE_TASKS =
+ EnumSet.of(ProfilerTask.CREATE_PACKAGE, ProfilerTask.PACKAGE_VALIDITY_CHECK,
+ ProfilerTask.CONFIGURED_TARGET, ProfilerTask.TRANSITIVE_CLOSURE,
+ ProfilerTask.EXCEPTION, ProfilerTask.INFO, ProfilerTask.UNKNOWN);
+
+ /** The tasks in the 'locks' category. */
+ private static Set<ProfilerTask> LOCK_TASKS =
+ EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT);
+
+ /** The tasks in the 'VFS' category. */
+ private static Set<ProfilerTask> VFS_TASKS =
+ EnumSet.of(ProfilerTask.VFS_STAT, ProfilerTask.VFS_DIR, ProfilerTask.VFS_LINK,
+ ProfilerTask.VFS_MD5, ProfilerTask.VFS_DELETE, ProfilerTask.VFS_OPEN,
+ ProfilerTask.VFS_READ, ProfilerTask.VFS_WRITE, ProfilerTask.VFS_GLOB,
+ ProfilerTask.VFS_XATTR);
+
+ /** The data of the profiled build. */
+ private final ProfileInfo info;
+
+ /**
+ * Statistics of the profiled build. This is expected to be a formatted
+ * string, ready to be printed out.
+ */
+ private final List<ProfilePhaseStatistics> statistics;
+
+ /** If true, VFS related information is added to the chart. */
+ private final boolean showVFS;
+
+ /** The type for bars of category 'blaze internal'. */
+ private ChartBarType blazeType;
+
+ /** The type for bars of category 'actions'. */
+ private ChartBarType actionType;
+
+ /** The type for bars of category 'locks'. */
+ private ChartBarType lockType;
+
+ /** The type for bars of category 'VFS'. */
+ private ChartBarType vfsType;
+
+ /**
+ * Creates the chart creator. The created {@link ChartCreator} does not add
+ * VFS related data to the generated chart.
+ *
+ * @param info the data of the profiled build
+ * @param statistics Statistics of the profiled build. This is expected to be
+ * a formatted string, ready to be printed out.
+ */
+ public AggregatingChartCreator(ProfileInfo info, List<ProfilePhaseStatistics> statistics) {
+ this(info, statistics, false);
+ }
+
+ /**
+ * Creates the chart creator.
+ *
+ * @param info the data of the profiled build
+ * @param statistics Statistics of the profiled build. This is expected to be
+ * a formatted string, ready to be printed out.
+ * @param showVFS if true, VFS related information is added to the chart
+ */
+ public AggregatingChartCreator(ProfileInfo info, List<ProfilePhaseStatistics> statistics,
+ boolean showVFS) {
+ this.info = info;
+ this.statistics = statistics;
+ this.showVFS = showVFS;
+ }
+
+ @Override
+ public Chart create() {
+ Chart chart = new Chart(info.comment, statistics);
+ CommonChartCreator.createCommonChartItems(chart, info);
+ createTypes(chart);
+
+ for (ProfileInfo.Task task : info.allTasksById) {
+ if (ACTION_TASKS.contains(task.type)) {
+ createBar(chart, task, actionType);
+ } else if (LOCK_TASKS.contains(task.type)) {
+ createBar(chart, task, lockType);
+ } else if (BLAZE_TASKS.contains(task.type)) {
+ createBar(chart, task, blazeType);
+ } else if (showVFS && VFS_TASKS.contains(task.type)) {
+ createBar(chart, task, vfsType);
+ }
+ }
+
+ return chart;
+ }
+
+ /**
+ * Creates a bar and adds it to the chart.
+ *
+ * @param chart the chart to add the types to
+ * @param task the profiler task from which the bar is created
+ * @param type the type of the bar
+ */
+ private void createBar(Chart chart, Task task, ChartBarType type) {
+ String label = task.type.description + ": " + task.getDescription();
+ chart.addBar(task.threadId, task.startTime, task.startTime + task.duration, type, label);
+ }
+
+ /**
+ * Creates the {@link ChartBarType}s and adds them to the chart.
+ *
+ * @param chart the chart to add the types to
+ */
+ private void createTypes(Chart chart) {
+ actionType = chart.createType("Action processing", new Color(0x000099));
+ blazeType = chart.createType("Blaze internal processing", new Color(0x999999));
+ lockType = chart.createType("Waiting for resources", new Color(0x990000));
+ if (showVFS) {
+ vfsType = chart.createType("File system access", new Color(0x009900));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/Chart.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/Chart.java
new file mode 100644
index 0000000..93c7c81
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/Chart.java
@@ -0,0 +1,233 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Data of a Gantt Chart to visualize the data of a profiled build.
+ */
+public class Chart {
+
+ /** The type that is returned when an unknown type is looked up. */
+ public static final ChartBarType UNKNOWN_TYPE = new ChartBarType("Unknown type", Color.RED);
+
+ /** The title of the chart. */
+ private final String title;
+
+ /** Statistics of the profiled build. */
+ private final List<ProfilePhaseStatistics> statistics;
+
+ /** The rows of the chart. */
+ private final Map<Long, ChartRow> rows = new HashMap<>();
+
+ /** The columns on the chart. */
+ private final List<ChartColumn> columns = new ArrayList<>();
+
+ /** The lines on the chart. */
+ private final List<ChartLine> lines = new ArrayList<>();
+
+ /** The types of the bars in the chart. */
+ private final Map<String, ChartBarType> types = new HashMap<>();
+
+ /** The running index of the rows in the chart. */
+ private int rowIndex = 0;
+
+ /** The maximum stop value of any bar in the chart. */
+ private long maxStop;
+
+ /**
+ * Creates a chart.
+ *
+ * @param title the title of the chart
+ * @param statistics Statistics of the profiled build. This is expected to be
+ * a formatted string, ready to be printed out.
+ */
+ public Chart(String title, List<ProfilePhaseStatistics> statistics) {
+ Preconditions.checkNotNull(title);
+ Preconditions.checkNotNull(statistics);
+ this.title = title;
+ this.statistics = statistics;
+ }
+
+ /**
+ * Adds a bar to a row of the chart. If a row with the given id already
+ * exists, the bar is added to the row, otherwise a new row is created and the
+ * bar is added to it.
+ *
+ * @param id the id of the row the new bar belongs to
+ * @param start the start value of the bar
+ * @param stop the stop value of the bar
+ * @param type the type of the bar
+ * @param highlight emphasize the bar
+ * @param label the label of the bar
+ */
+ public void addBar(long id, long start, long stop, ChartBarType type, boolean highlight,
+ String label) {
+ ChartRow slot = addSlotIfAbsent(id);
+ ChartBar bar = new ChartBar(slot, start, stop, type, highlight, label);
+ slot.addBar(bar);
+ maxStop = Math.max(maxStop, stop);
+ }
+
+ /**
+ * Adds a bar to a row of the chart. If a row with the given id already
+ * exists, the bar is added to the row, otherwise a new row is created and the
+ * bar is added to it.
+ *
+ * @param id the id of the row the new bar belongs to
+ * @param start the start value of the bar
+ * @param stop the stop value of the bar
+ * @param type the type of the bar
+ * @param label the label of the bar
+ */
+ public void addBar(long id, long start, long stop, ChartBarType type, String label) {
+ addBar(id, start, stop, type, false, label);
+ }
+
+ /**
+ * Adds a vertical line to the chart.
+ */
+ public void addVerticalLine(long startId, long stopId, long pos) {
+ ChartRow startSlot = addSlotIfAbsent(startId);
+ ChartRow stopSlot = addSlotIfAbsent(stopId);
+ ChartLine line = new ChartLine(startSlot, stopSlot, pos, pos);
+ lines.add(line);
+ }
+
+ /**
+ * Adds a column to the chart.
+ *
+ * @param start the start value of the bar
+ * @param stop the stop value of the bar
+ * @param type the type of the bar
+ * @param label the label of the bar
+ */
+ public void addTimeRange(long start, long stop, ChartBarType type, String label) {
+ ChartColumn column = new ChartColumn(start, stop, type, label);
+ columns.add(column);
+ maxStop = Math.max(maxStop, stop);
+ }
+
+ /**
+ * Creates a new {@link ChartBarType} and adds it to the list of types of the
+ * chart.
+ *
+ * @param name the name of the type
+ * @param color the color of the chart
+ * @return the newly created type
+ */
+ public ChartBarType createType(String name, Color color) {
+ ChartBarType type = new ChartBarType(name, color);
+ types.put(name, type);
+ return type;
+ }
+
+ /**
+ * Returns the type with the given name. If no type with the given name
+ * exists, a type with name 'Unknown type' is added to the chart and returned.
+ *
+ * @param name the name of the type to look up
+ */
+ public ChartBarType lookUpType(String name) {
+ ChartBarType type = types.get(name);
+ if (type == null) {
+ type = UNKNOWN_TYPE;
+ types.put(type.getName(), type);
+ }
+ return type;
+ }
+
+ /**
+ * Creates a new row with the given id if no row with this id existed.
+ * Otherwise the existing row with the given id is returned.
+ *
+ * @param id the ID of the row
+ * @return the existing row, if it was already present, the newly created one
+ * otherwise
+ */
+ private ChartRow addSlotIfAbsent(long id) {
+ ChartRow slot = rows.get(id);
+ if (slot == null) {
+ slot = new ChartRow(Long.toString(id), rowIndex++);
+ rows.put(id, slot);
+ }
+ return slot;
+ }
+
+ /**
+ * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(Chart)},
+ * delegates the visitor to the rows of the chart and calls
+ * {@link ChartVisitor#endVisit(Chart)}.
+ *
+ * @param visitor the visitor to accept
+ */
+ public void accept(ChartVisitor visitor) {
+ visitor.visit(this);
+ for (ChartRow slot : rows.values()) {
+ slot.accept(visitor);
+ }
+ int rowCount = getRowCount();
+ for (ChartColumn column : columns) {
+ column.setRowCount(rowCount);
+ column.accept(visitor);
+ }
+ for (ChartLine line : lines) {
+ line.accept(visitor);
+ }
+ visitor.endVisit(this);
+ }
+
+ /**
+ * Returns the {@link ChartBarType}s, sorted by name.
+ */
+ public List<ChartBarType> getSortedTypes() {
+ List<ChartBarType> list = new ArrayList<>(types.values());
+ Collections.sort(list);
+ return list;
+ }
+
+ /**
+ * Returns the {@link ChartRow}s, sorted by their index.
+ */
+ public List<ChartRow> getSortedRows() {
+ List<ChartRow> list = new ArrayList<>(rows.values());
+ Collections.sort(list);
+ return list;
+ }
+
+ public String getTitle() {
+ return title;
+ }
+
+ public List<ProfilePhaseStatistics> getStatistics() {
+ return statistics;
+ }
+
+ public int getRowCount() {
+ return rows.size();
+ }
+
+ public long getMaxStop() {
+ return maxStop;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBar.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBar.java
new file mode 100644
index 0000000..20d92f0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBar.java
@@ -0,0 +1,106 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A bar in a row of a Gantt Chart.
+ */
+public class ChartBar {
+
+ /**
+ * The start value of the bar. This value has no unit. The interpretation of
+ * the value is up to the user of the class.
+ */
+ private final long start;
+
+ /**
+ * The stop value of the bar. This value has no unit. The interpretation of
+ * the value is up to the user of the class.
+ */
+ private final long stop;
+
+ /** The type of the bar. */
+ private final ChartBarType type;
+
+ /** Emphasize the bar */
+ private boolean highlight;
+
+ /** The label of the bar. */
+ private final String label;
+
+ /** The chart row this bar belongs to. */
+ private final ChartRow row;
+
+ /**
+ * Creates a chart bar.
+ *
+ * @param row the chart row this bar belongs to
+ * @param start the start value of the bar
+ * @param stop the stop value of the bar
+ * @param type the type of the bar
+ * @param label the label of the bar
+ */
+ public ChartBar(ChartRow row, long start, long stop, ChartBarType type, boolean highlight,
+ String label) {
+ Preconditions.checkNotNull(row);
+ Preconditions.checkNotNull(type);
+ Preconditions.checkNotNull(label);
+ this.row = row;
+ this.start = start;
+ this.stop = stop;
+ this.type = type;
+ this.highlight = highlight;
+ this.label = label;
+ }
+
+ /**
+ * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartBar)}.
+ *
+ * @param visitor the visitor to accept
+ */
+ public void accept(ChartVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getStop() {
+ return stop;
+ }
+
+ public long getWidth() {
+ return stop - start;
+ }
+
+ public ChartBarType getType() {
+ return type;
+ }
+
+ public boolean getHighlight() {
+ return highlight;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public ChartRow getRow() {
+ return row;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBarType.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBarType.java
new file mode 100644
index 0000000..e0f3885
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBarType.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * The type of a bar in a Gantt Chart. A type consists of a name and a color.
+ * Types are used to create the legend of a Gantt Chart.
+ */
+public class ChartBarType implements Comparable<ChartBarType> {
+
+ /** The name of the type. */
+ private final String name;
+
+ /** The color of the type. */
+ private final Color color;
+
+ /**
+ * Creates a {@link ChartBarType}.
+ *
+ * @param name the name of the type
+ * @param color the color of the type
+ */
+ public ChartBarType(String name, Color color) {
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(color);
+ this.name = name;
+ this.color = color;
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Equality of two types is defined by the equality of their names.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ return name.equals(((ChartBarType) obj).name);
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Compares types by their names.
+ */
+ @Override
+ public int compareTo(ChartBarType o) {
+ return name.compareTo(o.name);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Color getColor() {
+ return color;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartColumn.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartColumn.java
new file mode 100644
index 0000000..25fffe8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartColumn.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A chart column. The column can be used to highlight a time-range.
+ */
+public class ChartColumn {
+
+ /**
+ * The start value of the bar. This value has no unit. The interpretation of
+ * the value is up to the user of the class.
+ */
+ private final long start;
+
+ /**
+ * The stop value of the bar. This value has no unit. The interpretation of
+ * the value is up to the user of the class.
+ */
+ private final long stop;
+
+ /** The type of the bar. */
+ private final ChartBarType type;
+
+ /** The label of the bar. */
+ private final String label;
+
+ private int rowCount;
+
+ /**
+ * Creates a chart column.
+ *
+ * @param start the start value of the bar
+ * @param stop the stop value of the bar
+ * @param type the type of the bar
+ * @param label the label of the bar
+ */
+ public ChartColumn(long start, long stop, ChartBarType type, String label) {
+ Preconditions.checkNotNull(type);
+ Preconditions.checkNotNull(label);
+ this.start = start;
+ this.stop = stop;
+ this.type = type;
+ this.label = label;
+ }
+
+ /**
+ * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartBar)}.
+ *
+ * @param visitor the visitor to accept
+ */
+ public void accept(ChartVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getWidth() {
+ return stop - start;
+ }
+
+ public ChartBarType getType() {
+ return type;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ public void setRowCount(int rowCount) {
+ this.rowCount = rowCount;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartCreator.java
new file mode 100644
index 0000000..31781c8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartCreator.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.devtools.build.lib.profiler.chart.Chart;
+
+/**
+ * Interface for classes that are capable of creating {@link Chart}s.
+ */
+public interface ChartCreator {
+
+ /**
+ * Creates a {@link Chart}.
+ */
+ Chart create();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartLine.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartLine.java
new file mode 100644
index 0000000..d9bd755
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartLine.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A chart line. Such lines can be used to connect boxes.
+ */
+public class ChartLine {
+ private final ChartRow startRow;
+ private final ChartRow stopRow;
+ private final long startTime;
+ /**
+ * Creates a chart line.
+ *
+ * @param startRow the start row
+ * @param stopRow the end row
+ * @param startTime the start time
+ * @param stopTime the end time
+ */
+ public ChartLine(ChartRow startRow, ChartRow stopRow, long startTime, long stopTime) {
+ Preconditions.checkNotNull(startRow);
+ Preconditions.checkNotNull(stopRow);
+ this.startRow = startRow;
+ this.stopRow = stopRow;
+ this.startTime = startTime;
+ }
+
+ /**
+ * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartBar)}.
+ *
+ * @param visitor the visitor to accept
+ */
+ public void accept(ChartVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ public ChartRow getStartRow() {
+ return startRow;
+ }
+
+ public ChartRow getStopRow() {
+ return stopRow;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartRow.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartRow.java
new file mode 100644
index 0000000..96597c9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartRow.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.common.base.Preconditions;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A row of a Gantt Chart. A chart row is identified by its id and has an index that
+ * determines its location in the chart.
+ */
+public class ChartRow implements Comparable<ChartRow> {
+
+ /** The unique id of this row. */
+ private final String id;
+
+ /** The index, i.e., the row number of the row in the chart. */
+ private final int index;
+
+ /** The list of bars in this row. */
+ private final List<ChartBar> bars = new ArrayList<>();
+
+ /**
+ * Creates a chart row.
+ *
+ * @param id the unique id of this row
+ * @param index the index, i.e., the row number, of the row in the chart
+ */
+ public ChartRow(String id, int index) {
+ Preconditions.checkNotNull(id);
+ this.id = id;
+ this.index = index;
+ }
+
+ /**
+ * Adds a bar to the chart row.
+ *
+ * @param bar the {@link ChartBar} to add
+ */
+ public void addBar(ChartBar bar) {
+ bars.add(bar);
+ }
+
+ /**
+ * Returns the bars of the row as an unmodifieable list.
+ */
+ public List<ChartBar> getBars() {
+ return Collections.unmodifiableList(bars);
+ }
+
+ /**
+ * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartRow)}
+ * and delegates the visitor to the bars of the chart row.
+ *
+ * @param visitor the visitor to accept
+ */
+ public void accept(ChartVisitor visitor) {
+ visitor.visit(this);
+ for (ChartBar bar : bars) {
+ bar.accept(visitor);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Compares to rows by their index.
+ */
+ @Override
+ public int compareTo(ChartRow other) {
+ return index - other.index;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public String getId() {
+ return id;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartVisitor.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartVisitor.java
new file mode 100644
index 0000000..b06b3ab
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartVisitor.java
@@ -0,0 +1,65 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+/**
+ * Visitor for {@link Chart} objects.
+ */
+public interface ChartVisitor {
+
+ /**
+ * Visits a {@link Chart} object before its children, i.e., rows and bars, are
+ * visited.
+ *
+ * @param chart the {@link Chart} to visit
+ */
+ void visit(Chart chart);
+
+ /**
+ * Visits a {@link Chart} object after its children, i.e., rows and bars, are
+ * visited.
+ *
+ * @param chart the {@link Chart} to visit
+ */
+ void endVisit(Chart chart);
+
+ /**
+ * Visits a {@link ChartRow} object.
+ *
+ * @param chartRow the {@link ChartRow} to visit
+ */
+ void visit(ChartRow chartRow);
+
+ /**
+ * Visits a {@link ChartBar} object.
+ *
+ * @param chartBar the {@link ChartBar} to visit
+ */
+ void visit(ChartBar chartBar);
+
+ /**
+ * Visits a {@link ChartColumn} object.
+ *
+ * @param chartColumn the {@link ChartColumn} to visit
+ */
+ void visit(ChartColumn chartColumn);
+
+ /**
+ * Visits a {@link ChartLine} object.
+ *
+ * @param chartLine the {@link ChartLine} to visit
+ */
+ void visit(ChartLine chartLine);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/Color.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/Color.java
new file mode 100644
index 0000000..d527093
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/Color.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+/**
+ * Represents a color in ARGB format, 8 bits per channel.
+ */
+public final class Color {
+ public static final Color RED = new Color(0xff0000);
+ public static final Color GREEN = new Color(0x00ff00);
+ public static final Color GRAY = new Color(0x808080);
+ public static final Color BLACK = new Color(0x000000);
+
+ private final int argb;
+
+ public Color(int rgb) {
+ this.argb = rgb | 0xff000000;
+ }
+
+ public Color(int argb, boolean hasAlpha) {
+ this.argb = argb;
+ }
+
+ public int getRed() {
+ return (argb >> 16) & 0xFF;
+ }
+
+ public int getGreen() {
+ return (argb >> 8) & 0xFF;
+ }
+
+ public int getBlue() {
+ return argb & 0xFF;
+ }
+
+ public int getAlpha() {
+ return (argb >> 24) & 0xFF;
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/CommonChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/CommonChartCreator.java
new file mode 100644
index 0000000..ed1da20
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/CommonChartCreator.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.devtools.build.lib.profiler.ProfileInfo;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+
+/**
+ * Provides some common functions for {@link ChartCreator}s.
+ */
+public final class CommonChartCreator {
+
+ static void createCommonChartItems(Chart chart, ProfileInfo info) {
+ createTypes(chart);
+
+ // add common info
+ for (ProfilePhase phase : ProfilePhase.values()) {
+ addColumn(chart,info,phase);
+ }
+ }
+
+ private static void addColumn(Chart chart, ProfileInfo info, ProfilePhase phase) {
+ ProfileInfo.Task task = info.getPhaseTask(phase);
+ if (task != null) {
+ String label = task.type.description + ": " + task.getDescription();
+ ChartBarType type = chart.lookUpType(task.getDescription());
+ long stop = task.startTime + info.getPhaseDuration(task);
+ chart.addTimeRange(task.startTime, stop, type, label);
+ }
+ }
+
+ /**
+ * Creates the {@link ChartBarType}s and adds them to the chart.
+ */
+ private static void createTypes(Chart chart) {
+ for (ProfilePhase phase : ProfilePhase.values()) {
+ chart.createType(phase.description, new Color(phase.color, true));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/DetailedChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/DetailedChartCreator.java
new file mode 100644
index 0000000..1e097c3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/DetailedChartCreator.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.devtools.build.lib.profiler.ProfileInfo;
+import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry;
+import com.google.devtools.build.lib.profiler.ProfileInfo.Task;
+import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+
+import java.util.EnumSet;
+import java.util.List;
+
+/**
+ * Implementation of {@link ChartCreator} that creates Gantt Charts that contain
+ * bars for all tasks in the profile.
+ */
+public class DetailedChartCreator implements ChartCreator {
+
+ /** The data of the profiled build. */
+ private final ProfileInfo info;
+
+ /**
+ * Statistics of the profiled build. This is expected to be a formatted
+ * string, ready to be printed out.
+ */
+ private final List<ProfilePhaseStatistics> statistics;
+
+ /**
+ * Creates the chart creator.
+ *
+ * @param info the data of the profiled build
+ * @param statistics Statistics of the profiled build. This is expected to be
+ * a formatted string, ready to be printed out.
+ */
+ public DetailedChartCreator(ProfileInfo info, List<ProfilePhaseStatistics> statistics) {
+ this.info = info;
+ this.statistics = statistics;
+ }
+
+ @Override
+ public Chart create() {
+ Chart chart = new Chart(info.comment, statistics);
+ CommonChartCreator.createCommonChartItems(chart, info);
+ createTypes(chart);
+
+ // calculate the critical path
+ EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class);
+ CriticalPathEntry criticalPath = info.getCriticalPath(typeFilter);
+ info.analyzeCriticalPath(typeFilter, criticalPath);
+
+ for (Task task : info.allTasksById) {
+ String label = task.type.description + ": " + task.getDescription();
+ ChartBarType type = chart.lookUpType(task.type.description);
+ long stop = task.startTime + task.duration;
+ CriticalPathEntry entry = null;
+
+ // for top level tasks, check if they are on the critical path
+ if (task.parentId == 0 && criticalPath != null) {
+ entry = info.getNextCriticalPathEntryForTask(criticalPath, task);
+ // find next top-level entry
+ if (entry != null) {
+ CriticalPathEntry nextEntry = entry.next;
+ while (nextEntry != null && nextEntry.task.parentId != 0) {
+ nextEntry = nextEntry.next;
+ }
+ if (nextEntry != null) {
+ // time is start and not stop as we traverse the critical back backwards
+ chart.addVerticalLine(task.threadId, nextEntry.task.threadId, task.startTime);
+ }
+ }
+ }
+
+ chart.addBar(task.threadId, task.startTime, stop, type, (entry != null), label);
+ }
+
+ return chart;
+ }
+
+ /**
+ * Creates a {@link ChartBarType} for every known {@link ProfilerTask} and
+ * adds it to the chart.
+ *
+ * @param chart the chart to add the types to
+ */
+ private void createTypes(Chart chart) {
+ for (ProfilerTask task : ProfilerTask.values()) {
+ chart.createType(task.description, new Color(task.color));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/HtmlChartVisitor.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/HtmlChartVisitor.java
new file mode 100644
index 0000000..8fa3d0e4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/HtmlChartVisitor.java
@@ -0,0 +1,368 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.profiler.chart;
+
+import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics;
+
+import java.io.PrintStream;
+import java.util.List;
+
+/**
+ * {@link ChartVisitor} that builds HTML from the visited chart and prints it
+ * out to the given {@link PrintStream}.
+ */
+public class HtmlChartVisitor implements ChartVisitor {
+
+ /** The default width of a second in the chart. */
+ private static final int DEFAULT_PIXEL_PER_SECOND = 50;
+
+ /** The horizontal offset of second zero. */
+ private static final int H_OFFSET = 40;
+
+ /** The font size of the row labels. */
+ private static final int ROW_LABEL_FONT_SIZE = 7;
+
+ /** The height of a bar in pixels. */
+ private static final int BAR_HEIGHT = 8;
+
+ /** The space between twp bars in pixels. */
+ private static final int BAR_SPACE = 2;
+
+ /** The height of a row. */
+ private static final int ROW_HEIGHT = BAR_HEIGHT + BAR_SPACE;
+
+ /** The {@link PrintStream} to output the HTML to. */
+ private final PrintStream out;
+
+ /** The maxmimum stop time of any bar in the chart. */
+ private long maxStop;
+
+ /** The width of a second in the chart. */
+ private final int pixelsPerSecond;
+
+ /**
+ * Creates the visitor, with a default width of a second of 50 pixels.
+ *
+ * @param out the {@link PrintStream} to output the HTML to
+ */
+ public HtmlChartVisitor(PrintStream out) {
+ this(out, DEFAULT_PIXEL_PER_SECOND);
+ }
+
+ /**
+ * Creates the visitor.
+ *
+ * @param out the {@link PrintStream} to output the HTML to
+ * @param pixelsPerSecond The width of a second in the chart. (In pixels)
+ */
+ public HtmlChartVisitor(PrintStream out, int pixelsPerSecond) {
+ this.out = out;
+ this.pixelsPerSecond = pixelsPerSecond;
+ }
+
+ @Override
+ public void visit(Chart chart) {
+ maxStop = chart.getMaxStop();
+ out.println("<html><head>");
+ out.printf("<title>%s</title>", chart.getTitle());
+ out.println("<style type=\"text/css\"><!--");
+
+ printCss(chart.getSortedTypes());
+
+ out.println("--></style>");
+ out.println("</head>");
+ out.println("<body>");
+
+ heading(chart.getTitle(), 1);
+
+ printContentBox();
+
+ heading("Tasks", 2);
+ out.println("<p>To get more information about a task point the mouse at one of the bars.</p>");
+
+ out.printf("<div style='position:relative; height: %dpx; margin: %dpx'>\n",
+ chart.getRowCount() * ROW_HEIGHT, H_OFFSET + 10);
+ }
+
+ @Override
+ public void endVisit(Chart chart) {
+ printTimeAxis(chart);
+ out.println("</div>");
+
+ heading("Legend", 2);
+ printLegend(chart.getSortedTypes());
+
+ heading("Statistics", 2);
+ printStatistics(chart.getStatistics());
+
+ out.println("</body>");
+ out.println("</html>");
+}
+
+ @Override
+ public void visit(ChartColumn column) {
+ int width = scale(column.getWidth());
+ if (width == 0) {
+ return;
+ }
+ int left = scale(column.getStart());
+ int height = column.getRowCount() * ROW_HEIGHT;
+ String style = chartTypeNameAsCSSClass(column.getType().getName());
+ box(left, 0, width, height, style, column.getLabel(), 10);
+ }
+
+
+ @Override
+ public void visit(ChartRow slot) {
+ String style = slot.getIndex() % 2 == 0 ? "shade-even" : "shade-odd";
+ int top = slot.getIndex() * ROW_HEIGHT;
+ int width = scale(maxStop) + 1;
+
+ label(-H_OFFSET, top, width + H_OFFSET, ROW_HEIGHT, ROW_LABEL_FONT_SIZE, slot.getId());
+ box(0, top, width, ROW_HEIGHT, style, "", 0);
+ }
+
+ @Override
+ public void visit(ChartBar bar) {
+ int width = scale(bar.getWidth());
+ if (width == 0) {
+ return;
+ }
+ int left = scale(bar.getStart());
+ int top = bar.getRow().getIndex() * ROW_HEIGHT;
+ String style = chartTypeNameAsCSSClass(bar.getType().getName());
+ if (bar.getHighlight()) {
+ style += "-highlight";
+ }
+ box(left, top + 2, width, BAR_HEIGHT, style, bar.getLabel(), 20);
+ }
+
+ @Override
+ public void visit(ChartLine chartLine) {
+ int start = chartLine.getStartRow().getIndex() * ROW_HEIGHT;
+ int stop = chartLine.getStopRow().getIndex() * ROW_HEIGHT;
+ int time = scale(chartLine.getStartTime());
+
+ if (start < stop) {
+ verticalLine(time, start + 1, 1, (stop - start) + ROW_HEIGHT, Color.RED);
+ } else {
+ verticalLine(time, stop + 1, 1, (start - stop) + ROW_HEIGHT, Color.RED);
+ }
+ }
+
+ /**
+ * Converts the given value from the bar of the chart to pixels.
+ */
+ private int scale(long value) {
+ return (int) (value / (1000000000L / pixelsPerSecond));
+ }
+
+ /**
+ * Prints a box with links to the sections of the generated HTML document.
+ */
+ private void printContentBox() {
+ out.println("<div style='position:fixed; top:1em; right:1em; z-index:50; padding: 1ex;"
+ + "border:1px solid #888; background-color:#eee; width:100px'><h3>Content</h3>");
+ out.println("<p style='text-align:left;font-size:small;margin:2px'>"
+ + "<a href='#Tasks'>Tasks</a></p>");
+ out.println("<p style='text-align:left;font-size:small;margin:2px'>"
+ + "<a href='#Legend'>Legend</a></p>");
+ out.println("<p style='text-align:left;font-size:small;margin:2px'>"
+ + "<a href='#Statistics'>Statistics</a></p></div>");
+ }
+
+ /**
+ * Prints the time axis of the chart and vertical lines for every second.
+ */
+ private void printTimeAxis(Chart chart) {
+ int location = 0;
+ int second = 0;
+ int end = scale(chart.getMaxStop());
+ while (location < end) {
+ label(location + 4, -17, pixelsPerSecond, ROW_HEIGHT, 0, second + "s");
+ verticalLine(location, -20, 1, chart.getRowCount() * ROW_HEIGHT + 20, Color.GRAY);
+ location += pixelsPerSecond;
+ second += 1;
+ }
+ }
+
+ private void printCss(List<ChartBarType> types) {
+ out.println("body { font-family: Sans; }");
+ out.printf("div.shade-even { position:absolute; border: 0px; background-color:#dddddd }\n");
+ out.printf("div.shade-odd { position:absolute; border: 0px; background-color:#eeeeee }\n");
+ for (ChartBarType type : types) {
+ String name = chartTypeNameAsCSSClass(type.getName());
+ String color = formatColor(type.getColor());
+
+ out.printf(
+ "div.%s-border { position:absolute; border:1px solid grey; background-color:%s }\n",
+ name, color);
+ out.printf(
+ "div.%s-highlight { position:absolute; border:1px solid red; background-color:%s }\n",
+ name, color);
+ out.printf("div.%s { position:absolute; border:0px; margin:1px; background-color:%s }\n",
+ name, color);
+ }
+ }
+
+ /**
+ * Prints the legend for the chart at the current position in the document. The
+ * legend is printed in columns of 10 rows each.
+ *
+ * @param types the list of {@link ChartBarType}s to print in the legend.
+ */
+ private void printLegend(List<ChartBarType> types) {
+ final int boxHeight = 20;
+ final int lineHeight = 25;
+ final int entriesPerColumn = 10;
+ final int legendWidth = 350;
+ int legendHeight;
+ if (types.size() / entriesPerColumn >= 1) {
+ legendHeight = entriesPerColumn;
+ } else {
+ legendHeight = types.size() % entriesPerColumn;
+ }
+
+ out.printf("<div style='position:relative; height: %dpx;'>",
+ (legendHeight + 1) * lineHeight);
+
+ int left = -legendWidth;
+ int top;
+ int i = 0;
+ for (ChartBarType type : types) {
+ if (i % entriesPerColumn == 0) {
+ left += legendWidth;
+ i = 0;
+ }
+ top = lineHeight * i;
+ String style = chartTypeNameAsCSSClass(type.getName()) + "-border";
+ box(left, top, boxHeight, boxHeight, style, type.getName(), 0);
+ label(left + lineHeight + 10, top, legendWidth - 10, boxHeight, 0, type.getName());
+ i++;
+ }
+ out.println("</div>");
+ }
+
+ private void printStatistics(List<ProfilePhaseStatistics> statistics) {
+ boolean first = true;
+
+ out.println("<table border=\"0\" width=\"100%\"><tr>");
+ for (ProfilePhaseStatistics stat : statistics) {
+ if (!first) {
+ out.println("<td><div style=\"width:20px;\"> </div></td>");
+ } else {
+ first = false;
+ }
+ out.println("<td valign=\"top\">");
+ String title = stat.getTitle();
+ if (title != "") {
+ heading(title, 3);
+ }
+ out.println("<pre>" + stat.getStatistics() + "</pre></td>");
+ }
+ out.println("</tr></table>");
+ }
+
+ /**
+ * Prints a head-line at the current position in the document.
+ *
+ * @param text the text to print
+ * @param level the headline level
+ */
+ private void heading(String text, int level) {
+ anchor(text);
+ out.printf("<h%d >%s</h%d>\n", level, text, level);
+ }
+
+ /**
+ * Prints a box with the given location, size, background color and border.
+ *
+ * @param x the x location of the top left corner of the box
+ * @param y the y location of the top left corner of the box
+ * @param width the width location of the box
+ * @param height the height location of the box
+ * @param style the CSS style class to use for the box
+ * @param title the text displayed when the mouse hovers over the box
+ */
+ private void box(int x, int y, int width, int height, String style, String title, int zIndex) {
+ out.printf("<div class=\"%s\" title=\"%s\" "
+ + "style=\"left:%dpx; top:%dpx; width:%dpx; height:%dpx; z-index:%d\"></div>\n",
+ style, title, x, y, width, height, zIndex);
+ }
+
+ /**
+ * Prints a label with the given location, size, background color and border.
+ *
+ * @param x the x location of the top left corner of the box
+ * @param y the y location of the top left corner of the box
+ * @param width the width location of the box
+ * @param height the height location of the box
+ * @param fontSize the font size of text in the box, 0 for default
+ * @param text the text displayed in the box
+ */
+ private void label(int x, int y, int width, int height, int fontSize, String text) {
+ if (fontSize > 0) {
+ out.printf("<div style=\"position:absolute; left:%dpx; top:%dpx; width:%dpx; "
+ + "height:%dpx; font-size:%dpt\">%s</div>\n",
+ x, y, width, height, fontSize, text);
+ } else {
+ out.printf("<div style=\"position:absolute; left:%dpx; top:%dpx; width:%dpx; "
+ + "height:%dpx\">%s</div>\n",
+ x, y, width, height, text);
+ }
+ }
+
+ /**
+ * Prints a vertical line of given width, height and color at the given
+ * location.
+ *
+ * @param x the x location of the start point of the line
+ * @param y the y location of the start point of the line
+ * @param width the width of the line
+ * @param length the length of the line
+ * @param color the color of the line
+ */
+ private void verticalLine(int x, int y, int width, int length, Color color) {
+ out.printf("<div style='position: absolute; left: %dpx; top: %dpx; width: %dpx; "
+ + "height: %dpx; border-left: %dpx solid %s'" + "></div>\n",
+ x, y, width, length, width, formatColor(color));
+ }
+
+ /**
+ * Prints an HTML anchor with the given name,
+ */
+ private void anchor(String name) {
+ out.println("<a name='" + name + "'/>");
+ }
+
+ /**
+ * Formats the given {@link Color} to a css style color string.
+ */
+ private String formatColor(Color color) {
+ int r = color.getRed();
+ int g = color.getGreen();
+ int b = color.getBlue();
+ int a = color.getAlpha();
+
+ return String.format("rgba(%d,%d,%d,%f)", r, g, b, (a / 255.0));
+ }
+
+ /**
+ * Transform the name into a form suitable as a css class.
+ */
+ private String chartTypeNameAsCSSClass(String name) {
+ return name.replace(' ', '_');
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java
new file mode 100644
index 0000000..d6c2992
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java
@@ -0,0 +1,633 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2;
+
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.events.ErrorSensingEventHandler;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.graph.Digraph;
+import com.google.devtools.build.lib.graph.Node;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.PackageProvider;
+import com.google.devtools.build.lib.pkgcache.TargetEdgeObserver;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.pkgcache.TargetProvider;
+import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment;
+import com.google.devtools.build.lib.query2.engine.QueryException;
+import com.google.devtools.build.lib.query2.engine.QueryExpression;
+import com.google.devtools.build.lib.query2.engine.SkyframeRestartQueryException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.BinaryPredicate;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The environment of a Blaze query. Not thread-safe.
+ */
+public class BlazeQueryEnvironment implements QueryEnvironment<Target> {
+ protected final ErrorSensingEventHandler eventHandler;
+ private final TargetProvider targetProvider;
+ private final TargetPatternEvaluator targetPatternEvaluator;
+ private final Digraph<Target> graph = new Digraph<>();
+ private final ErrorPrintingTargetEdgeErrorObserver errorObserver;
+ private final LabelVisitor labelVisitor;
+ private final Map<String, Set<Target>> letBindings = new HashMap<>();
+ private final Map<String, ResolvedTargets<Target>> resolvedTargetPatterns = new HashMap<>();
+ protected final boolean keepGoing;
+ private final boolean strictScope;
+ protected final int loadingPhaseThreads;
+
+ private final BinaryPredicate<Rule, Attribute> dependencyFilter;
+ private final Predicate<Label> labelFilter;
+
+ private final Set<Setting> settings;
+ private final List<QueryFunction> extraFunctions;
+ private final BlazeTargetAccessor accessor = new BlazeTargetAccessor();
+
+ /**
+ * Note that the correct operation of this class critically depends on the Reporter being a
+ * singleton object, shared by all cooperating classes contributing to Query.
+ * @param strictScope if true, fail the whole query if a label goes out of scope.
+ * @param loadingPhaseThreads the number of threads to use during loading
+ * the packages for the query.
+ * @param labelFilter a predicate that determines if a specific label is
+ * allowed to be visited during query execution. If it returns false,
+ * the query execution is stopped with an error message.
+ * @param settings a set of enabled settings
+ */
+ public BlazeQueryEnvironment(PackageProvider packageProvider,
+ TargetPatternEvaluator targetPatternEvaluator,
+ boolean keepGoing,
+ boolean strictScope,
+ int loadingPhaseThreads,
+ Predicate<Label> labelFilter,
+ EventHandler eventHandler,
+ Set<Setting> settings,
+ Iterable<QueryFunction> extraFunctions) {
+ this.eventHandler = new ErrorSensingEventHandler(eventHandler);
+ this.targetProvider = packageProvider;
+ this.targetPatternEvaluator = targetPatternEvaluator;
+ this.errorObserver = new ErrorPrintingTargetEdgeErrorObserver(this.eventHandler);
+ this.keepGoing = keepGoing;
+ this.strictScope = strictScope;
+ this.loadingPhaseThreads = loadingPhaseThreads;
+ this.dependencyFilter = constructDependencyFilter(settings);
+ this.labelVisitor = new LabelVisitor(packageProvider, dependencyFilter);
+ this.labelFilter = labelFilter;
+ this.settings = Sets.immutableEnumSet(settings);
+ this.extraFunctions = ImmutableList.copyOf(extraFunctions);
+ }
+
+ /**
+ * Note that the correct operation of this class critically depends on the Reporter being a
+ * singleton object, shared by all cooperating classes contributing to Query.
+ * @param loadingPhaseThreads the number of threads to use during loading
+ * the packages for the query.
+ * @param settings a set of enabled settings
+ */
+ public BlazeQueryEnvironment(PackageProvider packageProvider,
+ TargetPatternEvaluator targetPatternEvaluator,
+ boolean keepGoing,
+ int loadingPhaseThreads,
+ EventHandler eventHandler,
+ Set<Setting> settings,
+ Iterable<QueryFunction> extraFunctions) {
+ this(packageProvider, targetPatternEvaluator, keepGoing, /*strictScope=*/true,
+ loadingPhaseThreads, Rule.ALL_LABELS, eventHandler, settings, extraFunctions);
+ }
+
+ private static BinaryPredicate<Rule, Attribute> constructDependencyFilter(Set<Setting> settings) {
+ BinaryPredicate<Rule, Attribute> specifiedFilter =
+ settings.contains(Setting.NO_HOST_DEPS) ? Rule.NO_HOST_DEPS : Rule.ALL_DEPS;
+ if (settings.contains(Setting.NO_IMPLICIT_DEPS)) {
+ specifiedFilter = Rule.and(specifiedFilter, Rule.NO_IMPLICIT_DEPS);
+ }
+ if (settings.contains(Setting.NO_NODEP_DEPS)) {
+ specifiedFilter = Rule.and(specifiedFilter, Rule.NO_NODEP_ATTRIBUTES);
+ }
+ return specifiedFilter;
+ }
+
+ /**
+ * Evaluate the specified query expression in this environment.
+ *
+ * @return a {@link BlazeQueryEvalResult} object that contains the resulting set of targets, the
+ * partial graph, and a bit to indicate whether errors occured during evaluation; note that the
+ * success status can only be false if {@code --keep_going} was in effect
+ * @throws QueryException if the evaluation failed and {@code --nokeep_going} was in
+ * effect
+ */
+ public BlazeQueryEvalResult<Target> evaluateQuery(QueryExpression expr) throws QueryException {
+ // Some errors are reported as QueryExceptions and others as ERROR events
+ // (if --keep_going).
+ eventHandler.resetErrors();
+ resolvedTargetPatterns.clear();
+
+ // In the --nokeep_going case, errors are reported in the order in which the patterns are
+ // specified; using a linked hash set here makes sure that the left-most error is reported.
+ Set<String> targetPatternSet = new LinkedHashSet<>();
+ expr.collectTargetPatterns(targetPatternSet);
+ try {
+ resolvedTargetPatterns.putAll(preloadOrThrow(targetPatternSet));
+ } catch (TargetParsingException e) {
+ // Unfortunately, by evaluating the patterns in parallel, we lose some location information.
+ throw new QueryException(expr, e.getMessage());
+ }
+
+ Set<Target> resultNodes;
+ try {
+ resultNodes = expr.eval(this);
+ } catch (QueryException e) {
+ throw new QueryException(e, expr);
+ }
+
+ if (eventHandler.hasErrors()) {
+ if (!keepGoing) {
+ // This case represents loading-phase errors reported during evaluation
+ // of target patterns that don't cause evaluation to fail per se.
+ throw new QueryException("Evaluation of query \"" + expr
+ + "\" failed due to BUILD file errors");
+ } else {
+ eventHandler.handle(Event.warn("--keep_going specified, ignoring errors. "
+ + "Results may be inaccurate"));
+ }
+ }
+
+ return new BlazeQueryEvalResult<>(!eventHandler.hasErrors(), resultNodes, graph);
+ }
+
+ public BlazeQueryEvalResult<Target> evaluateQuery(String query) throws QueryException {
+ return evaluateQuery(QueryExpression.parse(query, this));
+ }
+
+ @Override
+ public void reportBuildFileError(QueryExpression caller, String message) throws QueryException {
+ if (!keepGoing) {
+ throw new QueryException(caller, message);
+ } else {
+ // Keep consistent with evaluateQuery() above.
+ eventHandler.handle(Event.error("Evaluation of query \"" + caller + "\" failed: " + message));
+ }
+ }
+
+ @Override
+ public Set<Target> getTargetsMatchingPattern(QueryExpression caller,
+ String pattern) throws QueryException {
+ // We can safely ignore the boolean error flag. The evaluateQuery() method above wraps the
+ // entire query computation in an error sensor.
+
+ Set<Target> targets = new LinkedHashSet<>(resolvedTargetPatterns.get(pattern).getTargets());
+
+ // Sets.filter would be more convenient here, but can't deal with exceptions.
+ Iterator<Target> targetIterator = targets.iterator();
+ while (targetIterator.hasNext()) {
+ Target target = targetIterator.next();
+ if (!validateScope(target.getLabel(), strictScope)) {
+ targetIterator.remove();
+ }
+ }
+
+ Set<PathFragment> packages = new HashSet<>();
+ for (Target target : targets) {
+ packages.add(target.getLabel().getPackageFragment());
+ }
+
+ Set<Target> result = new LinkedHashSet<>();
+ for (Target target : targets) {
+ result.add(getOrCreate(target));
+
+ // Preservation of graph order: it is important that targets obtained via
+ // a wildcard such as p:* are correctly ordered w.r.t. each other, so to
+ // ensure this, we add edges between any pair of directly connected
+ // targets in this set.
+ if (target instanceof OutputFile) {
+ OutputFile outputFile = (OutputFile) target;
+ if (targets.contains(outputFile.getGeneratingRule())) {
+ makeEdge(outputFile, outputFile.getGeneratingRule());
+ }
+ } else if (target instanceof Rule) {
+ Rule rule = (Rule) target;
+ for (Label label : rule.getLabels(dependencyFilter)) {
+ if (!packages.contains(label.getPackageFragment())) {
+ continue; // don't cause additional package loading
+ }
+ try {
+ if (!validateScope(label, strictScope)) {
+ continue; // Don't create edges to targets which are out of scope.
+ }
+ Target to = getTargetOrThrow(label);
+ if (targets.contains(to)) {
+ makeEdge(rule, to);
+ }
+ } catch (NoSuchThingException e) {
+ /* ignore */
+ } catch (InterruptedException e) {
+ throw new QueryException("interrupted");
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ public Node<Target> getTarget(Label label) throws TargetNotFoundException, QueryException {
+ // Can't use strictScope here because we are expecting a target back.
+ validateScope(label, true);
+ try {
+ return getNode(getTargetOrThrow(label));
+ } catch (NoSuchThingException e) {
+ throw new TargetNotFoundException(e);
+ } catch (InterruptedException e) {
+ throw new QueryException("interrupted");
+ }
+ }
+
+ private Node<Target> getNode(Target target) {
+ return graph.createNode(target);
+ }
+
+ private Collection<Node<Target>> getNodes(Iterable<Target> target) {
+ Set<Node<Target>> result = new LinkedHashSet<>();
+ for (Target t : target) {
+ result.add(getNode(t));
+ }
+ return result;
+ }
+
+ @Override
+ public Target getOrCreate(Target target) {
+ return getNode(target).getLabel();
+ }
+
+ @Override
+ public Collection<Target> getFwdDeps(Target target) {
+ return getTargetsFromNodes(getNode(target).getSuccessors());
+ }
+
+ @Override
+ public Collection<Target> getReverseDeps(Target target) {
+ return getTargetsFromNodes(getNode(target).getPredecessors());
+ }
+
+ @Override
+ public Set<Target> getTransitiveClosure(Set<Target> targetNodes) {
+ for (Target node : targetNodes) {
+ checkBuilt(node);
+ }
+ return getTargetsFromNodes(graph.getFwdReachable(getNodes(targetNodes)));
+ }
+
+ /**
+ * Checks that the graph rooted at 'targetNode' has been completely built;
+ * fails if not. Callers of {@link #getTransitiveClosure} must ensure that
+ * {@link #buildTransitiveClosure} has been called before.
+ *
+ * <p>It would be inefficient and failure-prone to make getTransitiveClosure
+ * call buildTransitiveClosure directly. Also, it would cause
+ * nondeterministic behavior of the operators, since the set of packages
+ * loaded (and hence errors reported) would depend on the ordering details of
+ * the query operators' implementations.
+ */
+ private void checkBuilt(Target targetNode) {
+ Preconditions.checkState(
+ labelVisitor.hasVisited(targetNode.getLabel()),
+ "getTransitiveClosure(%s) called without prior call to buildTransitiveClosure()",
+ targetNode);
+ }
+
+ protected void preloadTransitiveClosure(Set<Target> targets, int maxDepth) throws QueryException {
+ }
+
+ @Override
+ public void buildTransitiveClosure(QueryExpression caller,
+ Set<Target> targetNodes,
+ int maxDepth) throws QueryException {
+ Set<Target> targets = targetNodes;
+ preloadTransitiveClosure(targets, maxDepth);
+
+ try {
+ labelVisitor.syncWithVisitor(eventHandler, targets, keepGoing,
+ loadingPhaseThreads, maxDepth, errorObserver, new GraphBuildingObserver());
+ } catch (InterruptedException e) {
+ throw new QueryException(caller, "transitive closure computation was interrupted");
+ }
+
+ if (errorObserver.hasErrors()) {
+ reportBuildFileError(caller, "errors were encountered while computing transitive closure");
+ }
+ }
+
+ @Override
+ public Set<Target> getNodesOnPath(Target from, Target to) {
+ return getTargetsFromNodes(graph.getShortestPath(getNode(from), getNode(to)));
+ }
+
+ @Override
+ public Set<Target> getVariable(String name) {
+ return letBindings.get(name);
+ }
+
+ @Override
+ public Set<Target> setVariable(String name, Set<Target> value) {
+ return letBindings.put(name, value);
+ }
+
+ /**
+ * It suffices to synchronize the modifications of this.graph from within the
+ * GraphBuildingObserver, because that's the only concurrent part.
+ * Concurrency is always encapsulated within the evaluation of a single query
+ * operator (e.g. deps(), somepath(), etc).
+ */
+ private class GraphBuildingObserver implements TargetEdgeObserver {
+
+ @Override
+ public synchronized void edge(Target from, Attribute attribute, Target to) {
+ Preconditions.checkState(attribute == null ||
+ dependencyFilter.apply(((Rule) from), attribute),
+ "Disallowed edge from LabelVisitor: %s --> %s", from, to);
+ makeEdge(from, to);
+ }
+
+ @Override
+ public synchronized void node(Target node) {
+ graph.createNode(node);
+ }
+
+ @Override
+ public void missingEdge(Target target, Label to, NoSuchThingException e) {
+ // No - op.
+ }
+ }
+
+ private void makeEdge(Target from, Target to) {
+ graph.addEdge(from, to);
+ }
+
+ private boolean validateScope(Label label, boolean strict) throws QueryException {
+ if (!labelFilter.apply(label)) {
+ String error = String.format("target '%s' is not within the scope of the query", label);
+ if (strict) {
+ throw new QueryException(error);
+ } else {
+ eventHandler.handle(Event.warn(error + ". Skipping"));
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public Set<Target> evalTargetPattern(QueryExpression caller, String pattern)
+ throws QueryException {
+ if (!resolvedTargetPatterns.containsKey(pattern)) {
+ try {
+ resolvedTargetPatterns.putAll(preloadOrThrow(ImmutableList.of(pattern)));
+ } catch (TargetParsingException e) {
+ // Will skip the target and keep going if -k is specified.
+ resolvedTargetPatterns.put(pattern, ResolvedTargets.<Target>empty());
+ reportBuildFileError(caller, e.getMessage());
+ }
+ }
+ return getTargetsMatchingPattern(caller, pattern);
+ }
+
+ private Map<String, ResolvedTargets<Target>> preloadOrThrow(Collection<String> patterns)
+ throws TargetParsingException {
+ try {
+ // Note that this may throw a RuntimeException if deps are missing in Skyframe.
+ return targetPatternEvaluator.preloadTargetPatterns(
+ eventHandler, patterns, keepGoing);
+ } catch (InterruptedException e) {
+ // TODO(bazel-team): Propagate the InterruptedException from here [skyframe-loading].
+ throw new TargetParsingException("interrupted");
+ }
+ }
+
+ private Target getTargetOrThrow(Label label)
+ throws NoSuchThingException, SkyframeRestartQueryException, InterruptedException {
+ Target target = targetProvider.getTarget(eventHandler, label);
+ if (target == null) {
+ throw new SkyframeRestartQueryException();
+ }
+ return target;
+ }
+
+ // TODO(bazel-team): rename this to getDependentFiles when all implementations
+ // of QueryEnvironment is fixed.
+ @Override
+ public Set<Target> getBuildFiles(final QueryExpression caller, Set<Target> nodes)
+ throws QueryException {
+ Set<Target> dependentFiles = new LinkedHashSet<>();
+ Set<Package> seenPackages = new HashSet<>();
+ // Keep track of seen labels, to avoid adding a fake subinclude label that also exists as a
+ // real target.
+ Set<Label> seenLabels = new HashSet<>();
+
+ // Adds all the package definition files (BUILD files and build
+ // extensions) for package "pkg", to "buildfiles".
+ for (Target x : nodes) {
+ Package pkg = x.getPackage();
+ if (seenPackages.add(pkg)) {
+ addIfUniqueLabel(getNode(pkg.getBuildFile()), seenLabels, dependentFiles);
+ for (Label subinclude
+ : Iterables.concat(pkg.getSubincludeLabels(), pkg.getSkylarkFileDependencies())) {
+ addIfUniqueLabel(getSubincludeTarget(subinclude, pkg), seenLabels, dependentFiles);
+
+ // Also add the BUILD file of the subinclude.
+ try {
+ addIfUniqueLabel(getSubincludeTarget(
+ subinclude.getLocalTargetLabel("BUILD"), pkg), seenLabels, dependentFiles);
+ } catch (Label.SyntaxException e) {
+ throw new AssertionError("BUILD should always parse as a target name", e);
+ }
+ }
+ }
+ }
+ return dependentFiles;
+ }
+
+ private static void addIfUniqueLabel(Node<Target> node, Set<Label> labels, Set<Target> nodes) {
+ if (labels.add(node.getLabel().getLabel())) {
+ nodes.add(node.getLabel());
+ }
+ }
+
+ private Node<Target> getSubincludeTarget(final Label label, Package pkg) {
+ return getNode(new FakeSubincludeTarget(label, pkg.getBuildFile().getLocation()));
+ }
+
+ @Override
+ public TargetAccessor<Target> getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public boolean isSettingEnabled(Setting setting) {
+ return settings.contains(Preconditions.checkNotNull(setting));
+ }
+
+ @Override
+ public Iterable<QueryFunction> getFunctions() {
+ ImmutableList.Builder<QueryFunction> builder = ImmutableList.builder();
+ builder.addAll(DEFAULT_QUERY_FUNCTIONS);
+ builder.addAll(extraFunctions);
+ return builder.build();
+ }
+
+ private final class BlazeTargetAccessor implements TargetAccessor<Target> {
+
+ @Override
+ public String getTargetKind(Target target) {
+ return target.getTargetKind();
+ }
+
+ @Override
+ public String getLabel(Target target) {
+ return target.getLabel().toString();
+ }
+
+ @Override
+ public List<Target> getLabelListAttr(QueryExpression caller, Target target, String attrName,
+ String errorMsgPrefix) throws QueryException {
+ Preconditions.checkArgument(target instanceof Rule);
+
+ List<Target> result = new ArrayList<>();
+ Rule rule = (Rule) target;
+
+ AggregatingAttributeMapper attrMap = AggregatingAttributeMapper.of(rule);
+ Type<?> attrType = attrMap.getAttributeType(attrName);
+ if (attrType == null) {
+ // Return an empty list if the attribute isn't defined for this rule.
+ return ImmutableList.of();
+ }
+ for (Object value : attrMap.visitAttribute(attrName, attrType)) {
+ // Computed defaults may have null values.
+ if (value != null) {
+ for (Label label : attrType.getLabels(value)) {
+ try {
+ result.add(getTarget(label).getLabel());
+ } catch (TargetNotFoundException e) {
+ reportBuildFileError(caller, errorMsgPrefix + e.getMessage());
+ }
+ }
+ }
+ }
+
+ return result;
+ }
+
+ @Override
+ public List<String> getStringListAttr(Target target, String attrName) {
+ Preconditions.checkArgument(target instanceof Rule);
+ return NonconfigurableAttributeMapper.of((Rule) target).get(attrName, Type.STRING_LIST);
+ }
+
+ @Override
+ public String getStringAttr(Target target, String attrName) {
+ Preconditions.checkArgument(target instanceof Rule);
+ return NonconfigurableAttributeMapper.of((Rule) target).get(attrName, Type.STRING);
+ }
+
+ @Override
+ public Iterable<String> getAttrAsString(Target target, String attrName) {
+ Preconditions.checkArgument(target instanceof Rule);
+ List<String> values = new ArrayList<>(); // May hold null values.
+ Attribute attribute = ((Rule) target).getAttributeDefinition(attrName);
+ if (attribute != null) {
+ Type<?> attributeType = attribute.getType();
+ for (Object attrValue : AggregatingAttributeMapper.of((Rule) target).visitAttribute(
+ attribute.getName(), attributeType)) {
+
+ // Ugly hack to maintain backward 'attr' query compatibility for BOOLEAN and TRISTATE
+ // attributes. These are internally stored as actual Boolean or TriState objects but were
+ // historically queried as integers. To maintain compatibility, we inspect their actual
+ // value and return the integer equivalent represented as a String. This code is the
+ // opposite of the code in BooleanType and TriStateType respectively.
+ if (attributeType == BOOLEAN) {
+ values.add(Type.BOOLEAN.cast(attrValue) ? "1" : "0");
+ } else if (attributeType == TRISTATE) {
+ switch (Type.TRISTATE.cast(attrValue)) {
+ case AUTO :
+ values.add("-1");
+ break;
+ case NO :
+ values.add("0");
+ break;
+ case YES :
+ values.add("1");
+ break;
+ default :
+ throw new AssertionError("This can't happen!");
+ }
+ } else {
+ values.add(attrValue == null ? null : attrValue.toString());
+ }
+ }
+ }
+ return values;
+ }
+
+ @Override
+ public boolean isRule(Target target) {
+ return target instanceof Rule;
+ }
+
+ @Override
+ public boolean isTestRule(Target target) {
+ return TargetUtils.isTestRule(target);
+ }
+
+ @Override
+ public boolean isTestSuite(Target target) {
+ return TargetUtils.isTestSuiteRule(target);
+ }
+ }
+
+ /** Given a set of target nodes, returns the targets. */
+ private static Set<Target> getTargetsFromNodes(Iterable<Node<Target>> input) {
+ Set<Target> result = new LinkedHashSet<>();
+ for (Node<Target> node : input) {
+ result.add(node.getLabel());
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/ErrorPrintingTargetEdgeErrorObserver.java b/src/main/java/com/google/devtools/build/lib/query2/ErrorPrintingTargetEdgeErrorObserver.java
new file mode 100644
index 0000000..d47b8f3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/ErrorPrintingTargetEdgeErrorObserver.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.query2;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Record errors, such as missing package/target or rules containing errors,
+ * encountered during visitation. Emit an error message upon encountering
+ * missing edges
+ *
+ * The accessor {@link #hasErrors}) may not be called until the concurrent phase
+ * is over, i.e. all external calls to visit() methods have completed.
+ */
+@ThreadSafety.ConditionallyThreadSafe // condition: only call hasErrors
+ // once the visitation is complete.
+class ErrorPrintingTargetEdgeErrorObserver extends TargetEdgeErrorObserver {
+
+ private final EventHandler eventHandler;
+
+ /**
+ * @param eventHandler eventHandler to route exceptions to as errors.
+ */
+ public ErrorPrintingTargetEdgeErrorObserver(EventHandler eventHandler) {
+ this.eventHandler = eventHandler;
+ }
+
+ @ThreadSafety.ThreadSafe
+ @Override
+ public void missingEdge(Target target, Label label, NoSuchThingException e) {
+ eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target),
+ TargetUtils.formatMissingEdge(target, label, e)));
+ super.missingEdge(target, label, e);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/FakeSubincludeTarget.java b/src/main/java/com/google/devtools/build/lib/query2/FakeSubincludeTarget.java
new file mode 100644
index 0000000..f1a6469
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/FakeSubincludeTarget.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Set;
+
+/**
+ * A fake Target - Use only so that "blaze query" can report subincluded files as Targets.
+ */
+public class FakeSubincludeTarget implements Target {
+
+ private final Label label;
+ private final Location location;
+
+ FakeSubincludeTarget(Label label, Location location) {
+ this.label = Preconditions.checkNotNull(label);
+ this.location = Preconditions.checkNotNull(location);
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public String getName() {
+ return label.getName();
+ }
+
+ @Override
+ public Package getPackage() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getTargetKind() {
+ return "source file";
+ }
+
+ @Override
+ public Rule getAssociatedRule() {
+ return null;
+ }
+
+ @Override
+ public License getLicense() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public Set<License.DistributionType> getDistributions() {
+ return ImmutableSet.of();
+ }
+
+ @Override
+ public RuleVisibility getVisibility() {
+ return ConstantRuleVisibility.PUBLIC;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/LabelVisitor.java b/src/main/java/com/google/devtools/build/lib/query2/LabelVisitor.java
new file mode 100644
index 0000000..b72e3aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/LabelVisitor.java
@@ -0,0 +1,481 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.query2;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.SetMultimap;
+import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageProvider;
+import com.google.devtools.build.lib.pkgcache.TargetEdgeObserver;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.BinaryPredicate;
+
+import java.util.Collection;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * <p>Visit the transitive closure of a label. Primarily used to "fault in"
+ * packages to the packageProvider and ensure the necessary targets exists, in
+ * advance of the configuration step, which is intolerant of missing
+ * packages/targets.
+ *
+ * <p>LabelVisitor loads packages concurrently where possible, to increase I/O
+ * parallelism. However, the public interface is not thread-safe: calls to
+ * public methods should not be made concurrently.
+ *
+ * <p>LabelVisitor is stateful: It remembers the previous visitation and can
+ * check its validity on subsequent calls to sync() instead of doing the normal
+ * visitation.
+ *
+ * <p>TODO(bazel-team): (2009) a small further optimization could be achieved if we
+ * create tasks at the package (not individual label) level, since package
+ * loading is the expensive step. This would require additional bookkeeping to
+ * maintain the list of labels that we need to visit once a package becomes
+ * available. Profiling suggests that there is still a potential benefit to be
+ * gained: when the set of packages is known a-priori, loading a set of packages
+ * that took 20 seconds can be done under 5 in the sequential case or 7 in the
+ * current (parallel) case.
+ *
+ * <h4>Concurrency</h4>
+ *
+ * <p>The sync() methods of this class is thread-compatible. The accessor
+ * ({@link #hasVisited} and similar must not be called until the concurrent phase
+ * is over, i.e. all external calls to visit() methods have completed.
+ */
+final class LabelVisitor {
+
+ /**
+ * Attributes of a visitation which determine whether it is up-to-date or not.
+ */
+ private class VisitationAttributes {
+ private Collection<Target> targetsToVisit;
+ private boolean success = false;
+ private boolean visitSubincludes = true;
+ private int maxDepth = 0;
+
+ /**
+ * Returns true if and only if this visitation attribute is still up-to-date.
+ */
+ boolean current() {
+ return targetsToVisit.equals(lastVisitation.targetsToVisit)
+ && maxDepth <= lastVisitation.maxDepth
+ && visitSubincludes == lastVisitation.visitSubincludes;
+ }
+ }
+
+ /*
+ * Interrupts during the loading phase ===================================
+ *
+ * Bazel can be interrupted in the middle of the loading phase. The mechanics
+ * of this are far from trivial, so there is an explanation of how they are
+ * supposed to work. For a description how the same thing works in the
+ * execution phase, see ParallelBuilder.java .
+ *
+ * The sequence of events that happen when the user presses Ctrl-C is the
+ * following:
+ *
+ * 1. A SIGINT gets delivered to the Bazel client process.
+ *
+ * 2. The client process delivers the SIGINT to the server process.
+ *
+ * 3. The interruption state of the main thread is set to true.
+ *
+ * 4. Sooner or later, this results in an InterruptedException being thrown.
+ * Usually this takes place because the main thread is interrupted during
+ * AbstractQueueVisitor.awaitTermination(). The only exception to this is when
+ * the interruption occurs during the loading of a package of a label
+ * specified on the command line; in this case, the InterruptedException is
+ * thrown during the loading of an individual package (see below where this
+ * can occur)
+ *
+ * 5. The main thread calls ThreadPoolExecutor.shutdown(), which in turn
+ * interrupts every worker thread. Then the main thread waits for their
+ * termination.
+ *
+ * 6. An InterruptedException is thrown during the loading of an individual
+ * package in the worker threads.
+ *
+ * 7. All worker threads terminate.
+ *
+ * 8. An InterruptedException is thrown from
+ * AbstractQueueVisitor.awaitTermination()
+ *
+ * 9. This exception causes the execution of the currently running command to
+ * terminate prematurely.
+ *
+ * The interruption of the loading of an individual package can happen in two
+ * different ways depending on whether Python preprocessing is in effect or
+ * not.
+ *
+ * If there is no Python preprocessing:
+ *
+ * 1. We periodically check the interruption state of the thread in
+ * UnixGlob.reallyGlob(). If it is interrupted, an InterruptedException is
+ * thrown.
+ *
+ * 2. The stack is unwound until we are out of the part of the call stack
+ * responsible for package loading. This either means that the worker thread
+ * terminates or that the label parsing terminates if the package that is
+ * being loaded was specified on the command line.
+ *
+ * If there is Python preprocessing, events are a bit more complicated. In
+ * this case, the real work happens on the thread the Python preprocessor is
+ * called from, but in a bit more convoluted way: a new thread is spawned by
+ * to handle the input from the Python process and
+ * the output to the Python process is handled on the main thread. The reading
+ * thread parses requests from the preprocessor, and passes them using a queue
+ * to the writing thread (that is, the main thread), so that we can do the
+ * work there. This is important because this way, we don't have any work that
+ * we need to interrupt in a thread that is not spawned by us. So:
+ *
+ * 1. The interrupted state of the main thread is set.
+ *
+ * 2. This results in an InterruptedException during the execution of the task
+ * in PythonStdinInputStream.getNextMessage().
+ *
+ * 3. We exit from RequestParser.Request.run() prematurely, set a flag to
+ * signal that we were interrupted, and throw an InterruptedIOException.
+ *
+ * 4. The Python child process and reading thread are terminated.
+ *
+ * 5. Based on the flag we set in step 3, we realize that the termination was
+ * due to an interruption, and an InterruptedException is thrown. This can
+ * either raise an AbnormalTerminationException, or make Command.execute()
+ * return normally, so we check for both cases.
+ *
+ * 6. This InterruptedException causes the loading of the package to terminate
+ * prematurely.
+ *
+ * Life is not simple.
+ */
+ private final PackageProvider packageProvider;
+ private final BinaryPredicate<Rule, Attribute> edgeFilter;
+ private final SetMultimap<Package, Target> visitedMap =
+ Multimaps.synchronizedSetMultimap(HashMultimap.<Package, Target>create());
+ private final ConcurrentMap<Label, Integer> visitedTargets = new MapMaker().makeMap();
+
+ private VisitationAttributes lastVisitation;
+
+ /**
+ * Constant for limiting the permitted depth of recursion.
+ */
+ private static final int RECURSION_LIMIT = 100;
+
+ /**
+ * Construct a LabelVisitor.
+ *
+ * @param packageProvider how to resolve labels to targets.
+ * @param edgeFilter which edges may be traversed.
+ */
+ public LabelVisitor(PackageProvider packageProvider,
+ BinaryPredicate<Rule, Attribute> edgeFilter) {
+ this.packageProvider = packageProvider;
+ this.lastVisitation = new VisitationAttributes();
+ this.edgeFilter = edgeFilter;
+ }
+
+ boolean syncWithVisitor(EventHandler eventHandler, Collection<Target> targetsToVisit,
+ boolean keepGoing, int parallelThreads, int maxDepth, TargetEdgeObserver... observers)
+ throws InterruptedException {
+ VisitationAttributes nextVisitation = new VisitationAttributes();
+ nextVisitation.targetsToVisit = targetsToVisit;
+ nextVisitation.maxDepth = maxDepth;
+
+ if (!lastVisitation.success || !nextVisitation.current()) {
+ try {
+ nextVisitation.success = redoVisitation(eventHandler, nextVisitation, keepGoing,
+ parallelThreads, maxDepth, observers);
+ return nextVisitation.success;
+ } finally {
+ lastVisitation = nextVisitation;
+ }
+ } else {
+ return true;
+ }
+ }
+
+ // Does a bounded transitive visitation starting at the given top-level targets.
+ private boolean redoVisitation(EventHandler eventHandler,
+ VisitationAttributes visitation,
+ boolean keepGoing,
+ int parallelThreads,
+ int maxDepth,
+ TargetEdgeObserver... observers)
+ throws InterruptedException {
+ visitedMap.clear();
+ visitedTargets.clear();
+
+ Visitor visitor = new Visitor(eventHandler, keepGoing, parallelThreads, maxDepth, observers);
+
+ Throwable uncaught = null;
+ boolean result;
+ try {
+ visitor.visitTargets(visitation.targetsToVisit);
+ } catch (Throwable t) {
+ visitor.stopNewActions();
+ uncaught = t;
+ } finally {
+ // Run finish() in finally block to ensure we don't leak threads on exceptions.
+ result = visitor.finish();
+ }
+ Throwables.propagateIfPossible(uncaught);
+ return result;
+ }
+
+ boolean hasVisited(Label target) {
+ return visitedTargets.containsKey(target);
+ }
+
+ @VisibleForTesting class Visitor extends AbstractQueueVisitor {
+
+ private final static String THREAD_NAME = "LabelVisitor";
+
+ private final EventHandler eventHandler;
+ private final boolean keepGoing;
+ private final int maxDepth;
+ private final Iterable<TargetEdgeObserver> observers;
+ private final TargetEdgeErrorObserver errorObserver;
+ private final AtomicBoolean stopNewActions = new AtomicBoolean(false);
+ private static final boolean CONCURRENT = true;
+
+
+ public Visitor(EventHandler eventHandler, boolean keepGoing, int parallelThreads,
+ int maxDepth, TargetEdgeObserver... observers) {
+ // Observing the loading phase of a typical large package (with all subpackages) shows
+ // maximum thread-level concurrency of ~20. Limiting the total number of threads to 200 is
+ // therefore conservative and should help us avoid hitting native limits.
+ super(CONCURRENT, parallelThreads, parallelThreads, 1L, TimeUnit.SECONDS, !keepGoing,
+ THREAD_NAME);
+ this.eventHandler = eventHandler;
+ this.maxDepth = maxDepth;
+ this.errorObserver = new TargetEdgeErrorObserver();
+ ImmutableList.Builder<TargetEdgeObserver> builder = ImmutableList.builder();
+ for (TargetEdgeObserver observer : observers) {
+ builder.add(observer);
+ }
+ builder.add(errorObserver);
+ this.observers = builder.build();
+ this.keepGoing = keepGoing;
+ }
+
+ /**
+ * Visit the specified labels and follow the transitive closure of their
+ * outbound dependencies.
+ *
+ * @param targets the targets to visit
+ */
+ @ThreadSafe
+ public void visitTargets(Iterable<Target> targets) {
+ for (Target target : targets) {
+ visit(null, null, target, 0, 0);
+ }
+ }
+
+ @ThreadSafe
+ public boolean finish() throws InterruptedException {
+ work(true);
+ return !errorObserver.hasErrors();
+ }
+
+ @Override
+ protected boolean blockNewActions() {
+ return (!keepGoing && errorObserver.hasErrors()) || super.blockNewActions() ||
+ stopNewActions.get();
+ }
+
+ public void stopNewActions() {
+ stopNewActions.set(true);
+ }
+
+ private void enqueueTarget(
+ final Target from, final Attribute attr, final Label label, final int depth,
+ final int count) {
+ // Don't perform the targetProvider lookup if at the maximum depth already.
+ if (depth >= maxDepth) {
+ return;
+ } else if (attr != null && from instanceof Rule) {
+ if (!edgeFilter.apply((Rule) from, attr)) {
+ return;
+ }
+ }
+
+ // Avoid thread-related overhead when not crossing packages.
+ // Can start a new thread when count reaches 100, to prevent infinite recursion.
+ if (from != null && from.getLabel().getPackageFragment() == label.getPackageFragment() &&
+ !blockNewActions() && count < RECURSION_LIMIT) {
+ newVisitRunnable(from, attr, label, depth, count + 1).run();
+ } else {
+ enqueue(newVisitRunnable(from, attr, label, depth, 0));
+ }
+ }
+
+ private Runnable newVisitRunnable(final Target from, final Attribute attr, final Label label,
+ final int depth, final int count) {
+ return new Runnable () {
+ @Override
+ public void run() {
+ try {
+ Target target = packageProvider.getTarget(eventHandler, label);
+ if (target == null) {
+ // Let target visitation continue so we can discover additional unknown inputs.
+ return;
+ }
+ visit(from, attr, packageProvider.getTarget(eventHandler, label), depth + 1, count);
+ } catch (NoSuchThingException e) {
+ observeError(from, label, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+ }
+
+ private void visitTargetVisibility(Target target, int depth, int count) {
+ Attribute attribute = null;
+ if (target instanceof Rule) {
+ attribute = ((Rule) target).getRuleClassObject().getAttributeByName("visibility");
+ }
+
+ for (Label label : target.getVisibility().getDependencyLabels()) {
+ enqueueTarget(target, attribute, label, depth, count);
+ }
+ }
+
+ /**
+ * Visit all the labels in a given rule.
+ *
+ * <p>Called in a worker thread if CONCURRENT.
+ *
+ * @param rule the rule to visit
+ */
+ @ThreadSafe
+ private void visitRule(final Rule rule, final int depth, final int count) {
+ // Follow all labels defined by this rule:
+ AggregatingAttributeMapper.of(rule).visitLabels(new AttributeMap.AcceptsLabelAttribute() {
+ @Override
+ public void acceptLabelAttribute(Label label, Attribute attribute) {
+ enqueueTarget(rule, attribute, label, depth, count);
+ }
+ });
+ }
+
+ @ThreadSafe
+ private void visitPackageGroup(PackageGroup packageGroup, int depth, int count) {
+ for (final Label include : packageGroup.getIncludes()) {
+ enqueueTarget(packageGroup, null, include, depth, count);
+ }
+ }
+
+ /**
+ * Visits the target and its package.
+ *
+ * <p>Potentially blocking invocations into the package cache are
+ * enqueued in the worker pool if CONCURRENT.
+ */
+ private void visit(
+ Target from, Attribute attribute, final Target target, int depth, int count) {
+ if (depth > maxDepth) {
+ return;
+ }
+
+ if (from != null) {
+ observeEdge(from, attribute, target);
+ }
+
+ visitedMap.put(target.getPackage(), target);
+ visitTargetNode(target, depth, count);
+ }
+
+ /**
+ * Visit the specified target.
+ * Called in a worker thread if CONCURRENT.
+ *
+ * @param target the target to visit
+ */
+ private void visitTargetNode(Target target, int depth, int count) {
+ Integer minTargetDepth = visitedTargets.putIfAbsent(target.getLabel(), depth);
+ if (minTargetDepth != null) {
+ // The target was already visited at a greater depth.
+ // The closure we are about to build is therefore a subset of what
+ // has already been built, and we can skip it.
+ // Also special case MAX_VALUE, where we never want to revisit targets.
+ // (This avoids loading phase overhead outside of queries).
+ if (maxDepth == Integer.MAX_VALUE || minTargetDepth <= depth) {
+ return;
+ }
+ // Check again in case it was overwritten by another thread.
+ synchronized (visitedTargets) {
+ if (visitedTargets.get(target.getLabel()) <= depth) {
+ return;
+ }
+ visitedTargets.put(target.getLabel(), depth);
+ }
+ }
+
+ observeNode(target);
+ if (target instanceof OutputFile) {
+ Rule rule = ((OutputFile) target).getGeneratingRule();
+ observeEdge(target, null, rule);
+ // This is the only recursive call to visit which doesn't pass through enqueueTarget().
+ visit(null, null, rule, depth + 1, count + 1);
+ visitTargetVisibility(target, depth, count);
+ } else if (target instanceof InputFile) {
+ visitTargetVisibility(target, depth, count);
+ } else if (target instanceof Rule) {
+ visitTargetVisibility(target, depth, count);
+ visitRule((Rule) target, depth, count);
+ } else if (target instanceof PackageGroup) {
+ visitPackageGroup((PackageGroup) target, depth, count);
+ }
+ }
+
+ private void observeEdge(Target from, Attribute attribute, Target to) {
+ for (TargetEdgeObserver observer : observers) {
+ observer.edge(from, attribute, to);
+ }
+ }
+
+ private void observeNode(Target target) {
+ for (TargetEdgeObserver observer : observers) {
+ observer.node(target);
+ }
+ }
+
+ private void observeError(Target from, Label label, NoSuchThingException e) {
+ for (TargetEdgeObserver observer : observers) {
+ observer.missingEdge(from, label, e);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/SkyframeQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/SkyframeQueryEnvironment.java
new file mode 100644
index 0000000..318d844
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/SkyframeQueryEnvironment.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageProvider;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.query2.engine.QueryException;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Set;
+
+/**
+ * A BlazeQueryEnvironment for Skyframe builds. Currently, this is used to preload transitive
+ * closures of targets quickly.
+ */
+public final class SkyframeQueryEnvironment extends BlazeQueryEnvironment {
+
+ private final TransitivePackageLoader transitivePackageLoader;
+ private static final int MAX_DEPTH_FULL_SCAN_LIMIT = 20;
+
+ public SkyframeQueryEnvironment(
+ TransitivePackageLoader transitivePackageLoader, PackageProvider packageProvider,
+ TargetPatternEvaluator targetPatternEvaluator, boolean keepGoing, int loadingPhaseThreads,
+ EventHandler eventHandler, Set<Setting> settings, Iterable<QueryFunction> functions) {
+ super(packageProvider, targetPatternEvaluator, keepGoing, loadingPhaseThreads, eventHandler,
+ settings, functions);
+ this.transitivePackageLoader = transitivePackageLoader;
+ }
+
+ @Override
+ protected void preloadTransitiveClosure(Set<Target> targets, int maxDepth) throws QueryException {
+ if (maxDepth >= MAX_DEPTH_FULL_SCAN_LIMIT) {
+ // Only do the full visitation if "maxDepth" is large enough. Otherwise, the benefits of
+ // preloading will be outweighed by the cost of doing more work than necessary.
+ try {
+ transitivePackageLoader.sync(eventHandler, targets, ImmutableSet.<Label>of(), keepGoing,
+ loadingPhaseThreads, -1);
+ } catch (InterruptedException e) {
+ throw new QueryException("interrupted");
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/TargetEdgeErrorObserver.java b/src/main/java/com/google/devtools/build/lib/query2/TargetEdgeErrorObserver.java
new file mode 100644
index 0000000..5a075d2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/TargetEdgeErrorObserver.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.query2;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.TargetEdgeObserver;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Record errors, such as missing package/target or rules containing errors,
+ * encountered during visitation. Emit an error message upon encountering
+ * missing edges
+ *
+ * The accessor {@link #hasErrors}) may not be called until the concurrent phase
+ * is over, i.e. all external calls to visit() methods have completed.
+ *
+ * If you need to report errors to the console during visitation, use the
+ * subclass {@link com.google.devtools.build.lib.query2.ErrorPrintingTargetEdgeErrorObserver}.
+ */
+class TargetEdgeErrorObserver implements TargetEdgeObserver {
+
+ /**
+ * True iff errors were encountered. Note, may be set to "true" during the
+ * concurrent phase. Volatile, because it is assigned by worker threads and
+ * read by the main thread without monitor synchronization.
+ */
+ private volatile boolean hasErrors = false;
+
+ /**
+ * Reports an unresolved label error and records the fact that an error was
+ * encountered.
+ * @param target the target that referenced the unresolved label
+ * @param label the label that could not be resolved
+ * @param e the exception that was thrown when the label could not be resolved
+ */
+ @ThreadSafety.ThreadSafe
+ @Override
+ public void missingEdge(Target target, Label label, NoSuchThingException e) {
+ hasErrors = true;
+ }
+
+ /**
+ * Returns true iff any errors (such as missing targets or packages, or rules
+ * with errors) have been encountered during any work.
+ *
+ * <p>Not thread-safe; do not call during visitation.
+ *
+ * @return true iff no errors (such as missing targets or packages, or rules
+ * with errors) have been encountered during any work.
+ */
+ public boolean hasErrors() {
+ return hasErrors;
+ }
+
+ @Override
+ public void edge(Target from, Attribute attribute, Target to) {
+ // No-op.
+ }
+
+ @Override
+ public void node(Target node) {
+ if (node.getPackage().containsErrors() ||
+ ((node instanceof Rule) && ((Rule) node).containsErrors())) {
+ this.hasErrors = true; // Note, this is thread-safe.
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java
new file mode 100644
index 0000000..d2d5f05
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Implementation of the <code>allpaths()</code> function.
+ */
+public class AllPathsFunction implements QueryFunction {
+ AllPathsFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "allpaths";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 2;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ QueryExpression from = args.get(0).getExpression();
+ QueryExpression to = args.get(1).getExpression();
+
+ Set<T> fromValue = from.eval(env);
+ Set<T> toValue = to.eval(env);
+
+ // Algorithm: compute "reachableFromX", the forward transitive closure of
+ // the "from" set, then find the intersection of "reachableFromX" with the
+ // reverse transitive closure of the "to" set. The reverse transitive
+ // closure and intersection operations are interleaved for efficiency.
+ // "result" holds the intersection.
+
+ env.buildTransitiveClosure(expression, fromValue, Integer.MAX_VALUE);
+
+ Set<T> reachableFromX = env.getTransitiveClosure(fromValue);
+ Set<T> result = intersection(reachableFromX, toValue);
+ LinkedList<T> worklist = new LinkedList<>(result);
+
+ T n;
+ while ((n = worklist.poll()) != null) {
+ for (T np : env.getReverseDeps(n)) {
+ if (reachableFromX.contains(np)) {
+ if (result.add(np)) {
+ worklist.add(np);
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns a (new, mutable, unordered) set containing the intersection of the
+ * two specified sets.
+ */
+ private static <T> Set<T> intersection(Set<T> x, Set<T> y) {
+ Set<T> result = new HashSet<>();
+ if (x.size() > y.size()) {
+ Sets.intersection(y, x).copyInto(result);
+ } else {
+ Sets.intersection(x, y).copyInto(result);
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AttrFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AttrFunction.java
new file mode 100644
index 0000000..054cf78b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AttrFunction.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+
+import java.util.List;
+
+/**
+ * An attr(attribute, pattern, argument) filter expression, which computes
+ * the set of subset of nodes in 'argument' which correspond to rules with
+ * defined attribute 'attribute' with attribute value matching the unanchored
+ * regexp 'pattern'. For list attributes, the attribute value will be defined as
+ * a usual List.toString() representation (using '[' as first character, ']' as
+ * last character and ", " as a delimiter between multiple values). Also, all
+ * label-based attributes will use fully-qualified label names instead of
+ * original value specified in the BUILD file.
+ *
+ * <pre>expr ::= ATTR '(' ATTRNAME ',' WORD ',' expr ')'</pre>
+ *
+ * Examples
+ * <pre>
+ * attr(linkshared,1,//project/...) find all rules under in the //project/... that
+ * have attribute linkshared set to 1.
+ * </pre>
+ */
+class AttrFunction extends RegexFilterExpression {
+ AttrFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "attr";
+ }
+
+ @Override
+ protected String getPattern(List<Argument> args) {
+ return args.get(1).getWord();
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 3;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.WORD, ArgumentType.WORD, ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ protected <T> String getFilterString(QueryEnvironment<T> env, List<Argument> args, T target) {
+ throw new IllegalStateException(
+ "The 'attr' regex filter gets its match values directly from getFilterStrings");
+ }
+
+ @Override
+ protected <T> Iterable<String> getFilterStrings(QueryEnvironment<T> env,
+ List<Argument> args, T target) {
+ if (env.getAccessor().isRule(target)) {
+ return env.getAccessor().getAttrAsString(target, args.get(0).getWord());
+ }
+ return ImmutableList.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java
new file mode 100644
index 0000000..e5e600f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A binary algebraic set operation.
+ *
+ * <pre>
+ * expr ::= expr (INTERSECT expr)+
+ * | expr ('^' expr)+
+ * | expr (UNION expr)+
+ * | expr ('+' expr)+
+ * | expr (EXCEPT expr)+
+ * | expr ('-' expr)+
+ * </pre>
+ */
+class BinaryOperatorExpression extends QueryExpression {
+
+ private final Lexer.TokenKind operator; // ::= INTERSECT/CARET | UNION/PLUS | EXCEPT/MINUS
+ private final ImmutableList<QueryExpression> operands;
+
+ BinaryOperatorExpression(Lexer.TokenKind operator,
+ List<QueryExpression> operands) {
+ Preconditions.checkState(operands.size() > 1);
+ this.operator = operator;
+ this.operands = ImmutableList.copyOf(operands);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException {
+ Set<T> lhsValue = new LinkedHashSet<>(operands.get(0).eval(env));
+
+ for (int i = 1; i < operands.size(); i++) {
+ Set<T> rhsValue = operands.get(i).eval(env);
+ switch (operator) {
+ case INTERSECT:
+ case CARET:
+ lhsValue.retainAll(rhsValue);
+ break;
+ case UNION:
+ case PLUS:
+ lhsValue.addAll(rhsValue);
+ break;
+ case EXCEPT:
+ case MINUS:
+ lhsValue.removeAll(rhsValue);
+ break;
+ default:
+ throw new IllegalStateException("operator=" + operator);
+ }
+ }
+ return lhsValue;
+ }
+
+ @Override
+ public void collectTargetPatterns(Collection<String> literals) {
+ for (QueryExpression subExpression : operands) {
+ subExpression.collectTargetPatterns(literals);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ for (int i = 1; i < operands.size(); i++) {
+ result.append("(");
+ }
+ result.append(operands.get(0));
+ for (int i = 1; i < operands.size(); i++) {
+ result.append(" " + operator.getPrettyName() + " " + operands.get(i) + ")");
+ }
+ return result.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BlazeQueryEvalResult.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BlazeQueryEvalResult.java
new file mode 100644
index 0000000..7f2060f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BlazeQueryEvalResult.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.graph.Digraph;
+
+import java.util.Set;
+
+/** {@link QueryEvalResult} along with a digraph giving the structure of the results. */
+public class BlazeQueryEvalResult<T> extends QueryEvalResult<T> {
+
+ private final Digraph<T> graph;
+
+ public BlazeQueryEvalResult(boolean success, Set<T> resultSet, Digraph<T> graph) {
+ super(success, resultSet);
+ this.graph = Preconditions.checkNotNull(graph);
+ }
+
+ /** Returns the result as a directed graph over elements. */
+ public Digraph<T> getResultGraph() {
+ return graph.extractSubgraph(resultSet);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java
new file mode 100644
index 0000000..d606a6a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A buildfiles(x) query expression, which computes the set of BUILD files and
+ * subincluded files for each target in set x. The result is unordered. This
+ * operator is typically used for determinining what files or packages to check
+ * out.
+ *
+ * <pre>expr ::= BUILDFILES '(' expr ')'</pre>
+ */
+class BuildFilesFunction implements QueryFunction {
+ BuildFilesFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "buildfiles";
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ return env.getBuildFiles(expression, args.get(0).getExpression().eval(env));
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 1;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.EXPRESSION);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java
new file mode 100644
index 0000000..2b693eb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A "deps" query expression, which computes the dependencies of the argument. An optional
+ * integer-literal second argument may be specified; its value bounds the search from the arguments.
+ *
+ * <pre>expr ::= DEPS '(' expr ')'</pre>
+ * <pre> | DEPS '(' expr ',' WORD ')'</pre>
+ */
+final class DepsFunction implements QueryFunction {
+ DepsFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "deps";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 1; // last argument is optional
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.INTEGER);
+ }
+
+ /**
+ * Breadth-first search from the arguments.
+ */
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Set<T> argumentValue = args.get(0).getExpression().eval(env);
+ int depthBound = args.size() > 1 ? args.get(1).getInteger() : Integer.MAX_VALUE;
+ env.buildTransitiveClosure(expression, argumentValue, depthBound);
+
+ Set<T> visited = new LinkedHashSet<>();
+ Collection<T> current = argumentValue;
+
+ // We need to iterate depthBound + 1 times.
+ for (int i = 0; i <= depthBound; i++) {
+ List<T> next = new ArrayList<>();
+ for (T node : current) {
+ if (!visited.add(node)) {
+ // Already visited; if we see a node in a later round, then we don't need to visit it
+ // again, because the depth at which we see it at must be greater than or equal to the
+ // last visit.
+ continue;
+ }
+
+ next.addAll(env.getFwdDeps(node));
+ }
+ if (next.isEmpty()) {
+ // Exit when there are no more nodes to visit.
+ break;
+ }
+ current = next;
+ }
+
+ return visited;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/FilterFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/FilterFunction.java
new file mode 100644
index 0000000..bd89950
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/FilterFunction.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+
+import java.util.List;
+
+/**
+ * A label(pattern, argument) filter expression, which computes the set of subset
+ * of nodes in 'argument' whose label matches the unanchored regexp 'pattern'.
+ *
+ * <pre>expr ::= FILTER '(' WORD ',' expr ')'</pre>
+ *
+ * Example patterns:
+ * <pre>
+ * '//third_party' Match all targets in the //third_party/...
+ * (equivalent to 'intersect //third_party/...)
+ * '\.jar$' Match all *.jar targets.
+ * </pre>
+ */
+class FilterFunction extends RegexFilterExpression {
+ FilterFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "filter";
+ }
+
+ @Override
+ protected String getPattern(List<Argument> args) {
+ return args.get(0).getWord();
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 2;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.WORD, ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ protected <T> String getFilterString(QueryEnvironment<T> env, List<Argument> args, T target) {
+ return env.getAccessor().getLabel(target);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java
new file mode 100644
index 0000000..62734fd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A query expression for user-defined query functions.
+ */
+public class FunctionExpression extends QueryExpression {
+ QueryFunction function;
+ List<Argument> args;
+
+ public FunctionExpression(QueryFunction function, List<Argument> args) {
+ this.function = function;
+ this.args = ImmutableList.copyOf(args);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException {
+ return function.<T>eval(env, this, args);
+ }
+
+ @Override
+ public void collectTargetPatterns(Collection<String> literals) {
+ for (Argument arg : args) {
+ if (arg.getType() == ArgumentType.EXPRESSION) {
+ arg.getExpression().collectTargetPatterns(literals);
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return function.getName() +
+ "(" + Joiner.on(", ").join(Iterables.transform(args, Functions.toStringFunction())) + ")";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/KindFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/KindFunction.java
new file mode 100644
index 0000000..7ae80b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/KindFunction.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+
+import java.util.List;
+
+/**
+ * A kind(pattern, argument) filter expression, which computes the set of subset
+ * of nodes in 'argument' whose kind matches the unanchored regexp 'pattern'.
+ *
+ * <pre>expr ::= KIND '(' WORD ',' expr ')'</pre>
+ *
+ * Example patterns:
+ * <pre>
+ * ' file' Match all file targets.
+ * 'source file' Match all test source file targets.
+ * 'generated file' Match all test generated file targets.
+ * ' rule' Match all rule targets.
+ * 'foo_*' Match all rules starting with "foo_",
+ * 'test' Match all test (rule) targets.
+ * </pre>
+ *
+ * Note, the space before "file" is needed to prevent unwanted matches against
+ * (e.g.) "filegroup rule".
+ */
+class KindFunction extends RegexFilterExpression {
+
+ KindFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "kind";
+ }
+
+ @Override
+ protected String getPattern(List<Argument> args) {
+ return args.get(0).getWord();
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 2;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.WORD, ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ protected <T> String getFilterString(QueryEnvironment<T> env, List<Argument> args, T target) {
+ return env.getAccessor().getTargetKind(target);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java
new file mode 100644
index 0000000..1093d85
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A label(attr_name, argument) expression, which computes the set of targets
+ * whose labels appear in the specified attribute of some rule in 'argument'.
+ *
+ * <pre>expr ::= LABELS '(' WORD ',' expr ')'</pre>
+ *
+ * Example:
+ * <pre>
+ * labels(srcs, //foo) The 'srcs' source files to the //foo rule.
+ * </pre>
+ */
+class LabelsFunction implements QueryFunction {
+ LabelsFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "labels";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 2;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.WORD, ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Set<T> inputs = args.get(1).getExpression().eval(env);
+ Set<T> result = new LinkedHashSet<>();
+ String attrName = args.get(0).getWord();
+ for (T input : inputs) {
+ if (env.getAccessor().isRule(input)) {
+ List<T> targets = env.getAccessor().getLabelListAttr(expression, input, attrName,
+ "in '" + attrName + "' of rule " + env.getAccessor().getLabel(input) + ": ");
+ for (T target : targets) {
+ result.add(env.getOrCreate(target));
+ }
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java
new file mode 100644
index 0000000..3e17cce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * A let expression.
+ *
+ * <pre>expr ::= LET WORD = expr IN expr</pre>
+ */
+class LetExpression extends QueryExpression {
+
+ private static final String VAR_NAME_PATTERN = "[a-zA-Z_][a-zA-Z0-9_]*$";
+
+ // Variables names may be any legal identifier in the C programming language
+ private static final Pattern NAME_PATTERN = Pattern.compile("^" + VAR_NAME_PATTERN);
+
+ // Variable references are prepended with the "$" character.
+ // A variable named "x" is referenced as "$x".
+ private static final Pattern REF_PATTERN = Pattern.compile("^\\$" + VAR_NAME_PATTERN);
+
+ static boolean isValidVarReference(String varName) {
+ return REF_PATTERN.matcher(varName).matches();
+ }
+
+ static String getNameFromReference(String reference) {
+ return reference.substring(1);
+ }
+
+ private final String varName;
+ private final QueryExpression varExpr;
+ private final QueryExpression bodyExpr;
+
+ LetExpression(String varName, QueryExpression varExpr, QueryExpression bodyExpr) {
+ this.varName = varName;
+ this.varExpr = varExpr;
+ this.bodyExpr = bodyExpr;
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException {
+ if (!NAME_PATTERN.matcher(varName).matches()) {
+ throw new QueryException(this, "invalid variable name '" + varName + "' in let expression");
+ }
+ Set<T> varValue = varExpr.eval(env);
+ Set<T> prevValue = env.setVariable(varName, varValue);
+ try {
+ return bodyExpr.eval(env);
+ } finally {
+ env.setVariable(varName, prevValue); // restore
+ }
+ }
+
+ @Override
+ public void collectTargetPatterns(Collection<String> literals) {
+ varExpr.collectTargetPatterns(literals);
+ bodyExpr.collectTargetPatterns(literals);
+ }
+
+ @Override
+ public String toString() {
+ return "let " + varName + " = " + varExpr + " in " + bodyExpr;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/Lexer.java b/src/main/java/com/google/devtools/build/lib/query2/engine/Lexer.java
new file mode 100644
index 0000000..45b6f61
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/Lexer.java
@@ -0,0 +1,281 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A tokenizer for the Blaze query language, revision 2.
+ *
+ * Note, we can avoid a lot of quoting by noting that the characters [() ,] do
+ * not appear in any label, filename, function name, or regular expression we care about.
+ *
+ * No string escapes are allowed ("\"). Given the domain, that's not currently
+ * a problem.
+ */
+final class Lexer {
+
+ /**
+ * Discriminator for different kinds of tokens.
+ */
+ public enum TokenKind {
+ WORD("word"),
+ EOF("EOF"),
+
+ COMMA(","),
+ EQUALS("="),
+ LPAREN("("),
+ MINUS("-"),
+ PLUS("+"),
+ RPAREN(")"),
+ CARET("^"),
+
+ __ALL_IDENTIFIERS_FOLLOW(""), // See below
+
+ IN("in"),
+ LET("let"),
+ SET("set"),
+
+ INTERSECT("intersect"),
+ EXCEPT("except"),
+ UNION("union");
+
+ private final String prettyName;
+
+ private TokenKind(String prettyName) {
+ this.prettyName = prettyName;
+ }
+
+ public String getPrettyName() {
+ return prettyName;
+ }
+ }
+
+ public static final Set<TokenKind> BINARY_OPERATORS = EnumSet.of(
+ TokenKind.INTERSECT,
+ TokenKind.CARET,
+ TokenKind.UNION,
+ TokenKind.PLUS,
+ TokenKind.EXCEPT,
+ TokenKind.MINUS);
+
+ private static final Map<String, TokenKind> keywordMap = new HashMap<>();
+ static {
+ for (TokenKind kind : EnumSet.allOf(TokenKind.class)) {
+ if (kind.ordinal() > TokenKind.__ALL_IDENTIFIERS_FOLLOW.ordinal()) {
+ keywordMap.put(kind.getPrettyName(), kind);
+ }
+ }
+ }
+
+ /**
+ * Returns true iff 'word' is a reserved word of the language.
+ */
+ static boolean isReservedWord(String word) {
+ return keywordMap.containsKey(word);
+ }
+
+ /**
+ * Tokens returned by the Lexer.
+ */
+ static class Token {
+
+ public final TokenKind kind;
+ public final String word;
+
+ Token(TokenKind kind) {
+ this.kind = kind;
+ this.word = null;
+ }
+
+ Token(String word) {
+ this.kind = TokenKind.WORD;
+ this.word = word;
+ }
+
+ @Override
+ public String toString() {
+ return kind == TokenKind.WORD ? word : kind.getPrettyName();
+ }
+ }
+
+ /**
+ * Entry point to the lexer. Returns the list of tokens for the specified
+ * input, or throws QueryException.
+ */
+ public static List<Token> scan(char[] buffer) throws QueryException {
+ Lexer lexer = new Lexer(buffer);
+ lexer.tokenize();
+ return lexer.tokens;
+ }
+
+ // Input buffer and position
+ private char[] buffer;
+ private int pos;
+
+ private final List<Token> tokens = new ArrayList<>();
+
+ private Lexer(char[] buffer) {
+ this.buffer = buffer;
+ this.pos = 0;
+ }
+
+ private void addToken(Token s) {
+ tokens.add(s);
+ }
+
+ /**
+ * Scans a quoted word delimited by 'quot'.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first delimiter
+ * ON EXIT: 'pos' is 1 + the index of the last delimiter.
+ *
+ * @return the word token.
+ */
+ private Token quotedWord(char quot) throws QueryException {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ char c = buffer[pos++];
+ switch (c) {
+ case '\'':
+ case '"':
+ if (c == quot) {
+ // close-quote, all done.
+ return new Token(bufferSlice(oldPos + 1, pos - 1));
+ }
+ }
+ }
+ throw new QueryException("unclosed quotation");
+ }
+
+ private TokenKind getTokenKindForWord(String word) {
+ TokenKind kind = keywordMap.get(word);
+ return kind == null ? TokenKind.WORD : kind;
+ }
+
+ // Unquoted words may contain [-*$], but not start with them. For user convenience, unquoted
+ // words must include UNIX filenames, labels and target label patterns, and simple regexps
+ // (e.g. cc_.*). Keep consistent with TargetLiteral.toString()!
+ private String scanWord() {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ switch (buffer[pos]) {
+ case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
+ case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
+ case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
+ case 's': case 't': case 'u': case 'v': case 'w': case 'x':
+ case 'y': case 'z':
+ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
+ case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
+ case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
+ case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
+ case 'Y': case 'Z':
+ case '0': case '1': case '2': case '3': case '4': case '5':
+ case '6': case '7': case '8': case '9':
+ case '*': case '/': case '@': case '.': case '-': case '_':
+ case ':': case '$':
+ pos++;
+ break;
+ default:
+ return bufferSlice(oldPos, pos);
+ }
+ }
+ return bufferSlice(oldPos, pos);
+ }
+
+ /**
+ * Scans a word or keyword.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first char in the word.
+ * ON EXIT: 'pos' is 1 + the index of the last char in the word.
+ *
+ * @return the word or keyword token.
+ */
+ private Token wordOrKeyword() {
+ String word = scanWord();
+ TokenKind kind = getTokenKindForWord(word);
+ return kind == TokenKind.WORD ? new Token(word) : new Token(kind);
+ }
+
+ /**
+ * Performs tokenization of the character buffer of file contents provided to
+ * the constructor.
+ */
+ private void tokenize() throws QueryException {
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '(': {
+ addToken(new Token(TokenKind.LPAREN));
+ break;
+ }
+ case ')': {
+ addToken(new Token(TokenKind.RPAREN));
+ break;
+ }
+ case ',': {
+ addToken(new Token(TokenKind.COMMA));
+ break;
+ }
+ case '+': {
+ addToken(new Token(TokenKind.PLUS));
+ break;
+ }
+ case '-': {
+ addToken(new Token(TokenKind.MINUS));
+ break;
+ }
+ case '=': {
+ addToken(new Token(TokenKind.EQUALS));
+ break;
+ }
+ case '^': {
+ addToken(new Token(TokenKind.CARET));
+ break;
+ }
+ case '\n':
+ case ' ':
+ case '\t':
+ case '\r': {
+ /* ignore */
+ break;
+ }
+ case '\'':
+ case '\"': {
+ addToken(quotedWord(c));
+ break;
+ }
+ default: {
+ addToken(wordOrKeyword());
+ break;
+ } // default
+ } // switch
+ } // while
+
+ addToken(new Token(TokenKind.EOF));
+
+ this.buffer = null; // release buffer now that we have our tokens
+ }
+
+ private String bufferSlice(int start, int end) {
+ return new String(this.buffer, start, end - start);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java
new file mode 100644
index 0000000..46a7afd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java
@@ -0,0 +1,351 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nonnull;
+
+/**
+ * The environment of a Blaze query. Implementations do not need to be thread-safe. The generic type
+ * T represents a node of the graph on which the query runs; as such, there is no restriction on T.
+ * However, query assumes a certain graph model, and the {@link TargetAccessor} class is used to
+ * access properties of these nodes.
+ *
+ * @param <T> the node type of the dependency graph
+ */
+public interface QueryEnvironment<T> {
+ /**
+ * Type of an argument of a user-defined query function.
+ */
+ public enum ArgumentType {
+ EXPRESSION, WORD, INTEGER;
+ }
+
+ /**
+ * Value of an argument of a user-defined query function.
+ */
+ public static class Argument {
+ private final ArgumentType type;
+ private final QueryExpression expression;
+ private final String word;
+ private final int integer;
+
+ private Argument(ArgumentType type, QueryExpression expression, String word, int integer) {
+ this.type = type;
+ this.expression = expression;
+ this.word = word;
+ this.integer = integer;
+ }
+
+ static Argument of(QueryExpression expression) {
+ return new Argument(ArgumentType.EXPRESSION, expression, null, 0);
+ }
+
+ static Argument of(String word) {
+ return new Argument(ArgumentType.WORD, null, word, 0);
+ }
+
+ static Argument of(int integer) {
+ return new Argument(ArgumentType.INTEGER, null, null, integer);
+ }
+
+ public ArgumentType getType() {
+ return type;
+ }
+
+ public QueryExpression getExpression() {
+ return expression;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public int getInteger() {
+ return integer;
+ }
+
+ @Override
+ public String toString() {
+ switch (type) {
+ case WORD: return "'" + word + "'";
+ case EXPRESSION: return expression.toString();
+ case INTEGER: return Integer.toString(integer);
+ default: throw new IllegalStateException();
+ }
+ }
+ }
+
+ /**
+ * A user-defined query function.
+ */
+ public interface QueryFunction {
+ /**
+ * Name of the function as it appears in the query language.
+ */
+ String getName();
+
+ /**
+ * The number of arguments that are required. The rest is optional.
+ *
+ * <p>This should be greater than or equal to zero and at smaller than or equal to the length
+ * of the list returned by {@link #getArgumentTypes}.
+ */
+ int getMandatoryArguments();
+
+ /**
+ * The types of the arguments of the function.
+ */
+ List<ArgumentType> getArgumentTypes();
+
+ /**
+ * Called when a user-defined function is to be evaluated.
+ *
+ * @param env the query environment this function is evaluated in.
+ * @param expression the expression being evaluated.
+ * @param args the input arguments. These are type-checked against the specification returned
+ * by {@link #getArgumentTypes} and {@link #getMandatoryArguments}
+ */
+ <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException;
+ }
+
+ /**
+ * Exception type for the case where a target cannot be found. It's basically a wrapper for
+ * whatever exception is internally thrown.
+ */
+ public static final class TargetNotFoundException extends Exception {
+ public TargetNotFoundException(String msg) {
+ super(msg);
+ }
+
+ public TargetNotFoundException(Throwable cause) {
+ super(cause.getMessage(), cause);
+ }
+ }
+
+ /**
+ * Returns the set of target nodes in the graph for the specified target
+ * pattern, in 'blaze build' syntax.
+ */
+ Set<T> getTargetsMatchingPattern(QueryExpression owner, String pattern)
+ throws QueryException;
+
+ /** Ensures the specified target exists. */
+ // NOTE(bazel-team): this method is left here as scaffolding from a previous refactoring. It may
+ // be possible to remove it.
+ T getOrCreate(T target);
+
+ /** Returns the direct forward dependencies of the specified target. */
+ Collection<T> getFwdDeps(T target);
+
+ /** Returns the direct reverse dependencies of the specified target. */
+ Collection<T> getReverseDeps(T target);
+
+ /**
+ * Returns the forward transitive closure of all of the targets in
+ * "targets". Callers must ensure that {@link #buildTransitiveClosure}
+ * has been called for the relevant subgraph.
+ */
+ Set<T> getTransitiveClosure(Set<T> targets);
+
+ /**
+ * Construct the dependency graph for a depth-bounded forward transitive closure
+ * of all nodes in "targetNodes". The identity of the calling expression is
+ * required to produce error messages.
+ *
+ * <p>If a larger transitive closure was already built, returns it to
+ * improve incrementality, since all depth-constrained methods filter it
+ * after it is built anyway.
+ */
+ void buildTransitiveClosure(QueryExpression caller,
+ Set<T> targetNodes,
+ int maxDepth) throws QueryException;
+
+ /**
+ * Returns the set of nodes on some path from "from" to "to".
+ */
+ Set<T> getNodesOnPath(T from, T to);
+
+ /**
+ * Returns the value of the specified variable, or null if it is undefined.
+ */
+ Set<T> getVariable(String name);
+
+ /**
+ * Sets the value of the specified variable. If value is null the variable
+ * becomes undefined. Returns the previous value, if any.
+ */
+ Set<T> setVariable(String name, Set<T> value);
+
+ void reportBuildFileError(QueryExpression expression, String msg) throws QueryException;
+
+ /**
+ * Returns the set of BUILD, included, sub-included and Skylark files that define the given set of
+ * targets. Each such file is itself represented as a target in the result.
+ */
+ Set<T> getBuildFiles(QueryExpression caller, Set<T> nodes) throws QueryException;
+
+ /**
+ * Returns an object that can be used to query information about targets. Implementations should
+ * create a single instance and return that for all calls. A class can implement both {@code
+ * QueryEnvironment} and {@code TargetAccessor} at the same time, in which case this method simply
+ * returns {@code this}.
+ */
+ TargetAccessor<T> getAccessor();
+
+ /**
+ * Whether the given setting is enabled. The code should default to return {@code false} for all
+ * unknown settings. The enum is used rather than a method for each setting so that adding more
+ * settings is backwards-compatible.
+ *
+ * @throws NullPointerException if setting is null
+ */
+ boolean isSettingEnabled(@Nonnull Setting setting);
+
+ /**
+ * Returns the set of query functions implemented by this query environment.
+ */
+ Iterable<QueryFunction> getFunctions();
+
+ /**
+ * Settings for the query engine. See {@link QueryEnvironment#isSettingEnabled}.
+ */
+ public static enum Setting {
+
+ /**
+ * Whether to evaluate tests() expressions in strict mode. If {@link #isSettingEnabled} returns
+ * true for this setting, then the tests() expression will give an error when expanding tests
+ * suites, if the test suite contains any non-test targets.
+ */
+ TESTS_EXPRESSION_STRICT,
+
+ /**
+ * Do not consider implicit deps (any label that was not explicitly specified in the BUILD file)
+ * when traversing dependency edges.
+ */
+ NO_IMPLICIT_DEPS,
+
+ /**
+ * Do not consider host dependencies when traversing dependency edges.
+ */
+ NO_HOST_DEPS,
+
+ /**
+ * Do not consider nodep attributes when traversing dependency edges.
+ */
+ NO_NODEP_DEPS;
+ }
+
+ /**
+ * An adapter interface giving access to properties of T. There are four types of targets: rules,
+ * package groups, source files, and generated files. Of these, only rules can have attributes.
+ */
+ public static interface TargetAccessor<T> {
+ /**
+ * Returns the target type represented as a string of the form {@code <type> rule} or
+ * {@code package group} or {@code source file} or {@code generated file}. This is widely used
+ * for target filtering, so implementations must use the Blaze rule class naming scheme.
+ */
+ String getTargetKind(T target);
+
+ /**
+ * Returns the full label of the target as a string, e.g. {@code //some:target}.
+ */
+ String getLabel(T target);
+
+ /**
+ * Returns whether the given target is a rule.
+ */
+ boolean isRule(T target);
+
+ /**
+ * Returns whether the given target is a test target. If this returns true, then {@link #isRule}
+ * must also return true for the target.
+ */
+ boolean isTestRule(T target);
+
+ /**
+ * Returns whether the given target is a test suite target. If this returns true, then {@link
+ * #isRule} must also return true for the target, but {@link #isTestRule} must return false;
+ * test suites are not test rules, and vice versa.
+ */
+ boolean isTestSuite(T target);
+
+ /**
+ * If the attribute of the given name on the given target is a label or label list, then this
+ * method returns the list of corresponding target instances. Otherwise returns an empty list.
+ * If an error occurs during resolution, it throws a {@link QueryException} using the caller and
+ * error message prefix.
+ *
+ * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule})
+ */
+ List<T> getLabelListAttr(QueryExpression caller, T target, String attrName,
+ String errorMsgPrefix) throws QueryException;
+
+ /**
+ * If the attribute of the given name on the given target is a string list, then this method
+ * returns it.
+ *
+ * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule}), or
+ * if the target does not have an attribute of type string list
+ * with the given name
+ */
+ List<String> getStringListAttr(T target, String attrName);
+
+ /**
+ * If the attribute of the given name on the given target is a string, then this method returns
+ * it.
+ *
+ * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule}), or
+ * if the target does not have an attribute of type string with
+ * the given name
+ */
+ String getStringAttr(T target, String attrName);
+
+ /**
+ * Returns the given attribute represented as a list of strings. For "normal" attributes,
+ * this should just be a list of size one containing the attribute's value. For configurable
+ * attributes, there should be one entry for each possible value the attribute may take.
+ *
+ *<p>Note that for backwards compatibility, tristate and boolean attributes are returned as
+ * int using the values {@code 0, 1} and {@code -1}. If there is no such attribute, this
+ * method returns an empty list.
+ *
+ * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule})
+ */
+ Iterable<String> getAttrAsString(T target, String attrName);
+ }
+
+ /** List of the default query functions. */
+ public static final List<QueryFunction> DEFAULT_QUERY_FUNCTIONS =
+ ImmutableList.<QueryFunction>of(
+ new AllPathsFunction(),
+ new BuildFilesFunction(),
+ new AttrFunction(),
+ new FilterFunction(),
+ new LabelsFunction(),
+ new KindFunction(),
+ new SomeFunction(),
+ new SomePathFunction(),
+ new TestsFunction(),
+ new DepsFunction(),
+ new RdepsFunction()
+ );
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEvalResult.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEvalResult.java
new file mode 100644
index 0000000..5bcea7e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEvalResult.java
@@ -0,0 +1,51 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Set;
+
+/**
+ * The result of a query evaluation, containing a set of elements.
+ *
+ * @param <T> the node type of the elements.
+ */
+public class QueryEvalResult<T> {
+
+ protected final boolean success;
+ protected final Set<T> resultSet;
+
+ public QueryEvalResult(
+ boolean success, Set<T> resultSet) {
+ this.success = success;
+ this.resultSet = Preconditions.checkNotNull(resultSet);
+ }
+
+ /**
+ * Whether the query was successful. This can only be false if the query was run with
+ * <code>keep_going</code>, otherwise evaluation will throw a {@link QueryException}.
+ */
+ public boolean getSuccess() {
+ return success;
+ }
+
+ /**
+ * Returns the result as a set of targets.
+ */
+ public Set<T> getResultSet() {
+ return resultSet;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryException.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryException.java
new file mode 100644
index 0000000..71c1a8a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryException.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+/**
+ */
+public class QueryException extends Exception {
+
+ /**
+ * Returns a better error message for the query.
+ */
+ static String describeFailedQuery(QueryException e, QueryExpression toplevel) {
+ QueryExpression badQuery = e.getFailedExpression();
+ if (badQuery == null) {
+ return "Evaluation failed: " + e.getMessage();
+ }
+ return badQuery == toplevel
+ ? "Evaluation of query \"" + toplevel + "\" failed: " + e.getMessage()
+ : "Evaluation of subquery \"" + badQuery
+ + "\" failed (did you want to use --keep_going?): " + e.getMessage();
+ }
+
+ private final QueryExpression expression;
+
+ public QueryException(QueryException e, QueryExpression toplevel) {
+ super(describeFailedQuery(e, toplevel), e);
+ this.expression = null;
+ }
+
+ public QueryException(QueryExpression expression, String message) {
+ super(message);
+ this.expression = expression;
+ }
+
+ public QueryException(String message) {
+ this(null, message);
+ }
+
+ /**
+ * Returns the subexpression for which evaluation failed, or null if
+ * the failure occurred during lexing/parsing.
+ */
+ public QueryExpression getFailedExpression() {
+ return expression;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java
new file mode 100644
index 0000000..23603f1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * Base class for expressions in the Blaze query language, revision 2.
+ *
+ * <p>All queries return a subgraph of the dependency graph, represented
+ * as a set of target nodes.
+ *
+ * <p>All queries must ensure that sufficient graph edges are created in the
+ * QueryEnvironment so that all nodes in the result are correctly ordered
+ * according to the type of query. For example, "deps" queries require that
+ * all the nodes in the transitive closure of its argument set are correctly
+ * ordered w.r.t. each other; "somepath" queries require that the order of the
+ * nodes on the resulting path are correctly ordered; algebraic set operations
+ * such as intersect and union are inherently unordered.
+ *
+ * <h2>Package overview</h2>
+ *
+ * <p>This package consists of two basic class hierarchies. The first, {@code
+ * QueryExpression}, is the set of different query expressions in the language,
+ * and the {@link #eval} method of each defines the semantics. The result of
+ * evaluating a query is set of Blaze {@code Target}s (a file or rule). The
+ * set may be interpreted as either a set or as nodes of a DAG, depending on
+ * the context.
+ *
+ * <p>The second hierarchy is {@code OutputFormatter}. Its subclasses define
+ * different ways of printing out the result of a query. Each accepts a {@code
+ * Digraph} of {@code Target}s, and an output stream.
+ */
+public abstract class QueryExpression {
+
+ /**
+ * Scan and parse the specified query expression.
+ */
+ public static QueryExpression parse(String query, QueryEnvironment<?> env)
+ throws QueryException {
+ return QueryParser.parse(query, env);
+ }
+
+ protected QueryExpression() {}
+
+ /**
+ * Evaluates this query in the specified environment, and returns a subgraph,
+ * concretely represented a new (possibly-immutable) set of target nodes.
+ *
+ * Failures resulting from evaluation of an ill-formed query cause
+ * QueryException to be thrown.
+ *
+ * The reporting of failures arising from errors in BUILD files depends on
+ * the --keep_going flag. If enabled (the default), then QueryException is
+ * thrown. If disabled, evaluation will stumble on to produce a (possibly
+ * inaccurate) result, but a result nonetheless.
+ */
+ public abstract <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException;
+
+ /**
+ * Collects all target patterns that are referenced anywhere within this query expression and adds
+ * them to the given collection, which must be mutable.
+ */
+ public abstract void collectTargetPatterns(Collection<String> literals);
+
+ /**
+ * Returns this query expression pretty-printed.
+ */
+ @Override
+ public abstract String toString();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryParser.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryParser.java
new file mode 100644
index 0000000..bcd89cc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryParser.java
@@ -0,0 +1,261 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import static com.google.devtools.build.lib.query2.engine.Lexer.BINARY_OPERATORS;
+
+import com.google.devtools.build.lib.query2.engine.Lexer.TokenKind;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * LL(1) recursive descent parser for the Blaze query language, revision 2.
+ *
+ * In the grammar below, non-terminals are lowercase and terminals are
+ * uppercase, or character literals.
+ *
+ * <pre>
+ * expr ::= WORD
+ * | LET WORD = expr IN expr
+ * | '(' expr ')'
+ * | WORD '(' expr ( ',' expr ) * ')'
+ * | expr INTERSECT expr
+ * | expr '^' expr
+ * | expr UNION expr
+ * | expr '+' expr
+ * | expr EXCEPT expr
+ * | expr '-' expr
+ * | SET '(' WORD * ')'
+ * </pre>
+ */
+final class QueryParser {
+
+ private Lexer.Token token; // current lookahead token
+ private final List<Lexer.Token> tokens;
+ private final Iterator<Lexer.Token> tokenIterator;
+ private final Map<String, QueryFunction> functions;
+
+ /**
+ * Scan and parse the specified query expression.
+ */
+ static QueryExpression parse(String query, QueryEnvironment<?> env) throws QueryException {
+ QueryParser parser = new QueryParser(
+ Lexer.scan(query.toCharArray()), env);
+ QueryExpression expr = parser.parseExpression();
+ if (parser.token.kind != TokenKind.EOF) {
+ throw new QueryException("unexpected token '" + parser.token
+ + "' after query expression '" + expr + "'");
+ }
+ return expr;
+ }
+
+ private QueryParser(List<Lexer.Token> tokens, QueryEnvironment<?> env) {
+ this.functions = new HashMap<>();
+ for (QueryFunction queryFunction : env.getFunctions()) {
+ this.functions.put(queryFunction.getName(), queryFunction);
+ }
+ this.tokens = tokens;
+ this.tokenIterator = tokens.iterator();
+ nextToken();
+ }
+
+ /**
+ * Returns an exception. Don't forget to throw it.
+ */
+ private QueryException syntaxError(Lexer.Token token) {
+ String message = "premature end of input";
+ if (token.kind != TokenKind.EOF) {
+ StringBuilder buf = new StringBuilder("syntax error at '");
+ String sep = "";
+ for (int index = tokens.indexOf(token),
+ max = Math.min(tokens.size() - 1, index + 3); // 3 tokens of context
+ index < max; ++index) {
+ buf.append(sep).append(tokens.get(index));
+ sep = " ";
+ }
+ buf.append("'");
+ message = buf.toString();
+ }
+ return new QueryException(message);
+ }
+
+ /**
+ * Consumes the current token. If it is not of the specified (expected)
+ * kind, throws QueryException. Returns the value associated with the
+ * consumed token, if any.
+ */
+ private String consume(TokenKind kind) throws QueryException {
+ if (token.kind != kind) {
+ throw syntaxError(token);
+ }
+ String word = token.word;
+ nextToken();
+ return word;
+ }
+
+ /**
+ * Consumes the current token, which must be a WORD containing an integer
+ * literal. Returns that integer, or throws a QueryException otherwise.
+ */
+ private int consumeIntLiteral() throws QueryException {
+ String intString = consume(TokenKind.WORD);
+ try {
+ return Integer.parseInt(intString);
+ } catch (NumberFormatException e) {
+ throw new QueryException("expected an integer literal: '" + intString + "'");
+ }
+ }
+
+ private void nextToken() {
+ if (token == null || token.kind != TokenKind.EOF) {
+ token = tokenIterator.next();
+ }
+ }
+
+ /**
+ * expr ::= primary
+ * | expr INTERSECT expr
+ * | expr '^' expr
+ * | expr UNION expr
+ * | expr '+' expr
+ * | expr EXCEPT expr
+ * | expr '-' expr
+ */
+ private QueryExpression parseExpression() throws QueryException {
+ // All operators are left-associative and of equal precedence.
+ return parseBinaryOperatorTail(parsePrimary());
+ }
+
+ /**
+ * tail ::= ( <op> <primary> )*
+ * All operators have equal precedence.
+ * This factoring is required for left-associative binary operators in LL(1).
+ */
+ private QueryExpression parseBinaryOperatorTail(QueryExpression lhs) throws QueryException {
+ if (!BINARY_OPERATORS.contains(token.kind)) {
+ return lhs;
+ }
+
+ List<QueryExpression> operands = new ArrayList<>();
+ operands.add(lhs);
+ TokenKind lastOperator = token.kind;
+
+ while (BINARY_OPERATORS.contains(token.kind)) {
+ TokenKind operator = token.kind;
+ consume(operator);
+ if (operator != lastOperator) {
+ lhs = new BinaryOperatorExpression(lastOperator, operands);
+ operands.clear();
+ operands.add(lhs);
+ lastOperator = operator;
+ }
+ QueryExpression rhs = parsePrimary();
+ operands.add(rhs);
+ }
+ return new BinaryOperatorExpression(lastOperator, operands);
+ }
+
+ /**
+ * primary ::= WORD
+ * | LET WORD = expr IN expr
+ * | '(' expr ')'
+ * | WORD '(' expr ( ',' expr ) * ')'
+ * | DEPS '(' expr ')'
+ * | DEPS '(' expr ',' WORD ')'
+ * | RDEPS '(' expr ',' expr ')'
+ * | RDEPS '(' expr ',' expr ',' WORD ')'
+ * | SET '(' WORD * ')'
+ */
+ private QueryExpression parsePrimary() throws QueryException {
+ switch (token.kind) {
+ case WORD: {
+ String word = consume(TokenKind.WORD);
+ if (token.kind == TokenKind.LPAREN) {
+ QueryFunction function = functions.get(word);
+ if (function == null) {
+ throw syntaxError(token);
+ }
+ List<Argument> args = new ArrayList<>();
+ TokenKind tokenKind = TokenKind.LPAREN;
+ int argsSeen = 0;
+ for (ArgumentType type : function.getArgumentTypes()) {
+ if (token.kind == TokenKind.RPAREN && argsSeen >= function.getMandatoryArguments()) {
+ break;
+ }
+
+ consume(tokenKind);
+ tokenKind = TokenKind.COMMA;
+ switch (type) {
+ case EXPRESSION:
+ args.add(Argument.of(parseExpression()));
+ break;
+
+ case WORD:
+ args.add(Argument.of(consume(TokenKind.WORD)));
+ break;
+
+ case INTEGER:
+ args.add(Argument.of(consumeIntLiteral()));
+ break;
+
+ default:
+ throw new IllegalStateException();
+ }
+
+ argsSeen++;
+ }
+
+ consume(TokenKind.RPAREN);
+ return new FunctionExpression(function, args);
+ } else {
+ return new TargetLiteral(word);
+ }
+ }
+ case LET: {
+ consume(TokenKind.LET);
+ String name = consume(TokenKind.WORD);
+ consume(TokenKind.EQUALS);
+ QueryExpression varExpr = parseExpression();
+ consume(TokenKind.IN);
+ QueryExpression bodyExpr = parseExpression();
+ return new LetExpression(name, varExpr, bodyExpr);
+ }
+ case LPAREN: {
+ consume(TokenKind.LPAREN);
+ QueryExpression expr = parseExpression();
+ consume(TokenKind.RPAREN);
+ return expr;
+ }
+ case SET: {
+ nextToken();
+ consume(TokenKind.LPAREN);
+ List<TargetLiteral> words = new ArrayList<>();
+ while (token.kind == TokenKind.WORD) {
+ words.add(new TargetLiteral(consume(TokenKind.WORD)));
+ }
+ consume(TokenKind.RPAREN);
+ return new SetExpression(words);
+ }
+ default:
+ throw syntaxError(token);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java
new file mode 100644
index 0000000..6a8734b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java
@@ -0,0 +1,99 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * An "rdeps" query expression, which computes the reverse dependencies of the argument within the
+ * transitive closure of the universe. An optional integer-literal third argument may be
+ * specified; its value bounds the search from the arguments.
+ *
+ * <pre>expr ::= RDEPS '(' expr ',' expr ')'</pre>
+ * <pre> | RDEPS '(' expr ',' expr ',' WORD ')'</pre>
+ */
+final class RdepsFunction implements QueryFunction {
+ RdepsFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "rdeps";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 2; // last argument is optional
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(
+ ArgumentType.EXPRESSION, ArgumentType.EXPRESSION, ArgumentType.INTEGER);
+ }
+
+ /**
+ * Compute the transitive closure of the universe, then breadth-first search from the argument
+ * towards the universe while staying within the transitive closure.
+ */
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Set<T> universeValue = args.get(0).getExpression().eval(env);
+ Set<T> argumentValue = args.get(1).getExpression().eval(env);
+ int depthBound = args.size() > 2 ? args.get(2).getInteger() : Integer.MAX_VALUE;
+
+ env.buildTransitiveClosure(expression, universeValue, Integer.MAX_VALUE);
+
+ Set<T> visited = new LinkedHashSet<>();
+ Set<T> reachableFromUniverse = env.getTransitiveClosure(universeValue);
+ Collection<T> current = argumentValue;
+
+ // We need to iterate depthBound + 1 times.
+ for (int i = 0; i <= depthBound; i++) {
+ List<T> next = new ArrayList<>();
+ for (T node : current) {
+ if (!reachableFromUniverse.contains(node)) {
+ // Traversed outside the transitive closure of the universe.
+ continue;
+ }
+
+ if (!visited.add(node)) {
+ // Already visited; if we see a node in a later round, then we don't need to visit it
+ // again, because the depth at which we see it at must be greater than or equal to the
+ // last visit.
+ continue;
+ }
+
+ next.addAll(env.getReverseDeps(node));
+ }
+ if (next.isEmpty()) {
+ // Exit when there are no more nodes to visit.
+ break;
+ }
+ current = next;
+ }
+
+ return visited;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java
new file mode 100644
index 0000000..1dbe5e6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * An abstract class that provides generic regex filter expression. Actual
+ * expression are implemented by the subclasses.
+ */
+abstract class RegexFilterExpression implements QueryFunction {
+ protected RegexFilterExpression() {
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Pattern compiledPattern;
+ try {
+ compiledPattern = Pattern.compile(getPattern(args));
+ } catch (IllegalArgumentException e) {
+ throw new QueryException(expression, "illegal pattern regexp in '" + this + "': "
+ + e.getMessage());
+ }
+
+ QueryExpression argument = args.get(args.size() - 1).getExpression();
+
+ Set<T> result = new LinkedHashSet<>();
+ for (T target : argument.eval(env)) {
+ for (String str : getFilterStrings(env, args, target)) {
+ if ((str != null) && compiledPattern.matcher(str).find()) {
+ result.add(target);
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns string for the given target that must be matched against pattern.
+ * May return null, in which case matching is guaranteed to fail.
+ */
+ protected abstract <T> String getFilterString(
+ QueryEnvironment<T> env, List<Argument> args, T target);
+
+ /**
+ * Returns a list of strings for the given target that must be matched against
+ * pattern. The filter matches if *any* of these strings matches.
+ *
+ * <p>Unless subclasses have an explicit reason to override this method, it's fine
+ * to keep the default implementation that just delegates to {@link #getFilterString}.
+ * Overriding this method is useful for subclasses that want to match against a
+ * universe of possible values. For example, with configurable attributes, an
+ * attribute might have different values depending on the build configuration. One
+ * may wish the filter to match if *any* of those values matches.
+ */
+ protected <T> Iterable<String> getFilterStrings(
+ QueryEnvironment<T> env, List<Argument> args, T target) {
+ String filterString = getFilterString(env, args, target);
+ return filterString == null ? ImmutableList.<String>of() : ImmutableList.of(filterString);
+ }
+
+ protected abstract String getPattern(List<Argument> args);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java
new file mode 100644
index 0000000..a28c679
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.base.Joiner;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A set(word, ..., word) expression, which computes the union of zero or more
+ * target patterns separated by whitespace. This is intended to support the
+ * use-case in which a set of labels written to a file by a previous query
+ * expression can be modified externally, then used as input to another query,
+ * like so:
+ *
+ * <pre>
+ * % blaze query 'somepath(foo, bar)' | grep ... | sed ... | awk ... >file
+ * % blaze query "kind(qux_library, set($(<file)))"
+ * </pre>
+ *
+ * <p>The grammar currently restricts the operands of set() to being zero or
+ * more words (target patterns), with no intervening punctuation. In principle
+ * this could be extended to arbitrary expressions without grammatical
+ * ambiguity, but this seems excessively general for now.
+ *
+ * <pre>expr ::= SET '(' WORD * ')'</pre>
+ */
+class SetExpression extends QueryExpression {
+
+ private final List<TargetLiteral> words;
+
+ SetExpression(List<TargetLiteral> words) {
+ this.words = words;
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException {
+ Set<T> result = new LinkedHashSet<>();
+ for (TargetLiteral expr : words) {
+ result.addAll(expr.eval(env));
+ }
+ return result;
+ }
+
+ @Override
+ public void collectTargetPatterns(Collection<String> literals) {
+ for (TargetLiteral expr : words) {
+ expr.collectTargetPatterns(literals);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "set(" + Joiner.on(' ').join(words) + ")";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SkyframeRestartQueryException.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SkyframeRestartQueryException.java
new file mode 100644
index 0000000..d720ec9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SkyframeRestartQueryException.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+/**
+ * This exception is thrown when a query operation was unable to complete because of a Skyframe
+ * missing dependency.
+ */
+public class SkyframeRestartQueryException extends RuntimeException {
+ public SkyframeRestartQueryException() {
+ super("need skyframe retry. missing dep");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java
new file mode 100644
index 0000000..384b474
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A some(x) filter expression, which returns an arbitrary node in set x, or
+ * fails if x is empty.
+ *
+ * <pre>expr ::= SOME '(' expr ')'</pre>
+ */
+class SomeFunction implements QueryFunction {
+ SomeFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "some";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 1;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Set<T> argumentValue = args.get(0).getExpression().eval(env);
+ if (argumentValue.isEmpty()) {
+ throw new QueryException(expression, "argument set is empty");
+ }
+ return ImmutableSet.of(argumentValue.iterator().next());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java
new file mode 100644
index 0000000..b90bcdf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Sets.SetView;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A somepath(x, y) query expression, which computes the set of nodes
+ * on some arbitrary path from a target in set x to a target in set y.
+ *
+ * <pre>expr ::= SOMEPATH '(' expr ',' expr ')'</pre>
+ */
+class SomePathFunction implements QueryFunction {
+ SomePathFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "somepath";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 2;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Set<T> fromValue = args.get(0).getExpression().eval(env);
+ Set<T> toValue = args.get(1).getExpression().eval(env);
+
+ // Implementation strategy: for each x in "from", compute its forward
+ // transitive closure. If it intersects "to", then do a path search from x
+ // to an arbitrary node in the intersection, and return the path. This
+ // avoids computing the full transitive closure of "from" in some cases.
+
+ env.buildTransitiveClosure(expression, fromValue, Integer.MAX_VALUE);
+
+ // This set contains all nodes whose TC does not intersect "toValue".
+ Set<T> done = new HashSet<>();
+
+ for (T x : fromValue) {
+ if (done.contains(x)) {
+ continue;
+ }
+ Set<T> xtc = env.getTransitiveClosure(ImmutableSet.of(x));
+ SetView<T> result;
+ if (xtc.size() > toValue.size()) {
+ result = Sets.intersection(toValue, xtc);
+ } else {
+ result = Sets.intersection(xtc, toValue);
+ }
+ if (!result.isEmpty()) {
+ return env.getNodesOnPath(x, result.iterator().next());
+ }
+ done.addAll(xtc);
+ }
+ return ImmutableSet.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java
new file mode 100644
index 0000000..b6a57cc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * A literal set of targets, using 'blaze build' syntax. Or, a reference to a
+ * variable name. (The syntax of the string "pattern" determines which.)
+ *
+ * TODO(bazel-team): Perhaps we should distinguish NAME from WORD in the parser,
+ * based on the characters in it? Also, perhaps we should not allow NAMEs to
+ * be quoted like WORDs can be.
+ *
+ * <pre>expr ::= NAME | WORD</pre>
+ */
+final class TargetLiteral extends QueryExpression {
+
+ private final String pattern;
+
+ TargetLiteral(String pattern) {
+ this.pattern = Preconditions.checkNotNull(pattern);
+ }
+
+ public boolean isVariableReference() {
+ return LetExpression.isValidVarReference(pattern);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException {
+ if (isVariableReference()) {
+ String varName = LetExpression.getNameFromReference(pattern);
+ Set<T> value = env.getVariable(varName);
+ if (value == null) {
+ throw new QueryException(this, "undefined variable '" + varName + "'");
+ }
+ return env.getVariable(varName);
+ }
+
+ return env.getTargetsMatchingPattern(this, pattern);
+ }
+
+ @Override
+ public void collectTargetPatterns(Collection<String> literals) {
+ if (!isVariableReference()) {
+ literals.add(pattern);
+ }
+ }
+
+ @Override
+ public String toString() {
+ // Keep predicate consistent with Lexer.scanWord!
+ boolean needsQuoting = Lexer.isReservedWord(pattern)
+ || pattern.isEmpty()
+ || "$-*".indexOf(pattern.charAt(0)) != -1;
+ return needsQuoting ? ("\"" + pattern + "\"") : pattern;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java
new file mode 100644
index 0000000..c902609
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java
@@ -0,0 +1,257 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.engine;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A tests(x) filter expression, which returns all the tests in set x,
+ * expanding test_suite rules into their constituents.
+ *
+ * <p>Unfortunately this class reproduces a substantial amount of logic from
+ * {@code TestSuiteConfiguredTarget}, albeit in a somewhat simplified form.
+ * This is basically inevitable since the expansion of test_suites cannot be
+ * done during the loading phase, because it involves inter-package references.
+ * We make no attempt to validate the input, or report errors or warnings other
+ * than missing target.
+ *
+ * <pre>expr ::= TESTS '(' expr ')'</pre>
+ */
+class TestsFunction implements QueryFunction {
+ TestsFunction() {
+ }
+
+ @Override
+ public String getName() {
+ return "tests";
+ }
+
+ @Override
+ public int getMandatoryArguments() {
+ return 1;
+ }
+
+ @Override
+ public List<ArgumentType> getArgumentTypes() {
+ return ImmutableList.of(ArgumentType.EXPRESSION);
+ }
+
+ @Override
+ public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args)
+ throws QueryException {
+ Closure<T> closure = new Closure<>(expression, env);
+ Set<T> result = new HashSet<>();
+ for (T target : args.get(0).getExpression().eval(env)) {
+ if (env.getAccessor().isTestRule(target)) {
+ result.add(target);
+ } else if (env.getAccessor().isTestSuite(target)) {
+ for (T test : closure.getTestsInSuite(target)) {
+ result.add(env.getOrCreate(test));
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Decides whether to include a test in a test_suite or not.
+ * @param testTags Collection of all tags exhibited by a given test.
+ * @param positiveTags Tags declared by the suite. A test must match ALL of these.
+ * @param negativeTags Tags declared by the suite. A test must match NONE of these.
+ * @return false is the test is to be removed.
+ */
+ private static boolean includeTest(Collection<String> testTags,
+ Collection<String> positiveTags, Collection<String> negativeTags) {
+ // Add this test if it matches ALL of the positive tags and NONE of the
+ // negative tags in the tags attribute.
+ for (String tag : negativeTags) {
+ if (testTags.contains(tag)) {
+ return false;
+ }
+ }
+ for (String tag : positiveTags) {
+ if (!testTags.contains(tag)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Separates a list of text "tags" into a Pair of Collections, where
+ * the first element are the required or positive tags and the second element
+ * are the excluded or negative tags.
+ * This should work on tag list provided from the command line
+ * --test_tags_filters flag or on tag filters explicitly declared in the
+ * suite.
+ *
+ * Keep this function in sync with the version in
+ * java.com.google.devtools.build.lib.view.packages.TestTargetUtils.sortTagsBySense
+ *
+ * @param tagList A collection of text tags to separate.
+ */
+ private static void sortTagsBySense(
+ Collection<String> tagList, Set<String> requiredTags, Set<String> excludedTags) {
+ for (String tag : tagList) {
+ if (tag.startsWith("-")) {
+ excludedTags.add(tag.substring(1));
+ } else if (tag.startsWith("+")) {
+ requiredTags.add(tag.substring(1));
+ } else if (tag.equals("manual")) {
+ // Ignore manual attribute because it is an exception: it is not a filter
+ // but a property of test_suite
+ continue;
+ } else {
+ requiredTags.add(tag);
+ }
+ }
+ }
+
+ /**
+ * A closure over the temporary state needed to compute the expression. This makes the evaluation
+ * thread-safe, as long as instances of this class are used only within a single thread.
+ */
+ private final class Closure<T> {
+ private final QueryExpression expression;
+ /** A dynamically-populated mapping from test_suite rules to their tests. */
+ private final Map<T, Set<T>> testsInSuite = new HashMap<>();
+
+ /** The environment in which this query is being evaluated. */
+ private final QueryEnvironment<T> env;
+
+ private final boolean strict;
+
+ private Closure(QueryExpression expression, QueryEnvironment<T> env) {
+ this.expression = expression;
+ this.env = env;
+ this.strict = env.isSettingEnabled(Setting.TESTS_EXPRESSION_STRICT);
+ }
+
+ /**
+ * Computes and returns the set of test rules in a particular suite. Uses
+ * dynamic programming---a memoized version of {@link #computeTestsInSuite}.
+ *
+ * @precondition env.getAccessor().isTestSuite(testSuite)
+ */
+ private Set<T> getTestsInSuite(T testSuite) throws QueryException {
+ Set<T> tests = testsInSuite.get(testSuite);
+ if (tests == null) {
+ tests = Sets.newHashSet();
+ testsInSuite.put(testSuite, tests); // break cycles by inserting empty set early.
+ computeTestsInSuite(testSuite, tests);
+ }
+ return tests;
+ }
+
+ /**
+ * Populates 'result' with all the tests associated with the specified
+ * 'testSuite'. Throws an exception if any target is missing.
+ *
+ * <p>CAUTION! Keep this logic consistent with {@code TestsSuiteConfiguredTarget}!
+ *
+ * @precondition env.getAccessor().isTestSuite(testSuite)
+ */
+ private void computeTestsInSuite(T testSuite, Set<T> result) throws QueryException {
+ List<T> testsAndSuites = new ArrayList<>();
+ // Note that testsAndSuites can contain input file targets; the test_suite rule does not
+ // restrict the set of targets that can appear in tests or suites.
+ testsAndSuites.addAll(getPrerequisites(testSuite, "tests"));
+ testsAndSuites.addAll(getPrerequisites(testSuite, "suites"));
+
+ // 1. Add all tests
+ for (T test : testsAndSuites) {
+ if (env.getAccessor().isTestRule(test)) {
+ result.add(test);
+ } else if (strict && !env.getAccessor().isTestSuite(test)) {
+ // If strict mode is enabled, then give an error for any non-test, non-test-suite targets.
+ env.reportBuildFileError(expression, "The label '"
+ + env.getAccessor().getLabel(test) + "' in the test_suite '"
+ + env.getAccessor().getLabel(testSuite) + "' does not refer to a test or test_suite "
+ + "rule!");
+ }
+ }
+
+ // 2. Add implicit dependencies on tests in same package, if any.
+ for (T target : getPrerequisites(testSuite, "$implicit_tests")) {
+ // The Package construction of $implicit_tests ensures that this check never fails, but we
+ // add it here anyway for compatibility with future code.
+ if (env.getAccessor().isTestRule(target)) {
+ result.add(target);
+ }
+ }
+
+ // 3. Filter based on tags, size, env.
+ filterTests(testSuite, result);
+
+ // 4. Expand all suites recursively.
+ for (T suite : testsAndSuites) {
+ if (env.getAccessor().isTestSuite(suite)) {
+ result.addAll(getTestsInSuite(suite));
+ }
+ }
+ }
+
+ /**
+ * Returns the set of rules named by the attribute 'attrName' of test_suite rule 'testSuite'.
+ * The attribute must be a list of labels. If a target cannot be resolved, then an error is
+ * reported to the environment (which may throw an exception if {@code keep_going} is disabled).
+ *
+ * @precondition env.getAccessor().isTestSuite(testSuite)
+ */
+ private List<T> getPrerequisites(T testSuite, String attrName) throws QueryException {
+ return env.getAccessor().getLabelListAttr(expression, testSuite, attrName,
+ "couldn't expand '" + attrName
+ + "' attribute of test_suite " + env.getAccessor().getLabel(testSuite) + ": ");
+ }
+
+ /**
+ * Filters 'tests' (by mutation) according to the 'tags' attribute, specifically those that
+ * match ALL of the tags in tagsAttribute.
+ *
+ * @precondition {@code env.getAccessor().isTestSuite(testSuite)}
+ * @precondition {@code env.getAccessor().isTestRule(test)} for all test in tests
+ */
+ private void filterTests(T testSuite, Set<T> tests) {
+ List<String> tagsAttribute = env.getAccessor().getStringListAttr(testSuite, "tags");
+ // Split the tags list into positive and negative tags
+ Set<String> requiredTags = new HashSet<>();
+ Set<String> excludedTags = new HashSet<>();
+ sortTagsBySense(tagsAttribute, requiredTags, excludedTags);
+
+ Iterator<T> it = tests.iterator();
+ while (it.hasNext()) {
+ T test = it.next();
+ List<String> testTags = new ArrayList<>(env.getAccessor().getStringListAttr(test, "tags"));
+ testTags.add(env.getAccessor().getStringAttr(test, "size"));
+ if (!includeTest(testTags, requiredTags, excludedTags)) {
+ it.remove();
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/GraphOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/GraphOutputFormatter.java
new file mode 100644
index 0000000..5ded5e2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/output/GraphOutputFormatter.java
@@ -0,0 +1,174 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.output;
+
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.EquivalenceRelation;
+import com.google.devtools.build.lib.graph.Digraph;
+import com.google.devtools.build.lib.graph.DotOutputVisitor;
+import com.google.devtools.build.lib.graph.LabelSerializer;
+import com.google.devtools.build.lib.graph.Node;
+import com.google.devtools.build.lib.packages.Target;
+
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * An output formatter that prints the result as factored graph in AT&T
+ * GraphViz format.
+ */
+class GraphOutputFormatter extends OutputFormatter {
+
+ private int graphNodeStringLimit;
+ private boolean graphFactored;
+
+ @Override
+ public String getName() {
+ return "graph";
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ this.graphNodeStringLimit = options.graphNodeStringLimit;
+ this.graphFactored = options.graphFactored;
+
+ if (graphFactored) {
+ outputFactored(result, new PrintWriter(out));
+ } else {
+ outputUnfactored(result, new PrintWriter(out));
+ }
+ }
+
+ private void outputUnfactored(Digraph<Target> result, PrintWriter out) {
+ result.visitNodesBeforeEdges(
+ new DotOutputVisitor<Target>(out, LABEL_STRINGIFIER) {
+ @Override
+ public void beginVisit() {
+ super.beginVisit();
+ // TODO(bazel-team): (2009) make this the default in Digraph.
+ out.println(" node [shape=box];");
+ }
+ });
+ }
+
+ private void outputFactored(Digraph<Target> result, PrintWriter out) {
+ EquivalenceRelation<Node<Target>> equivalenceRelation = createEquivalenceRelation();
+
+ // Notes on ordering:
+ // - Digraph.getNodes() returns nodes in no particular order
+ // - CollectionUtils.partition inserts elements into unordered sets
+ // This means partitions may contain nodes in a different order than perhaps expected.
+ // Example (package //foo):
+ // some_rule(
+ // name = 'foo',
+ // srcs = ['a', 'b', 'c'],
+ // )
+ // Querying for deps('foo') will return (among others) the 'foo' node with successors 'a', 'b'
+ // and 'c' (in this order), however when asking the Digraph for all of its nodes, the returned
+ // collection may be ordered differently.
+ Collection<Set<Node<Target>>> partition =
+ CollectionUtils.partition(result.getNodes(), equivalenceRelation);
+
+ Digraph<Set<Node<Target>>> factoredGraph = result.createImageUnderPartition(partition);
+
+ // Concatenate the labels of all topologically-equivalent nodes.
+ LabelSerializer<Set<Node<Target>>> labelSerializer = new LabelSerializer<Set<Node<Target>>>() {
+ @Override
+ public String serialize(Node<Set<Node<Target>>> node) {
+ int actualLimit = graphNodeStringLimit - RESERVED_LABEL_CHARS;
+ boolean firstItem = true;
+ StringBuffer buf = new StringBuffer();
+ int count = 0;
+ for (Node<Target> eqNode : node.getLabel()) {
+ String labelString = eqNode.getLabel().getLabel().toString();
+ if (!firstItem) {
+ buf.append("\\n");
+
+ // Use -1 to denote no limit, as it is easier than trying to pass MAX_INT on the cmdline
+ if (graphNodeStringLimit != -1 && (buf.length() + labelString.length() > actualLimit)) {
+ buf.append("...and ");
+ buf.append(node.getLabel().size() - count);
+ buf.append(" more items");
+ break;
+ }
+ }
+
+ buf.append(labelString);
+ count++;
+ firstItem = false;
+ }
+ return buf.toString();
+ }
+ };
+
+ factoredGraph.visitNodesBeforeEdges(
+ new DotOutputVisitor<Set<Node<Target>>>(out, labelSerializer) {
+ @Override
+ public void beginVisit() {
+ super.beginVisit();
+ // TODO(bazel-team): (2009) make this the default in Digraph.
+ out.println(" node [shape=box];");
+ }
+ });
+ }
+
+ /**
+ * Returns an equivalence relation for nodes in the specified graph.
+ *
+ * <p>Two nodes are considered equal iff they have equal topology (predecessors and successors).
+ *
+ * TODO(bazel-team): Make this a method of Digraph.
+ */
+ private static <LABEL> EquivalenceRelation<Node<LABEL>> createEquivalenceRelation() {
+ return new EquivalenceRelation<Node<LABEL>>() {
+ @Override
+ public int compare(Node<LABEL> x, Node<LABEL> y) {
+ if (x == y) {
+ return 0;
+ }
+
+ if (x.numPredecessors() != y.numPredecessors()
+ || x.numSuccessors() != y.numSuccessors()) {
+ return -1;
+ }
+
+ Set<Node<LABEL>> xpred = new HashSet<>(x.getPredecessors());
+ Set<Node<LABEL>> ypred = new HashSet<>(y.getPredecessors());
+ if (!xpred.equals(ypred)) {
+ return -1;
+ }
+
+ Set<Node<LABEL>> xsucc = new HashSet<>(x.getSuccessors());
+ Set<Node<LABEL>> ysucc = new HashSet<>(y.getSuccessors());
+ if (!xsucc.equals(ysucc)) {
+ return -1;
+ }
+
+ return 0;
+ }
+ };
+ }
+
+ private static final int RESERVED_LABEL_CHARS = "\\n...and 9999999 more items".length();
+
+ private static final LabelSerializer<Target> LABEL_STRINGIFIER = new LabelSerializer<Target>() {
+ @Override
+ public String serialize(Node<Target> node) {
+ return node.getLabel().getLabel().toString();
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java
new file mode 100644
index 0000000..b7a9d64
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java
@@ -0,0 +1,486 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.output;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.graph.Digraph;
+import com.google.devtools.build.lib.graph.Node;
+import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.BinaryPredicate;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.common.options.EnumConverter;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Interface for classes which order, format and print the result of a Blaze
+ * graph query.
+ */
+public abstract class OutputFormatter {
+
+ /**
+ * Discriminator for different kinds of OutputFormatter.
+ */
+ public enum Type {
+ LABEL,
+ LABEL_KIND,
+ BUILD,
+ MINRANK,
+ MAXRANK,
+ PACKAGE,
+ LOCATION,
+ GRAPH,
+ XML,
+ PROTO,
+ RECORD,
+ }
+
+ /**
+ * Where the value of an attribute comes from
+ */
+ protected enum AttributeValueSource {
+ RULE, // Explicitly specified on the rule
+ PACKAGE, // Package default
+ DEFAULT // Rule class default
+ }
+
+ public static final Function<Node<Target>, Target> EXTRACT_NODE_LABEL =
+ new Function<Node<Target>, Target>() {
+ @Override
+ public Target apply(Node<Target> input) {
+ return input.getLabel();
+ }
+ };
+
+ /**
+ * Converter from strings to OutputFormatter.Type.
+ */
+ public static class Converter extends EnumConverter<Type> {
+ public Converter() { super(Type.class, "output formatter"); }
+ }
+
+ public static ImmutableList<OutputFormatter> getDefaultFormatters() {
+ return ImmutableList.of(
+ new LabelOutputFormatter(false),
+ new LabelOutputFormatter(true),
+ new BuildOutputFormatter(),
+ new MinrankOutputFormatter(),
+ new MaxrankOutputFormatter(),
+ new PackageOutputFormatter(),
+ new LocationOutputFormatter(),
+ new GraphOutputFormatter(),
+ new XmlOutputFormatter(),
+ new ProtoOutputFormatter());
+ }
+
+ public static String formatterNames(Iterable<OutputFormatter> formatters) {
+ return Joiner.on(", ").join(Iterables.transform(formatters,
+ new Function<OutputFormatter, String>() {
+ @Override
+ public String apply(OutputFormatter input) {
+ return input.getName();
+ }
+ }));
+ }
+
+ /**
+ * Returns the output formatter for the specified command-line options.
+ */
+ public static OutputFormatter getFormatter(
+ Iterable<OutputFormatter> formatters, String type) {
+ for (OutputFormatter formatter : formatters) {
+ if (formatter.getName().equals(type)) {
+ return formatter;
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Given a set of query options, returns a BinaryPredicate suitable for
+ * passing to {@link Rule#getLabels()}, {@link XmlOutputFormatter}, etc.
+ */
+ public static BinaryPredicate<Rule, Attribute> getDependencyFilter(QueryOptions queryOptions) {
+ // TODO(bazel-team): Optimize: and(ALL_DEPS, x) -> x, etc.
+ return Rule.and(
+ queryOptions.includeHostDeps ? Rule.ALL_DEPS : Rule.NO_HOST_DEPS,
+ queryOptions.includeImplicitDeps ? Rule.ALL_DEPS : Rule.NO_IMPLICIT_DEPS);
+ }
+
+ /**
+ * Format the result (a set of target nodes implicitly ordered according to
+ * the graph maintained by the QueryEnvironment), and print it to "out".
+ */
+ public abstract void output(QueryOptions options, Digraph<Target> result, PrintStream out)
+ throws IOException;
+
+ /**
+ * Unordered output formatter (wrt. dependency ordering).
+ *
+ * <p>Formatters that support unordered output may be used when only the set of query results is
+ * requested but their ordering is irrelevant.
+ *
+ * <p>The benefit of using a unordered formatter is that we can save the potentially expensive
+ * subgraph extraction step before presenting the query results.
+ */
+ public interface UnorderedFormatter {
+ void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out)
+ throws IOException;
+ }
+
+ /**
+ * Returns the user-visible name of the output formatter.
+ */
+ public abstract String getName();
+
+ /**
+ * An output formatter that prints the labels of the resulting target set in
+ * topological order, optionally with the target's kind.
+ */
+ private static class LabelOutputFormatter extends OutputFormatter implements UnorderedFormatter{
+
+ private final boolean showKind;
+
+ public LabelOutputFormatter(boolean showKind) {
+ this.showKind = showKind;
+ }
+
+ @Override
+ public String getName() {
+ return showKind ? "label_kind" : "label";
+ }
+
+ @Override
+ public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) {
+ for (Target target : result) {
+ if (showKind) {
+ out.print(target.getTargetKind());
+ out.print(' ');
+ }
+ out.println(target.getLabel());
+ }
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ Iterable<Target> ordered = Iterables.transform(
+ result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL);
+ outputUnordered(options, ordered, out);
+ }
+ }
+
+ /**
+ * An ordering of Targets based on the ordering of their labels.
+ */
+ static class TargetOrdering implements Comparator<Target> {
+ @Override
+ public int compare(Target o1, Target o2) {
+ return o1.getLabel().compareTo(o2.getLabel());
+ }
+ }
+
+ /**
+ * An output formatter that prints the names of the packages of the target
+ * set, in lexicographical order without duplicates.
+ */
+ private static class PackageOutputFormatter extends OutputFormatter implements
+ UnorderedFormatter {
+ @Override
+ public String getName() {
+ return "package";
+ }
+
+ @Override
+ public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) {
+ Set<String> packageNames = Sets.newTreeSet();
+ for (Target target : result) {
+ packageNames.add(target.getLabel().getPackageName());
+ }
+ for (String packageName : packageNames) {
+ out.println(packageName);
+ }
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ Iterable<Target> ordered = Iterables.transform(
+ result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL);
+ outputUnordered(options, ordered, out);
+ }
+ }
+
+ /**
+ * An output formatter that prints the labels of the targets, preceded by
+ * their locations and kinds, in topological order. For output files, the
+ * location of the generating rule is given; for input files, the location of
+ * line 1 is given.
+ */
+ private static class LocationOutputFormatter extends OutputFormatter implements
+ UnorderedFormatter {
+ @Override
+ public String getName() {
+ return "location";
+ }
+
+ @Override
+ public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) {
+ for (Target target : result) {
+ Location location = target.getLocation();
+ out.println(location.print() + ": " + target.getTargetKind() + " " + target.getLabel());
+ }
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ Iterable<Target> ordered = Iterables.transform(
+ result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL);
+ outputUnordered(options, ordered, out);
+ }
+ }
+
+ /**
+ * An output formatter that prints the generating rules using the syntax of
+ * the BUILD files. If multiple targets are generated by the same rule, it is
+ * printed only once.
+ */
+ private static class BuildOutputFormatter extends OutputFormatter implements UnorderedFormatter {
+ @Override
+ public String getName() {
+ return "build";
+ }
+
+ private void outputRule(Rule rule, PrintStream out) {
+ out.println(String.format("# %s", rule.getLocation()));
+ out.println(String.format("%s(", rule.getRuleClass()));
+ out.println(String.format(" name = \"%s\",", rule.getName()));
+
+ for (Attribute attr : rule.getAttributes()) {
+ Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr);
+ if (Iterables.size(values.first) != 1) {
+ continue; // TODO(bazel-team): handle configurable attributes.
+ }
+ if (values.second != AttributeValueSource.RULE) {
+ continue; // Don't print default values.
+ }
+ Object value = Iterables.getOnlyElement(values.first);
+ out.print(String.format(" %s = ", attr.getName()));
+ if (value instanceof Label) {
+ value = value.toString();
+ } else if (value instanceof List<?> && EvalUtils.isImmutable(value)) {
+ // Display it as a list (and not as a tuple). Attributes can never be tuples.
+ value = new ArrayList<>((List<?>) value);
+ }
+ EvalUtils.prettyPrintValue(value, out);
+ out.println(",");
+ }
+ out.println(String.format(")\n"));
+ }
+
+ @Override
+ public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) {
+ Set<Label> printed = new HashSet<>();
+ for (Target target : result) {
+ Rule rule = target.getAssociatedRule();
+ if (rule == null || printed.contains(rule.getLabel())) {
+ continue;
+ }
+ outputRule(rule, out);
+ printed.add(rule.getLabel());
+ }
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ Iterable<Target> ordered = Iterables.transform(
+ result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL);
+ outputUnordered(options, ordered, out);
+ }
+ }
+
+ /**
+ * An output formatter that prints the labels in minimum rank order, preceded by
+ * their rank number. "Roots" have rank 0, their direct prerequisites have
+ * rank 1, etc. All nodes in a cycle are considered of equal rank. MINRANK
+ * shows the lowest rank for a given node, i.e. the length of the shortest
+ * path from a zero-rank node to it.
+ *
+ * If the result came from a <code>deps(x)</code> query, then the MINRANKs
+ * correspond to the shortest path from x to each of its prerequisites.
+ */
+ private static class MinrankOutputFormatter extends OutputFormatter {
+ @Override
+ public String getName() {
+ return "minrank";
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ // getRoots() isn't defined for cyclic graphs, so in order to handle
+ // cycles correctly, we need work on the strong component graph, as
+ // cycles should be treated a "clump" of nodes all on the same rank.
+ // Graphs may contain cycles because there are errors in BUILD files.
+
+ Digraph<Set<Node<Target>>> scGraph = result.getStrongComponentGraph();
+ Set<Node<Set<Node<Target>>>> rankNodes = scGraph.getRoots();
+ Set<Node<Set<Node<Target>>>> seen = new HashSet<>();
+ seen.addAll(rankNodes);
+ for (int rank = 0; !rankNodes.isEmpty(); rank++) {
+ // Print out this rank:
+ for (Node<Set<Node<Target>>> xScc : rankNodes) {
+ for (Node<Target> x : xScc.getLabel()) {
+ out.println(rank + " " + x.getLabel().getLabel());
+ }
+ }
+
+ // Find the next rank:
+ Set<Node<Set<Node<Target>>>> nextRankNodes = new LinkedHashSet<>();
+ for (Node<Set<Node<Target>>> x : rankNodes) {
+ for (Node<Set<Node<Target>>> y : x.getSuccessors()) {
+ if (seen.add(y)) {
+ nextRankNodes.add(y);
+ }
+ }
+ }
+ rankNodes = nextRankNodes;
+ }
+ }
+ }
+
+ /**
+ * An output formatter that prints the labels in maximum rank order, preceded
+ * by their rank number. "Roots" have rank 0, all other nodes have a rank
+ * which is one greater than the maximum rank of each of their predecessors.
+ * All nodes in a cycle are considered of equal rank. MAXRANK shows the
+ * highest rank for a given node, i.e. the length of the longest non-cyclic
+ * path from a zero-rank node to it.
+ *
+ * If the result came from a <code>deps(x)</code> query, then the MAXRANKs
+ * correspond to the longest path from x to each of its prerequisites.
+ */
+ private static class MaxrankOutputFormatter extends OutputFormatter {
+ @Override
+ public String getName() {
+ return "maxrank";
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ // In order to handle cycles correctly, we need work on the strong
+ // component graph, as cycles should be treated a "clump" of nodes all on
+ // the same rank. Graphs may contain cycles because there are errors in BUILD files.
+
+ // Dynamic programming algorithm:
+ // rank(x) = max(rank(p)) + 1 foreach p in preds(x)
+ // TODO(bazel-team): Move to Digraph.
+ class DP {
+ final Map<Node<Set<Node<Target>>>, Integer> ranks = new HashMap<>();
+
+ int rank(Node<Set<Node<Target>>> node) {
+ Integer rank = ranks.get(node);
+ if (rank == null) {
+ int maxPredRank = -1;
+ for (Node<Set<Node<Target>>> p : node.getPredecessors()) {
+ maxPredRank = Math.max(maxPredRank, rank(p));
+ }
+ rank = maxPredRank + 1;
+ ranks.put(node, rank);
+ }
+ return rank;
+ }
+ }
+ DP dp = new DP();
+
+ // Now sort by rank...
+ List<Pair<Integer, Label>> output = new ArrayList<>();
+ for (Node<Set<Node<Target>>> x : result.getStrongComponentGraph().getNodes()) {
+ int rank = dp.rank(x);
+ for (Node<Target> y : x.getLabel()) {
+ output.add(Pair.of(rank, y.getLabel().getLabel()));
+ }
+ }
+ Collections.sort(output, new Comparator<Pair<Integer, Label>>() {
+ @Override
+ public int compare(Pair<Integer, Label> x, Pair<Integer, Label> y) {
+ return x.first - y.first;
+ }
+ });
+
+ for (Pair<Integer, Label> pair : output) {
+ out.println(pair.first + " " + pair.second);
+ }
+ }
+ }
+
+ /**
+ * Returns the possible values of the specified attribute in the specified rule. For
+ * non-configured attributes, this is a single value. For configurable attributes, this
+ * may be multiple values.
+ *
+ * <p>This is needed because the visibility attribute is replaced with an empty list
+ * during package loading if it is public or private in order not to visit
+ * the package called 'visibility'.
+ *
+ * @return a pair, where the first value is the set of possible values and the
+ * second is an enum that tells where the values come from (declared on the
+ * rule, declared as a package level default or a
+ * global default)
+ */
+ protected static Pair<Iterable<Object>, AttributeValueSource> getAttributeValues(
+ Rule rule, Attribute attr) {
+ List<Object> values = new LinkedList<>(); // Not an ImmutableList: may host null values.
+ AttributeValueSource source;
+
+ if (attr.getName().equals("visibility")) {
+ values.add(rule.getVisibility().getDeclaredLabels());
+ if (rule.isVisibilitySpecified()) {
+ source = AttributeValueSource.RULE;
+ } else if (rule.getPackage().isDefaultVisibilitySet()) {
+ source = AttributeValueSource.PACKAGE;
+ } else {
+ source = AttributeValueSource.DEFAULT;
+ }
+ } else {
+ for (Object o :
+ AggregatingAttributeMapper.of(rule).visitAttribute(attr.getName(), attr.getType())) {
+ values.add(o);
+ }
+ source = rule.isAttributeValueExplicitlySpecified(attr)
+ ? AttributeValueSource.RULE : AttributeValueSource.DEFAULT;
+ }
+
+ return Pair.of((Iterable<Object>) values, source);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java
new file mode 100644
index 0000000..53fbb21
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java
@@ -0,0 +1,491 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.output;
+
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS;
+import static com.google.devtools.build.lib.packages.Type.FILESET_ENTRY_LIST;
+import static com.google.devtools.build.lib.packages.Type.INTEGER;
+import static com.google.devtools.build.lib.packages.Type.INTEGER_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL;
+import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT;
+import static com.google.devtools.build.lib.packages.Type.STRING_DICT_UNARY;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST_DICT;
+import static com.google.devtools.build.lib.packages.Type.TRISTATE;
+import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.GENERATED_FILE;
+import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.PACKAGE_GROUP;
+import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.RULE;
+import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.SOURCE_FILE;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.graph.Digraph;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.ProtoUtils;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TriState;
+import com.google.devtools.build.lib.query2.FakeSubincludeTarget;
+import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.syntax.GlobCriteria;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.util.BinaryPredicate;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An output formatter that outputs a protocol buffer representation
+ * of a query result and outputs the proto bytes to the output print stream.
+ * By taking the bytes and calling {@code mergeFrom()} on a
+ * {@code Build.QueryResult} object the full result can be reconstructed.
+ */
+public class ProtoOutputFormatter extends OutputFormatter implements UnorderedFormatter {
+
+ /**
+ * A special attribute name for the rule implementation hash code.
+ */
+ public static final String RULE_IMPLEMENTATION_HASH_ATTR_NAME = "$rule_implementation_hash";
+
+ private BinaryPredicate<Rule, Attribute> dependencyFilter;
+
+ protected void setDependencyFilter(QueryOptions options) {
+ this.dependencyFilter = OutputFormatter.getDependencyFilter(options);
+ }
+
+ @Override
+ public String getName() {
+ return "proto";
+ }
+
+ @Override
+ public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out)
+ throws IOException {
+ setDependencyFilter(options);
+
+ Build.QueryResult.Builder queryResult = Build.QueryResult.newBuilder();
+ for (Target target : result) {
+ addTarget(queryResult, target);
+ }
+
+ queryResult.build().writeTo(out);
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out)
+ throws IOException {
+ outputUnordered(options, result.getLabels(), out);
+ }
+
+ /**
+ * Add the target to the query result.
+ * @param queryResult The query result that contains all rule, input and
+ * output targets.
+ * @param target The query target being converted to a protocol buffer.
+ */
+ private void addTarget(Build.QueryResult.Builder queryResult, Target target) {
+ queryResult.addTarget(toTargetProtoBuffer(target));
+ }
+
+ /**
+ * Converts a logical Target object into a Target protobuffer.
+ */
+ protected Build.Target toTargetProtoBuffer(Target target) {
+ Build.Target.Builder targetPb = Build.Target.newBuilder();
+
+ String location = target.getLocation().print();
+ if (target instanceof Rule) {
+ Rule rule = (Rule) target;
+ Build.Rule.Builder rulePb = Build.Rule.newBuilder()
+ .setName(rule.getLabel().toString())
+ .setRuleClass(rule.getRuleClass())
+ .setLocation(location);
+
+ for (Attribute attr : rule.getAttributes()) {
+ addAttributeToProto(rulePb, attr, getAttributeValues(rule, attr).first, null,
+ rule.isAttributeValueExplicitlySpecified(attr), false);
+ }
+
+ SkylarkEnvironment env = rule.getRuleClassObject().getRuleDefinitionEnvironment();
+ if (env != null) {
+ // The RuleDefinitionEnvironment is always defined for Skylark rules and
+ // always null for non Skylark rules.
+ rulePb.addAttribute(
+ Build.Attribute.newBuilder()
+ .setName(RULE_IMPLEMENTATION_HASH_ATTR_NAME)
+ .setType(ProtoUtils.getDiscriminatorFromType(
+ com.google.devtools.build.lib.packages.Type.STRING))
+ .setStringValue(env.getTransitiveFileContentHashCode()));
+ }
+
+ // Include explicit elements for all direct inputs and outputs of a rule;
+ // this goes beyond what is available from the attributes above, since it
+ // may also (depending on options) include implicit outputs,
+ // host-configuration outputs, and default values.
+ for (Label label : rule.getLabels(dependencyFilter)) {
+ rulePb.addRuleInput(label.toString());
+ }
+ for (OutputFile outputFile : rule.getOutputFiles()) {
+ Label fileLabel = outputFile.getLabel();
+ rulePb.addRuleOutput(fileLabel.toString());
+ }
+ for (String feature : rule.getFeatures()) {
+ rulePb.addDefaultSetting(feature);
+ }
+
+ targetPb.setType(RULE);
+ targetPb.setRule(rulePb);
+ } else if (target instanceof OutputFile) {
+ OutputFile outputFile = (OutputFile) target;
+ Label label = outputFile.getLabel();
+
+ Rule generatingRule = outputFile.getGeneratingRule();
+ Build.GeneratedFile output = Build.GeneratedFile.newBuilder()
+ .setLocation(location)
+ .setGeneratingRule(generatingRule.getLabel().toString())
+ .setName(label.toString())
+ .build();
+
+ targetPb.setType(GENERATED_FILE);
+ targetPb.setGeneratedFile(output);
+ } else if (target instanceof InputFile) {
+ InputFile inputFile = (InputFile) target;
+ Label label = inputFile.getLabel();
+
+ Build.SourceFile.Builder input = Build.SourceFile.newBuilder()
+ .setLocation(location)
+ .setName(label.toString());
+
+ if (inputFile.getName().equals("BUILD")) {
+ for (Label subinclude : inputFile.getPackage().getSubincludeLabels()) {
+ input.addSubinclude(subinclude.toString());
+ }
+
+ for (Label skylarkFileDep : inputFile.getPackage().getSkylarkFileDependencies()) {
+ input.addSubinclude(skylarkFileDep.toString());
+ }
+
+ for (String feature : inputFile.getPackage().getFeatures()) {
+ input.addFeature(feature);
+ }
+ }
+
+ for (Label visibilityDependency : target.getVisibility().getDependencyLabels()) {
+ input.addPackageGroup(visibilityDependency.toString());
+ }
+
+ for (Label visibilityDeclaration : target.getVisibility().getDeclaredLabels()) {
+ input.addVisibilityLabel(visibilityDeclaration.toString());
+ }
+
+ targetPb.setType(SOURCE_FILE);
+ targetPb.setSourceFile(input);
+ } else if (target instanceof FakeSubincludeTarget) {
+ Label label = target.getLabel();
+ Build.SourceFile input = Build.SourceFile.newBuilder()
+ .setLocation(location)
+ .setName(label.toString())
+ .build();
+
+ targetPb.setType(SOURCE_FILE);
+ targetPb.setSourceFile(input);
+ } else if (target instanceof PackageGroup) {
+ PackageGroup packageGroup = (PackageGroup) target;
+ Build.PackageGroup.Builder packageGroupPb = Build.PackageGroup.newBuilder()
+ .setName(packageGroup.getLabel().toString());
+ for (String containedPackage : packageGroup.getContainedPackages()) {
+ packageGroupPb.addContainedPackage(containedPackage);
+ }
+ for (Label include : packageGroup.getIncludes()) {
+ packageGroupPb.addIncludedPackageGroup(include.toString());
+ }
+
+ targetPb.setType(PACKAGE_GROUP);
+ targetPb.setPackageGroup(packageGroupPb);
+ } else {
+ throw new IllegalArgumentException(target.toString());
+ }
+
+ return targetPb.build();
+ }
+
+ /**
+ * Adds the serialized version of the specified attribute to the specified message.
+ *
+ * @param rulePb the message to amend
+ * @param attr the attribute to add
+ * @param value the possible values of the attribute (can be a multi-value list for
+ * configurable attributes)
+ * @param location the location of the attribute in the source file
+ * @param explicitlySpecified whether the attribute was explicitly specified or not
+ * @param includeGlobs add glob expression for attributes that contain them
+ */
+ @SuppressWarnings("unchecked")
+ public static void addAttributeToProto(
+ Build.Rule.Builder rulePb, Attribute attr, Iterable<Object> values,
+ Location location, Boolean explicitlySpecified, boolean includeGlobs) {
+ // Get the attribute type. We need to convert and add appropriately
+ com.google.devtools.build.lib.packages.Type<?> type = attr.getType();
+
+ Build.Attribute.Builder attrPb = Build.Attribute.newBuilder();
+
+ // Set the type, name and source
+ attrPb.setName(attr.getName());
+ attrPb.setType(ProtoUtils.getDiscriminatorFromType(type));
+
+ if (location != null) {
+ attrPb.setParseableLocation(serialize(location));
+ }
+
+ if (explicitlySpecified != null) {
+ attrPb.setExplicitlySpecified(explicitlySpecified);
+ }
+
+ // Convenience binding for single-value attributes. Because those attributes can only
+ // have a single value, when we encounter configurable versions of them we need to
+ // react somehow to having multiple possible values to report. We currently just
+ // refrain from setting *any* value in that scenario. This variable is set to null
+ // to indicate that scenario.
+ Object singleAttributeValue = Iterables.size(values) == 1
+ ? Iterables.getOnlyElement(values)
+ : null;
+
+ /*
+ * Set the appropriate type and value. Since string and string list store
+ * values for multiple types, use the toString() method on the objects
+ * instead of casting them. Note that Boolean and TriState attributes have
+ * both an integer and string representation.
+ */
+ if (type == INTEGER) {
+ if (singleAttributeValue != null) {
+ attrPb.setIntValue((Integer) singleAttributeValue);
+ }
+ } else if (type == STRING || type == LABEL || type == NODEP_LABEL || type == OUTPUT) {
+ if (singleAttributeValue != null) {
+ attrPb.setStringValue(singleAttributeValue.toString());
+ }
+ } else if (type == STRING_LIST || type == LABEL_LIST || type == NODEP_LABEL_LIST
+ || type == OUTPUT_LIST || type == DISTRIBUTIONS) {
+ for (Object value : values) {
+ for (Object entry : (Collection<?>) value) {
+ attrPb.addStringListValue(entry.toString());
+ }
+ }
+ } else if (type == INTEGER_LIST) {
+ for (Object value : values) {
+ for (Integer entry : (Collection<Integer>) value) {
+ attrPb.addIntListValue(entry);
+ }
+ }
+ } else if (type == BOOLEAN) {
+ if (singleAttributeValue != null) {
+ if ((Boolean) singleAttributeValue) {
+ attrPb.setStringValue("true");
+ attrPb.setBooleanValue(true);
+ } else {
+ attrPb.setStringValue("false");
+ attrPb.setBooleanValue(false);
+ }
+ // This maintains partial backward compatibility for external users of the
+ // protobuf that were expecting an integer field and not a true boolean.
+ attrPb.setIntValue((Boolean) singleAttributeValue ? 1 : 0);
+ }
+ } else if (type == TRISTATE) {
+ if (singleAttributeValue != null) {
+ switch ((TriState) singleAttributeValue) {
+ case AUTO:
+ attrPb.setIntValue(-1);
+ attrPb.setStringValue("auto");
+ attrPb.setTristateValue(Build.Attribute.Tristate.AUTO);
+ break;
+ case NO:
+ attrPb.setIntValue(0);
+ attrPb.setStringValue("no");
+ attrPb.setTristateValue(Build.Attribute.Tristate.NO);
+ break;
+ case YES:
+ attrPb.setIntValue(1);
+ attrPb.setStringValue("yes");
+ attrPb.setTristateValue(Build.Attribute.Tristate.YES);
+ break;
+ default:
+ throw new IllegalStateException("Execpted AUTO/NO/YES to cover all possible cases");
+ }
+ }
+ } else if (type == LICENSE) {
+ if (singleAttributeValue != null) {
+ License license = (License) singleAttributeValue;
+ Build.License.Builder licensePb = Build.License.newBuilder();
+ for (License.LicenseType licenseType : license.getLicenseTypes()) {
+ licensePb.addLicenseType(licenseType.toString());
+ }
+ for (Label exception : license.getExceptions()) {
+ licensePb.addException(exception.toString());
+ }
+ attrPb.setLicense(licensePb);
+ }
+ } else if (type == STRING_DICT) {
+ // TODO(bazel-team): support better de-duping here and in other dictionaries.
+ for (Object value : values) {
+ Map<String, String> dict = (Map<String, String>) value;
+ for (Map.Entry<String, String> keyValueList : dict.entrySet()) {
+ Build.StringDictEntry entry = Build.StringDictEntry.newBuilder()
+ .setKey(keyValueList.getKey())
+ .setValue(keyValueList.getValue())
+ .build();
+ attrPb.addStringDictValue(entry);
+ }
+ }
+ } else if (type == STRING_DICT_UNARY) {
+ for (Object value : values) {
+ Map<String, String> dict = (Map<String, String>) value;
+ for (Map.Entry<String, String> dictEntry : dict.entrySet()) {
+ Build.StringDictUnaryEntry entry = Build.StringDictUnaryEntry.newBuilder()
+ .setKey(dictEntry.getKey())
+ .setValue(dictEntry.getValue())
+ .build();
+ attrPb.addStringDictUnaryValue(entry);
+ }
+ }
+ } else if (type == STRING_LIST_DICT) {
+ for (Object value : values) {
+ Map<String, List<String>> dict = (Map<String, List<String>>) value;
+ for (Map.Entry<String, List<String>> dictEntry : dict.entrySet()) {
+ Build.StringListDictEntry.Builder entry = Build.StringListDictEntry.newBuilder()
+ .setKey(dictEntry.getKey());
+ for (Object dictEntryValue : dictEntry.getValue()) {
+ entry.addValue(dictEntryValue.toString());
+ }
+ attrPb.addStringListDictValue(entry);
+ }
+ }
+ } else if (type == LABEL_LIST_DICT) {
+ for (Object value : values) {
+ Map<String, List<Label>> dict = (Map<String, List<Label>>) value;
+ for (Map.Entry<String, List<Label>> dictEntry : dict.entrySet()) {
+ Build.LabelListDictEntry.Builder entry = Build.LabelListDictEntry.newBuilder()
+ .setKey(dictEntry.getKey());
+ for (Object dictEntryValue : dictEntry.getValue()) {
+ entry.addValue(dictEntryValue.toString());
+ }
+ attrPb.addLabelListDictValue(entry);
+ }
+ }
+ } else if (type == FILESET_ENTRY_LIST) {
+ for (Object value : values) {
+ List<FilesetEntry> filesetEntries = (List<FilesetEntry>) value;
+ for (FilesetEntry filesetEntry : filesetEntries) {
+ Build.FilesetEntry.Builder filesetEntryPb = Build.FilesetEntry.newBuilder()
+ .setSource(filesetEntry.getSrcLabel().toString())
+ .setDestinationDirectory(filesetEntry.getDestDir().getPathString())
+ .setSymlinkBehavior(symlinkBehaviorToPb(filesetEntry.getSymlinkBehavior()))
+ .setStripPrefix(filesetEntry.getStripPrefix())
+ .setFilesPresent(filesetEntry.getFiles() != null);
+
+ if (filesetEntry.getFiles() != null) {
+ for (Label file : filesetEntry.getFiles()) {
+ filesetEntryPb.addFile(file.toString());
+ }
+ }
+
+ if (filesetEntry.getExcludes() != null) {
+ for (String exclude : filesetEntry.getExcludes()) {
+ filesetEntryPb.addExclude(exclude);
+ }
+ }
+
+ attrPb.addFilesetListValue(filesetEntryPb);
+ }
+ }
+ } else {
+ throw new IllegalStateException("Unknown type: " + type);
+ }
+
+ if (includeGlobs) {
+ for (Object value : values) {
+ if (value instanceof GlobList<?>) {
+ GlobList<?> globList = (GlobList<?>) value;
+
+ for (GlobCriteria criteria : globList.getCriteria()) {
+ Build.GlobCriteria.Builder criteriaPb = Build.GlobCriteria.newBuilder()
+ .setGlob(criteria.isGlob());
+ for (String include : criteria.getIncludePatterns()) {
+ criteriaPb.addInclude(include);
+ }
+ for (String exclude : criteria.getExcludePatterns()) {
+ criteriaPb.addExclude(exclude);
+ }
+
+ attrPb.addGlobCriteria(criteriaPb);
+ }
+ }
+ }
+ }
+
+ rulePb.addAttribute(attrPb);
+ }
+
+ // This is needed because I do not want to use the SymlinkBehavior from the
+ // protocol buffer all over the place, so there are two classes that do
+ // essentially the same thing.
+ private static Build.FilesetEntry.SymlinkBehavior symlinkBehaviorToPb(
+ FilesetEntry.SymlinkBehavior symlinkBehavior) {
+ switch (symlinkBehavior) {
+ case COPY:
+ return Build.FilesetEntry.SymlinkBehavior.COPY;
+ case DEREFERENCE:
+ return Build.FilesetEntry.SymlinkBehavior.DEREFERENCE;
+ default:
+ throw new AssertionError("Unhandled FilesetEntry.SymlinkBehavior");
+ }
+ }
+
+ private static Build.Location serialize(Location location) {
+ Build.Location.Builder result = Build.Location.newBuilder();
+
+ result.setStartOffset(location.getStartOffset());
+ if (location.getStartLineAndColumn() != null) {
+ result.setStartLine(location.getStartLineAndColumn().getLine());
+ result.setStartColumn(location.getStartLineAndColumn().getColumn());
+ }
+
+ result.setEndOffset(location.getEndOffset());
+ if (location.getEndLineAndColumn() != null) {
+ result.setEndLine(location.getEndLineAndColumn().getLine());
+ result.setEndColumn(location.getEndLineAndColumn().getColumn());
+ }
+
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/QueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/output/QueryOptions.java
new file mode 100644
index 0000000..810e8c7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/output/QueryOptions.java
@@ -0,0 +1,136 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.output;
+
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * Command-line options for the Blaze query language, revision 2.
+ */
+public class QueryOptions extends OptionsBase {
+
+ @Option(name = "output",
+ defaultValue = "label",
+ category = "query",
+ help = "The format in which the query results should be printed."
+ + " Allowed values are: label, label_kind, minrank, maxrank, package, location, graph,"
+ + " xml, proto, record.")
+ public String outputFormat;
+
+ @Option(name = "order_results",
+ defaultValue = "true",
+ category = "query",
+ help = "Output the results in dependency-ordered (default) or unordered fashion. The"
+ + " unordered output is faster but only supported when --output is one of label,"
+ + " label_kind, location, package, proto, record, xml.")
+ public boolean orderResults;
+
+ @Option(name = "keep_going",
+ abbrev = 'k',
+ defaultValue = "false",
+ category = "strategy",
+ help = "Continue as much as possible after an error. While the "
+ + "target that failed, and those that depend on it, cannot be "
+ + "analyzed, other prerequisites of these "
+ + "targets can be.")
+ public boolean keepGoing;
+
+ @Option(name = "loading_phase_threads",
+ defaultValue = "200",
+ category = "undocumented",
+ help = "Number of parallel threads to use for the loading phase.")
+ public int loadingPhaseThreads;
+
+ @Option(name = "host_deps",
+ defaultValue = "true",
+ category = "query",
+ help = "If enabled, dependencies on 'host configuration' targets will be included in "
+ + "the dependency graph over which the query operates. A 'host configuration' "
+ + "dependency edge, such as the one from any 'proto_library' rule to the Protocol "
+ + "Compiler, usually points to a tool executed during the build (on the host machine) "
+ + "rather than a part of the same 'target' program. Queries whose purpose is to "
+ + "discover the set of things needed during a build will typically enable this option; "
+ + "queries aimed at revealing the structure of a single program will typically disable "
+ + "this option.")
+ public boolean includeHostDeps;
+
+ @Option(name = "implicit_deps",
+ defaultValue = "true",
+ category = "query",
+ help = "If enabled, implicit dependencies will be included in the dependency graph over "
+ + "which the query operates. An implicit dependency is one that is not explicitly "
+ + "specified in the BUILD file but added by blaze.")
+ public boolean includeImplicitDeps;
+
+ @Option(name = "graph:node_limit",
+ defaultValue = "512",
+ category = "query",
+ help = "The maximum length of the label string for a graph node in the output. Longer labels"
+ + " will be truncated; -1 means no truncation. This option is only applicable to"
+ + " --output=graph.")
+ public int graphNodeStringLimit;
+
+ @Option(name = "graph:factored",
+ defaultValue = "true",
+ category = "query",
+ help = "If true, then the graph will be emitted 'factored', i.e. "
+ + "topologically-equivalent nodes will be merged together and their "
+ + "labels concatenated. This option is only applicable to "
+ + "--output=graph.")
+ public boolean graphFactored;
+
+ @Option(name = "xml:line_numbers",
+ defaultValue = "true",
+ category = "query",
+ help = "If true, XML output contains line numbers. Disabling this option "
+ + "may make diffs easier to read. This option is only applicable to "
+ + "--output=xml.")
+ public boolean xmlLineNumbers;
+
+ @Option(name = "xml:default_values",
+ defaultValue = "false",
+ category = "query",
+ help = "If true, rule attributes whose value is not explicitly specified "
+ + "in the BUILD file are printed; otherwise they are omitted.")
+ public boolean xmlShowDefaultValues;
+
+ @Option(name = "strict_test_suite",
+ defaultValue = "false",
+ category = "query",
+ help = "If true, the tests() expression gives an error if it encounters a test_suite "
+ + "containing non-test targets.")
+ public boolean strictTestSuite;
+
+ /**
+ * Return the current options as a set of QueryEnvironment settings.
+ */
+ public Set<Setting> toSettings() {
+ Set<Setting> settings = EnumSet.noneOf(Setting.class);
+ if (strictTestSuite) {
+ settings.add(Setting.TESTS_EXPRESSION_STRICT);
+ }
+ if (!includeHostDeps) {
+ settings.add(Setting.NO_HOST_DEPS);
+ }
+ if (!includeImplicitDeps) {
+ settings.add(Setting.NO_IMPLICIT_DEPS);
+ }
+ return settings;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java
new file mode 100644
index 0000000..c01c90e4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java
@@ -0,0 +1,352 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.query2.output;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.graph.Digraph;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.query2.FakeSubincludeTarget;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.BinaryPredicate;
+import com.google.devtools.build.lib.util.Pair;
+
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.TransformerFactoryConfigurationError;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+/**
+ * An output formatter that prints the result as XML.
+ */
+class XmlOutputFormatter extends OutputFormatter implements OutputFormatter.UnorderedFormatter {
+
+ private boolean xmlLineNumbers;
+ private boolean showDefaultValues;
+ private BinaryPredicate<Rule, Attribute> dependencyFilter;
+
+ @Override
+ public String getName() {
+ return "xml";
+ }
+
+ @Override
+ public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) {
+ this.xmlLineNumbers = options.xmlLineNumbers;
+ this.showDefaultValues = options.xmlShowDefaultValues;
+ this.dependencyFilter = OutputFormatter.getDependencyFilter(options);
+
+ Document doc;
+ try {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ doc = factory.newDocumentBuilder().newDocument();
+ } catch (ParserConfigurationException e) {
+ // This shouldn't be possible: all the configuration is hard-coded.
+ throw new IllegalStateException("XML output failed", e);
+ }
+ doc.setXmlVersion("1.1");
+ Element queryElem = doc.createElement("query");
+ queryElem.setAttribute("version", "2");
+ doc.appendChild(queryElem);
+ for (Target target : result) {
+ queryElem.appendChild(createTargetElement(doc, target));
+ }
+ try {
+ Transformer transformer = TransformerFactory.newInstance().newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.transform(new DOMSource(doc), new StreamResult(out));
+ } catch (TransformerFactoryConfigurationError | TransformerException e) {
+ // This shouldn't be possible: all the configuration is hard-coded.
+ throw new IllegalStateException("XML output failed", e);
+ }
+ }
+
+ @Override
+ public void output(QueryOptions options, Digraph<Target> result, PrintStream out) {
+ Iterable<Target> ordered = Iterables.transform(
+ result.getTopologicalOrder(new TargetOrdering()), OutputFormatter.EXTRACT_NODE_LABEL);
+ outputUnordered(options, ordered, out);
+ }
+
+ /**
+ * Creates and returns a new DOM tree for the specified build target.
+ *
+ * XML structure:
+ * - element tag is <source-file>, <generated-file> or <rule
+ * class="cc_library">, following the terminology of
+ * {@link Target#getTargetKind()}.
+ * - 'name' attribute is target's label.
+ * - 'location' attribute is consistent with output of --output location.
+ * - rule attributes are represented in the DOM structure.
+ */
+ private Element createTargetElement(Document doc, Target target) {
+ Element elem;
+ if (target instanceof Rule) {
+ Rule rule = (Rule) target;
+ elem = doc.createElement("rule");
+ elem.setAttribute("class", rule.getRuleClass());
+ for (Attribute attr: rule.getAttributes()) {
+ Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr);
+ if (values.second == AttributeValueSource.RULE || showDefaultValues) {
+ Element attrElem = createValueElement(doc, attr.getType(), values.first);
+ attrElem.setAttribute("name", attr.getName());
+ elem.appendChild(attrElem);
+ }
+ }
+
+ // Include explicit elements for all direct inputs and outputs of a rule;
+ // this goes beyond what is available from the attributes above, since it
+ // may also (depending on options) include implicit outputs,
+ // host-configuration outputs, and default values.
+ for (Label label : rule.getLabels(dependencyFilter)) {
+ Element inputElem = doc.createElement("rule-input");
+ inputElem.setAttribute("name", label.toString());
+ elem.appendChild(inputElem);
+ }
+ for (OutputFile outputFile: rule.getOutputFiles()) {
+ Element outputElem = doc.createElement("rule-output");
+ outputElem.setAttribute("name", outputFile.getLabel().toString());
+ elem.appendChild(outputElem);
+ }
+ for (String feature : rule.getFeatures()) {
+ Element outputElem = doc.createElement("rule-default-setting");
+ outputElem.setAttribute("name", feature);
+ elem.appendChild(outputElem);
+ }
+ } else if (target instanceof PackageGroup) {
+ PackageGroup packageGroup = (PackageGroup) target;
+ elem = doc.createElement("package-group");
+ elem.setAttribute("name", packageGroup.getName());
+ Element includes = createValueElement(doc,
+ com.google.devtools.build.lib.packages.Type.LABEL_LIST,
+ packageGroup.getIncludes());
+ includes.setAttribute("name", "includes");
+ elem.appendChild(includes);
+ Element packages = createValueElement(doc,
+ com.google.devtools.build.lib.packages.Type.STRING_LIST,
+ packageGroup.getContainedPackages());
+ packages.setAttribute("name", "packages");
+ elem.appendChild(packages);
+ } else if (target instanceof OutputFile) {
+ OutputFile outputFile = (OutputFile) target;
+ elem = doc.createElement("generated-file");
+ elem.setAttribute("generating-rule",
+ outputFile.getGeneratingRule().getLabel().toString());
+ } else if (target instanceof InputFile) {
+ elem = doc.createElement("source-file");
+ InputFile inputFile = (InputFile) target;
+ if (inputFile.getName().equals("BUILD")) {
+ addSubincludedFilesToElement(doc, elem, inputFile);
+ addSkylarkFilesToElement(doc, elem, inputFile);
+ addFeaturesToElement(doc, elem, inputFile);
+ }
+
+ addPackageGroupsToElement(doc, elem, inputFile);
+ } else if (target instanceof FakeSubincludeTarget) {
+ elem = doc.createElement("source-file");
+ } else {
+ throw new IllegalArgumentException(target.toString());
+ }
+
+ elem.setAttribute("name", target.getLabel().toString());
+ String location = target.getLocation().print();
+ if (!xmlLineNumbers) {
+ int firstColon = location.indexOf(":");
+ if (firstColon != -1) {
+ location = location.substring(0, firstColon);
+ }
+ }
+
+ elem.setAttribute("location", location);
+ return elem;
+ }
+
+ private void addPackageGroupsToElement(Document doc, Element parent, Target target) {
+ for (Label visibilityDependency : target.getVisibility().getDependencyLabels()) {
+ Element elem = doc.createElement("package-group");
+ elem.setAttribute("name", visibilityDependency.toString());
+ parent.appendChild(elem);
+ }
+
+ for (Label visibilityDeclaration : target.getVisibility().getDeclaredLabels()) {
+ Element elem = doc.createElement("visibility-label");
+ elem.setAttribute("name", visibilityDeclaration.toString());
+ parent.appendChild(elem);
+ }
+ }
+
+ private void addFeaturesToElement(Document doc, Element parent, InputFile inputFile) {
+ for (String feature : inputFile.getPackage().getFeatures()) {
+ Element elem = doc.createElement("feature");
+ elem.setAttribute("name", feature);
+ parent.appendChild(elem);
+ }
+ }
+
+ private void addSubincludedFilesToElement(Document doc, Element parent, InputFile inputFile) {
+ for (Label subinclude : inputFile.getPackage().getSubincludeLabels()) {
+ Element elem = doc.createElement("subinclude");
+ elem.setAttribute("name", subinclude.toString());
+ parent.appendChild(elem);
+ }
+ }
+
+ private void addSkylarkFilesToElement(Document doc, Element parent, InputFile inputFile) {
+ for (Label skylarkFileDep : inputFile.getPackage().getSkylarkFileDependencies()) {
+ Element elem = doc.createElement("load");
+ elem.setAttribute("name", skylarkFileDep.toString());
+ parent.appendChild(elem);
+ }
+ }
+
+ /**
+ * Creates and returns a new DOM tree for the specified attribute values.
+ * For non-configurable attributes, this is a single value. For configurable
+ * attributes, this contains one value for each configuration.
+ * (Only toplevel values are named attributes; list elements are unnamed.)
+ *
+ * <p>In the case of configurable attributes, multi-value attributes (e.g. lists)
+ * merge all configured lists into an aggregate flattened list. Single-value attributes
+ * simply refrain to set a value and annotate the DOM element as configurable.
+ *
+ * <P>(The ungainly qualified class name is required to avoid ambiguity with
+ * OutputFormatter.Type.)
+ */
+ private static Element createValueElement(Document doc,
+ com.google.devtools.build.lib.packages.Type<?> type, Iterable<Object> values) {
+ // "Import static" with method scope:
+ com.google.devtools.build.lib.packages.Type<?>
+ FILESET_ENTRY = com.google.devtools.build.lib.packages.Type.FILESET_ENTRY,
+ LABEL_LIST = com.google.devtools.build.lib.packages.Type.LABEL_LIST,
+ LICENSE = com.google.devtools.build.lib.packages.Type.LICENSE,
+ STRING_LIST = com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+ final Element elem;
+ final boolean hasMultipleValues = Iterables.size(values) > 1;
+ com.google.devtools.build.lib.packages.Type<?> elemType = type.getListElementType();
+ if (elemType != null) { // it's a list (includes "distribs")
+ elem = doc.createElement("list");
+ for (Object value : values) {
+ for (Object elemValue : (Collection<?>) value) {
+ elem.appendChild(createValueElement(doc, elemType, elemValue));
+ }
+ }
+ } else if (type instanceof com.google.devtools.build.lib.packages.Type.DictType) {
+ Set<Object> visitedValues = new HashSet<>();
+ elem = doc.createElement("dict");
+ com.google.devtools.build.lib.packages.Type.DictType<?, ?> dictType =
+ (com.google.devtools.build.lib.packages.Type.DictType<?, ?>) type;
+ for (Object value : values) {
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
+ if (visitedValues.add(entry.getKey())) {
+ Element pairElem = doc.createElement("pair");
+ elem.appendChild(pairElem);
+ pairElem.appendChild(createValueElement(doc,
+ dictType.getKeyType(), entry.getKey()));
+ pairElem.appendChild(createValueElement(doc,
+ dictType.getValueType(), entry.getValue()));
+ }
+ }
+ }
+ } else if (type == LICENSE) {
+ elem = createSingleValueElement(doc, "license", hasMultipleValues);
+ if (!hasMultipleValues) {
+ License license = (License) Iterables.getOnlyElement(values);
+
+ Element exceptions = createValueElement(doc, LABEL_LIST, license.getExceptions());
+ exceptions.setAttribute("name", "exceptions");
+ elem.appendChild(exceptions);
+
+ Element licenseTypes = createValueElement(doc, STRING_LIST, license.getLicenseTypes());
+ licenseTypes.setAttribute("name", "license-types");
+ elem.appendChild(licenseTypes);
+ }
+ } else if (type == FILESET_ENTRY) {
+ // Fileset entries: not configurable.
+ FilesetEntry filesetEntry = (FilesetEntry) Iterables.getOnlyElement(values);
+ elem = doc.createElement("fileset-entry");
+ elem.setAttribute("srcdir", filesetEntry.getSrcLabel().toString());
+ elem.setAttribute("destdir", filesetEntry.getDestDir().toString());
+ elem.setAttribute("symlinks", filesetEntry.getSymlinkBehavior().toString());
+ elem.setAttribute("strip_prefix", filesetEntry.getStripPrefix());
+
+ if (filesetEntry.getExcludes() != null) {
+ Element excludes =
+ createValueElement(doc, LABEL_LIST, filesetEntry.getExcludes());
+ excludes.setAttribute("name", "excludes");
+ elem.appendChild(excludes);
+ }
+ if (filesetEntry.getFiles() != null) {
+ Element files = createValueElement(doc, LABEL_LIST, filesetEntry.getFiles());
+ files.setAttribute("name", "files");
+ elem.appendChild(files);
+ }
+ } else { // INTEGER STRING LABEL DISTRIBUTION OUTPUT
+ elem = createSingleValueElement(doc, type.toString(), hasMultipleValues);
+ if (!hasMultipleValues && !Iterables.isEmpty(values)) {
+ Object value = Iterables.getOnlyElement(values);
+ // Values such as those of attribute "linkstamp" may be null.
+ if (value != null) {
+ try {
+ elem.setAttribute("value", value.toString());
+ } catch (DOMException e) {
+ elem.setAttribute("value", "[[[ERROR: could not be encoded as XML]]]");
+ }
+ }
+ }
+ }
+ return elem;
+ }
+
+ private static Element createValueElement(Document doc,
+ com.google.devtools.build.lib.packages.Type<?> type, Object value) {
+ return createValueElement(doc, type, ImmutableList.of(value));
+ }
+
+ /**
+ * Creates the given DOM element, adding <code>configurable="yes"</code> if it represents
+ * a configurable single-value attribute (configurable list attributes simply have their
+ * lists merged into an aggregate flat list).
+ */
+ private static Element createSingleValueElement(Document doc, String name,
+ boolean configurable) {
+ Element elem = doc.createElement(name);
+ if (configurable) {
+ elem.setAttribute("configurable", "yes");
+ }
+ return elem;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java
new file mode 100644
index 0000000..45df124
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules;
+
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.packages.RuleClass;
+
+/**
+ * A shortcut class to the appropriate specialization of {@code RuleClass.ConfiguredTargetFactory}.
+ */
+public interface RuleConfiguredTargetFactory
+ extends RuleClass.ConfiguredTargetFactory<ConfiguredTarget, RuleContext> {
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java
new file mode 100644
index 0000000..dfd6a92
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java
@@ -0,0 +1,350 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules;
+
+import static com.google.devtools.build.lib.syntax.SkylarkFunction.castList;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.Attribute.SkylarkLateBound;
+import com.google.devtools.build.lib.packages.SkylarkFileType;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param;
+import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.SkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.syntax.UserDefinedFunction;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.Map;
+
+/**
+ * A helper class to provide Attr module in Skylark.
+ */
+@SkylarkModule(name = "attr", namespace = true, onlyLoadingPhase = true,
+ doc = "Module for creating new attributes. "
+ + "They are only for use with the <code>rule</code> function.")
+public final class SkylarkAttr {
+
+ private static final String MANDATORY_DOC =
+ "set to true if users have to explicitely specify the value";
+
+ private static final String ALLOW_FILES_DOC =
+ "whether File targets are allowed. Can be True, False (default), or "
+ + "a FileType filter.";
+
+ private static final String ALLOW_RULES_DOC =
+ "which rule targets (name of the classes) are allowed."
+ + "This is deprecated (kept only for compatiblity), use providers instead.";
+
+ private static final String FLAGS_DOC =
+ "deprecated, will be removed";
+
+ private static final String DEFAULT_DOC =
+ "sets the default value of the attribute.";
+
+ private static final String CONFIGURATION_DOC =
+ "configuration of the attribute. "
+ + "For example, use DATA_CFG or HOST_CFG.";
+
+ private static final String EXECUTABLE_DOC =
+ "set to True if the labels have to be executable. Access the labels with "
+ + "ctx.executable.<attribute_name>";
+
+ private static Attribute.Builder<?> createAttribute(Type<?> type, Map<String, Object> arguments,
+ FuncallExpression ast, SkylarkEnvironment env) throws EvalException, ConversionException {
+ final Location loc = ast.getLocation();
+ // We use an empty name now so that we can set it later.
+ // This trick makes sense only in the context of Skylark (builtin rules should not use it).
+ Attribute.Builder<?> builder = Attribute.attr("", type);
+
+ Object defaultValue = arguments.get("default");
+ if (defaultValue != null) {
+ if (defaultValue instanceof UserDefinedFunction) {
+ // Late bound attribute. Non label type attributes already caused a type check error.
+ builder.value(new SkylarkLateBound(
+ new SkylarkCallbackFunction((UserDefinedFunction) defaultValue, ast, env)));
+ } else {
+ builder.defaultValue(defaultValue);
+ }
+ }
+
+ for (String flag : castList(arguments.get("flags"), String.class)) {
+ builder.setPropertyFlag(flag);
+ }
+
+ if (arguments.containsKey("mandatory") && (Boolean) arguments.get("mandatory")) {
+ builder.setPropertyFlag("MANDATORY");
+ }
+
+ if (arguments.containsKey("executable") && (Boolean) arguments.get("executable")) {
+ builder.setPropertyFlag("EXECUTABLE");
+ }
+
+ if (arguments.containsKey("single_file") && (Boolean) arguments.get("single_file")) {
+ builder.setPropertyFlag("SINGLE_ARTIFACT");
+ }
+
+ if (arguments.containsKey("allow_files")) {
+ Object fileTypesObj = arguments.get("allow_files");
+ if (fileTypesObj == Boolean.TRUE) {
+ builder.allowedFileTypes(FileTypeSet.ANY_FILE);
+ } else if (fileTypesObj == Boolean.FALSE) {
+ builder.allowedFileTypes(FileTypeSet.NO_FILE);
+ } else if (fileTypesObj instanceof SkylarkFileType) {
+ builder.allowedFileTypes(((SkylarkFileType) fileTypesObj).getFileTypeSet());
+ } else {
+ throw new EvalException(loc, "allow_files should be a boolean or a filetype object.");
+ }
+ } else if (type.equals(Type.LABEL) || type.equals(Type.LABEL_LIST)) {
+ builder.allowedFileTypes(FileTypeSet.NO_FILE);
+ }
+
+ Object ruleClassesObj = arguments.get("allow_rules");
+ if (ruleClassesObj != null) {
+ builder.allowedRuleClasses(castList(ruleClassesObj, String.class,
+ "allowed rule classes for attribute definition"));
+ }
+
+ if (arguments.containsKey("providers")) {
+ builder.mandatoryProviders(castList(arguments.get("providers"), String.class));
+ }
+
+ if (arguments.containsKey("cfg")) {
+ builder.cfg((ConfigurationTransition) arguments.get("cfg"));
+ }
+ return builder;
+ }
+
+ private static Object createAttribute(Map<String, Object> kwargs, Type<?> type,
+ FuncallExpression ast, Environment env) throws EvalException {
+ try {
+ return createAttribute(type, kwargs, ast, (SkylarkEnvironment) env);
+ } catch (ConversionException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+
+ @SkylarkBuiltin(name = "int", doc =
+ "Creates an attribute of type int.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = Integer.class,
+ doc = DEFAULT_DOC + " If not specified, default is 0."),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction integer = new SkylarkFunction("int") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.INTEGER, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "string", doc =
+ "Creates an attribute of type string.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = String.class,
+ doc = DEFAULT_DOC + " If not specified, default is \"\"."),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction string = new SkylarkFunction("string") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.STRING, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "label", doc =
+ "Creates an attribute of type Label. "
+ + "It is the only way to specify a dependency to another target. "
+ + "If you need a dependency that the user cannot overwrite, make the attribute "
+ + "private (starts with <code>_</code>).",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = Label.class, callbackEnabled = true,
+ doc = DEFAULT_DOC + " If not specified, default is None. "
+ + "Use the <code>Label</code> function to specify a default value."),
+ @Param(name = "executable", type = Boolean.class, doc = EXECUTABLE_DOC),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "allow_files", doc = ALLOW_FILES_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "providers", type = SkylarkList.class, generic1 = String.class,
+ doc = "mandatory providers every dependency has to have"),
+ @Param(name = "allow_rules", type = SkylarkList.class, generic1 = String.class,
+ doc = ALLOW_RULES_DOC),
+ @Param(name = "single_file", doc =
+ "if true, the label must correspond to a single File. "
+ + "Access it through ctx.file.<attribute_name>."),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction label = new SkylarkFunction("label") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.LABEL, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "string_list", doc =
+ "Creates an attribute of type list of strings",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = SkylarkList.class, generic1 = String.class,
+ doc = DEFAULT_DOC + " If not specified, default is []."),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class,
+ doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction stringList = new SkylarkFunction("string_list") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.STRING_LIST, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "label_list", doc =
+ "Creates an attribute of type list of labels. "
+ + "See <code>label</code> for more information.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = SkylarkList.class, generic1 = Label.class,
+ callbackEnabled = true,
+ doc = DEFAULT_DOC + " If not specified, default is []. "
+ + "Use the <code>Label</code> function to specify a default value."),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "allow_files", doc = ALLOW_FILES_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "allow_rules", type = SkylarkList.class, generic1 = String.class,
+ doc = ALLOW_RULES_DOC),
+ @Param(name = "providers", type = SkylarkList.class, generic1 = String.class,
+ doc = "mandatory providers every dependency has to have"),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction labelList = new SkylarkFunction("label_list") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.LABEL_LIST, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "bool", doc =
+ "Creates an attribute of type bool. Its default value is False.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = Boolean.class, doc = DEFAULT_DOC),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction bool = new SkylarkFunction("bool") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.BOOLEAN, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "output", doc =
+ "Creates an attribute of type output. Its default value is None. "
+ + "The user provides a file name (string) and the rule must create an action that "
+ + "generates the file.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = Label.class, doc = DEFAULT_DOC),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction output = new SkylarkFunction("output") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.OUTPUT, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "output_list", doc =
+ "Creates an attribute of type list of outputs. Its default value is []. "
+ + "See <code>output</code> above for more information.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = SkylarkList.class, generic1 = Label.class, doc = DEFAULT_DOC),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction outputList = new SkylarkFunction("output_list") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.OUTPUT_LIST, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "string_dict", doc =
+ "Creates an attribute of type dictionary, mapping from string to string. "
+ + "Its default value is {}.",
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", type = Map.class, doc = DEFAULT_DOC),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction stringDict = new SkylarkFunction("string_dict") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.STRING_DICT, ast, env);
+ }
+ };
+
+ @SkylarkBuiltin(name = "license", doc =
+ "Creates an attribute of type license. Its default value is NO_LICENSE.",
+ // TODO(bazel-team): Implement proper license support for Skylark.
+ objectType = SkylarkAttr.class,
+ returnType = Attribute.class,
+ optionalParams = {
+ @Param(name = "default", doc = DEFAULT_DOC),
+ @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC),
+ @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC),
+ @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)})
+ private static SkylarkFunction license = new SkylarkFunction("license") {
+ @Override
+ public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env)
+ throws EvalException {
+ return createAttribute(kwargs, Type.LICENSE, ast, env);
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java
new file mode 100644
index 0000000..e51805e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java
@@ -0,0 +1,89 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param;
+import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+
+import java.util.Map;
+
+/**
+ * A Skylark module class to create memory efficient command lines.
+ */
+@SkylarkModule(name = "cmd_helper", namespace = true,
+ doc = "Module for creating memory efficient command lines.")
+public class SkylarkCommandLine {
+
+ @SkylarkBuiltin(name = "join_paths",
+ doc = "Creates a single command line argument joining the paths of a set "
+ + "of files on the separator string.",
+ objectType = SkylarkCommandLine.class,
+ returnType = String.class,
+ mandatoryParams = {
+ @Param(name = "separator", type = String.class, doc = "the separator string to join on"),
+ @Param(name = "files", type = SkylarkNestedSet.class, generic1 = Artifact.class,
+ doc = "the files to concatenate")})
+ private static SimpleSkylarkFunction joinPaths =
+ new SimpleSkylarkFunction("join_paths") {
+ @Override
+ public Object call(Map<String, Object> params, Location loc)
+ throws EvalException {
+ final String separator = (String) params.get("separator");
+ final NestedSet<Artifact> artifacts =
+ ((SkylarkNestedSet) params.get("files")).getSet(Artifact.class);
+ // TODO(bazel-team): lazy evaluate
+ return Artifact.joinExecPaths(separator, artifacts);
+ }
+ };
+
+ // TODO(bazel-team): this method should support sets of objects and substitute all struct fields.
+ @SkylarkBuiltin(name = "template",
+ doc = "Transforms a set of files to a list of strings using the template string.",
+ objectType = SkylarkCommandLine.class,
+ returnType = SkylarkList.class,
+ mandatoryParams = {
+ @Param(name = "items", type = SkylarkNestedSet.class, generic1 = Artifact.class,
+ doc = "The set of structs to transform."),
+ @Param(name = "template", type = String.class,
+ doc = "The template to use for the transformation, %{path} and %{short_path} "
+ + "being substituted with the corresponding fields of each file.")})
+ private static SimpleSkylarkFunction template = new SimpleSkylarkFunction("template") {
+ @Override
+ public Object call(Map<String, Object> params, Location loc)
+ throws EvalException {
+ final String template = (String) params.get("template");
+ SkylarkNestedSet items = (SkylarkNestedSet) params.get("items");
+ return SkylarkList.lazyList(Iterables.transform(items, new Function<Object, String>() {
+ @Override
+ public String apply(Object input) {
+ Artifact artifact = (Artifact) input;
+ return template
+ .replace("%{path}", artifact.getExecPathString())
+ .replace("%{short_path}", artifact.getRootRelativePathString());
+ }
+ }), String.class);
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java
new file mode 100644
index 0000000..ae81f81
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java
@@ -0,0 +1,198 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.MethodLibrary;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.SkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.syntax.SkylarkType;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+import com.google.devtools.build.lib.syntax.ValidationEnvironment;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A class to handle all Skylark modules, to create and setup Validation and regular Environments.
+ */
+public class SkylarkModules {
+
+ public static final ImmutableList<Class<?>> MODULES = ImmutableList.of(
+ SkylarkAttr.class,
+ SkylarkCommandLine.class,
+ SkylarkRuleClassFunctions.class,
+ SkylarkRuleImplementationFunctions.class);
+
+ private static final ImmutableMap<Class<?>, ImmutableList<Function>> FUNCTION_MAP;
+ private static final ImmutableMap<String, Object> OBJECTS;
+
+ static {
+ try {
+ ImmutableMap.Builder<Class<?>, ImmutableList<Function>> functionMap = ImmutableMap.builder();
+ ImmutableMap.Builder<String, Object> objects = ImmutableMap.builder();
+ for (Class<?> moduleClass : MODULES) {
+ if (moduleClass.isAnnotationPresent(SkylarkModule.class)) {
+ objects.put(moduleClass.getAnnotation(SkylarkModule.class).name(),
+ moduleClass.newInstance());
+ }
+ ImmutableList.Builder<Function> functions = ImmutableList.builder();
+ collectSkylarkFunctionsAndObjectsFromFields(moduleClass, functions, objects);
+ functionMap.put(moduleClass, functions.build());
+ }
+ FUNCTION_MAP = functionMap.build();
+ OBJECTS = objects.build();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Returns a new SkylarkEnvironment with the elements of the Skylark modules.
+ */
+ public static SkylarkEnvironment getNewEnvironment(
+ EventHandler eventHandler, String astFileContentHashCode) {
+ SkylarkEnvironment env = new SkylarkEnvironment(eventHandler, astFileContentHashCode);
+ setupEnvironment(env);
+ return env;
+ }
+
+ @VisibleForTesting
+ public static SkylarkEnvironment getNewEnvironment(EventHandler eventHandler) {
+ return getNewEnvironment(eventHandler, null);
+ }
+
+ private static void setupEnvironment(Environment env) {
+ MethodLibrary.setupMethodEnvironment(env);
+ for (Map.Entry<Class<?>, ImmutableList<Function>> entry : FUNCTION_MAP.entrySet()) {
+ for (Function function : entry.getValue()) {
+ if (function.getObjectType() != null) {
+ env.registerFunction(function.getObjectType(), function.getName(), function);
+ } else {
+ env.update(function.getName(), function);
+ }
+ }
+ }
+ for (Map.Entry<String, Object> entry : OBJECTS.entrySet()) {
+ env.update(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Returns a new ValidationEnvironment with the elements of the Skylark modules.
+ */
+ public static ValidationEnvironment getValidationEnvironment() {
+ return getValidationEnvironment(ImmutableMap.<String, SkylarkType>of());
+ }
+
+ /**
+ * Returns a new ValidationEnvironment with the elements of the Skylark modules and extraObjects.
+ */
+ public static ValidationEnvironment getValidationEnvironment(
+ ImmutableMap<String, SkylarkType> extraObjects) {
+ Map<SkylarkType, Map<String, SkylarkType>> builtIn = new HashMap<>();
+ Map<String, SkylarkType> global = new HashMap<>();
+ builtIn.put(SkylarkType.GLOBAL, global);
+ collectSkylarkTypesFromFields(Environment.class, builtIn);
+ for (Class<?> moduleClass : MODULES) {
+ if (moduleClass.isAnnotationPresent(SkylarkModule.class)) {
+ global.put(moduleClass.getAnnotation(SkylarkModule.class).name(),
+ SkylarkType.of(moduleClass));
+ }
+ }
+ global.put("native", SkylarkType.UNKNOWN);
+ MethodLibrary.setupValidationEnvironment(builtIn);
+ for (Class<?> module : MODULES) {
+ collectSkylarkTypesFromFields(module, builtIn);
+ }
+ global.putAll(extraObjects);
+ return new ValidationEnvironment(CollectionUtils.toImmutable(builtIn));
+ }
+
+ /**
+ * Collects the SkylarkFunctions from the fields of the class of the object parameter
+ * and adds them into the builder.
+ */
+ private static void collectSkylarkFunctionsAndObjectsFromFields(Class<?> type,
+ ImmutableList.Builder<Function> functions, ImmutableMap.Builder<String, Object> objects) {
+ try {
+ for (Field field : type.getDeclaredFields()) {
+ if (field.isAnnotationPresent(SkylarkBuiltin.class)) {
+ // Fields in Skylark modules are sometimes private. Nevertheless they have to
+ // be annotated with SkylarkBuiltin.
+ field.setAccessible(true);
+ SkylarkBuiltin annotation = field.getAnnotation(SkylarkBuiltin.class);
+ if (SkylarkFunction.class.isAssignableFrom(field.getType())) {
+ SkylarkFunction function = (SkylarkFunction) field.get(null);
+ if (!function.isConfigured()) {
+ function.configure(annotation);
+ }
+ functions.add(function);
+ } else {
+ objects.put(annotation.name(), field.get(null));
+ }
+ }
+ }
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ // This should never happen.
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Collects the SkylarkFunctions from the fields of the class of the object parameter
+ * and adds their class and their corresponding return value to the builder.
+ */
+ private static void collectSkylarkTypesFromFields(Class<?> classObject,
+ Map<SkylarkType, Map<String, SkylarkType>> builtIn) {
+ for (Field field : classObject.getDeclaredFields()) {
+ if (field.isAnnotationPresent(SkylarkBuiltin.class)) {
+ SkylarkBuiltin annotation = field.getAnnotation(SkylarkBuiltin.class);
+ if (SkylarkFunction.class.isAssignableFrom(field.getType())) {
+ try {
+ // TODO(bazel-team): infer the correct types.
+ SkylarkType objectType = annotation.objectType().equals(Object.class)
+ ? SkylarkType.GLOBAL
+ : SkylarkType.of(annotation.objectType());
+ if (!builtIn.containsKey(objectType)) {
+ builtIn.put(objectType, new HashMap<String, SkylarkType>());
+ }
+ // TODO(bazel-team): add parameters to SkylarkFunctionType
+ SkylarkType returnType = SkylarkType.getReturnType(annotation);
+ builtIn.get(objectType).put(annotation.name(),
+ SkylarkFunctionType.of(annotation.name(), returnType));
+ } catch (IllegalArgumentException e) {
+ // This should never happen.
+ throw new RuntimeException(e);
+ }
+ } else if (Function.class.isAssignableFrom(field.getType())) {
+ builtIn.get(SkylarkType.GLOBAL).put(annotation.name(),
+ SkylarkFunctionType.of(annotation.name(), SkylarkType.UNKNOWN));
+ } else {
+ builtIn.get(SkylarkType.GLOBAL).put(annotation.name(), SkylarkType.of(field.getType()));
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java
new file mode 100644
index 0000000..39b8836
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java
@@ -0,0 +1,430 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA;
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.INTEGER;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunctionWithCallback;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunctionWithMap;
+import com.google.devtools.build.lib.packages.Package.NameConflictException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.PackageContext;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.RuleFactory;
+import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException;
+import com.google.devtools.build.lib.packages.SkylarkFileType;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.AbstractFunction;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Environment.NoSuchVariableException;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param;
+import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.SkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.UserDefinedFunction;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * A helper class to provide an easier API for Skylark rule definitions.
+ * This is experimental code.
+ */
+public class SkylarkRuleClassFunctions {
+
+ //TODO(bazel-team): proper enum support
+ @SkylarkBuiltin(name = "DATA_CFG", returnType = ConfigurationTransition.class,
+ doc = "The default runfiles collection state.")
+ private static final Object dataTransition = ConfigurationTransition.DATA;
+
+ @SkylarkBuiltin(name = "HOST_CFG", returnType = ConfigurationTransition.class,
+ doc = "The default runfiles collection state.")
+ private static final Object hostTransition = ConfigurationTransition.HOST;
+
+ private static final Attribute.ComputedDefault DEPRECATION =
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.getPackageDefaultDeprecation();
+ }
+ };
+
+ private static final Attribute.ComputedDefault TEST_ONLY =
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.getPackageDefaultTestOnly();
+ }
+ };
+
+ private static final LateBoundLabel<BuildConfiguration> RUN_UNDER =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ RunUnder runUnder = configuration.getRunUnder();
+ return runUnder == null ? null : runUnder.getLabel();
+ }
+ };
+
+ // TODO(bazel-team): Copied from ConfiguredRuleClassProvider for the transition from built-in
+ // rules to skylark extensions. Using the same instance would require a large refactoring.
+ // If we don't want to support old built-in rules and Skylark simultaneously
+ // (except for transition phase) it's probably OK.
+ private static LoadingCache<String, Label> labelCache =
+ CacheBuilder.newBuilder().build(new CacheLoader<String, Label>() {
+ @Override
+ public Label load(String from) throws Exception {
+ try {
+ return Label.parseAbsolute(from);
+ } catch (Label.SyntaxException e) {
+ throw new Exception(from);
+ }
+ }
+ });
+
+ // TODO(bazel-team): Remove the code duplication (BaseRuleClasses and this class).
+ private static final RuleClass baseRule =
+ BaseRuleClasses.commonCoreAndSkylarkAttributes(
+ new RuleClass.Builder("$base_rule", RuleClassType.ABSTRACT, true))
+ .add(attr("expect_failure", STRING))
+ .build();
+
+ private static final RuleClass testBaseRule =
+ new RuleClass.Builder("$test_base_rule", RuleClassType.ABSTRACT, true, baseRule)
+ .add(attr("size", STRING).value("medium").taggable()
+ .nonconfigurable("used in loading phase rule validation logic"))
+ .add(attr("timeout", STRING).taggable()
+ .nonconfigurable("used in loading phase rule validation logic").value(
+ new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ TestSize size = TestSize.getTestSize(rule.get("size", Type.STRING));
+ if (size != null) {
+ String timeout = size.getDefaultTimeout().toString();
+ if (timeout != null) {
+ return timeout;
+ }
+ }
+ return "illegal";
+ }
+ }))
+ .add(attr("flaky", BOOLEAN).value(false).taggable()
+ .nonconfigurable("taggable - called in Rule.getRuleTags"))
+ .add(attr("shard_count", INTEGER).value(-1))
+ .add(attr("local", BOOLEAN).value(false).taggable()
+ .nonconfigurable("policy decision: this should be consistent across configurations"))
+ .add(attr("$test_runtime", LABEL_LIST).cfg(HOST).value(ImmutableList.of(
+ labelCache.getUnchecked("//tools/test:runtime"))))
+ .add(attr(":run_under", LABEL).cfg(DATA).value(RUN_UNDER))
+ .build();
+
+ /**
+ * In native code, private values start with $.
+ * In Skylark, private values start with _, because of the grammar.
+ */
+ private static String attributeToNative(String oldName, Location loc, boolean isLateBound)
+ throws EvalException {
+ if (oldName.isEmpty()) {
+ throw new EvalException(loc, "Attribute name cannot be empty");
+ }
+ if (isLateBound) {
+ if (oldName.charAt(0) != '_') {
+ throw new EvalException(loc, "When an attribute value is a function, "
+ + "the attribute must be private (start with '_')");
+ }
+ return ":" + oldName.substring(1);
+ }
+ if (oldName.charAt(0) == '_') {
+ return "$" + oldName.substring(1);
+ }
+ return oldName;
+ }
+
+ // TODO(bazel-team): implement attribute copy and other rule properties
+
+ @SkylarkBuiltin(name = "rule", doc =
+ "Creates a new rule. Store it in a global value, so that it can be loaded and called "
+ + "from BUILD files.",
+ onlyLoadingPhase = true,
+ returnType = Function.class,
+ mandatoryParams = {
+ @Param(name = "implementation", type = UserDefinedFunction.class,
+ doc = "the function implementing this rule, has to have exactly one parameter: "
+ + "<code>ctx</code>. The function is called during analysis phase for each "
+ + "instance of the rule. It can access the attributes provided by the user. "
+ + "It must create actions to generate all the declared outputs.")
+ },
+ optionalParams = {
+ @Param(name = "test", type = Boolean.class, doc = "Whether this rule is a test rule. "
+ + "If True, the rule must end with <code>_test</code> (otherwise it cannot)."),
+ @Param(name = "attrs", doc =
+ "dictionary to declare all the attributes of the rule. It maps from an attribute name "
+ + "to an attribute object (see 'attr' module). Attributes starting with <code>_</code> "
+ + "are private, and can be used to add an implicit dependency on a label."),
+ @Param(name = "outputs", doc = "outputs of this rule. "
+ + "It is a dictionary mapping from string to a template name. For example: "
+ + "<code>{\"ext\": \"${name}.ext\"}</code>. <br>"
+ // TODO(bazel-team): Make doc more clear, wrt late-bound attributes.
+ + "It may also be a function (which receives <code>ctx.attr</code> as argument) "
+ + "returning such a dictionary."),
+ @Param(name = "executable", type = Boolean.class,
+ doc = "whether this rule always outputs an executable of the same name or not. If True, "
+ + "there must be an action that generates <code>ctx.outputs.executable</code>.")})
+ private static final SkylarkFunction rule = new SkylarkFunction("rule") {
+
+ @Override
+ public Object call(Map<String, Object> arguments, FuncallExpression ast,
+ Environment funcallEnv) throws EvalException, ConversionException {
+ final Location loc = ast.getLocation();
+
+ RuleClassType type = RuleClassType.NORMAL;
+ if (arguments.containsKey("test") && EvalUtils.toBoolean(arguments.get("test"))) {
+ type = RuleClassType.TEST;
+ }
+
+ // We'll set the name later, pass the empty string for now.
+ final RuleClass.Builder builder = type == RuleClassType.TEST
+ ? new RuleClass.Builder("", type, true, testBaseRule)
+ : new RuleClass.Builder("", type, true, baseRule);
+
+ for (Map.Entry<String, Attribute.Builder> attr : castMap(
+ arguments.get("attrs"), String.class, Attribute.Builder.class, "attrs")) {
+ Attribute.Builder<?> attrBuilder = attr.getValue();
+ String attrName = attributeToNative(attr.getKey(), loc,
+ attrBuilder.hasLateBoundValue());
+ builder.addOrOverrideAttribute(attrBuilder.build(attrName));
+ }
+ if (arguments.containsKey("executable") && (Boolean) arguments.get("executable")) {
+ builder.addOrOverrideAttribute(
+ attr("$is_executable", BOOLEAN).value(true)
+ .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target")
+ .build());
+ builder.setOutputsDefaultExecutable();
+ }
+
+ if (arguments.containsKey("outputs")) {
+ final Object implicitOutputs = arguments.get("outputs");
+ if (implicitOutputs instanceof UserDefinedFunction) {
+ UserDefinedFunction func = (UserDefinedFunction) implicitOutputs;
+ final SkylarkCallbackFunction callback =
+ new SkylarkCallbackFunction(func, ast, (SkylarkEnvironment) funcallEnv);
+ builder.setImplicitOutputsFunction(
+ new SkylarkImplicitOutputsFunctionWithCallback(callback, loc));
+ } else {
+ builder.setImplicitOutputsFunction(new SkylarkImplicitOutputsFunctionWithMap(
+ toMap(castMap(arguments.get("outputs"), String.class, String.class,
+ "implicit outputs of the rule class"))));
+ }
+ }
+
+ builder.setConfiguredTargetFunction(
+ (UserDefinedFunction) arguments.get("implementation"));
+ builder.setRuleDefinitionEnvironment((SkylarkEnvironment) funcallEnv);
+ return new RuleFunction(builder, type);
+ }
+ };
+
+ // This class is needed for testing
+ static final class RuleFunction extends AbstractFunction {
+ // Note that this means that we can reuse the same builder.
+ // This is fine since we don't modify the builder from here.
+ private final RuleClass.Builder builder;
+ private final RuleClassType type;
+
+ public RuleFunction(Builder builder, RuleClassType type) {
+ super("rule");
+ this.builder = builder;
+ this.type = type;
+ }
+
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ Environment env) throws EvalException, InterruptedException {
+ try {
+ String ruleClassName = ast.getFunction().getName();
+ if (ruleClassName.startsWith("_")) {
+ throw new EvalException(ast.getLocation(), "Invalid rule class name '" + ruleClassName
+ + "', cannot be private");
+ }
+ if (type == RuleClassType.TEST != TargetUtils.isTestRuleName(ruleClassName)) {
+ throw new EvalException(ast.getLocation(), "Invalid rule class name '" + ruleClassName
+ + "', test rule class names must end with '_test' and other rule classes must not");
+ }
+ RuleClass ruleClass = builder.build(ruleClassName);
+ PackageContext pkgContext = (PackageContext) env.lookup(PackageFactory.PKG_CONTEXT);
+ return RuleFactory.createAndAddRule(pkgContext, ruleClass, kwargs, ast);
+ } catch (InvalidRuleException | NameConflictException | NoSuchVariableException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+
+ @VisibleForTesting
+ RuleClass.Builder getBuilder() {
+ return builder;
+ }
+ }
+
+ @SkylarkBuiltin(name = "Label", doc = "Creates a Label referring to a BUILD target. Use "
+ + "this function only when you want to give a default value for the label attributes. "
+ + "Example: <br><pre class=language-python>Label(\"//tools:default\")</pre>",
+ returnType = Label.class,
+ mandatoryParams = {@Param(name = "label_string", type = String.class,
+ doc = "the label string")})
+ private static final SkylarkFunction label = new SimpleSkylarkFunction("Label") {
+ @Override
+ public Object call(Map<String, Object> arguments, Location loc) throws EvalException,
+ ConversionException {
+ String labelString = (String) arguments.get("label_string");
+ try {
+ return labelCache.get(labelString);
+ } catch (ExecutionException e) {
+ throw new EvalException(loc, "Illegal absolute label syntax: " + labelString);
+ }
+ }
+ };
+
+ @SkylarkBuiltin(name = "FileType",
+ doc = "Creates a file filter from a list of strings. For example, to match files ending "
+ + "with .cc or .cpp, use: <pre class=language-python>FileType([\".cc\", \".cpp\"])</pre>",
+ returnType = SkylarkFileType.class,
+ mandatoryParams = {
+ @Param(name = "types", type = SkylarkList.class, generic1 = String.class,
+ doc = "a list of the accepted file extensions")})
+ private static final SkylarkFunction fileType = new SimpleSkylarkFunction("FileType") {
+ @Override
+ public Object call(Map<String, Object> arguments, Location loc) throws EvalException,
+ ConversionException {
+ return SkylarkFileType.of(castList(arguments.get("types"), String.class));
+ }
+ };
+
+ @SkylarkBuiltin(name = "to_proto",
+ doc = "Creates a text message from the struct parameter. This method only works if all "
+ + "struct elements (recursively) are strings, ints, booleans, other structs or a "
+ + "list of these types. Quotes and new lines in strings are escaped. "
+ + "Examples:<br><pre class=language-python>"
+ + "struct(key=123).to_proto()\n# key: 123\n\n"
+ + "struct(key=True).to_proto()\n# key: true\n\n"
+ + "struct(key=[1, 2, 3]).to_proto()\n# key: 1\n# key: 2\n# key: 3\n\n"
+ + "struct(key='text').to_proto()\n# key: \"text\"\n\n"
+ + "struct(key=struct(inner_key='text')).to_proto()\n"
+ + "# key {\n# inner_key: \"text\"\n# }\n\n"
+ + "struct(key=[struct(inner_key=1), struct(inner_key=2)]).to_proto()\n"
+ + "# key {\n# inner_key: 1\n# }\n# key {\n# inner_key: 2\n# }\n\n"
+ + "struct(key=struct(inner_key=struct(inner_inner_key='text'))).to_proto()\n"
+ + "# key {\n# inner_key {\n# inner_inner_key: \"text\"\n# }\n# }\n</pre>",
+ objectType = SkylarkClassObject.class, returnType = String.class)
+ private static final SkylarkFunction toProto = new SimpleSkylarkFunction("to_proto") {
+ @Override
+ public Object call(Map<String, Object> arguments, Location loc) throws EvalException,
+ ConversionException {
+ ClassObject object = (ClassObject) arguments.get("self");
+ StringBuilder sb = new StringBuilder();
+ printTextMessage(object, sb, 0, loc);
+ return sb.toString();
+ }
+
+ private void printTextMessage(ClassObject object, StringBuilder sb,
+ int indent, Location loc) throws EvalException {
+ for (String key : object.getKeys()) {
+ printTextMessage(key, object.getValue(key), sb, indent, loc);
+ }
+ }
+
+ private void printSimpleTextMessage(String key, Object value, StringBuilder sb,
+ int indent, Location loc, String container) throws EvalException {
+ if (value instanceof ClassObject) {
+ print(sb, key + " {", indent);
+ printTextMessage((ClassObject) value, sb, indent + 1, loc);
+ print(sb, "}", indent);
+ } else if (value instanceof String) {
+ print(sb, key + ": \"" + escape((String) value) + "\"", indent);
+ } else if (value instanceof Integer) {
+ print(sb, key + ": " + value, indent);
+ } else if (value instanceof Boolean) {
+ // We're relying on the fact that Java converts Booleans to Strings in the same way
+ // as the protocol buffers do.
+ print(sb, key + ": " + value, indent);
+ } else {
+ throw new EvalException(loc,
+ "Invalid text format, expected a struct, a string, a bool, or an int but got a "
+ + EvalUtils.getDatatypeName(value) + " for " + container + " '" + key + "'");
+ }
+ }
+
+ private void printTextMessage(String key, Object value, StringBuilder sb,
+ int indent, Location loc) throws EvalException {
+ if (value instanceof SkylarkList) {
+ for (Object item : ((SkylarkList) value)) {
+ // TODO(bazel-team): There should be some constraint on the fields of the structs
+ // in the same list but we ignore that for now.
+ printSimpleTextMessage(key, item, sb, indent, loc, "list element in struct field");
+ }
+ } else {
+ printSimpleTextMessage(key, value, sb, indent, loc, "struct field");
+ }
+ }
+
+ private String escape(String string) {
+ // TODO(bazel-team): use guava's SourceCodeEscapers when it's released.
+ return string.replace("\"", "\\\"").replace("\n", "\\n");
+ }
+
+ private void print(StringBuilder sb, String text, int indent) {
+ for (int i = 0; i < indent; i++) {
+ sb.append(" ");
+ }
+ sb.append(text);
+ sb.append("\n");
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java
new file mode 100644
index 0000000..528e0f1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java
@@ -0,0 +1,213 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules;
+
+import static com.google.devtools.build.lib.syntax.SkylarkFunction.cast;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+
+/**
+ * A helper class to build Rule Configured Targets via runtime loaded rule implementations
+ * defined using the Skylark Build Extension Language. This is experimental code.
+ */
+public final class SkylarkRuleConfiguredTargetBuilder {
+
+ /**
+ * Create a Rule Configured Target from the ruleContext and the ruleImplementation.
+ */
+ public static ConfiguredTarget buildRule(RuleContext ruleContext,
+ Function ruleImplementation) {
+ String expectError = ruleContext.attributes().get("expect_failure", Type.STRING);
+ try {
+ SkylarkRuleContext skylarkRuleContext = new SkylarkRuleContext(ruleContext);
+ SkylarkEnvironment env = ruleContext.getRule().getRuleClassObject()
+ .getRuleDefinitionEnvironment().cloneEnv(
+ ruleContext.getAnalysisEnvironment().getEventHandler());
+ // Collect the symbols to disable statically and pass at the next call, so we don't need to
+ // clone the RuleDefinitionEnvironment.
+ env.disableOnlyLoadingPhaseObjects();
+ Object target = ruleImplementation.call(ImmutableList.<Object>of(skylarkRuleContext),
+ ImmutableMap.<String, Object>of(), null, env);
+
+ if (ruleContext.hasErrors()) {
+ return null;
+ } else if (!(target instanceof SkylarkClassObject) && target != Environment.NONE) {
+ ruleContext.ruleError("Rule implementation doesn't return a struct");
+ return null;
+ } else if (!expectError.isEmpty()) {
+ ruleContext.ruleError("Expected error not found: " + expectError);
+ return null;
+ }
+ ConfiguredTarget configuredTarget = createTarget(ruleContext, target);
+ checkOrphanArtifacts(ruleContext);
+ return configuredTarget;
+
+ } catch (InterruptedException e) {
+ ruleContext.ruleError(e.getMessage());
+ return null;
+ } catch (EvalException e) {
+ // If the error was expected, return an empty target.
+ if (!expectError.isEmpty() && e.getMessage().matches(expectError)) {
+ return new com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder(ruleContext)
+ .add(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .build();
+ }
+ ruleContext.ruleError("\n" + e.print());
+ return null;
+ }
+ }
+
+ private static void checkOrphanArtifacts(RuleContext ruleContext) throws EvalException {
+ ImmutableSet<Artifact> orphanArtifacts =
+ ruleContext.getAnalysisEnvironment().getOrphanArtifacts();
+ if (!orphanArtifacts.isEmpty()) {
+ throw new EvalException(null, "The following files have no generating action:\n"
+ + Joiner.on("\n").join(Iterables.transform(orphanArtifacts,
+ new com.google.common.base.Function<Artifact, String>() {
+ @Override
+ public String apply(Artifact artifact) {
+ return artifact.getRootRelativePathString();
+ }
+ })));
+ }
+ }
+
+ // TODO(bazel-team): this whole defaulting - overriding executable, runfiles and files_to_build
+ // is getting out of hand. Clean this whole mess up.
+ private static ConfiguredTarget createTarget(RuleContext ruleContext, Object target)
+ throws EvalException {
+ Artifact executable = getExecutable(ruleContext, target);
+ RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext);
+ // Set the default files to build.
+ NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder()
+ .addAll(ruleContext.getOutputArtifacts());
+ if (executable != null) {
+ filesToBuild.add(executable);
+ }
+ builder.setFilesToBuild(filesToBuild.build());
+ return addStructFields(ruleContext, builder, target, executable);
+ }
+
+ private static Artifact getExecutable(RuleContext ruleContext, Object target)
+ throws EvalException {
+ Artifact executable = ruleContext.getRule().getRuleClassObject().outputsDefaultExecutable()
+ // This doesn't actually create a new Artifact just returns the one
+ // created in SkylarkruleContext.
+ ? ruleContext.createOutputArtifact() : null;
+ if (target instanceof SkylarkClassObject) {
+ SkylarkClassObject struct = (SkylarkClassObject) target;
+ if (struct.getValue("executable") != null) {
+ // We need this because of genrule.bzl. This overrides the default executable.
+ executable = cast(
+ struct.getValue("executable"), Artifact.class, "executable", struct.getCreationLoc());
+ }
+ }
+ return executable;
+ }
+
+ private static ConfiguredTarget addStructFields(RuleContext ruleContext,
+ RuleConfiguredTargetBuilder builder, Object target, Artifact executable)
+ throws EvalException {
+ Location loc = null;
+ Runfiles statelessRunfiles = null;
+ Runfiles dataRunfiles = null;
+ Runfiles defaultRunfiles = null;
+ if (target instanceof SkylarkClassObject) {
+ SkylarkClassObject struct = (SkylarkClassObject) target;
+ loc = struct.getCreationLoc();
+ for (String key : struct.getKeys()) {
+ if (key.equals("files")) {
+ // If we specify files_to_build we don't have the executable in it by default.
+ builder.setFilesToBuild(cast(struct.getValue("files"),
+ SkylarkNestedSet.class, "files", loc).getSet(Artifact.class));
+ } else if (key.equals("runfiles")) {
+ statelessRunfiles = cast(struct.getValue("runfiles"), Runfiles.class, "runfiles", loc);
+ } else if (key.equals("data_runfiles")) {
+ dataRunfiles =
+ cast(struct.getValue("data_runfiles"), Runfiles.class, "data_runfiles", loc);
+ } else if (key.equals("default_runfiles")) {
+ defaultRunfiles =
+ cast(struct.getValue("default_runfiles"), Runfiles.class, "default_runfiles", loc);
+ } else if (!key.equals("executable")) {
+ // We handled executable already.
+ builder.addSkylarkTransitiveInfo(key, struct.getValue(key), loc);
+ }
+ }
+ }
+
+ if ((statelessRunfiles != null) && (dataRunfiles != null || defaultRunfiles != null)) {
+ throw new EvalException(loc, "Cannot specify the provider 'runfiles' "
+ + "together with 'data_runfiles' or 'default_runfiles'");
+ }
+
+ if (statelessRunfiles == null && dataRunfiles == null && defaultRunfiles == null) {
+ // No runfiles specified, set default
+ statelessRunfiles = Runfiles.EMPTY;
+ }
+
+ RunfilesProvider runfilesProvider = statelessRunfiles != null
+ ? RunfilesProvider.simple(merge(statelessRunfiles, executable))
+ : RunfilesProvider.withData(
+ // The executable doesn't get into the default runfiles if we have runfiles states.
+ // This is to keep skylark genrule consistent with the original genrule.
+ defaultRunfiles != null ? defaultRunfiles : Runfiles.EMPTY,
+ dataRunfiles != null ? dataRunfiles : Runfiles.EMPTY);
+ builder.addProvider(RunfilesProvider.class, runfilesProvider);
+
+ Runfiles computedDefaultRunfiles = runfilesProvider.getDefaultRunfiles();
+ // This works because we only allowed to call a rule *_test iff it's a test type rule.
+ boolean testRule = TargetUtils.isTestRuleName(ruleContext.getRule().getRuleClass());
+ if (testRule && computedDefaultRunfiles.isEmpty()) {
+ throw new EvalException(loc, "Test rules have to define runfiles");
+ }
+ if (executable != null || testRule) {
+ RunfilesSupport runfilesSupport = computedDefaultRunfiles.isEmpty()
+ ? null : RunfilesSupport.withExecutable(ruleContext, computedDefaultRunfiles, executable);
+ builder.setRunfilesSupport(runfilesSupport, executable);
+ }
+ try {
+ return builder.build();
+ } catch (IllegalArgumentException e) {
+ throw new EvalException(loc, e.getMessage());
+ }
+ }
+
+ private static Runfiles merge(Runfiles runfiles, Artifact executable) {
+ if (executable == null) {
+ return runfiles;
+ }
+ return new Runfiles.Builder().addArtifact(executable).merge(runfiles).build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java
new file mode 100644
index 0000000..fc06677
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java
@@ -0,0 +1,484 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.LabelExpander;
+import com.google.devtools.build.lib.analysis.LabelExpander.NotUniqueExpansionException;
+import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FuncallExpression.FuncallException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.syntax.SkylarkType;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A Skylark API for the ruleContext.
+ */
+@SkylarkModule(name = "ctx", doc = "The context of the rule containing helper functions and "
+ + "information about attributes, depending targets and outputs. "
+ + "You get a ctx object as an argument to the <code>implementation</code> function when "
+ + "you create a rule.")
+public final class SkylarkRuleContext {
+
+ public static final String PROVIDER_CLASS_PREFIX = "com.google.devtools.build.lib.";
+
+ static final LoadingCache<String, Class<?>> classCache = CacheBuilder.newBuilder()
+ .initialCapacity(10)
+ .maximumSize(100)
+ .build(new CacheLoader<String, Class<?>>() {
+
+ @Override
+ public Class<?> load(String key) throws Exception {
+ String classPath = SkylarkRuleContext.PROVIDER_CLASS_PREFIX + key;
+ return Class.forName(classPath);
+ }
+ });
+
+ private final RuleContext ruleContext;
+
+ // TODO(bazel-team): support configurable attributes.
+ private final SkylarkClassObject attrObject;
+
+ private final SkylarkClassObject outputsObject;
+
+ private final SkylarkClassObject executableObject;
+
+ private final SkylarkClassObject fileObject;
+
+ private final SkylarkClassObject filesObject;
+
+ private final SkylarkClassObject targetsObject;
+
+ private final SkylarkClassObject targetObject;
+
+ // TODO(bazel-team): we only need this because of the css_binary rule.
+ private final ImmutableMap<Artifact, Label> artifactLabelMap;
+
+ private final ImmutableMap<Artifact, FilesToRunProvider> executableRunfilesMap;
+
+ /**
+ * In native code, private values start with $.
+ * In Skylark, private values start with _, because of the grammar.
+ */
+ private String attributeToSkylark(String oldName) {
+ if (!oldName.isEmpty() && (oldName.charAt(0) == '$' || oldName.charAt(0) == ':')) {
+ return "_" + oldName.substring(1);
+ }
+ return oldName;
+ }
+
+ /**
+ * Creates a new SkylarkRuleContext using ruleContext.
+ */
+ public SkylarkRuleContext(RuleContext ruleContext) throws EvalException {
+ this.ruleContext = Preconditions.checkNotNull(ruleContext);
+
+ HashMap<String, Object> outputsBuilder = new HashMap<>();
+ if (ruleContext.getRule().getRuleClassObject().outputsDefaultExecutable()) {
+ addOutput(outputsBuilder, "executable", ruleContext.createOutputArtifact());
+ }
+ ImplicitOutputsFunction implicitOutputsFunction =
+ ruleContext.getRule().getRuleClassObject().getImplicitOutputsFunction();
+
+ if (implicitOutputsFunction instanceof SkylarkImplicitOutputsFunction) {
+ SkylarkImplicitOutputsFunction func = (SkylarkImplicitOutputsFunction)
+ ruleContext.getRule().getRuleClassObject().getImplicitOutputsFunction();
+ for (Map.Entry<String, String> entry : func.calculateOutputs(
+ RawAttributeMapper.of(ruleContext.getRule())).entrySet()) {
+ addOutput(outputsBuilder, entry.getKey(),
+ ruleContext.getImplicitOutputArtifact(entry.getValue()));
+ }
+ }
+
+ ImmutableMap.Builder<Artifact, Label> artifactLabelMapBuilder =
+ ImmutableMap.builder();
+ for (Attribute a : ruleContext.getRule().getAttributes()) {
+ String attrName = a.getName();
+ Type<?> type = a.getType();
+ if (type != Type.OUTPUT && type != Type.OUTPUT_LIST) {
+ continue;
+ }
+ ImmutableList.Builder<Artifact> artifactsBuilder = ImmutableList.builder();
+ for (OutputFile outputFile : ruleContext.getRule().getOutputFileMap().get(attrName)) {
+ Artifact artifact = ruleContext.createOutputArtifact(outputFile);
+ artifactsBuilder.add(artifact);
+ artifactLabelMapBuilder.put(artifact, outputFile.getLabel());
+ }
+ ImmutableList<Artifact> artifacts = artifactsBuilder.build();
+
+ if (type == Type.OUTPUT) {
+ if (artifacts.size() == 1) {
+ addOutput(outputsBuilder, attrName, Iterables.getOnlyElement(artifacts));
+ } else {
+ addOutput(outputsBuilder, attrName, Environment.NONE);
+ }
+ } else if (type == Type.OUTPUT_LIST) {
+ addOutput(outputsBuilder, attrName,
+ SkylarkList.list(artifacts, Artifact.class));
+ } else {
+ throw new IllegalArgumentException(
+ "Type of " + attrName + "(" + type + ") is not output type ");
+ }
+ }
+ artifactLabelMap = artifactLabelMapBuilder.build();
+ outputsObject = new SkylarkClassObject(outputsBuilder, "No such output '%s'");
+
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+ ImmutableMap.Builder<String, Object> executableBuilder = new ImmutableMap.Builder<>();
+ ImmutableMap.Builder<Artifact, FilesToRunProvider> executableRunfilesbuilder =
+ new ImmutableMap.Builder<>();
+ ImmutableMap.Builder<String, Object> fileBuilder = new ImmutableMap.Builder<>();
+ ImmutableMap.Builder<String, Object> filesBuilder = new ImmutableMap.Builder<>();
+ ImmutableMap.Builder<String, Object> targetBuilder = new ImmutableMap.Builder<>();
+ ImmutableMap.Builder<String, Object> targetsBuilder = new ImmutableMap.Builder<>();
+ for (Attribute a : ruleContext.getRule().getAttributes()) {
+ Type<?> type = a.getType();
+ Object val = ruleContext.attributes().get(a.getName(), type);
+ builder.put(attributeToSkylark(a.getName()), val == null ? Environment.NONE
+ // Attribute values should be type safe
+ : SkylarkType.convertToSkylark(val, null));
+ if (type != Type.LABEL && type != Type.LABEL_LIST) {
+ continue;
+ }
+ String skyname = attributeToSkylark(a.getName());
+ Mode mode = getMode(a.getName());
+ if (a.isExecutable()) {
+ // In Skylark only label (not label list) type attributes can have the Executable flag.
+ FilesToRunProvider provider = ruleContext.getExecutablePrerequisite(a.getName(), mode);
+ if (provider != null && provider.getExecutable() != null) {
+ Artifact executable = provider.getExecutable();
+ executableBuilder.put(skyname, executable);
+ executableRunfilesbuilder.put(executable, provider);
+ } else {
+ executableBuilder.put(skyname, Environment.NONE);
+ }
+ }
+ if (a.isSingleArtifact()) {
+ // In Skylark only label (not label list) type attributes can have the SingleArtifact flag.
+ Artifact artifact = ruleContext.getPrerequisiteArtifact(a.getName(), mode);
+ if (artifact != null) {
+ fileBuilder.put(skyname, artifact);
+ } else {
+ fileBuilder.put(skyname, Environment.NONE);
+ }
+ }
+ filesBuilder.put(skyname, ruleContext.getPrerequisiteArtifacts(a.getName(), mode).list());
+ targetsBuilder.put(skyname, SkylarkList.list(
+ ruleContext.getPrerequisites(a.getName(), mode), TransitiveInfoCollection.class));
+ if (type == Type.LABEL) {
+ Object prereq = ruleContext.getPrerequisite(a.getName(), mode);
+ if (prereq != null) {
+ targetBuilder.put(skyname, prereq);
+ } else {
+ targetBuilder.put(skyname, Environment.NONE);
+ }
+ }
+ }
+ attrObject = new SkylarkClassObject(builder.build(), "No such attribute '%s'");
+ executableObject = new SkylarkClassObject(executableBuilder.build(), "No such executable. "
+ + "Make sure there is a '%s' label type attribute marked as 'executable'");
+ fileObject = new SkylarkClassObject(fileBuilder.build(),
+ "No such file. Make sure there is a '%s' label type attribute marked as 'single_file'");
+ filesObject = new SkylarkClassObject(filesBuilder.build(),
+ "No such files. Make sure there is a '%s' label or label_list type attribute");
+ targetObject = new SkylarkClassObject(targetBuilder.build(),
+ "No such target. Make sure there is a '%s' label type attribute");
+ targetsObject = new SkylarkClassObject(targetsBuilder.build(),
+ "No such targets. Make sure there is a '%s' label or label_list type attribute");
+ executableRunfilesMap = executableRunfilesbuilder.build();
+ }
+
+ private void addOutput(HashMap<String, Object> outputsBuilder, String key, Object value)
+ throws EvalException {
+ if (outputsBuilder.containsKey(key)) {
+ throw new EvalException(null, "Multiple outputs with the same key: " + key);
+ }
+ outputsBuilder.put(key, value);
+ }
+
+ /**
+ * Returns the original ruleContext.
+ */
+ public RuleContext getRuleContext() {
+ return ruleContext;
+ }
+
+ private Mode getMode(String attributeName) {
+ return ruleContext.getAttributeMode(attributeName);
+ }
+
+ @SkylarkCallable(name = "attr", structField = true,
+ doc = "A struct to access the values of the attributes. The values are provided by "
+ + "the user (if not, a default value is used).")
+ public SkylarkClassObject getAttr() {
+ return attrObject;
+ }
+
+ /**
+ * <p>See {@link RuleContext#getExecutablePrerequisite(String, Mode)}.
+ */
+ @SkylarkCallable(name = "executable", structField = true,
+ doc = "A <code>struct</code> containing executable files defined in label type "
+ + "attributes marked as <code>executable=True</code>. The struct fields correspond "
+ + "to the attribute names. The struct value is always a <code>file</code>s or "
+ + "<code>None</code>. If a non-mandatory attribute is not specified in the rule "
+ + "the corresponding struct value is <code>None</code>. If a label type is not "
+ + "marked as <code>executable=True</code>, no corresponding struct field is generated.")
+ public SkylarkClassObject getExecutable() {
+ return executableObject;
+ }
+
+ /**
+ * See {@link RuleContext#getPrerequisiteArtifact(String, Mode)}.
+ */
+ @SkylarkCallable(name = "file", structField = true,
+ doc = "A <code>struct</code> containing files defined in label type "
+ + "attributes marked as <code>single_file=True</code>. The struct fields correspond "
+ + "to the attribute names. The struct value is always a <code>file</code> or "
+ + "<code>None</code>. If a non-mandatory attribute is not specified in the rule "
+ + "the corresponding struct value is <code>None</code>. If a label type is not "
+ + "marked as <code>single_file=True</code>, no corresponding struct field is generated.")
+ public SkylarkClassObject getFile() {
+ return fileObject;
+ }
+
+ /**
+ * See {@link RuleContext#getPrerequisiteArtifacts(String, Mode)}.
+ */
+ @SkylarkCallable(name = "files", structField = true,
+ doc = "A <code>struct</code> containing files defined in label or label list "
+ + "type attributes. The struct fields correspond to the attribute names. The struct "
+ + "values are <code>list</code> of <code>file</code>s. If a non-mandatory attribute is "
+ + "not specified in the rule, an empty list is generated.")
+ public SkylarkClassObject getFiles() {
+ return filesObject;
+ }
+
+ /**
+ * See {@link RuleContext#getPrerequisite(String, Mode)}.
+ */
+ @SkylarkCallable(name = "target", structField = true,
+ doc = "A <code>struct</code> containing prerequisite targets defined in label type "
+ + "attributes. The struct fields correspond to the attribute names. The struct value "
+ + "is always a <code>target</code> or <code>None</code>. If a non-mandatory attribute "
+ + "is not specified in the rule, the corresponding struct value is <code>None</code>.")
+ public SkylarkClassObject getTarget() {
+ return targetObject;
+ }
+
+ /**
+ * See {@link RuleContext#getPrerequisites(String, Mode)}.
+ */
+ @SkylarkCallable(name = "targets", structField = true,
+ doc = "A <code>struct</code> containing prerequisite targets defined in label or label list "
+ + "type attributes. The struct fields correspond to the attribute names. The struct "
+ + "values are <code>list</code> of <code>target</code>s. If a non-mandatory attribute is "
+ + "not specified in the rule, an empty list is generated.")
+ public SkylarkClassObject getTargets() {
+ return targetsObject;
+ }
+
+ @SkylarkCallable(name = "label", structField = true, doc = "The label of this rule.")
+ public Label getLabel() {
+ return ruleContext.getLabel();
+ }
+
+ @SkylarkCallable(name = "configuration", structField = true,
+ doc = "Returns the default configuration. See the <code>configuration</code> type for "
+ + "more details.")
+ public BuildConfiguration getConfiguration() {
+ return ruleContext.getConfiguration();
+ }
+
+ @SkylarkCallable(name = "host_configuration", structField = true,
+ doc = "Returns the host configuration. See the <code>configuration</code> type for "
+ + "more details.")
+ public BuildConfiguration getHostConfiguration() {
+ return ruleContext.getHostConfiguration();
+ }
+
+ @SkylarkCallable(name = "data_configuration", structField = true,
+ doc = "Returns the data configuration. See the <code>configuration</code> type for "
+ + "more details.")
+ public BuildConfiguration getDataConfiguration() {
+ return ruleContext.getConfiguration().getConfiguration(ConfigurationTransition.DATA);
+ }
+
+ @SkylarkCallable(structField = true,
+ doc = "A <code>struct</code> containing all the output files."
+ + " The struct is generated the following way:<br>"
+ + "<ul><li>If the rule is marked as <code>executable=True</code> the struct has an "
+ + "\"executable\" field with the rules default executable <code>file</code> value."
+ + "<li>For every entry in the rule's <code>outputs</code> dict a field is generated with "
+ + "the same name and the corresponding <code>file</code> value."
+ + "<li>For every output type attribute a struct field is generated with the "
+ + "same name and the corresponding <code>file</code> value or <code>None</code>, "
+ + "if no value is specified in the rule."
+ + "<li>For every output list type attribute a struct field is generated with the "
+ + "same name and corresponding <code>list</code> of <code>file</code>s value "
+ + "(an empty list if no value is specified in the rule.</ul>")
+ public SkylarkClassObject outputs() {
+ return outputsObject;
+ }
+
+ @Override
+ public String toString() {
+ return ruleContext.getLabel().toString();
+ }
+
+ @SkylarkCallable(doc = "Splits a shell command to a list of tokens.", hidden = true)
+ public List<String> tokenize(String optionString) throws FuncallException {
+ List<String> options = new ArrayList<String>();
+ try {
+ ShellUtils.tokenize(options, optionString);
+ } catch (TokenizationException e) {
+ throw new FuncallException(e.getMessage() + " while tokenizing '" + optionString + "'");
+ }
+ return ImmutableList.copyOf(options);
+ }
+
+ @SkylarkCallable(doc =
+ "Expands all references to labels embedded within a string for all files using a mapping "
+ + "from definition labels (i.e. the label in the output type attribute) to files. Deprecated.",
+ hidden = true)
+ public String expand(@Nullable String expression,
+ List<Artifact> artifacts, Label labelResolver) throws FuncallException {
+ try {
+ Map<Label, Iterable<Artifact>> labelMap = new HashMap<>();
+ for (Artifact artifact : artifacts) {
+ labelMap.put(artifactLabelMap.get(artifact), ImmutableList.of(artifact));
+ }
+ return LabelExpander.expand(expression, labelMap, labelResolver);
+ } catch (NotUniqueExpansionException e) {
+ throw new FuncallException(e.getMessage() + " while expanding '" + expression + "'");
+ }
+ }
+
+ @SkylarkCallable(doc =
+ "Creates a file with the given filename. You must create an action that generates "
+ + "the file. If the file should be publicly visible, declare a rule "
+ + "output instead when possible.")
+ public Artifact newFile(Root root, String filename) {
+ PathFragment fragment = ruleContext.getLabel().getPackageFragment();
+ for (String pathFragmentString : filename.split("/")) {
+ fragment = fragment.getRelative(pathFragmentString);
+ }
+ return ruleContext.getAnalysisEnvironment().getDerivedArtifact(fragment, root);
+ }
+
+ @SkylarkCallable(doc =
+ "Creates a new file, derived from the given file and suffix. "
+ + "You must create an action that generates "
+ + "the file. If the file should be publicly visible, declare a rule "
+ + "output instead when possible.")
+ public Artifact newFile(Root root, Artifact baseArtifact, String suffix) {
+ PathFragment original = baseArtifact.getRootRelativePath();
+ PathFragment fragment = original.replaceName(original.getBaseName() + suffix);
+ return ruleContext.getAnalysisEnvironment().getDerivedArtifact(fragment, root);
+ }
+
+ @SkylarkCallable(doc = "", hidden = true)
+ public NestedSet<Artifact> middleMan(String attribute) {
+ return AnalysisUtils.getMiddlemanFor(ruleContext, attribute);
+ }
+
+ @SkylarkCallable(doc = "", hidden = true)
+ public boolean checkPlaceholders(String template, List<String> allowedPlaceholders) {
+ List<String> actualPlaceHolders = new LinkedList<>();
+ Set<String> allowedPlaceholderSet = ImmutableSet.copyOf(allowedPlaceholders);
+ ImplicitOutputsFunction.createPlaceholderSubstitutionFormatString(template, actualPlaceHolders);
+ for (String placeholder : actualPlaceHolders) {
+ if (!allowedPlaceholderSet.contains(placeholder)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @SkylarkCallable(doc = "")
+ public String expandMakeVariables(String attributeName, String command,
+ final Map<String, String> additionalSubstitutions) {
+ return ruleContext.expandMakeVariables(attributeName,
+ command, new ConfigurationMakeVariableContext(ruleContext.getRule().getPackage(),
+ ruleContext.getConfiguration()) {
+ @Override
+ public String lookupMakeVariable(String name) throws ExpansionException {
+ if (additionalSubstitutions.containsKey(name)) {
+ return additionalSubstitutions.get(name);
+ } else {
+ return super.lookupMakeVariable(name);
+ }
+ }
+ });
+ }
+
+ FilesToRunProvider getExecutableRunfiles(Artifact executable) {
+ return executableRunfilesMap.get(executable);
+ }
+
+ @SkylarkCallable(name = "info_file", structField = true, hidden = true,
+ doc = "Returns the file that is used to hold the non-volatile workspace status for the "
+ + "current build request.")
+ public Artifact getStableWorkspaceStatus() {
+ return ruleContext.getAnalysisEnvironment().getStableWorkspaceStatusArtifact();
+ }
+
+ @SkylarkCallable(name = "version_file", structField = true, hidden = true,
+ doc = "Returns the file that is used to hold the volatile workspace status for the "
+ + "current build request.")
+ public Artifact getVolatileWorkspaceStatus() {
+ return ruleContext.getAnalysisEnvironment().getVolatileWorkspaceStatusArtifact();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java
new file mode 100644
index 0000000..1f7d160
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java
@@ -0,0 +1,367 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.CommandHelper;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.MakeVariableExpander;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction;
+import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.EvalUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
+import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param;
+import com.google.devtools.build.lib.syntax.SkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction;
+import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+// TODO(bazel-team): function argument names are often duplicated,
+// figure out a nicely readable way to get rid of the duplications.
+/**
+ * A helper class to provide an easier API for Skylark rule implementations
+ * and hide the original Java API. This is experimental code.
+ */
+public class SkylarkRuleImplementationFunctions {
+
+ // TODO(bazel-team): add all the remaining parameters
+ // TODO(bazel-team): merge executable and arguments
+ /**
+ * A Skylark built-in function to create and register a SpawnAction using a
+ * dictionary of parameters:
+ * createSpawnAction(
+ * inputs = [input1, input2, ...],
+ * outputs = [output1, output2, ...],
+ * executable = executable,
+ * arguments = [argument1, argument2, ...],
+ * mnemonic = 'mnemonic',
+ * command = 'command',
+ * register = 1
+ * )
+ */
+ @SkylarkBuiltin(name = "action",
+ doc = "Creates an action that runs an executable or a shell command.",
+ objectType = SkylarkRuleContext.class,
+ returnType = Environment.NoneType.class,
+ mandatoryParams = {
+ @Param(name = "outputs", type = SkylarkList.class, generic1 = Artifact.class,
+ doc = "list of the output files of the action")},
+ optionalParams = {
+ @Param(name = "inputs", type = SkylarkList.class, generic1 = Artifact.class,
+ doc = "list of the input files of the action"),
+ @Param(name = "executable", doc = "the executable file to be called by the action"),
+ @Param(name = "arguments", type = SkylarkList.class, generic1 = String.class,
+ doc = "command line arguments of the action"),
+ @Param(name = "mnemonic", type = String.class, doc = "mnemonic"),
+ @Param(name = "command", doc = "shell command to execute"),
+ @Param(name = "command_line", doc = "a command line to execute"),
+ @Param(name = "progress_message", type = String.class,
+ doc = "progress message to show to the user during the build"),
+ @Param(name = "use_default_shell_env", type = Boolean.class,
+ doc = "whether the action should use the built in shell environment or not"),
+ @Param(name = "env", type = Map.class, doc = "sets the dictionary of environment variables"),
+ @Param(name = "execution_requirements", type = Map.class,
+ doc = "information for scheduling the action"),
+ @Param(name = "input_manifests", type = Map.class,
+ doc = "sets the map of input manifests files; "
+ + "they are typicially generated by the command_helper")})
+ private static final SkylarkFunction createSpawnAction =
+ new SimpleSkylarkFunction("action") {
+
+ @Override
+ public Object call(Map<String, Object> params, Location loc) throws EvalException,
+ ConversionException {
+ SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self");
+ SpawnAction.Builder builder = new SpawnAction.Builder();
+ // TODO(bazel-team): builder still makes unnecessary copies of inputs, outputs and args.
+ builder.addInputs(castList(params.get("inputs"), Artifact.class));
+ builder.addOutputs(castList(params.get("outputs"), Artifact.class));
+ builder.addArguments(castList(params.get("arguments"), String.class));
+ if (params.containsKey("executable")) {
+ Object exe = params.get("executable");
+ if (exe instanceof Artifact) {
+ Artifact executable = (Artifact) exe;
+ builder.addInput(executable);
+ FilesToRunProvider provider = ctx.getExecutableRunfiles(executable);
+ if (provider == null) {
+ builder.setExecutable((Artifact) exe);
+ } else {
+ builder.setExecutable(provider);
+ }
+ } else if (exe instanceof PathFragment) {
+ builder.setExecutable((PathFragment) exe);
+ } else {
+ throw new EvalException(loc, "expected file or PathFragment for "
+ + "executable but got " + EvalUtils.getDatatypeName(exe) + " instead");
+ }
+ }
+ if (params.containsKey("command") == params.containsKey("executable")) {
+ throw new EvalException(loc, "You must specify either 'command' or 'executable' argument");
+ }
+ if (params.containsKey("command")) {
+ Object command = params.get("command");
+ if (command instanceof String) {
+ builder.setShellCommand((String) command);
+ } else if (command instanceof SkylarkList) {
+ SkylarkList commandList = (SkylarkList) command;
+ if (commandList.size() < 3) {
+ throw new EvalException(loc, "'command' list has to be of size at least 3");
+ }
+ builder.setShellCommand(castList(commandList, String.class, "command"));
+ } else {
+ throw new EvalException(loc, "expected string or list of strings for "
+ + "command instead of " + EvalUtils.getDatatypeName(command));
+ }
+ }
+ if (params.containsKey("command_line")) {
+ builder.setCommandLine(CommandLine.ofCharSequences(ImmutableList.copyOf(castList(
+ params.get("command_line"), CharSequence.class, "command line"))));
+ }
+ if (params.containsKey("mnemonic")) {
+ builder.setMnemonic((String) params.get("mnemonic"));
+ }
+ if (params.containsKey("env")) {
+ builder.setEnvironment(
+ toMap(castMap(params.get("env"), String.class, String.class, "env")));
+ }
+ if (params.containsKey("progress_message")) {
+ builder.setProgressMessage((String) params.get("progress_message"));
+ }
+ if (params.containsKey("use_default_shell_env")
+ && EvalUtils.toBoolean(params.get("use_default_shell_env"))) {
+ builder.useDefaultShellEnvironment();
+ }
+ if (params.containsKey("execution_requirements")) {
+ builder.setExecutionInfo(toMap(castMap(params.get("execution_requirements"),
+ String.class, String.class, "execution_requirements")));
+ }
+ if (params.containsKey("input_manifests")) {
+ for (Map.Entry<PathFragment, Artifact> entry : castMap(params.get("input_manifests"),
+ PathFragment.class, Artifact.class, "input manifest file map")) {
+ builder.addInputManifest(entry.getValue(), entry.getKey());
+ }
+ }
+ // Always register the action
+ ctx.getRuleContext().registerAction(builder.build(ctx.getRuleContext()));
+ return Environment.NONE;
+ }
+ };
+
+ // TODO(bazel-team): improve this method to be more memory friendly
+ @SkylarkBuiltin(name = "file_action",
+ doc = "Creates a file write action.",
+ objectType = SkylarkRuleContext.class,
+ returnType = Environment.NoneType.class,
+ optionalParams = {
+ @Param(name = "executable", type = Boolean.class,
+ doc = "whether the output file should be executable (default is False)"),
+ },
+ mandatoryParams = {
+ @Param(name = "output", type = Artifact.class, doc = "the output file"),
+ @Param(name = "content", type = String.class, doc = "the contents of the file")})
+ private static final SkylarkFunction createFileWriteAction =
+ new SimpleSkylarkFunction("file_action") {
+
+ @Override
+ public Object call(Map<String, Object> params, Location loc) throws EvalException,
+ ConversionException {
+ SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self");
+ boolean executable = params.containsKey("executable") && (Boolean) params.get("executable");
+ FileWriteAction action = new FileWriteAction(
+ ctx.getRuleContext().getActionOwner(),
+ (Artifact) params.get("output"),
+ (String) params.get("content"),
+ executable);
+ ctx.getRuleContext().registerAction(action);
+ return action;
+ }
+ };
+
+ @SkylarkBuiltin(name = "template_action",
+ doc = "Creates a template expansion action.",
+ objectType = SkylarkRuleContext.class,
+ returnType = Environment.NoneType.class,
+ mandatoryParams = {
+ @Param(name = "template", type = Artifact.class, doc = "the template file"),
+ @Param(name = "output", type = Artifact.class, doc = "the output file"),
+ @Param(name = "substitutions", type = Map.class,
+ doc = "substitutions to make when expanding the template")},
+ optionalParams = {
+ @Param(name = "executable", type = Boolean.class,
+ doc = "whether the output file should be executable (default is False)")})
+ private static final SkylarkFunction createTemplateAction =
+ new SimpleSkylarkFunction("template_action") {
+
+ @Override
+ public Object call(Map<String, Object> params, Location loc) throws EvalException,
+ ConversionException {
+ SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self");
+ ImmutableList.Builder<Substitution> substitutions = ImmutableList.builder();
+ for (Map.Entry<String, String> substitution
+ : castMap(params.get("substitutions"), String.class, String.class, "substitutions")) {
+ substitutions.add(Substitution.of(substitution.getKey(), substitution.getValue()));
+ }
+
+ boolean executable = params.containsKey("executable") && (Boolean) params.get("executable");
+ TemplateExpansionAction action = new TemplateExpansionAction(
+ ctx.getRuleContext().getActionOwner(),
+ (Artifact) params.get("template"),
+ (Artifact) params.get("output"),
+ substitutions.build(),
+ executable);
+ ctx.getRuleContext().registerAction(action);
+ return action;
+ }
+ };
+
+ /**
+ * A built in Skylark helper function to access the
+ * Transitive info providers of Transitive info collections.
+ */
+ @SkylarkBuiltin(name = "provider",
+ doc = "Returns the transitive info provider provided by the target.",
+ mandatoryParams = {
+ @Param(name = "target", type = TransitiveInfoCollection.class,
+ doc = "the configured target which provides the provider"),
+ @Param(name = "type", type = String.class, doc = "the class type of the provider")})
+ private static final SkylarkFunction provider = new SimpleSkylarkFunction("provider") {
+ @Override
+ public Object call(Map<String, Object> params, Location loc) throws EvalException {
+ TransitiveInfoCollection target = (TransitiveInfoCollection) params.get("target");
+ String type = (String) params.get("type");
+ try {
+ Class<?> classType = SkylarkRuleContext.classCache.get(type);
+ Class<? extends TransitiveInfoProvider> convertedClass =
+ classType.asSubclass(TransitiveInfoProvider.class);
+ Object result = target.getProvider(convertedClass);
+ return result == null ? Environment.NONE : result;
+ } catch (ExecutionException e) {
+ throw new EvalException(loc, "Unknown class type " + type);
+ } catch (ClassCastException e) {
+ throw new EvalException(loc, "Not a TransitiveInfoProvider " + type);
+ }
+ }
+ };
+
+ // TODO(bazel-team): Remove runfile states from Skylark.
+ @SkylarkBuiltin(name = "runfiles",
+ doc = "Creates a runfiles object.",
+ objectType = SkylarkRuleContext.class,
+ returnType = Runfiles.class,
+ optionalParams = {
+ @Param(name = "files", type = SkylarkList.class, generic1 = Artifact.class,
+ doc = "The list of files to be added to the runfiles."),
+ // TODO(bazel-team): If we have a memory efficient support for lazy list containing NestedSets
+ // we can remove this and just use files = [file] + list(set)
+ @Param(name = "transitive_files", type = SkylarkNestedSet.class, generic1 = Artifact.class,
+ doc = "The (transitive) set of files to be added to the runfiles."),
+ @Param(name = "collect_data", type = Boolean.class, doc = "Whether to collect the data "
+ + "runfiles from the dependencies in srcs, data and deps attributes."),
+ @Param(name = "collect_default", type = Boolean.class, doc = "Whether to collect the default "
+ + "runfiles from the dependencies in srcs, data and deps attributes.")})
+ private static final SkylarkFunction runfiles = new SimpleSkylarkFunction("runfiles") {
+ @Override
+ public Object call(Map<String, Object> params, Location loc) throws EvalException,
+ ConversionException {
+ SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self");
+ Runfiles.Builder builder = new Runfiles.Builder();
+ if (params.containsKey("collect_data") && (Boolean) params.get("collect_data")) {
+ builder.addRunfiles(ctx.getRuleContext(), RunfilesProvider.DATA_RUNFILES);
+ }
+ if (params.containsKey("collect_default") && (Boolean) params.get("collect_default")) {
+ builder.addRunfiles(ctx.getRuleContext(), RunfilesProvider.DEFAULT_RUNFILES);
+ }
+ if (params.containsKey("files")) {
+ builder.addArtifacts(castList(params.get("files"), Artifact.class));
+ }
+ if (params.containsKey("transitive_files")) {
+ builder.addTransitiveArtifacts(cast(params.get("transitive_files"),
+ SkylarkNestedSet.class, "files", loc).getSet(Artifact.class));
+ }
+ return builder.build();
+ }
+ };
+
+ @SkylarkBuiltin(name = "command_helper", doc = "Creates a command helper class.",
+ objectType = SkylarkRuleContext.class,
+ returnType = CommandHelper.class,
+ mandatoryParams = {
+ @Param(name = "tools", type = SkylarkList.class, generic1 = TransitiveInfoCollection.class,
+ doc = "list of tools (list of targets)"),
+ @Param(name = "label_dict", type = Map.class,
+ doc = "dictionary of resolved labels and the corresponding list of artifacts "
+ + "(a dict of Label : list of files)")})
+ private static final SkylarkFunction createCommandHelper =
+ new SimpleSkylarkFunction("command_helper") {
+ @SuppressWarnings("unchecked")
+ @Override
+ protected Object call(Map<String, Object> params, Location loc)
+ throws ConversionException, EvalException {
+ SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self");
+ return new CommandHelper(ctx.getRuleContext(),
+ AnalysisUtils.getProviders(
+ castList(params.get("tools"), TransitiveInfoCollection.class),
+ FilesToRunProvider.class),
+ // TODO(bazel-team): this cast to Map is unchecked and is not safe.
+ // The best way to fix this probably is to convert CommandHelper to Skylark.
+ ImmutableMap.copyOf((Map<Label, Iterable<Artifact>>) params.get("label_dict")));
+ }
+ };
+
+
+ @SkylarkBuiltin(name = "var",
+ doc = "get the value bound to a configuration variable in the context",
+ objectType = SkylarkRuleContext.class,
+ mandatoryParams = {
+ @Param(name = "name", type = String.class, doc = "the name of the variable")
+ },
+ returnType = String.class)
+ private static final SkylarkFunction configurationMakeVariableContext =
+ new SimpleSkylarkFunction("var") {
+ @SuppressWarnings("unchecked")
+ @Override
+ protected Object call(Map<String, Object> params, Location loc)
+ throws ConversionException, EvalException {
+ SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self");
+ String name = (String) params.get("name");
+ try {
+ return ctx.getRuleContext().getConfigurationMakeVariableContext()
+ .lookupMakeVariable(name);
+ } catch (MakeVariableExpander.ExpansionException e) {
+ throw new EvalException(loc, "configuration variable "
+ + ShellEscaper.escapeString(name) + " not defined");
+ }
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java
new file mode 100644
index 0000000..6efcd9d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java
@@ -0,0 +1,635 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ParameterFile;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.Util;
+import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.rules.test.BaselineCoverageAction;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A ConfiguredTarget for <code>cc_binary</code> rules.
+ */
+public abstract class CcBinary implements RuleConfiguredTargetFactory {
+
+ private final CppSemantics semantics;
+
+ protected CcBinary(CppSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES?
+ private static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of(
+ CppFileTypes.SHARED_LIBRARY,
+ CppFileTypes.VERSIONED_SHARED_LIBRARY);
+
+ /**
+ * The maximum number of inputs for any single .dwp generating action. For cases where
+ * this value is exceeded, the action is split up into "batches" that fall under the limit.
+ * See {@link #createDebugPackagerActions} for details.
+ */
+ @VisibleForTesting
+ public static final int MAX_INPUTS_PER_DWP_ACTION = 100;
+
+ /**
+ * Intermediate dwps are written to this subdirectory under the main dwp's output path.
+ */
+ @VisibleForTesting
+ public static final String INTERMEDIATE_DWP_DIR = "_dwps";
+
+ private static Runfiles collectRunfiles(RuleContext context,
+ CcCommon common,
+ CcLinkingOutputs linkingOutputs,
+ CppCompilationContext cppCompilationContext,
+ LinkStaticness linkStaticness,
+ NestedSet<Artifact> filesToBuild,
+ Iterable<Artifact> fakeLinkerInputs,
+ boolean fake) {
+ Runfiles.Builder builder = new Runfiles.Builder();
+ Function<TransitiveInfoCollection, Runfiles> runfilesMapping =
+ CppRunfilesProvider.runfilesFunction(linkStaticness != LinkStaticness.DYNAMIC);
+ boolean linkshared = isLinkShared(context);
+ builder.addTransitiveArtifacts(filesToBuild);
+ // Add the shared libraries to the runfiles. This adds any shared libraries that are in the
+ // srcs of this target.
+ builder.addArtifacts(linkingOutputs.getLibrariesForRunfiles(true));
+ builder.addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES);
+ builder.add(context, runfilesMapping);
+ CcToolchainProvider toolchain = CppHelper.getToolchain(context);
+ // Add the C++ runtime libraries if linking them dynamically.
+ if (linkStaticness == LinkStaticness.DYNAMIC) {
+ builder.addTransitiveArtifacts(toolchain.getDynamicRuntimeLinkInputs());
+ }
+ // For cc_binary and cc_test rules, there is an implicit dependency on
+ // the malloc library package, which is specified by the "malloc" attribute.
+ // As the BUILD encyclopedia says, the "malloc" attribute should be ignored
+ // if linkshared=1.
+ if (!linkshared) {
+ TransitiveInfoCollection malloc = CppHelper.mallocForTarget(context);
+ builder.addTarget(malloc, RunfilesProvider.DEFAULT_RUNFILES);
+ builder.addTarget(malloc, runfilesMapping);
+ }
+
+ if (fake) {
+ // Add the object files, libraries, and linker scripts that are used to
+ // link this executable.
+ builder.addSymlinksToArtifacts(Iterables.filter(fakeLinkerInputs, Artifact.MIDDLEMAN_FILTER));
+ // The crosstool inputs for the link action are not sufficient; we also need the crosstool
+ // inputs for compilation. Node that these cannot be middlemen because Runfiles does not
+ // know how to expand them.
+ builder.addTransitiveArtifacts(toolchain.getCrosstool());
+ builder.addTransitiveArtifacts(toolchain.getLibcLink());
+ // Add the sources files that are used to compile the object files.
+ // We add the headers in the transitive closure and our own sources in the srcs
+ // attribute. We do not provide the auxiliary inputs, because they are only used when we
+ // do FDO compilation, and cc_fake_binary does not support FDO.
+ builder.addSymlinksToArtifacts(
+ Iterables.transform(common.getCAndCppSources(), Pair.<Artifact, Label>firstFunction()));
+ builder.addSymlinksToArtifacts(cppCompilationContext.getDeclaredIncludeSrcs());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext context) {
+ return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ false);
+ }
+
+ public static ConfiguredTarget init(CppSemantics semantics, RuleContext ruleContext, boolean fake,
+ boolean useTestOnlyFlags) {
+ ruleContext.checkSrcsSamePackage(true);
+ CcCommon common = new CcCommon(ruleContext);
+ CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class);
+
+ LinkTargetType linkType =
+ isLinkShared(ruleContext) ? LinkTargetType.DYNAMIC_LIBRARY : LinkTargetType.EXECUTABLE;
+
+ CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics)
+ .setLinkType(linkType)
+ .setHeadersCheckingMode(common.determineHeadersCheckingMode())
+ .addCopts(common.getCopts())
+ .setNoCopts(common.getNoCopts())
+ .addLinkopts(common.getLinkopts())
+ .addDefines(common.getDefines())
+ .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs())
+ .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs())
+ .addSources(common.getCAndCppSources())
+ .addPrivateHeaders(FileType.filter(
+ ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(),
+ CppFileTypes.CPP_HEADER))
+ .addObjectFiles(common.getObjectFilesFromSrcs(false))
+ .addPicObjectFiles(common.getObjectFilesFromSrcs(true))
+ .addPicIndependentObjectFiles(common.getLinkerScripts())
+ .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET))
+ .addDeps(ImmutableList.of(CppHelper.mallocForTarget(ruleContext)))
+ .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK))
+ .addSystemIncludeDirs(common.getSystemIncludeDirs())
+ .addIncludeDirs(common.getIncludeDirs())
+ .addLooseIncludeDirs(common.getLooseIncludeDirs())
+ .setFake(fake);
+
+ CcLibraryHelper.Info info = helper.build();
+ CppCompilationContext cppCompilationContext = info.getCppCompilationContext();
+ CcCompilationOutputs ccCompilationOutputs = info.getCcCompilationOutputs();
+
+ // if cc_binary includes "linkshared=1", then gcc will be invoked with
+ // linkopt "-shared", which causes the result of linking to be a shared
+ // library. In this case, the name of the executable target should end
+ // in ".so".
+ PathFragment executableName = Util.getWorkspaceRelativePath(
+ ruleContext.getTarget(), "", OsUtils.executableExtension());
+ CppLinkAction.Builder linkActionBuilder = determineLinkerArguments(
+ ruleContext, common, cppConfiguration, ccCompilationOutputs,
+ cppCompilationContext.getCompilationPrerequisites(), fake, executableName);
+ linkActionBuilder.setUseTestOnlyFlags(useTestOnlyFlags);
+ linkActionBuilder.addNonLibraryInputs(ccCompilationOutputs.getHeaderTokenFiles());
+
+ CcToolchainProvider ccToolchain = CppHelper.getToolchain(ruleContext);
+ LinkStaticness linkStaticness = getLinkStaticness(ruleContext, common, cppConfiguration);
+ if (linkStaticness == LinkStaticness.DYNAMIC) {
+ linkActionBuilder.setRuntimeInputs(
+ ccToolchain.getDynamicRuntimeLinkMiddleman(),
+ ccToolchain.getDynamicRuntimeLinkInputs());
+ } else {
+ linkActionBuilder.setRuntimeInputs(
+ ccToolchain.getStaticRuntimeLinkMiddleman(),
+ ccToolchain.getStaticRuntimeLinkInputs());
+ // Only force a static link of libgcc if static runtime linking is enabled (which
+ // can't be true if runtimeInputs is empty).
+ // TODO(bazel-team): Move this to CcToolchain.
+ if (!ccToolchain.getStaticRuntimeLinkInputs().isEmpty()) {
+ linkActionBuilder.addLinkopt("-static-libgcc");
+ }
+ }
+
+ linkActionBuilder.setLinkType(linkType);
+ linkActionBuilder.setLinkStaticness(linkStaticness);
+ linkActionBuilder.setFake(fake);
+
+ // store immutable context now, recreate builder later
+ CppLinkAction.Context linkContext = new CppLinkAction.Context(linkActionBuilder);
+
+ CppLinkAction linkAction = linkActionBuilder.build();
+ ruleContext.registerAction(linkAction);
+ LibraryToLink outputLibrary = linkAction.getOutputLibrary();
+ Iterable<Artifact> fakeLinkerInputs =
+ fake ? linkAction.getInputs() : ImmutableList.<Artifact>of();
+ Artifact executable = outputLibrary.getArtifact();
+ CcLinkingOutputs.Builder linkingOutputsBuilder = new CcLinkingOutputs.Builder();
+ if (isLinkShared(ruleContext)) {
+ if (CppFileTypes.SHARED_LIBRARY.matches(executableName)) {
+ linkingOutputsBuilder.addDynamicLibrary(outputLibrary);
+ linkingOutputsBuilder.addExecutionDynamicLibrary(outputLibrary);
+ } else {
+ ruleContext.attributeError("linkshared", "'linkshared' used in non-shared library");
+ }
+ }
+ // Also add all shared libraries from srcs.
+ for (Artifact library : common.getSharedLibrariesFromSrcs()) {
+ LibraryToLink symlink = common.getDynamicLibrarySymlink(library, true);
+ linkingOutputsBuilder.addDynamicLibrary(symlink);
+ linkingOutputsBuilder.addExecutionDynamicLibrary(symlink);
+ }
+ CcLinkingOutputs linkingOutputs = linkingOutputsBuilder.build();
+ NestedSet<Artifact> filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, executable);
+
+ // Create the stripped binary, but don't add it to filesToBuild; it's only built when requested.
+ Artifact strippedFile = ruleContext.getImplicitOutputArtifact(
+ CppRuleClasses.CC_BINARY_STRIPPED);
+ createStripAction(ruleContext, cppConfiguration, executable, strippedFile);
+
+ DwoArtifactsCollector dwoArtifacts =
+ collectTransitiveDwoArtifacts(ruleContext, common, cppConfiguration, ccCompilationOutputs);
+ Artifact dwpFile =
+ ruleContext.getImplicitOutputArtifact(CppRuleClasses.CC_BINARY_DEBUG_PACKAGE);
+ createDebugPackagerActions(ruleContext, cppConfiguration, dwpFile, dwoArtifacts);
+
+ // The debug package should include the dwp file only if it was explicitly requested.
+ Artifact explicitDwpFile = dwpFile;
+ if (!cppConfiguration.useFission()) {
+ explicitDwpFile = null;
+ }
+
+ // TODO(bazel-team): Do we need to put original shared libraries (along with
+ // mangled symlinks) into the RunfilesSupport object? It does not seem
+ // logical since all symlinked libraries will be linked anyway and would
+ // not require manual loading but if we do, then we would need to collect
+ // their names and use a different constructor below.
+ Runfiles runfiles = collectRunfiles(ruleContext, common, linkingOutputs,
+ cppCompilationContext, linkStaticness, filesToBuild, fakeLinkerInputs, fake);
+ RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable(
+ ruleContext, runfiles, executable, ruleContext.getConfiguration().buildRunfiles());
+
+ TransitiveLipoInfoProvider transitiveLipoInfo;
+ if (cppConfiguration.isLipoContextCollector()) {
+ transitiveLipoInfo = common.collectTransitiveLipoLabels(ccCompilationOutputs);
+ } else {
+ transitiveLipoInfo = TransitiveLipoInfoProvider.EMPTY;
+ }
+
+ RuleConfiguredTargetBuilder ruleBuilder = new RuleConfiguredTargetBuilder(ruleContext);
+ common.addTransitiveInfoProviders(
+ ruleBuilder, filesToBuild, ccCompilationOutputs, cppCompilationContext, linkingOutputs,
+ dwoArtifacts, transitiveLipoInfo);
+
+ Map<Artifact, IncludeScannable> scannableMap = new LinkedHashMap<>();
+ if (cppConfiguration.isLipoContextCollector()) {
+ for (IncludeScannable scannable : transitiveLipoInfo.getTransitiveIncludeScannables()) {
+ // These should all be CppCompileActions, which should have only one source file.
+ // This is also checked when they are put into the nested set.
+ Artifact source =
+ Iterables.getOnlyElement(scannable.getIncludeScannerSources());
+ scannableMap.put(source, scannable);
+ }
+ }
+
+ return ruleBuilder
+ .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles))
+ .add(
+ CppDebugPackageProvider.class,
+ new CppDebugPackageProvider(strippedFile, executable, explicitDwpFile))
+ .setRunfilesSupport(runfilesSupport, executable)
+ .setBaselineCoverageArtifacts(createBaselineCoverageArtifacts(
+ ruleContext, common, ccCompilationOutputs, fake))
+ .addProvider(LipoContextProvider.class, new LipoContextProvider(
+ cppCompilationContext, ImmutableMap.copyOf(scannableMap)))
+ .addProvider(CppLinkAction.Context.class, linkContext)
+ .build();
+ }
+
+ /**
+ * Creates an action to strip an executable.
+ */
+ private static void createStripAction(RuleContext context,
+ CppConfiguration cppConfiguration, Artifact input, Artifact output) {
+ context.registerAction(new SpawnAction.Builder()
+ .addInput(input)
+ .addTransitiveInputs(CppHelper.getToolchain(context).getStrip())
+ .addOutput(output)
+ .useDefaultShellEnvironment()
+ .setExecutable(cppConfiguration.getStripExecutable())
+ .addArguments("-S", "-p", "-o", output.getExecPathString())
+ .addArguments("-R", ".gnu.switches.text.quote_paths")
+ .addArguments("-R", ".gnu.switches.text.bracket_paths")
+ .addArguments("-R", ".gnu.switches.text.system_paths")
+ .addArguments("-R", ".gnu.switches.text.cpp_defines")
+ .addArguments("-R", ".gnu.switches.text.cpp_includes")
+ .addArguments("-R", ".gnu.switches.text.cl_args")
+ .addArguments("-R", ".gnu.switches.text.lipo_info")
+ .addArguments("-R", ".gnu.switches.text.annotation")
+ .addArguments(cppConfiguration.getStripOpts())
+ .addArgument(input.getExecPathString())
+ .setProgressMessage("Stripping " + output.prettyPrint() + " for " + context.getLabel())
+ .setMnemonic("CcStrip")
+ .build(context));
+ }
+
+ /**
+ * Given 'temps', traverse this target and its dependencies and collect up all
+ * the object files, libraries, linker options, linkstamps attributes and linker scripts.
+ */
+ private static CppLinkAction.Builder determineLinkerArguments(RuleContext context,
+ CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs,
+ ImmutableSet<Artifact> compilationPrerequisites,
+ boolean fake, PathFragment executableName) {
+ CppLinkAction.Builder builder = new CppLinkAction.Builder(context, executableName)
+ .setCrosstoolInputs(CppHelper.getToolchain(context).getLink())
+ .addNonLibraryInputs(compilationPrerequisites);
+
+ // Determine the object files to link in.
+ boolean usePic = CppHelper.usePic(context, !isLinkShared(context)) && !fake;
+ Iterable<Artifact> compiledObjectFiles = compilationOutputs.getObjectFiles(usePic);
+
+ if (fake) {
+ builder.addFakeNonLibraryInputs(compiledObjectFiles);
+ } else {
+ builder.addNonLibraryInputs(compiledObjectFiles);
+ }
+
+ builder.addNonLibraryInputs(common.getObjectFilesFromSrcs(usePic));
+ builder.addNonLibraryInputs(common.getLinkerScripts());
+
+ // Determine the libraries to link in.
+ // First libraries from srcs. Shared library artifacts here are substituted with mangled symlink
+ // artifacts generated by getDynamicLibraryLink(). This is done to minimize number of -rpath
+ // entries during linking process.
+ for (Artifact library : common.getLibrariesFromSrcs()) {
+ if (SHARED_LIBRARY_FILETYPES.matches(library.getFilename())) {
+ builder.addLibrary(common.getDynamicLibrarySymlink(library, true));
+ } else {
+ builder.addLibrary(LinkerInputs.opaqueLibraryToLink(library));
+ }
+ }
+
+ // Then libraries from the closure of deps.
+ List<String> linkopts = new ArrayList<>();
+ Map<Artifact, ImmutableList<Artifact>> linkstamps = new LinkedHashMap<>();
+
+ NestedSet<LibraryToLink> librariesInDepsClosure =
+ findLibrariesToLinkInDepsClosure(context, common, cppConfiguration, linkopts, linkstamps);
+ builder.addLinkopts(linkopts);
+ builder.addLinkstamps(linkstamps);
+
+ builder.addLibraries(librariesInDepsClosure);
+ return builder;
+ }
+
+ /**
+ * Explore the transitive closure of our deps to collect linking information.
+ */
+ private static NestedSet<LibraryToLink> findLibrariesToLinkInDepsClosure(
+ RuleContext context,
+ CcCommon common,
+ CppConfiguration cppConfiguration,
+ List<String> linkopts,
+ Map<Artifact,
+ ImmutableList<Artifact>> linkstamps) {
+ // This is true for both FULLY STATIC and MOSTLY STATIC linking.
+ boolean linkingStatically =
+ getLinkStaticness(context, common, cppConfiguration) != LinkStaticness.DYNAMIC;
+
+ CcLinkParams linkParams = collectCcLinkParams(
+ context, common, linkingStatically, isLinkShared(context));
+ linkopts.addAll(linkParams.flattenedLinkopts());
+ linkstamps.putAll(CppHelper.resolveLinkstamps(context, linkParams));
+ return linkParams.getLibraries();
+ }
+
+ /**
+ * Gets the linkopts to use for this binary. These options are NOT used when
+ * linking other binaries that depend on this binary.
+ *
+ * @return a new List instance that contains the linkopts for this binary
+ * target.
+ */
+ private static ImmutableList<String> getBinaryLinkopts(RuleContext context,
+ CcCommon common) {
+ List<String> linkopts = new ArrayList<>();
+ if (isLinkShared(context)) {
+ linkopts.add("-shared");
+ }
+ linkopts.addAll(common.getLinkopts());
+ return ImmutableList.copyOf(linkopts);
+ }
+
+ private static boolean linkstaticAttribute(RuleContext context) {
+ return context.attributes().get("linkstatic", Type.BOOLEAN);
+ }
+
+ /**
+ * Returns "true" if the {@code linkshared} attribute exists and is set.
+ */
+ private static final boolean isLinkShared(RuleContext context) {
+ return context.getRule().getRuleClassObject().hasAttr("linkshared", Type.BOOLEAN)
+ && context.attributes().get("linkshared", Type.BOOLEAN);
+ }
+
+ private static final boolean dashStaticInLinkopts(CcCommon common,
+ CppConfiguration cppConfiguration) {
+ return common.getLinkopts().contains("-static")
+ || cppConfiguration.getLinkOptions().contains("-static");
+ }
+
+ private static final LinkStaticness getLinkStaticness(RuleContext context,
+ CcCommon common, CppConfiguration cppConfiguration) {
+ if (cppConfiguration.getDynamicMode() == DynamicMode.FULLY) {
+ return LinkStaticness.DYNAMIC;
+ } else if (dashStaticInLinkopts(common, cppConfiguration)) {
+ return LinkStaticness.FULLY_STATIC;
+ } else if (cppConfiguration.getDynamicMode() == DynamicMode.OFF
+ || linkstaticAttribute(context)) {
+ return LinkStaticness.MOSTLY_STATIC;
+ } else {
+ return LinkStaticness.DYNAMIC;
+ }
+ }
+
+ /**
+ * Collects .dwo artifacts either transitively or directly, depending on the link type.
+ *
+ * <p>For a cc_binary, we only include the .dwo files corresponding to the .o files that are
+ * passed into the link. For static linking, this includes all transitive dependencies. But
+ * for dynamic linking, dependencies are separately linked into their own shared libraries,
+ * so we don't need them here.
+ */
+ private static DwoArtifactsCollector collectTransitiveDwoArtifacts(RuleContext context,
+ CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs) {
+ if (getLinkStaticness(context, common, cppConfiguration) == LinkStaticness.DYNAMIC) {
+ return DwoArtifactsCollector.directCollector(compilationOutputs);
+ } else {
+ return CcCommon.collectTransitiveDwoArtifacts(context, compilationOutputs);
+ }
+ }
+
+ @VisibleForTesting
+ public static Iterable<Artifact> getDwpInputs(
+ RuleContext context, NestedSet<Artifact> picDwoArtifacts, NestedSet<Artifact> dwoArtifacts) {
+ return CppHelper.usePic(context, !isLinkShared(context)) ? picDwoArtifacts : dwoArtifacts;
+ }
+
+ /**
+ * Creates the actions needed to generate this target's "debug info package"
+ * (i.e. its .dwp file).
+ */
+ private static void createDebugPackagerActions(RuleContext context,
+ CppConfiguration cppConfiguration, Artifact dwpOutput,
+ DwoArtifactsCollector dwoArtifactsCollector) {
+ Iterable<Artifact> allInputs = getDwpInputs(context,
+ dwoArtifactsCollector.getPicDwoArtifacts(),
+ dwoArtifactsCollector.getDwoArtifacts());
+
+ // No inputs? Just generate a trivially empty .dwp.
+ //
+ // Note this condition automatically triggers for any build where fission is disabled.
+ // Because rules referencing .dwp targets may be invoked with or without fission, we need
+ // to support .dwp generation even when fission is disabled. Since no actual functionality
+ // is expected then, an empty file is appropriate.
+ if (Iterables.isEmpty(allInputs)) {
+ context.registerAction(
+ new FileWriteAction(context.getActionOwner(), dwpOutput, "", false));
+ return;
+ }
+
+ // Get the tool inputs necessary to run the dwp command.
+ NestedSet<Artifact> dwpTools = CppHelper.getToolchain(context).getDwp();
+ Preconditions.checkState(!dwpTools.isEmpty());
+
+ // We apply a hierarchical action structure to limit the maximum number of inputs to any
+ // single action.
+ //
+ // While the dwp tools consumes .dwo files, it can also consume intermediate .dwp files,
+ // allowing us to split a large input set into smaller batches of arbitrary size and order.
+ // Aside from the parallelism performance benefits this offers, this also reduces input
+ // size requirements: if a.dwo, b.dwo, c.dwo, and e.dwo are each 1 KB files, we can apply
+ // two intermediate actions DWP(a.dwo, b.dwo) --> i1.dwp and DWP(c.dwo, e.dwo) --> i2.dwp.
+ // When we then apply the final action DWP(i1.dwp, i2.dwp) --> finalOutput.dwp, the inputs
+ // to this action will usually total far less than 4 KB.
+ //
+ // This list tracks every action we'll need to generate the output .dwp with batching.
+ List<SpawnAction.Builder> packagers = new ArrayList<>();
+
+ // Step 1: generate our batches. We currently break into arbitrary batches of fixed maximum
+ // input counts, but we can always apply more intelligent heuristics if the need arises.
+ SpawnAction.Builder currentPackager = newDwpAction(cppConfiguration, dwpTools);
+ int inputsForCurrentPackager = 0;
+
+ for (Artifact dwoInput : allInputs) {
+ if (inputsForCurrentPackager == MAX_INPUTS_PER_DWP_ACTION) {
+ packagers.add(currentPackager);
+ currentPackager = newDwpAction(cppConfiguration, dwpTools);
+ inputsForCurrentPackager = 0;
+ }
+ currentPackager.addInputArgument(dwoInput);
+ inputsForCurrentPackager++;
+ }
+ packagers.add(currentPackager);
+
+ // Step 2: given the batches, create the actions.
+ if (packagers.size() == 1) {
+ // If we only have one batch, make a single "original inputs --> final output" action.
+ context.registerAction(Iterables.getOnlyElement(packagers)
+ .addArgument("-o")
+ .addOutputArgument(dwpOutput)
+ .setMnemonic("CcGenerateDwp")
+ .build(context));
+ } else {
+ // If we have multiple batches, make them all intermediate actions, then pipe their outputs
+ // into an additional action that outputs the final artifact.
+ //
+ // Note this only creates a hierarchy one level deep (i.e. we don't check if the number of
+ // intermediate outputs exceeds the maximum batch size). This is okay for current needs,
+ // which shouldn't stress those limits.
+ List<Artifact> intermediateOutputs = new ArrayList<>();
+
+ int count = 1;
+ for (SpawnAction.Builder packager : packagers) {
+ Artifact intermediateOutput =
+ getIntermediateDwpFile(context.getAnalysisEnvironment(), dwpOutput, count++);
+ context.registerAction(packager
+ .addArgument("-o")
+ .addOutputArgument(intermediateOutput)
+ .setMnemonic("CcGenerateIntermediateDwp")
+ .build(context));
+ intermediateOutputs.add(intermediateOutput);
+ }
+
+ // Now create the final action.
+ context.registerAction(newDwpAction(cppConfiguration, dwpTools)
+ .addInputArguments(intermediateOutputs)
+ .addArgument("-o")
+ .addOutputArgument(dwpOutput)
+ .setMnemonic("CcGenerateDwp")
+ .build(context));
+ }
+ }
+
+ /**
+ * Returns a new SpawnAction builder for generating dwp files, pre-initialized with
+ * standard settings.
+ */
+ private static SpawnAction.Builder newDwpAction(CppConfiguration cppConfiguration,
+ NestedSet<Artifact> dwpTools) {
+ return new SpawnAction.Builder()
+ .addTransitiveInputs(dwpTools)
+ .setExecutable(cppConfiguration.getDwpExecutable())
+ .useParameterFile(ParameterFile.ParameterFileType.UNQUOTED);
+ }
+
+ /**
+ * Creates an intermediate dwp file keyed off the name and path of the final output.
+ */
+ private static Artifact getIntermediateDwpFile(AnalysisEnvironment env, Artifact dwpOutput,
+ int orderNumber) {
+ PathFragment outputPath = dwpOutput.getRootRelativePath();
+ PathFragment intermediatePath =
+ FileSystemUtils.appendWithoutExtension(outputPath, "-" + orderNumber);
+ return env.getDerivedArtifact(
+ outputPath.getParentDirectory().getRelative(
+ INTERMEDIATE_DWP_DIR + "/" + intermediatePath.getPathString()),
+ dwpOutput.getRoot());
+ }
+
+ /**
+ * Collect link parameters from the transitive closure.
+ */
+ private static CcLinkParams collectCcLinkParams(RuleContext context, CcCommon common,
+ boolean linkingStatically, boolean linkShared) {
+ CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared);
+
+ if (isLinkShared(context)) {
+ // CcLinkingOutputs is empty because this target is not configured yet
+ builder.addCcLibrary(context, common, false, CcLinkingOutputs.EMPTY);
+ } else {
+ builder.addTransitiveTargets(
+ context.getPrerequisites("deps", Mode.TARGET),
+ CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS);
+ builder.addTransitiveTarget(CppHelper.mallocForTarget(context));
+ builder.addLinkOpts(getBinaryLinkopts(context, common));
+ }
+ return builder.build();
+ }
+
+ private static ImmutableList<Artifact> createBaselineCoverageArtifacts(
+ RuleContext context, CcCommon common, CcCompilationOutputs compilationOutputs,
+ boolean fake) {
+ if (!TargetUtils.isTestRule(context.getRule()) && !fake) {
+ Iterable<Artifact> objectFiles = compilationOutputs.getObjectFiles(
+ CppHelper.usePic(context, !isLinkShared(context)));
+ return BaselineCoverageAction.getBaselineCoverageArtifacts(context,
+ common.getInstrumentedFilesProvider(objectFiles).getInstrumentedFiles());
+ } else {
+ return ImmutableList.of();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java
new file mode 100644
index 0000000..3f5ff76
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java
@@ -0,0 +1,678 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToCompileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TempsProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl;
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Common parts of the implementation of cc rules.
+ */
+public final class CcCommon {
+
+ private static final String NO_COPTS_ATTRIBUTE = "nocopts";
+
+ private static final FileTypeSet SOURCE_TYPES = FileTypeSet.of(
+ CppFileTypes.CPP_SOURCE,
+ CppFileTypes.CPP_HEADER,
+ CppFileTypes.C_SOURCE,
+ CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR);
+
+ /**
+ * Collects all metadata files generated by C++ compilation actions that output the .o files
+ * on the input.
+ */
+ private static final LocalMetadataCollector CC_METADATA_COLLECTOR =
+ new LocalMetadataCollector() {
+ @Override
+ public void collectMetadataArtifacts(Iterable<Artifact> objectFiles,
+ AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) {
+ for (Artifact artifact : objectFiles) {
+ Action action = analysisEnvironment.getLocalGeneratingAction(artifact);
+ if (action instanceof CppCompileAction) {
+ addOutputs(metadataFilesBuilder, action, CppFileTypes.COVERAGE_NOTES);
+ }
+ }
+ }
+ };
+
+ /** C++ configuration */
+ private final CppConfiguration cppConfiguration;
+
+ /** The Artifacts from srcs. */
+ private final ImmutableList<Artifact> sources;
+
+ private final ImmutableList<Pair<Artifact, Label>> cAndCppSources;
+
+ /** Expanded and tokenized copts attribute. Set by initCopts(). */
+ private final ImmutableList<String> copts;
+
+ /**
+ * The expanded linkopts for this rule.
+ */
+ private final ImmutableList<String> linkopts;
+
+ private final RuleContext ruleContext;
+
+ public CcCommon(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class);
+ this.sources = hasAttribute("srcs", Type.LABEL_LIST)
+ ? ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list()
+ : ImmutableList.<Artifact>of();
+
+ this.cAndCppSources = collectCAndCppSources();
+ copts = initCopts();
+ linkopts = initLinkopts();
+ }
+
+ ImmutableList<Artifact> getTemps(CcCompilationOutputs compilationOutputs) {
+ return cppConfiguration.isLipoContextCollector()
+ ? ImmutableList.<Artifact>of()
+ : compilationOutputs.getTemps();
+ }
+
+ /**
+ * Returns our own linkopts from the rule attribute. This determines linker
+ * options to use when building this target and anything that depends on it.
+ */
+ public ImmutableList<String> getLinkopts() {
+ return linkopts;
+ }
+
+ public ImmutableList<String> getCopts() {
+ return copts;
+ }
+
+ private boolean hasAttribute(String name, Type<?> type) {
+ return ruleContext.getRule().getRuleClassObject().hasAttr(name, type);
+ }
+
+ private static NestedSet<Artifact> collectExecutionDynamicLibraryArtifacts(
+ RuleContext ruleContext,
+ List<LibraryToLink> executionDynamicLibraries) {
+ Iterable<Artifact> artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries);
+ if (!Iterables.isEmpty(artifacts)) {
+ return NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts);
+ }
+
+ Iterable<CcExecutionDynamicLibrariesProvider> deps = ruleContext
+ .getPrerequisites("deps", Mode.TARGET, CcExecutionDynamicLibrariesProvider.class);
+
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (CcExecutionDynamicLibrariesProvider dep : deps) {
+ builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Collects all .dwo artifacts in this target's transitive closure.
+ */
+ public static DwoArtifactsCollector collectTransitiveDwoArtifacts(
+ RuleContext ruleContext,
+ CcCompilationOutputs compilationOutputs) {
+ ImmutableList.Builder<TransitiveInfoCollection> deps =
+ ImmutableList.<TransitiveInfoCollection>builder();
+
+ deps.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET));
+
+ if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) {
+ deps.add(CppHelper.mallocForTarget(ruleContext));
+ }
+ if (ruleContext.getRule().getRuleClassObject().hasAttr("implementation", Type.LABEL_LIST)) {
+ deps.addAll(ruleContext.getPrerequisites("implementation", Mode.TARGET));
+ }
+
+ return compilationOutputs == null // Possible in LIPO collection mode (see initializationHook).
+ ? DwoArtifactsCollector.emptyCollector()
+ : DwoArtifactsCollector.transitiveCollector(compilationOutputs, deps.build());
+ }
+
+ public TransitiveLipoInfoProvider collectTransitiveLipoLabels(CcCompilationOutputs outputs) {
+ if (cppConfiguration.getFdoSupport().getFdoRoot() == null
+ || !cppConfiguration.isLipoContextCollector()) {
+ return TransitiveLipoInfoProvider.EMPTY;
+ }
+
+ NestedSetBuilder<IncludeScannable> scannableBuilder = NestedSetBuilder.stableOrder();
+ CppHelper.addTransitiveLipoInfoForCommonAttributes(ruleContext, outputs, scannableBuilder);
+ if (hasAttribute("implementation", Type.LABEL_LIST)) {
+ for (TransitiveLipoInfoProvider impl : AnalysisUtils.getProviders(
+ ruleContext.getPrerequisites("implementation", Mode.TARGET),
+ TransitiveLipoInfoProvider.class)) {
+ scannableBuilder.addTransitive(impl.getTransitiveIncludeScannables());
+ }
+ }
+
+ return new TransitiveLipoInfoProvider(scannableBuilder.build());
+ }
+
+ private NestedSet<LinkerInput> collectTransitiveCcNativeLibraries(
+ RuleContext ruleContext,
+ List<? extends LinkerInput> dynamicLibraries) {
+ NestedSetBuilder<LinkerInput> builder = NestedSetBuilder.linkOrder();
+ builder.addAll(dynamicLibraries);
+ for (CcNativeLibraryProvider dep :
+ ruleContext.getPrerequisites("deps", Mode.TARGET, CcNativeLibraryProvider.class)) {
+ builder.addTransitive(dep.getTransitiveCcNativeLibraries());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns a list of ({@link Artifact}, {@link Label}) pairs. Each pair represents an input
+ * source file and the label of the rule that generates it (or the label of the source file
+ * itself if it is an input file)
+ */
+ ImmutableList<Pair<Artifact, Label>> getCAndCppSources() {
+ return cAndCppSources;
+ }
+
+ private boolean shouldProcessHeaders() {
+ boolean crosstoolSupportsHeaderParsing =
+ CppHelper.getToolchain(ruleContext).supportsHeaderParsing();
+ return crosstoolSupportsHeaderParsing && (
+ ruleContext.getFeatures().contains(CppRuleClasses.PREPROCESS_HEADERS)
+ || ruleContext.getFeatures().contains(CppRuleClasses.PARSE_HEADERS));
+ }
+
+ private ImmutableList<Pair<Artifact, Label>> collectCAndCppSources() {
+ Map<Artifact, Label> map = Maps.newLinkedHashMap();
+ if (!hasAttribute("srcs", Type.LABEL_LIST)) {
+ return ImmutableList.<Pair<Artifact, Label>>of();
+ }
+ Iterable<FileProvider> providers =
+ ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class);
+ // TODO(bazel-team): Move header processing logic down in the stack (to CcLibraryHelper or
+ // such).
+ boolean processHeaders = shouldProcessHeaders();
+ if (processHeaders && hasAttribute("hdrs", Type.LABEL_LIST)) {
+ providers = Iterables.concat(providers,
+ ruleContext.getPrerequisites("hdrs", Mode.TARGET, FileProvider.class));
+ }
+ for (FileProvider provider : providers) {
+ for (Artifact artifact : FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)) {
+ boolean isHeader = CppFileTypes.CPP_HEADER.matches(artifact.getExecPath());
+ if ((isHeader && !processHeaders)
+ || CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(artifact.getExecPath())) {
+ continue;
+ }
+ Label oldLabel = map.put(artifact, provider.getLabel());
+ // TODO(bazel-team): We currently do not warn for duplicate headers with
+ // different labels, as that would require cleaning up the code base
+ // without significant benefit; we should eventually make this
+ // consistent one way or the other.
+ if (!isHeader && oldLabel != null && !oldLabel.equals(provider.getLabel())) {
+ ruleContext.attributeError("srcs", String.format(
+ "Artifact '%s' is duplicated (through '%s' and '%s')",
+ artifact.getExecPathString(), oldLabel, provider.getLabel()));
+ }
+ }
+ }
+
+ ImmutableList.Builder<Pair<Artifact, Label>> result = ImmutableList.builder();
+ for (Map.Entry<Artifact, Label> entry : map.entrySet()) {
+ result.add(Pair.of(entry.getKey(), entry.getValue()));
+ }
+
+ return result.build();
+ }
+
+ Iterable<Artifact> getLibrariesFromSrcs() {
+ return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE,
+ CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY,
+ CppFileTypes.SHARED_LIBRARY,
+ CppFileTypes.VERSIONED_SHARED_LIBRARY);
+ }
+
+ Iterable<Artifact> getSharedLibrariesFromSrcs() {
+ return getSharedLibrariesFrom(sources);
+ }
+
+ static Iterable<Artifact> getSharedLibrariesFrom(Iterable<Artifact> collection) {
+ return FileType.filter(collection, CppFileTypes.SHARED_LIBRARY,
+ CppFileTypes.VERSIONED_SHARED_LIBRARY);
+ }
+
+ Iterable<Artifact> getStaticLibrariesFromSrcs() {
+ return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.ALWAYS_LINK_LIBRARY);
+ }
+
+ Iterable<LibraryToLink> getPicStaticLibrariesFromSrcs() {
+ return LinkerInputs.opaqueLibrariesToLink(
+ FileType.filter(sources, CppFileTypes.PIC_ARCHIVE,
+ CppFileTypes.ALWAYS_LINK_PIC_LIBRARY));
+ }
+
+ Iterable<Artifact> getObjectFilesFromSrcs(final boolean usePic) {
+ if (usePic) {
+ return Iterables.filter(sources, new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact artifact) {
+ String filename = artifact.getExecPathString();
+
+ // For compatibility with existing BUILD files, any ".o" files listed
+ // in srcs are assumed to be position-independent code, or
+ // at least suitable for inclusion in shared libraries, unless they
+ // end with ".nopic.o". (The ".nopic.o" extension is an undocumented
+ // feature to give users at least some control over this.) Note that
+ // some target platforms do not require shared library code to be PIC.
+ return CppFileTypes.PIC_OBJECT_FILE.matches(filename) ||
+ (CppFileTypes.OBJECT_FILE.matches(filename) && !filename.endsWith(".nopic.o"));
+ }
+ });
+ } else {
+ return FileType.filter(sources, CppFileTypes.OBJECT_FILE);
+ }
+ }
+
+ /**
+ * Returns the files from headers and does some sanity checks. Note that this method reports
+ * warnings to the {@link RuleContext} as a side effect, and so should only be called once for any
+ * given rule.
+ */
+ public static List<Artifact> getHeaders(RuleContext ruleContext) {
+ List<Artifact> hdrs = new ArrayList<>();
+ for (TransitiveInfoCollection target :
+ ruleContext.getPrerequisitesIf("hdrs", Mode.TARGET, FileProvider.class)) {
+ FileProvider provider = target.getProvider(FileProvider.class);
+ for (Artifact artifact : provider.getFilesToBuild()) {
+ if (!CppRuleClasses.DISALLOWED_HDRS_FILES.matches(artifact.getFilename())) {
+ hdrs.add(artifact);
+ } else {
+ ruleContext.attributeWarning("hdrs", "file '" + artifact.getFilename()
+ + "' from target '" + target.getLabel() + "' is not allowed in hdrs");
+ }
+ }
+ }
+ return hdrs;
+ }
+
+ /**
+ * Uses {@link #getHeaders(RuleContext)} to get the {@code hdrs} on this target. This method will
+ * return an empty list if there is no {@code hdrs} attribute on this rule type.
+ */
+ List<Artifact> getHeaders() {
+ if (!hasAttribute("hdrs", Type.LABEL_LIST)) {
+ return ImmutableList.of();
+ }
+ return getHeaders(ruleContext);
+ }
+
+ HeadersCheckingMode determineHeadersCheckingMode() {
+ HeadersCheckingMode headersCheckingMode = cppConfiguration.getHeadersCheckingMode();
+
+ // Package default overrides command line option.
+ if (ruleContext.getRule().getPackage().isDefaultHdrsCheckSet()) {
+ String value =
+ ruleContext.getRule().getPackage().getDefaultHdrsCheck().toUpperCase(Locale.ENGLISH);
+ headersCheckingMode = HeadersCheckingMode.valueOf(value);
+ }
+
+ // 'hdrs_check' attribute overrides package default.
+ if (hasAttribute("hdrs_check", Type.STRING)
+ && ruleContext.getRule().isAttributeValueExplicitlySpecified("hdrs_check")) {
+ try {
+ String value = ruleContext.attributes().get("hdrs_check", Type.STRING)
+ .toUpperCase(Locale.ENGLISH);
+ headersCheckingMode = HeadersCheckingMode.valueOf(value);
+ } catch (IllegalArgumentException e) {
+ ruleContext.attributeError("hdrs_check", "must be one of: 'loose', 'warn' or 'strict'");
+ }
+ }
+
+ return headersCheckingMode;
+ }
+
+ /**
+ * Expand and tokenize the copts and nocopts attributes.
+ */
+ private ImmutableList<String> initCopts() {
+ if (!hasAttribute("copts", Type.STRING_LIST)) {
+ return ImmutableList.<String>of();
+ }
+ // TODO(bazel-team): getAttributeCopts should not tokenize the strings.
+ // Make a warning for now.
+ List<String> tokens = new ArrayList<>();
+ for (String str : ruleContext.attributes().get("copts", Type.STRING_LIST)) {
+ tokens.clear();
+ try {
+ ShellUtils.tokenize(tokens, str);
+ if (tokens.size() > 1) {
+ ruleContext.attributeWarning("copts",
+ "each item in the list should contain only one option");
+ }
+ } catch (ShellUtils.TokenizationException e) {
+ // ignore, the error is reported in the getAttributeCopts call
+ }
+ }
+
+ Pattern nocopts = getNoCopts(ruleContext);
+ if (nocopts != null && nocopts.matcher("-Wno-future-warnings").matches()) {
+ ruleContext.attributeWarning("nocopts",
+ "Regular expression '" + nocopts.pattern() + "' is too general; for example, it matches "
+ + "'-Wno-future-warnings'. Thus it might *re-enable* compiler warnings we wish to "
+ + "disable globally. To disable all compiler warnings, add '-w' to copts instead");
+ }
+
+ return ImmutableList.<String>builder()
+ .addAll(getPackageCopts(ruleContext))
+ .addAll(CppHelper.getAttributeCopts(ruleContext, "copts"))
+ .build();
+ }
+
+ private static ImmutableList<String> getPackageCopts(RuleContext ruleContext) {
+ List<String> unexpanded = ruleContext.getRule().getPackage().getDefaultCopts();
+ return ImmutableList.copyOf(CppHelper.expandMakeVariables(ruleContext, "copts", unexpanded));
+ }
+
+ Pattern getNoCopts() {
+ return getNoCopts(ruleContext);
+ }
+
+ /**
+ * Returns nocopts pattern built from the make variable expanded nocopts
+ * attribute.
+ */
+ private static Pattern getNoCopts(RuleContext ruleContext) {
+ Pattern nocopts = null;
+ if (ruleContext.getRule().isAttrDefined(NO_COPTS_ATTRIBUTE, Type.STRING)) {
+ String nocoptsAttr = ruleContext.expandMakeVariables(NO_COPTS_ATTRIBUTE,
+ ruleContext.attributes().get(NO_COPTS_ATTRIBUTE, Type.STRING));
+ try {
+ nocopts = Pattern.compile(nocoptsAttr);
+ } catch (PatternSyntaxException e) {
+ ruleContext.attributeError(NO_COPTS_ATTRIBUTE,
+ "invalid regular expression '" + nocoptsAttr + "': " + e.getMessage());
+ }
+ }
+ return nocopts;
+ }
+
+ // TODO(bazel-team): calculating nocopts every time is not very efficient,
+ // fix this after the rule migration. The problem is that in some cases we call this after
+ // the RCT is created (so RuleContext is not accessible), in some cases during the creation.
+ // It would probably make more sense to use TransitiveInfoProviders.
+ /**
+ * Returns true if the rule context has a nocopts regex that matches the given value, false
+ * otherwise.
+ */
+ static boolean noCoptsMatches(String option, RuleContext ruleContext) {
+ Pattern nocopts = getNoCopts(ruleContext);
+ return nocopts == null ? false : nocopts.matcher(option).matches();
+ }
+
+ private static final String DEFINES_ATTRIBUTE = "defines";
+
+ /**
+ * Returns a list of define tokens from "defines" attribute.
+ *
+ * <p>We tokenize the "defines" attribute, to ensure that the handling of
+ * quotes and backslash escapes is consistent Bazel's treatment of the "copts" attribute.
+ *
+ * <p>But we require that the "defines" attribute consists of a single token.
+ */
+ public List<String> getDefines() {
+ List<String> defines = new ArrayList<>();
+ for (String define :
+ ruleContext.attributes().get(DEFINES_ATTRIBUTE, Type.STRING_LIST)) {
+ List<String> tokens = new ArrayList<>();
+ try {
+ ShellUtils.tokenize(tokens, ruleContext.expandMakeVariables(DEFINES_ATTRIBUTE, define));
+ if (tokens.size() == 1) {
+ defines.add(tokens.get(0));
+ } else if (tokens.isEmpty()) {
+ ruleContext.attributeError(DEFINES_ATTRIBUTE, "empty definition not allowed");
+ } else {
+ ruleContext.attributeError(DEFINES_ATTRIBUTE,
+ "definition contains too many tokens (found " + tokens.size()
+ + ", expecting exactly one)");
+ }
+ } catch (ShellUtils.TokenizationException e) {
+ ruleContext.attributeError(DEFINES_ATTRIBUTE, e.getMessage());
+ }
+ }
+ return defines;
+ }
+
+ /**
+ * Collects our own linkopts from the rule attribute. This determines linker
+ * options to use when building this library and anything that depends on it.
+ */
+ private final ImmutableList<String> initLinkopts() {
+ if (!hasAttribute("linkopts", Type.STRING_LIST)) {
+ return ImmutableList.<String>of();
+ }
+ List<String> ourLinkopts = ruleContext.attributes().get("linkopts", Type.STRING_LIST);
+ List<String> result = new ArrayList<>();
+ if (ourLinkopts != null) {
+ boolean allowDashStatic = !cppConfiguration.forceIgnoreDashStatic()
+ && (cppConfiguration.getDynamicMode() != DynamicMode.FULLY);
+ for (String linkopt : ourLinkopts) {
+ if (linkopt.equals("-static") && !allowDashStatic) {
+ continue;
+ }
+ CppHelper.expandAttribute(ruleContext, result, "linkopts", linkopt, true);
+ }
+ }
+ return ImmutableList.copyOf(result);
+ }
+
+ /**
+ * Determines a list of loose include directories that are only allowed to be referenced when
+ * headers checking is {@link HeadersCheckingMode#LOOSE} or {@link HeadersCheckingMode#WARN}.
+ */
+ List<PathFragment> getLooseIncludeDirs() {
+ List<PathFragment> result = new ArrayList<>();
+ // The package directory of the rule contributes includes. Note that this also covers all
+ // non-subpackage sub-directories.
+ PathFragment rulePackage = ruleContext.getLabel().getPackageFragment();
+ result.add(rulePackage);
+
+ // Gather up all the dirs from the rule's srcs as well as any of the srcs outputs.
+ if (hasAttribute("srcs", Type.LABEL_LIST)) {
+ for (FileProvider src :
+ ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) {
+ PathFragment packageDir = src.getLabel().getPackageFragment();
+ for (Artifact a : src.getFilesToBuild()) {
+ result.add(packageDir);
+ // Attempt to gather subdirectories that might contain include files.
+ result.add(a.getRootRelativePath().getParentDirectory());
+ }
+ }
+ }
+
+ // Add in any 'includes' attribute values as relative path fragments
+ if (ruleContext.getRule().isAttributeValueExplicitlySpecified("includes")) {
+ PathFragment packageFragment = ruleContext.getLabel().getPackageFragment();
+ // For now, anything with an 'includes' needs a blanket declaration
+ result.add(packageFragment.getRelative("**"));
+ }
+ return result;
+ }
+
+ List<PathFragment> getSystemIncludeDirs() {
+ // Add in any 'includes' attribute values as relative path fragments
+ if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes")
+ || !cppConfiguration.useIsystemForIncludes()) {
+ return ImmutableList.of();
+ }
+ return getIncludeDirsFromIncludesAttribute();
+ }
+
+ List<PathFragment> getIncludeDirs() {
+ if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes")
+ || cppConfiguration.useIsystemForIncludes()) {
+ return ImmutableList.of();
+ }
+ return getIncludeDirsFromIncludesAttribute();
+ }
+
+ private List<PathFragment> getIncludeDirsFromIncludesAttribute() {
+ List<PathFragment> result = new ArrayList<>();
+ PathFragment packageFragment = ruleContext.getLabel().getPackageFragment();
+ for (String includesAttr : ruleContext.attributes().get("includes", Type.STRING_LIST)) {
+ includesAttr = ruleContext.expandMakeVariables("includes", includesAttr);
+ if (includesAttr.startsWith("/")) {
+ ruleContext.attributeWarning("includes",
+ "ignoring invalid absolute path '" + includesAttr + "'");
+ continue;
+ }
+ PathFragment includesPath = packageFragment.getRelative(includesAttr).normalize();
+ if (!includesPath.isNormalized()) {
+ ruleContext.attributeError("includes",
+ "Path references a path above the execution root.");
+ }
+ result.add(includesPath);
+ result.add(ruleContext.getConfiguration().getGenfilesFragment().getRelative(includesPath));
+ }
+ return result;
+ }
+
+ /**
+ * Collects compilation prerequisite artifacts.
+ */
+ static CompilationPrerequisitesProvider collectCompilationPrerequisites(
+ RuleContext ruleContext, CppCompilationContext context) {
+ // TODO(bazel-team): Use context.getCompilationPrerequisites() instead.
+ NestedSetBuilder<Artifact> prerequisites = NestedSetBuilder.stableOrder();
+ if (ruleContext.getRule().getRuleClassObject().hasAttr("srcs", Type.LABEL_LIST)) {
+ for (FileProvider provider : ruleContext
+ .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) {
+ prerequisites.addAll(FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES));
+ }
+ }
+ prerequisites.addTransitive(context.getDeclaredIncludeSrcs());
+ return new CompilationPrerequisitesProvider(prerequisites.build());
+ }
+
+ /**
+ * Replaces shared library artifact with mangled symlink and creates related
+ * symlink action. For artifacts that should retain filename (e.g. libraries
+ * with SONAME tag), link is created to the parent directory instead.
+ *
+ * This action is performed to minimize number of -rpath entries used during
+ * linking process (by essentially "collecting" as many shared libraries as
+ * possible in the single directory), since we will be paying quadratic price
+ * for each additional entry on the -rpath.
+ *
+ * @param library Shared library artifact that needs to be mangled
+ * @param preserveName true if filename should be preserved, false - mangled.
+ * @return mangled symlink artifact.
+ */
+ public LibraryToLink getDynamicLibrarySymlink(Artifact library, boolean preserveName) {
+ return SolibSymlinkAction.getDynamicLibrarySymlink(
+ ruleContext, library, preserveName, true, ruleContext.getConfiguration());
+ }
+
+ /**
+ * Returns any linker scripts found in the dependencies of the rule.
+ */
+ Iterable<Artifact> getLinkerScripts() {
+ return FileType.filter(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET).list(),
+ CppFileTypes.LINKER_SCRIPT);
+ }
+
+ ImmutableList<Artifact> getFilesToCompile(CcCompilationOutputs compilationOutputs) {
+ return cppConfiguration.isLipoContextCollector()
+ ? ImmutableList.<Artifact>of()
+ : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false));
+ }
+
+ InstrumentedFilesProvider getInstrumentedFilesProvider(Iterable<Artifact> files) {
+ return cppConfiguration.isLipoContextCollector()
+ ? InstrumentedFilesProviderImpl.EMPTY
+ : new InstrumentedFilesProviderImpl(new InstrumentedFilesCollector(
+ ruleContext, CppRuleClasses.INSTRUMENTATION_SPEC, CC_METADATA_COLLECTOR, files));
+ }
+
+ public static FeatureConfiguration configureFeatures(RuleContext ruleContext) {
+ CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext);
+ Set<String> requestedFeatures = ImmutableSet.of(CppRuleClasses.MODULE_MAP_HOME_CWD);
+ return toolchain.getFeatures().getFeatureConfiguration(requestedFeatures);
+ }
+
+ public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder,
+ NestedSet<Artifact> filesToBuild,
+ CcCompilationOutputs ccCompilationOutputs,
+ CppCompilationContext cppCompilationContext,
+ CcLinkingOutputs linkingOutputs,
+ DwoArtifactsCollector dwoArtifacts,
+ TransitiveLipoInfoProvider transitiveLipoInfo) {
+ List<Artifact> instrumentedObjectFiles = new ArrayList<>();
+ instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(false));
+ instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(true));
+ builder
+ .setFilesToBuild(filesToBuild)
+ .add(CppCompilationContext.class, cppCompilationContext)
+ .add(TransitiveLipoInfoProvider.class, transitiveLipoInfo)
+ .add(CcExecutionDynamicLibrariesProvider.class,
+ new CcExecutionDynamicLibrariesProvider(collectExecutionDynamicLibraryArtifacts(
+ ruleContext, linkingOutputs.getExecutionDynamicLibraries())))
+ .add(CcNativeLibraryProvider.class, new CcNativeLibraryProvider(
+ collectTransitiveCcNativeLibraries(ruleContext, linkingOutputs.getDynamicLibraries())))
+ .add(InstrumentedFilesProvider.class, getInstrumentedFilesProvider(
+ instrumentedObjectFiles))
+ .add(FilesToCompileProvider.class, new FilesToCompileProvider(
+ getFilesToCompile(ccCompilationOutputs)))
+ .add(CompilationPrerequisitesProvider.class,
+ collectCompilationPrerequisites(ruleContext, cppCompilationContext))
+ .add(TempsProvider.class, new TempsProvider(getTemps(ccCompilationOutputs)))
+ .add(CppDebugFileProvider.class, new CppDebugFileProvider(
+ dwoArtifacts.getDwoArtifacts(),
+ dwoArtifacts.getPicDwoArtifacts()));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java
new file mode 100644
index 0000000..b9fa4e8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java
@@ -0,0 +1,207 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A structured representation of the compilation outputs of a C++ rule.
+ */
+public class CcCompilationOutputs {
+ /**
+ * All .o files built by the target.
+ */
+ private final ImmutableList<Artifact> objectFiles;
+
+ /**
+ * All .pic.o files built by the target.
+ */
+ private final ImmutableList<Artifact> picObjectFiles;
+
+ /**
+ * All .dwo files built by the target, corresponding to .o outputs.
+ */
+ private final ImmutableList<Artifact> dwoFiles;
+
+ /**
+ * All .pic.dwo files built by the target, corresponding to .pic.o outputs.
+ */
+ private final ImmutableList<Artifact> picDwoFiles;
+
+ /**
+ * All artifacts that are created if "--save_temps" is true.
+ */
+ private final ImmutableList<Artifact> temps;
+
+ /**
+ * All token .h.processed files created when preprocessing or parsing headers.
+ */
+ private final ImmutableList<Artifact> headerTokenFiles;
+
+ private final List<IncludeScannable> lipoScannables;
+
+ private CcCompilationOutputs(ImmutableList<Artifact> objectFiles,
+ ImmutableList<Artifact> picObjectFiles, ImmutableList<Artifact> dwoFiles,
+ ImmutableList<Artifact> picDwoFiles, ImmutableList<Artifact> temps,
+ ImmutableList<Artifact> headerTokenFiles,
+ ImmutableList<IncludeScannable> lipoScannables) {
+ this.objectFiles = objectFiles;
+ this.picObjectFiles = picObjectFiles;
+ this.dwoFiles = dwoFiles;
+ this.picDwoFiles = picDwoFiles;
+ this.temps = temps;
+ this.headerTokenFiles = headerTokenFiles;
+ this.lipoScannables = lipoScannables;
+ }
+
+ /**
+ * Returns an unmodifiable view of the .o or .pic.o files set.
+ *
+ * @param usePic whether to return .pic.o files
+ */
+ public ImmutableList<Artifact> getObjectFiles(boolean usePic) {
+ return usePic ? picObjectFiles : objectFiles;
+ }
+
+ /**
+ * Returns an unmodifiable view of the .dwo files set.
+ */
+ public ImmutableList<Artifact> getDwoFiles() {
+ return dwoFiles;
+ }
+
+ /**
+ * Returns an unmodifiable view of the .pic.dwo files set.
+ */
+ public ImmutableList<Artifact> getPicDwoFiles() {
+ return picDwoFiles;
+ }
+
+ /**
+ * Returns an unmodifiable view of the temp files set.
+ */
+ public ImmutableList<Artifact> getTemps() {
+ return temps;
+ }
+
+ /**
+ * Returns an unmodifiable view of the .h.processed files.
+ */
+ public Iterable<Artifact> getHeaderTokenFiles() {
+ return headerTokenFiles;
+ }
+
+ /**
+ * Returns the {@link IncludeScannable} objects this C++ compile action contributes to a
+ * LIPO context collector.
+ */
+ public List<IncludeScannable> getLipoScannables() {
+ return lipoScannables;
+ }
+
+ public static final class Builder {
+ private final Set<Artifact> objectFiles = new LinkedHashSet<>();
+ private final Set<Artifact> picObjectFiles = new LinkedHashSet<>();
+ private final Set<Artifact> dwoFiles = new LinkedHashSet<>();
+ private final Set<Artifact> picDwoFiles = new LinkedHashSet<>();
+ private final Set<Artifact> temps = new LinkedHashSet<>();
+ private final Set<Artifact> headerTokenFiles = new LinkedHashSet<>();
+ private final List<IncludeScannable> lipoScannables = new ArrayList<>();
+
+ public CcCompilationOutputs build() {
+ return new CcCompilationOutputs(ImmutableList.copyOf(objectFiles),
+ ImmutableList.copyOf(picObjectFiles), ImmutableList.copyOf(dwoFiles),
+ ImmutableList.copyOf(picDwoFiles), ImmutableList.copyOf(temps),
+ ImmutableList.copyOf(headerTokenFiles),
+ ImmutableList.copyOf(lipoScannables));
+ }
+
+ public Builder merge(CcCompilationOutputs outputs) {
+ this.objectFiles.addAll(outputs.objectFiles);
+ this.picObjectFiles.addAll(outputs.picObjectFiles);
+ this.dwoFiles.addAll(outputs.dwoFiles);
+ this.picDwoFiles.addAll(outputs.picDwoFiles);
+ this.temps.addAll(outputs.temps);
+ this.headerTokenFiles.addAll(outputs.headerTokenFiles);
+ this.lipoScannables.addAll(outputs.lipoScannables);
+ return this;
+ }
+
+ /**
+ * Adds an .o file.
+ */
+ public Builder addObjectFile(Artifact artifact) {
+ objectFiles.add(artifact);
+ return this;
+ }
+
+ public Builder addObjectFiles(Iterable<Artifact> artifacts) {
+ Iterables.addAll(objectFiles, artifacts);
+ return this;
+ }
+
+ /**
+ * Adds a .pic.o file.
+ */
+ public Builder addPicObjectFile(Artifact artifact) {
+ picObjectFiles.add(artifact);
+ return this;
+ }
+
+ public Builder addPicObjectFiles(Iterable<Artifact> artifacts) {
+ Iterables.addAll(picObjectFiles, artifacts);
+ return this;
+ }
+
+ public Builder addDwoFile(Artifact artifact) {
+ dwoFiles.add(artifact);
+ return this;
+ }
+
+ public Builder addPicDwoFile(Artifact artifact) {
+ picDwoFiles.add(artifact);
+ return this;
+ }
+
+ /**
+ * Adds temp files.
+ */
+ public Builder addTemps(Iterable<Artifact> artifacts) {
+ Iterables.addAll(temps, artifacts);
+ return this;
+ }
+
+ public Builder addHeaderTokenFile(Artifact artifact) {
+ headerTokenFiles.add(artifact);
+ return this;
+ }
+
+ /**
+ * Adds an {@link IncludeScannable} that this compilation output object contributes to a
+ * LIPO context collector.
+ */
+ public Builder addLipoScannable(IncludeScannable scannable) {
+ lipoScannables.add(scannable);
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java
new file mode 100644
index 0000000..39ce942
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A target that provides the execution-time dynamic libraries of a C++ rule.
+ */
+@Immutable
+public final class CcExecutionDynamicLibrariesProvider implements TransitiveInfoProvider {
+ public static final CcExecutionDynamicLibrariesProvider EMPTY =
+ new CcExecutionDynamicLibrariesProvider(
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER));
+
+ private final NestedSet<Artifact> ccExecutionDynamicLibraries;
+
+ public CcExecutionDynamicLibrariesProvider(NestedSet<Artifact> ccExecutionDynamicLibraries) {
+ this.ccExecutionDynamicLibraries = ccExecutionDynamicLibraries;
+ }
+
+ /**
+ * Returns the execution-time dynamic libraries.
+ *
+ * <p>This normally returns the dynamic library created by the rule itself. However, if the rule
+ * does not create any dynamic libraries, then it returns the combined results of calling
+ * getExecutionDynamicLibraryArtifacts on all the rule's deps. This behaviour is so that this
+ * method is useful for a cc_library with deps but no srcs.
+ */
+ public NestedSet<Artifact> getExecutionDynamicLibraryArtifacts() {
+ return ccExecutionDynamicLibraries;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java
new file mode 100644
index 0000000..428eedb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java
@@ -0,0 +1,395 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AlwaysBuiltArtifactsProvider;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.rules.test.BaselineCoverageAction;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A ConfiguredTarget for <code>cc_library</code> rules.
+ */
+public abstract class CcLibrary implements RuleConfiguredTargetFactory {
+
+ private final CppSemantics semantics;
+
+ protected CcLibrary(CppSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ // These file extensions don't generate object files.
+ private static final FileTypeSet NO_OBJECT_GENERATING_FILETYPES = FileTypeSet.of(
+ CppFileTypes.CPP_HEADER, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE,
+ CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY,
+ CppFileTypes.SHARED_LIBRARY);
+
+ private static final Predicate<LibraryToLink> PIC_STATIC_FILTER = new Predicate<LibraryToLink>() {
+ @Override
+ public boolean apply(LibraryToLink input) {
+ String name = input.getArtifact().getExecPath().getBaseName();
+ return !name.endsWith(".nopic.a") && !name.endsWith(".nopic.lo");
+ }
+ };
+
+ private static Runfiles collectRunfiles(RuleContext context,
+ CcLinkingOutputs ccLinkingOutputs,
+ boolean neverLink, boolean addDynamicRuntimeInputArtifactsToRunfiles,
+ boolean linkingStatically) {
+ Runfiles.Builder builder = new Runfiles.Builder();
+
+ // neverlink= true creates a library that will never be linked into any binary that depends on
+ // it, but instead be loaded as an extension. So we need the dynamic library for this in the
+ // runfiles.
+ builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically && !neverLink));
+ builder.add(context, CppRunfilesProvider.runfilesFunction(linkingStatically));
+ if (context.getRule().isAttrDefined("implements", Type.LABEL_LIST)) {
+ builder.addTargets(context.getPrerequisites("implements", Mode.TARGET),
+ RunfilesProvider.DEFAULT_RUNFILES);
+ builder.addTargets(context.getPrerequisites("implements", Mode.TARGET),
+ CppRunfilesProvider.runfilesFunction(linkingStatically));
+ }
+ if (context.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) {
+ builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET),
+ RunfilesProvider.DEFAULT_RUNFILES);
+ builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET),
+ CppRunfilesProvider.runfilesFunction(linkingStatically));
+ }
+
+ builder.addDataDeps(context);
+
+ if (addDynamicRuntimeInputArtifactsToRunfiles) {
+ builder.addTransitiveArtifacts(CppHelper.getToolchain(context).getDynamicRuntimeLinkInputs());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext context) {
+ RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(context);
+ LinkTargetType linkType = getStaticLinkType(context);
+ boolean linkStatic = context.attributes().get("linkstatic", Type.BOOLEAN);
+ init(semantics, context, builder, linkType,
+ /*neverLink =*/ false,
+ linkStatic,
+ /*collectLinkstamp =*/ true,
+ /*addDynamicRuntimeInputArtifactsToRunfiles =*/ false);
+ return builder.build();
+ }
+
+ public static void init(CppSemantics semantics, RuleContext ruleContext,
+ RuleConfiguredTargetBuilder targetBuilder, LinkTargetType linkType,
+ boolean neverLink,
+ boolean linkStatic,
+ boolean collectLinkstamp,
+ boolean addDynamicRuntimeInputArtifactsToRunfiles) {
+ final CcCommon common = new CcCommon(ruleContext);
+
+ CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics)
+ .setLinkType(linkType)
+ .enableCcNativeLibrariesProvider()
+ .enableInterfaceSharedObjects()
+ .enableCompileProviders()
+ .setNeverLink(neverLink)
+ .setHeadersCheckingMode(common.determineHeadersCheckingMode())
+ .addCopts(common.getCopts())
+ .setNoCopts(common.getNoCopts())
+ .addLinkopts(common.getLinkopts())
+ .addDefines(common.getDefines())
+ .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs())
+ .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs())
+ .addSources(common.getCAndCppSources())
+ .addPublicHeaders(common.getHeaders())
+ .addObjectFiles(common.getObjectFilesFromSrcs(false))
+ .addPicObjectFiles(common.getObjectFilesFromSrcs(true))
+ .addPicIndependentObjectFiles(common.getLinkerScripts())
+ .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET))
+ .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK))
+ .setCompileHeaderModules(ruleContext.getFeatures().contains(CppRuleClasses.HEADER_MODULES))
+ .addSystemIncludeDirs(common.getSystemIncludeDirs())
+ .addIncludeDirs(common.getIncludeDirs())
+ .addLooseIncludeDirs(common.getLooseIncludeDirs())
+ .setEmitHeaderTargetModuleMaps(
+ ruleContext.getRule().getRuleClass().equals("cc_public_library"));
+
+ if (collectLinkstamp) {
+ helper.addLinkstamps(ruleContext.getPrerequisites("linkstamp", Mode.TARGET));
+ }
+
+ if (ruleContext.getRule().isAttrDefined("implements", Type.LABEL_LIST)) {
+ helper.addDeps(ruleContext.getPrerequisites("implements", Mode.TARGET));
+ }
+
+ if (ruleContext.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) {
+ helper.addDeps(ruleContext.getPrerequisites("implementation", Mode.TARGET));
+ }
+
+ PathFragment soImplFilename = null;
+ if (ruleContext.getRule().isAttrDefined("outs", Type.STRING_LIST)) {
+ List<String> outs = ruleContext.attributes().get("outs", Type.STRING_LIST);
+ if (outs.size() > 1) {
+ ruleContext.attributeError("outs", "must be a singleton list");
+ } else if (outs.size() == 1) {
+ soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY);
+ soImplFilename = soImplFilename.replaceName(outs.get(0));
+ if (!soImplFilename.getPathString().endsWith(".so")) { // Sanity check.
+ ruleContext.attributeError("outs", "file name must end in '.so'");
+ }
+ }
+ }
+
+ if (ruleContext.getRule().isAttrDefined("srcs", Type.LABEL_LIST)) {
+ helper.addPrivateHeaders(FileType.filter(
+ ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(),
+ CppFileTypes.CPP_HEADER));
+ ruleContext.checkSrcsSamePackage(true);
+ }
+
+ if (common.getLinkopts().contains("-static")) {
+ ruleContext.attributeWarning("linkopts", "Using '-static' here won't work. "
+ + "Did you mean to use 'linkstatic=1' instead?");
+ }
+
+ boolean createDynamicLibrary =
+ !linkStatic && !appearsToHaveNoObjectFiles(ruleContext.attributes());
+ helper.setCreateDynamicLibrary(createDynamicLibrary);
+ helper.setDynamicLibraryPath(soImplFilename);
+
+ /*
+ * Add the libraries from srcs, if any. For static/mostly static
+ * linking we setup the dynamic libraries if there are no static libraries
+ * to choose from. Path to the libraries will be mangled to avoid using
+ * absolute path names on the -rpath, but library filenames will be
+ * preserved (since some libraries might have SONAME tag) - symlink will
+ * be created to the parent directory instead.
+ *
+ * For compatibility with existing BUILD files, any ".a" or ".lo" files listed in
+ * srcs are assumed to be position-independent code, or at least suitable for
+ * inclusion in shared libraries, unless they end with ".nopic.a" or ".nopic.lo".
+ *
+ * Note that some target platforms do not require shared library code to be PIC.
+ */
+ Iterable<LibraryToLink> staticLibrariesFromSrcs =
+ LinkerInputs.opaqueLibrariesToLink(common.getStaticLibrariesFromSrcs());
+ helper.addStaticLibraries(staticLibrariesFromSrcs);
+ helper.addPicStaticLibraries(Iterables.filter(staticLibrariesFromSrcs, PIC_STATIC_FILTER));
+ helper.addPicStaticLibraries(common.getPicStaticLibrariesFromSrcs());
+ helper.addDynamicLibraries(Iterables.transform(common.getSharedLibrariesFromSrcs(),
+ new Function<Artifact, LibraryToLink>() {
+ @Override
+ public LibraryToLink apply(Artifact library) {
+ return common.getDynamicLibrarySymlink(library, true);
+ }
+ }));
+ CcLibraryHelper.Info info = helper.build();
+
+ /*
+ * We always generate a static library, even if there aren't any source files.
+ * This keeps things simpler by avoiding special cases when making use of the library.
+ * For example, this is needed to ensure that building a library with "bazel build"
+ * will also build all of the library's "deps".
+ * However, we only generate a dynamic library if there are source files.
+ */
+ // For now, we don't add the precompiled libraries to the files to build.
+ CcLinkingOutputs linkedLibraries = info.getCcLinkingOutputsExcludingPrecompiledLibraries();
+
+ NestedSet<Artifact> artifactsToForce =
+ collectArtifactsToForce(ruleContext, common, info.getCcCompilationOutputs());
+
+ NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder();
+ filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getStaticLibraries()));
+ filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getPicStaticLibraries()));
+ filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkedLibraries.getDynamicLibraries()));
+ filesBuilder.addAll(
+ LinkerInputs.toNonSolibArtifacts(linkedLibraries.getExecutionDynamicLibraries()));
+
+ CcLinkingOutputs linkingOutputs = info.getCcLinkingOutputs();
+ warnAboutEmptyLibraries(
+ ruleContext, info.getCcCompilationOutputs(), linkType, linkStatic);
+ NestedSet<Artifact> filesToBuild = filesBuilder.build();
+
+ Runfiles staticRunfiles = collectRunfiles(ruleContext,
+ linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, true);
+ Runfiles sharedRunfiles = collectRunfiles(ruleContext,
+ linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, false);
+
+ List<Artifact> instrumentedObjectFiles = new ArrayList<>();
+ instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(false));
+ instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(true));
+ InstrumentedFilesProvider instrumentedFilesProvider =
+ common.getInstrumentedFilesProvider(instrumentedObjectFiles);
+ targetBuilder
+ .setFilesToBuild(filesToBuild)
+ .addProviders(info.getProviders())
+ .add(InstrumentedFilesProvider.class, instrumentedFilesProvider)
+ .add(RunfilesProvider.class, RunfilesProvider.withData(staticRunfiles, sharedRunfiles))
+ // Remove this?
+ .add(CppRunfilesProvider.class, new CppRunfilesProvider(staticRunfiles, sharedRunfiles))
+ .setBaselineCoverageArtifacts(BaselineCoverageAction.getBaselineCoverageArtifacts(
+ ruleContext, instrumentedFilesProvider.getInstrumentedFiles()))
+ .add(ImplementedCcPublicLibrariesProvider.class,
+ new ImplementedCcPublicLibrariesProvider(getImplementedCcPublicLibraries(ruleContext)))
+ .add(AlwaysBuiltArtifactsProvider.class,
+ new AlwaysBuiltArtifactsProvider(artifactsToForce));
+ }
+
+ private static NestedSet<Artifact> collectArtifactsToForce(RuleContext ruleContext,
+ CcCommon common, CcCompilationOutputs ccCompilationOutputs) {
+ // Ensure that we build all the dependencies, otherwise users may get confused.
+ NestedSetBuilder<Artifact> artifactsToForceBuilder = NestedSetBuilder.stableOrder();
+ artifactsToForceBuilder.addTransitive(
+ NestedSetBuilder.wrap(Order.STABLE_ORDER, common.getFilesToCompile(ccCompilationOutputs)));
+ for (AlwaysBuiltArtifactsProvider dep :
+ ruleContext.getPrerequisites("deps", Mode.TARGET, AlwaysBuiltArtifactsProvider.class)) {
+ artifactsToForceBuilder.addTransitive(dep.getArtifactsToAlwaysBuild());
+ }
+ return artifactsToForceBuilder.build();
+ }
+
+ /**
+ * Returns the type of the generated static library.
+ */
+ private static LinkTargetType getStaticLinkType(RuleContext context) {
+ return context.attributes().get("alwayslink", Type.BOOLEAN)
+ ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY
+ : LinkTargetType.STATIC_LIBRARY;
+ }
+
+ private static void warnAboutEmptyLibraries(RuleContext ruleContext,
+ CcCompilationOutputs ccCompilationOutputs, LinkTargetType linkType,
+ boolean linkstaticAttribute) {
+ if (ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) {
+ // Do not signal warnings in the lipo context collector configuration. These will be duly
+ // signaled in the target configuration, and there can be spurious warnings since targets in
+ // the LIPO context collector configuration do not compile anything.
+ return;
+ }
+ if (ccCompilationOutputs.getObjectFiles(false).isEmpty()
+ && ccCompilationOutputs.getObjectFiles(true).isEmpty()) {
+ if (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY
+ || linkType == LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY) {
+ ruleContext.attributeWarning("alwayslink",
+ "'alwayslink' has no effect if there are no 'srcs'");
+ }
+ if (!linkstaticAttribute && !appearsToHaveNoObjectFiles(ruleContext.attributes())) {
+ ruleContext.attributeWarning("linkstatic",
+ "setting 'linkstatic=1' is recommended if there are no object files");
+ }
+ } else {
+ if (!linkstaticAttribute && appearsToHaveNoObjectFiles(ruleContext.attributes())) {
+ Artifact element = ccCompilationOutputs.getObjectFiles(false).isEmpty()
+ ? ccCompilationOutputs.getObjectFiles(true).get(0)
+ : ccCompilationOutputs.getObjectFiles(false).get(0);
+ ruleContext.attributeWarning("srcs",
+ "this library appears at first glance to have no object files, "
+ + "but on closer inspection it does have something to link, e.g. "
+ + element.prettyPrint() + ". "
+ + "(You may have used some very confusing rule names in srcs? "
+ + "Or the library consists entirely of a linker script?) "
+ + "Bazel assumed linkstatic=1, but this may be inappropriate. "
+ + "You may need to add an explicit '.cc' file to 'srcs'. "
+ + "Alternatively, add 'linkstatic=1' to suppress this warning");
+ }
+ }
+ }
+
+ private static ImmutableList<Label> getImplementedCcPublicLibraries(RuleContext context) {
+ if (context.getRule().getRuleClassObject().hasAttr("implements", Type.LABEL_LIST)) {
+ return ImmutableList.copyOf(context.attributes().get("implements", Type.LABEL_LIST));
+ } else {
+ return ImmutableList.of();
+ }
+ }
+
+ /**
+ * Returns true if the rule (which must be a cc_library rule)
+ * appears to have no object files. This only looks at the rule
+ * itself, not at any other rules (from this package or other
+ * packages) that it might reference.
+ *
+ * <p>
+ * In some cases, this may return "false" even
+ * though the rule actually has no object files.
+ * For example, it will return false for a rule such as
+ * <code>cc_library(name = 'foo', srcs = [':bar'])</code>
+ * because we can't tell what ':bar' is; it might
+ * be a genrule that generates a source file, or it might
+ * be a genrule that generates a header file.
+ *
+ * <p>
+ * In other cases, this may return "true" even
+ * though the rule actually does have object files.
+ * For example, it will return true for a rule such as
+ * <code>cc_library(name = 'foo', srcs = ['bar.h'])</code>
+ * but as in the other example above, we can't tell whether
+ * 'bar.h' is a file name or a rule name, and 'bar.h' could
+ * in fact be the name of a genrule that generates a source file.
+ */
+ public static boolean appearsToHaveNoObjectFiles(AttributeMap rule) {
+ // Temporary hack while configurable attributes is under development. This has no effect
+ // for any rule that doesn't use configurable attributes.
+ // TODO(bazel-team): remove this hack for a more principled solution.
+ try {
+ rule.get("srcs", Type.LABEL_LIST);
+ } catch (ClassCastException e) {
+ // "srcs" is actually a configurable selector. Assume object files are possible somewhere.
+ return false;
+ }
+
+ List<Label> srcs = rule.get("srcs", Type.LABEL_LIST);
+ if (srcs != null) {
+ for (Label srcfile : srcs) {
+ /*
+ * We cheat a little bit here by looking at the file extension
+ * of the Label treated as file name. In general that might
+ * not necessarily work, because of the possibility that the
+ * user might give a rule a funky name ending in one of these
+ * extensions, e.g.
+ * genrule(name = 'foo.h', outs = ['foo.cc'], ...) // Funky rule name!
+ * cc_library(name = 'bar', srcs = ['foo.h']) // This DOES have object files.
+ */
+ if (!NO_OBJECT_GENERATING_FILETYPES.matches(srcfile.getName())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java
new file mode 100644
index 0000000..3812bf5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java
@@ -0,0 +1,905 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToCompileProvider;
+import com.google.devtools.build.lib.analysis.LanguageDependentFragment;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TempsProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * A class to create C/C++ compile and link actions in a way that is consistent with cc_library.
+ * Rules that generate source files and emulate cc_library on top of that should use this class
+ * instead of the lower-level APIs in CppHelper and CppModel.
+ *
+ * <p>Rules that want to use this class are required to have implicit dependencies on the
+ * toolchain, the STL, the lipo context, and so on. Optionally, they can also have copts,
+ * and malloc attributes, but note that these require explicit calls to the corresponding setter
+ * methods.
+ */
+public final class CcLibraryHelper {
+ /** Function for extracting module maps from CppCompilationDependencies. */
+ public static final Function<TransitiveInfoCollection, CppModuleMap> CPP_DEPS_TO_MODULES =
+ new Function<TransitiveInfoCollection, CppModuleMap>() {
+ @Override
+ @Nullable
+ public CppModuleMap apply(TransitiveInfoCollection dep) {
+ CppCompilationContext context = dep.getProvider(CppCompilationContext.class);
+ return context == null ? null : context.getCppModuleMap();
+ }
+ };
+
+ /**
+ * Contains the providers as well as the compilation and linking outputs, and the compilation
+ * context.
+ */
+ public static final class Info {
+ private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers;
+ private final CcCompilationOutputs compilationOutputs;
+ private final CcLinkingOutputs linkingOutputs;
+ private final CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries;
+ private final CppCompilationContext context;
+
+ private Info(Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers,
+ CcCompilationOutputs compilationOutputs, CcLinkingOutputs linkingOutputs,
+ CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries,
+ CppCompilationContext context) {
+ this.providers = Collections.unmodifiableMap(providers);
+ this.compilationOutputs = compilationOutputs;
+ this.linkingOutputs = linkingOutputs;
+ this.linkingOutputsExcludingPrecompiledLibraries =
+ linkingOutputsExcludingPrecompiledLibraries;
+ this.context = context;
+ }
+
+ public Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> getProviders() {
+ return providers;
+ }
+
+ public CcCompilationOutputs getCcCompilationOutputs() {
+ return compilationOutputs;
+ }
+
+ public CcLinkingOutputs getCcLinkingOutputs() {
+ return linkingOutputs;
+ }
+
+ /**
+ * Returns the linking outputs before adding the pre-compiled libraries. Avoid using this -
+ * pre-compiled and locally compiled libraries should be treated identically. This method only
+ * exists for backwards compatibility.
+ */
+ public CcLinkingOutputs getCcLinkingOutputsExcludingPrecompiledLibraries() {
+ return linkingOutputsExcludingPrecompiledLibraries;
+ }
+
+ public CppCompilationContext getCppCompilationContext() {
+ return context;
+ }
+
+ /**
+ * Adds the static, pic-static, and dynamic (both compile-time and execution-time) libraries to
+ * the given builder.
+ */
+ public void addLinkingOutputsTo(NestedSetBuilder<Artifact> filesBuilder) {
+ filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getStaticLibraries()));
+ filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getPicStaticLibraries()));
+ filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkingOutputs.getDynamicLibraries()));
+ filesBuilder.addAll(
+ LinkerInputs.toNonSolibArtifacts(linkingOutputs.getExecutionDynamicLibraries()));
+ }
+ }
+
+ private final RuleContext ruleContext;
+ private final BuildConfiguration configuration;
+ private final CppSemantics semantics;
+
+ private final List<Artifact> publicHeaders = new ArrayList<>();
+ private final List<Artifact> privateHeaders = new ArrayList<>();
+ private final List<PathFragment> additionalExportedHeaders = new ArrayList<>();
+ private final List<Pair<Artifact, Label>> sources = new ArrayList<>();
+ private final List<Artifact> objectFiles = new ArrayList<>();
+ private final List<Artifact> picObjectFiles = new ArrayList<>();
+ private final List<String> copts = new ArrayList<>();
+ @Nullable private Pattern nocopts;
+ private final List<String> linkopts = new ArrayList<>();
+ private final Set<String> defines = new LinkedHashSet<>();
+ private final List<TransitiveInfoCollection> deps = new ArrayList<>();
+ private final List<Artifact> linkstamps = new ArrayList<>();
+ private final List<Artifact> prerequisites = new ArrayList<>();
+ private final List<PathFragment> looseIncludeDirs = new ArrayList<>();
+ private final List<PathFragment> systemIncludeDirs = new ArrayList<>();
+ private final List<PathFragment> includeDirs = new ArrayList<>();
+ @Nullable private PathFragment dynamicLibraryPath;
+ private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY;
+ private HeadersCheckingMode headersCheckingMode = HeadersCheckingMode.LOOSE;
+ private boolean neverlink;
+ private boolean fake;
+
+ private final List<LibraryToLink> staticLibraries = new ArrayList<>();
+ private final List<LibraryToLink> picStaticLibraries = new ArrayList<>();
+ private final List<LibraryToLink> dynamicLibraries = new ArrayList<>();
+
+ private boolean emitCppModuleMaps = true;
+ private boolean enableLayeringCheck;
+ private boolean compileHeaderModules;
+ private boolean emitCompileActionsIfEmpty = true;
+ private boolean emitCcNativeLibrariesProvider;
+ private boolean emitCcSpecificLinkParamsProvider;
+ private boolean emitInterfaceSharedObjects;
+ private boolean emitDynamicLibrary = true;
+ private boolean checkDepsGenerateCpp = true;
+ private boolean emitCompileProviders;
+ private boolean emitHeaderTargetModuleMaps = false;
+
+ public CcLibraryHelper(RuleContext ruleContext, CppSemantics semantics) {
+ this.ruleContext = Preconditions.checkNotNull(ruleContext);
+ this.configuration = ruleContext.getConfiguration();
+ this.semantics = Preconditions.checkNotNull(semantics);
+ }
+
+ /**
+ * Add the corresponding files as header files, i.e., these files will not be compiled, but are
+ * made visible as includes to dependent rules.
+ */
+ public CcLibraryHelper addPublicHeaders(Collection<Artifact> headers) {
+ this.publicHeaders.addAll(headers);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as public header files, i.e., these files will not be compiled, but
+ * are made visible as includes to dependent rules in module maps.
+ */
+ public CcLibraryHelper addPublicHeaders(Artifact... headers) {
+ return addPublicHeaders(Arrays.asList(headers));
+ }
+
+ /**
+ * Add the corresponding files as private header files, i.e., these files will not be compiled,
+ * but are not made visible as includes to dependent rules in module maps.
+ */
+ public CcLibraryHelper addPrivateHeaders(Iterable<Artifact> privateHeaders) {
+ Iterables.addAll(this.privateHeaders, privateHeaders);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as public header files, i.e., these files will not be compiled, but
+ * are made visible as includes to dependent rules in module maps.
+ */
+ public CcLibraryHelper addAdditionalExportedHeaders(
+ Iterable<PathFragment> additionalExportedHeaders) {
+ Iterables.addAll(this.additionalExportedHeaders, additionalExportedHeaders);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as source files. These may also be header files, in which case
+ * they will not be compiled, but also not made visible as includes to dependent rules.
+ */
+ // TODO(bazel-team): This is inconsistent with the documentation on CppModel.
+ public CcLibraryHelper addSources(Collection<Artifact> sources) {
+ for (Artifact source : sources) {
+ this.sources.add(Pair.of(source, ruleContext.getLabel()));
+ }
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as source files. These may also be header files, in which case
+ * they will not be compiled, but also not made visible as includes to dependent rules.
+ */
+ // TODO(bazel-team): This is inconsistent with the documentation on CppModel.
+ public CcLibraryHelper addSources(Iterable<Pair<Artifact, Label>> sources) {
+ Iterables.addAll(this.sources, sources);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as source files. These may also be header files, in which case
+ * they will not be compiled, but also not made visible as includes to dependent rules.
+ */
+ public CcLibraryHelper addSources(Artifact... sources) {
+ return addSources(Arrays.asList(sources));
+ }
+
+ /**
+ * Add the corresponding files as linker inputs for non-PIC links. If the corresponding files are
+ * compiled with PIC, the final link may or may not fail. Note that the final link may not happen
+ * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively
+ * depends on the current rule.
+ */
+ public CcLibraryHelper addObjectFiles(Iterable<Artifact> objectFiles) {
+ Iterables.addAll(this.objectFiles, objectFiles);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as linker inputs for PIC links. If the corresponding files are not
+ * compiled with PIC, the final link may or may not fail. Note that the final link may not happen
+ * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively
+ * depends on the current rule.
+ */
+ public CcLibraryHelper addPicObjectFiles(Iterable<Artifact> picObjectFiles) {
+ Iterables.addAll(this.picObjectFiles, picObjectFiles);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as linker inputs for both PIC and non-PIC links.
+ */
+ public CcLibraryHelper addPicIndependentObjectFiles(Iterable<Artifact> objectFiles) {
+ addPicObjectFiles(objectFiles);
+ return addObjectFiles(objectFiles);
+ }
+
+ /**
+ * Add the corresponding files as linker inputs for both PIC and non-PIC links.
+ */
+ public CcLibraryHelper addPicIndependentObjectFiles(Artifact... objectFiles) {
+ return addPicIndependentObjectFiles(Arrays.asList(objectFiles));
+ }
+
+ /**
+ * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker
+ * action) - this makes them available for linking to binary rules that depend on this rule.
+ */
+ public CcLibraryHelper addStaticLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(staticLibraries, libraries);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker
+ * action) - this makes them available for linking to binary rules that depend on this rule.
+ */
+ public CcLibraryHelper addPicStaticLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(picStaticLibraries, libraries);
+ return this;
+ }
+
+ /**
+ * Add the corresponding files as dynamic libraries into the linker outputs (i.e., after the
+ * linker action) - this makes them available for linking to binary rules that depend on this
+ * rule.
+ */
+ public CcLibraryHelper addDynamicLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(dynamicLibraries, libraries);
+ return this;
+ }
+
+ /**
+ * Adds the copts to the compile command line.
+ */
+ public CcLibraryHelper addCopts(Iterable<String> copts) {
+ Iterables.addAll(this.copts, copts);
+ return this;
+ }
+
+ /**
+ * Sets a pattern that is used to filter copts; set to {@code null} for no filtering.
+ */
+ public CcLibraryHelper setNoCopts(@Nullable Pattern nocopts) {
+ this.nocopts = nocopts;
+ return this;
+ }
+
+ /**
+ * Adds the given options as linker options to the link command.
+ */
+ public CcLibraryHelper addLinkopts(Iterable<String> linkopts) {
+ Iterables.addAll(this.linkopts, linkopts);
+ return this;
+ }
+
+ /**
+ * Adds the given defines to the compiler command line.
+ */
+ public CcLibraryHelper addDefines(Iterable<String> defines) {
+ Iterables.addAll(this.defines, defines);
+ return this;
+ }
+
+ /**
+ * Adds the given targets as dependencies - this can include explicit dependencies on other
+ * rules (like from a "deps" attribute) and also implicit dependencies on runtime libraries.
+ */
+ public CcLibraryHelper addDeps(Iterable<? extends TransitiveInfoCollection> deps) {
+ for (TransitiveInfoCollection dep : deps) {
+ Preconditions.checkArgument(dep.getConfiguration() == null
+ || dep.getConfiguration().equals(configuration));
+ this.deps.add(dep);
+ }
+ return this;
+ }
+
+ /**
+ * Adds the given linkstamps. Note that linkstamps are usually not compiled at the library level,
+ * but only in the dependent binary rules.
+ */
+ public CcLibraryHelper addLinkstamps(Iterable<? extends TransitiveInfoCollection> linkstamps) {
+ for (TransitiveInfoCollection linkstamp : linkstamps) {
+ Iterables.addAll(this.linkstamps,
+ linkstamp.getProvider(FileProvider.class).getFilesToBuild());
+ }
+ return this;
+ }
+
+ /**
+ * Adds the given prerequisites as prerequisites for the generated compile actions. This ensures
+ * that the corresponding files exist - otherwise the action fails. Note that these dependencies
+ * add edges to the action graph, and can therefore increase the length of the critical path,
+ * i.e., make the build slower.
+ */
+ public CcLibraryHelper addCompilationPrerequisites(Iterable<Artifact> prerequisites) {
+ Iterables.addAll(this.prerequisites, prerequisites);
+ return this;
+ }
+
+ /**
+ * Adds the given directories to the loose include directories that are only allowed to be
+ * referenced when headers checking is {@link HeadersCheckingMode#LOOSE} or {@link
+ * HeadersCheckingMode#WARN}.
+ */
+ public CcLibraryHelper addLooseIncludeDirs(Iterable<PathFragment> looseIncludeDirs) {
+ Iterables.addAll(this.looseIncludeDirs, looseIncludeDirs);
+ return this;
+ }
+
+ /**
+ * Adds the given directories to the system include directories (they are passed with {@code
+ * "-isystem"} to the compiler); these are also passed to dependent rules.
+ */
+ public CcLibraryHelper addSystemIncludeDirs(Iterable<PathFragment> systemIncludeDirs) {
+ Iterables.addAll(this.systemIncludeDirs, systemIncludeDirs);
+ return this;
+ }
+
+ /**
+ * Adds the given directories to the quote include directories (they are passed with {@code
+ * "-iquote"} to the compiler); these are also passed to dependent rules.
+ */
+ public CcLibraryHelper addIncludeDirs(Iterable<PathFragment> includeDirs) {
+ Iterables.addAll(this.includeDirs, includeDirs);
+ return this;
+ }
+
+ /**
+ * Overrides the path for the generated dynamic library - this should only be called if the
+ * dynamic library is an implicit or explicit output of the rule, i.e., if it is accessible by
+ * name from other rules in the same package. Set to {@code null} to use the default computation.
+ */
+ public CcLibraryHelper setDynamicLibraryPath(@Nullable PathFragment dynamicLibraryPath) {
+ this.dynamicLibraryPath = dynamicLibraryPath;
+ return this;
+ }
+
+ /**
+ * Marks the output of this rule as alwayslink, i.e., the corresponding symbols will be retained
+ * by the linker even if they are not otherwise used. This is useful for libraries that register
+ * themselves somewhere during initialization.
+ *
+ * <p>This only sets the link type (see {@link #setLinkType}), either to a static library or to
+ * an alwayslink static library (blaze uses a different file extension to signal alwayslink to
+ * downstream code).
+ */
+ public CcLibraryHelper setAlwayslink(boolean alwayslink) {
+ linkType = alwayslink
+ ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY
+ : LinkTargetType.STATIC_LIBRARY;
+ return this;
+ }
+
+ /**
+ * Directly set the link type. This can be used instead of {@link #setAlwayslink}. Setting
+ * anything other than a static link causes this class to skip the link action creation.
+ */
+ public CcLibraryHelper setLinkType(LinkTargetType linkType) {
+ this.linkType = Preconditions.checkNotNull(linkType);
+ return this;
+ }
+
+ /**
+ * Marks the resulting code as neverlink, i.e., the code will not be linked into dependent
+ * libraries or binaries - the header files are still available.
+ */
+ public CcLibraryHelper setNeverLink(boolean neverlink) {
+ this.neverlink = neverlink;
+ return this;
+ }
+
+ /**
+ * Sets the given headers checking mode. The default is {@link HeadersCheckingMode#LOOSE}.
+ */
+ public CcLibraryHelper setHeadersCheckingMode(HeadersCheckingMode headersCheckingMode) {
+ this.headersCheckingMode = Preconditions.checkNotNull(headersCheckingMode);
+ return this;
+ }
+
+ /**
+ * Marks the resulting code as fake, i.e., the code will not actually be compiled or linked, but
+ * instead, the compile command is written to a file and added to the runfiles. This is currently
+ * used for non-compilation tests. Unfortunately, the design is problematic, so please don't add
+ * any further uses.
+ */
+ public CcLibraryHelper setFake(boolean fake) {
+ this.fake = fake;
+ return this;
+ }
+
+ /**
+ * This adds the {@link CcNativeLibraryProvider} to the providers created by this class.
+ */
+ public CcLibraryHelper enableCcNativeLibrariesProvider() {
+ this.emitCcNativeLibrariesProvider = true;
+ return this;
+ }
+
+ /**
+ * This adds the {@link CcSpecificLinkParamsProvider} to the providers created by this class.
+ * Otherwise the result will contain an instance of {@link CcLinkParamsProvider}.
+ */
+ public CcLibraryHelper enableCcSpecificLinkParamsProvider() {
+ this.emitCcSpecificLinkParamsProvider = true;
+ return this;
+ }
+
+ /**
+ * This disables C++ module map generation for the current rule. Don't call this unless you know
+ * what you are doing.
+ */
+ public CcLibraryHelper disableCppModuleMapGeneration() {
+ this.emitCppModuleMaps = false;
+ return this;
+ }
+
+ /**
+ * This enables or disables use of module maps during compilation, i.e., layering checks.
+ */
+ public CcLibraryHelper setEnableLayeringCheck(boolean enableLayeringCheck) {
+ this.enableLayeringCheck = enableLayeringCheck;
+ return this;
+ }
+
+ /**
+ * This enabled or disables compilation of C++ header modules.
+ * TODO(bazel-team): Add a cc_toolchain flag that allows fully disabling this feature and document
+ * this feature.
+ * See http://clang.llvm.org/docs/Modules.html.
+ */
+ public CcLibraryHelper setCompileHeaderModules(boolean compileHeaderModules) {
+ this.compileHeaderModules = compileHeaderModules;
+ return this;
+ }
+
+ /**
+ * Enables or disables generation of compile actions if there are no sources. Some rules declare a
+ * .a or .so implicit output, which requires that these files are created even if there are no
+ * source files, so be careful when calling this.
+ */
+ public CcLibraryHelper setGenerateCompileActionsIfEmpty(boolean emitCompileActionsIfEmpty) {
+ this.emitCompileActionsIfEmpty = emitCompileActionsIfEmpty;
+ return this;
+ }
+
+ /**
+ * Enables the optional generation of interface dynamic libraries - this is only used when the
+ * linker generates a dynamic library, and only if the crosstool supports it. The default is not
+ * to generate interface dynamic libraries.
+ */
+ public CcLibraryHelper enableInterfaceSharedObjects() {
+ this.emitInterfaceSharedObjects = true;
+ return this;
+ }
+
+ /**
+ * This enables or disables the generation of a dynamic library link action. The default is to
+ * generate a dynamic library. Note that the selection between dynamic or static linking is
+ * performed at the binary rule level.
+ */
+ public CcLibraryHelper setCreateDynamicLibrary(boolean emitDynamicLibrary) {
+ this.emitDynamicLibrary = emitDynamicLibrary;
+ return this;
+ }
+
+ /**
+ * Disables checking that the deps actually are C++ rules. By default, the {@link #build} method
+ * uses {@link LanguageDependentFragment.Checker#depSupportsLanguage} to check that all deps
+ * provide C++ providers.
+ */
+ public CcLibraryHelper setCheckDepsGenerateCpp(boolean checkDepsGenerateCpp) {
+ this.checkDepsGenerateCpp = checkDepsGenerateCpp;
+ return this;
+ }
+
+ /**
+ * Enables the output of {@link FilesToCompileProvider} and {@link
+ * CompilationPrerequisitesProvider}.
+ */
+ // TODO(bazel-team): We probably need to adjust this for the multi-language rules.
+ public CcLibraryHelper enableCompileProviders() {
+ this.emitCompileProviders = true;
+ return this;
+ }
+
+ /**
+ * Sets whether to emit the transitive module map references of a public library headers target.
+ */
+ public CcLibraryHelper setEmitHeaderTargetModuleMaps(boolean emitHeaderTargetModuleMaps) {
+ this.emitHeaderTargetModuleMaps = emitHeaderTargetModuleMaps;
+ return this;
+ }
+
+ /**
+ * Create the C++ compile and link actions, and the corresponding C++-related providers.
+ */
+ public Info build() {
+ // Fail early if there is no lipo context collector on the rule - otherwise we end up failing
+ // in lipo optimization.
+ Preconditions.checkState(
+ // 'cc_inc_library' rules do not compile, and thus are not affected by LIPO.
+ ruleContext.getRule().getRuleClass().equals("cc_inc_library")
+ || ruleContext.getRule().isAttrDefined(":lipo_context_collector", Type.LABEL));
+
+ if (checkDepsGenerateCpp) {
+ for (LanguageDependentFragment dep :
+ AnalysisUtils.getProviders(deps, LanguageDependentFragment.class)) {
+ LanguageDependentFragment.Checker.depSupportsLanguage(
+ ruleContext, dep, CppRuleClasses.LANGUAGE);
+ }
+ }
+
+ CcLinkingOutputs ccLinkingOutputs = CcLinkingOutputs.EMPTY;
+ CcCompilationOutputs ccOutputs = new CcCompilationOutputs.Builder().build();
+ FeatureConfiguration featureConfiguration = CcCommon.configureFeatures(ruleContext);
+
+ CppModel model = new CppModel(ruleContext, semantics)
+ .addSources(sources)
+ .addCopts(copts)
+ .setLinkTargetType(linkType)
+ .setNeverLink(neverlink)
+ .setFake(fake)
+ .setAllowInterfaceSharedObjects(emitInterfaceSharedObjects)
+ .setCreateDynamicLibrary(emitDynamicLibrary)
+ // Note: this doesn't actually save the temps, it just makes the CppModel use the
+ // configurations --save_temps setting to decide whether to actually save the temps.
+ .setSaveTemps(true)
+ .setEnableLayeringCheck(enableLayeringCheck)
+ .setCompileHeaderModules(compileHeaderModules)
+ .setNoCopts(nocopts)
+ .setDynamicLibraryPath(dynamicLibraryPath)
+ .addLinkopts(linkopts)
+ .setFeatureConfiguration(featureConfiguration);
+ CppCompilationContext cppCompilationContext =
+ initializeCppCompilationContext(model, featureConfiguration);
+ model.setContext(cppCompilationContext);
+ if (emitCompileActionsIfEmpty || !sources.isEmpty() || compileHeaderModules) {
+ Preconditions.checkState(
+ !compileHeaderModules || cppCompilationContext.getCppModuleMap() != null,
+ "All cc rules must support module maps.");
+ ccOutputs = model.createCcCompileActions();
+ if (!objectFiles.isEmpty() || !picObjectFiles.isEmpty()) {
+ // Merge the pre-compiled object files into the compiler outputs.
+ ccOutputs = new CcCompilationOutputs.Builder()
+ .merge(ccOutputs)
+ .addObjectFiles(objectFiles)
+ .addPicObjectFiles(picObjectFiles)
+ .build();
+ }
+ if (linkType.isStaticLibraryLink()) {
+ // TODO(bazel-team): This can't create the link action for a cc_binary yet.
+ ccLinkingOutputs = model.createCcLinkActions(ccOutputs);
+ }
+ }
+ CcLinkingOutputs originalLinkingOutputs = ccLinkingOutputs;
+ if (!(
+ staticLibraries.isEmpty() && picStaticLibraries.isEmpty() && dynamicLibraries.isEmpty())) {
+ // Merge the pre-compiled libraries (static & dynamic) into the linker outputs.
+ ccLinkingOutputs = new CcLinkingOutputs.Builder()
+ .merge(ccLinkingOutputs)
+ .addStaticLibraries(staticLibraries)
+ .addPicStaticLibraries(picStaticLibraries)
+ .addDynamicLibraries(dynamicLibraries)
+ .addExecutionDynamicLibraries(dynamicLibraries)
+ .build();
+ }
+
+ DwoArtifactsCollector dwoArtifacts = DwoArtifactsCollector.transitiveCollector(ccOutputs, deps);
+ Runfiles cppStaticRunfiles = collectCppRunfiles(ccLinkingOutputs, true);
+ Runfiles cppSharedRunfiles = collectCppRunfiles(ccLinkingOutputs, false);
+
+ // By very careful when adding new providers here - it can potentially affect a lot of rules.
+ // We should consider merging most of these providers into a single provider.
+ Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers =
+ new LinkedHashMap<>();
+ providers.put(CppRunfilesProvider.class,
+ new CppRunfilesProvider(cppStaticRunfiles, cppSharedRunfiles));
+ providers.put(CppCompilationContext.class, cppCompilationContext);
+ providers.put(CppDebugFileProvider.class, new CppDebugFileProvider(
+ dwoArtifacts.getDwoArtifacts(), dwoArtifacts.getPicDwoArtifacts()));
+ providers.put(TransitiveLipoInfoProvider.class, collectTransitiveLipoInfo(ccOutputs));
+ providers.put(TempsProvider.class, getTemps(ccOutputs));
+ if (emitCompileProviders) {
+ providers.put(FilesToCompileProvider.class, new FilesToCompileProvider(
+ getFilesToCompile(ccOutputs)));
+ providers.put(CompilationPrerequisitesProvider.class,
+ CcCommon.collectCompilationPrerequisites(ruleContext, cppCompilationContext));
+ }
+
+ // TODO(bazel-team): Maybe we can infer these from other data at the places where they are
+ // used.
+ if (emitCcNativeLibrariesProvider) {
+ providers.put(CcNativeLibraryProvider.class,
+ new CcNativeLibraryProvider(collectNativeCcLibraries(ccLinkingOutputs)));
+ }
+ providers.put(CcExecutionDynamicLibrariesProvider.class,
+ collectExecutionDynamicLibraryArtifacts(ccLinkingOutputs.getExecutionDynamicLibraries()));
+
+ boolean forcePic = ruleContext.getFragment(CppConfiguration.class).forcePic();
+ if (emitCcSpecificLinkParamsProvider) {
+ providers.put(CcSpecificLinkParamsProvider.class, new CcSpecificLinkParamsProvider(
+ createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic)));
+ } else {
+ providers.put(CcLinkParamsProvider.class, new CcLinkParamsProvider(
+ createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic)));
+ }
+ return new Info(providers, ccOutputs, ccLinkingOutputs, originalLinkingOutputs,
+ cppCompilationContext);
+ }
+
+ /**
+ * Create context for cc compile action from generated inputs.
+ */
+ private CppCompilationContext initializeCppCompilationContext(CppModel model,
+ FeatureConfiguration featureConfiguration) {
+ CppCompilationContext.Builder contextBuilder =
+ new CppCompilationContext.Builder(ruleContext);
+
+ // Setup the include path; local include directories come before those inherited from deps or
+ // from the toolchain; in case of aliasing (same include file found on different entries),
+ // prefer the local include rather than the inherited one.
+
+ // Add in the roots for well-formed include names for source files and
+ // generated files. It is important that the execRoot (EMPTY_FRAGMENT) comes
+ // before the genfilesFragment to preferably pick up source files. Otherwise
+ // we might pick up stale generated files.
+ contextBuilder.addQuoteIncludeDir(PathFragment.EMPTY_FRAGMENT);
+ contextBuilder.addQuoteIncludeDir(ruleContext.getConfiguration().getGenfilesFragment());
+
+ for (PathFragment systemIncludeDir : systemIncludeDirs) {
+ contextBuilder.addSystemIncludeDir(systemIncludeDir);
+ }
+ for (PathFragment includeDir : includeDirs) {
+ contextBuilder.addIncludeDir(includeDir);
+ }
+
+ contextBuilder.mergeDependentContexts(
+ AnalysisUtils.getProviders(deps, CppCompilationContext.class));
+ CppHelper.mergeToolchainDependentContext(ruleContext, contextBuilder);
+
+ // But defines come after those inherited from deps.
+ contextBuilder.addDefines(defines);
+
+ // There are no ordering constraints for declared include dirs/srcs, or the pregrepped headers.
+ contextBuilder.addDeclaredIncludeSrcs(publicHeaders);
+ contextBuilder.addDeclaredIncludeSrcs(privateHeaders);
+ contextBuilder.addPregreppedHeaderMap(
+ CppHelper.createExtractInclusions(ruleContext, publicHeaders));
+ contextBuilder.addPregreppedHeaderMap(
+ CppHelper.createExtractInclusions(ruleContext, privateHeaders));
+ contextBuilder.addCompilationPrerequisites(prerequisites);
+
+ // Add this package's dir to declaredIncludeDirs, & this rule's headers to declaredIncludeSrcs
+ // Note: no include dir for STRICT mode.
+ if (headersCheckingMode == HeadersCheckingMode.WARN) {
+ contextBuilder.addDeclaredIncludeWarnDir(ruleContext.getLabel().getPackageFragment());
+ for (PathFragment looseIncludeDir : looseIncludeDirs) {
+ contextBuilder.addDeclaredIncludeWarnDir(looseIncludeDir);
+ }
+ } else if (headersCheckingMode == HeadersCheckingMode.LOOSE) {
+ contextBuilder.addDeclaredIncludeDir(ruleContext.getLabel().getPackageFragment());
+ for (PathFragment looseIncludeDir : looseIncludeDirs) {
+ contextBuilder.addDeclaredIncludeDir(looseIncludeDir);
+ }
+ }
+
+ if (emitCppModuleMaps) {
+ CppModuleMap cppModuleMap = CppHelper.addCppModuleMapToContext(ruleContext, contextBuilder);
+ // TODO(bazel-team): addCppModuleMapToContext second-guesses whether module maps should
+ // actually be enabled, so we need to double-check here. Who would write code like this?
+ if (cppModuleMap != null) {
+ CppModuleMapAction action = new CppModuleMapAction(ruleContext.getActionOwner(),
+ cppModuleMap,
+ privateHeaders,
+ publicHeaders,
+ collectModuleMaps(),
+ additionalExportedHeaders,
+ compileHeaderModules,
+ featureConfiguration.isEnabled(CppRuleClasses.MODULE_MAP_HOME_CWD));
+ ruleContext.registerAction(action);
+ }
+ if (model.getGeneratesPicHeaderModule()) {
+ contextBuilder.setPicHeaderModule(model.getPicHeaderModule(cppModuleMap.getArtifact()));
+ }
+ if (model.getGeratesNoPicHeaderModule()) {
+ contextBuilder.setHeaderModule(model.getHeaderModule(cppModuleMap.getArtifact()));
+ }
+ }
+
+ semantics.setupCompilationContext(ruleContext, contextBuilder);
+ return contextBuilder.build();
+ }
+
+ private Iterable<CppModuleMap> collectModuleMaps() {
+ // Cpp module maps may be null for some rules. We filter the nulls out at the end.
+ List<CppModuleMap> result = new ArrayList<>();
+ Iterables.addAll(result, Iterables.transform(deps, CPP_DEPS_TO_MODULES));
+ CppCompilationContext stl =
+ ruleContext.getPrerequisite(":stl", Mode.TARGET, CppCompilationContext.class);
+ if (stl != null) {
+ result.add(stl.getCppModuleMap());
+ }
+
+ CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext);
+ if (toolchain != null) {
+ result.add(toolchain.getCppCompilationContext().getCppModuleMap());
+ }
+
+ if (emitHeaderTargetModuleMaps) {
+ for (HeaderTargetModuleMapProvider provider : AnalysisUtils.getProviders(
+ deps, HeaderTargetModuleMapProvider.class)) {
+ result.addAll(provider.getCppModuleMaps());
+ }
+ }
+
+ return Iterables.filter(result, Predicates.<CppModuleMap>notNull());
+ }
+
+ private TransitiveLipoInfoProvider collectTransitiveLipoInfo(CcCompilationOutputs outputs) {
+ if (ruleContext.getFragment(CppConfiguration.class).getFdoSupport().getFdoRoot() == null) {
+ return TransitiveLipoInfoProvider.EMPTY;
+ }
+ NestedSetBuilder<IncludeScannable> scannableBuilder = NestedSetBuilder.stableOrder();
+ // TODO(bazel-team): Only fetch the STL prerequisite in one place.
+ TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET);
+ if (stl != null) {
+ TransitiveLipoInfoProvider provider = stl.getProvider(TransitiveLipoInfoProvider.class);
+ if (provider != null) {
+ scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables());
+ }
+ }
+
+ for (TransitiveLipoInfoProvider dep :
+ AnalysisUtils.getProviders(deps, TransitiveLipoInfoProvider.class)) {
+ scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables());
+ }
+
+ for (IncludeScannable scannable : outputs.getLipoScannables()) {
+ Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1);
+ scannableBuilder.add(scannable);
+ }
+ return new TransitiveLipoInfoProvider(scannableBuilder.build());
+ }
+
+ private Runfiles collectCppRunfiles(
+ CcLinkingOutputs ccLinkingOutputs, boolean linkingStatically) {
+ Runfiles.Builder builder = new Runfiles.Builder();
+ builder.addTargets(deps, RunfilesProvider.DEFAULT_RUNFILES);
+ builder.addTargets(deps, CppRunfilesProvider.runfilesFunction(linkingStatically));
+ // Add the shared libraries to the runfiles.
+ builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically));
+ return builder.build();
+ }
+
+ private CcLinkParamsStore createCcLinkParamsStore(
+ final CcLinkingOutputs ccLinkingOutputs, final CppCompilationContext cppCompilationContext,
+ final boolean forcePic) {
+ return new CcLinkParamsStore() {
+ @Override
+ protected void collect(CcLinkParams.Builder builder, boolean linkingStatically,
+ boolean linkShared) {
+ builder.addLinkstamps(linkstamps, cppCompilationContext);
+ builder.addTransitiveTargets(deps,
+ CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS);
+ if (!neverlink) {
+ builder.addLibraries(ccLinkingOutputs.getPreferredLibraries(linkingStatically,
+ /*preferPic=*/linkShared || forcePic));
+ builder.addLinkOpts(linkopts);
+ }
+ }
+ };
+ }
+
+ private NestedSet<LinkerInput> collectNativeCcLibraries(CcLinkingOutputs ccLinkingOutputs) {
+ NestedSetBuilder<LinkerInput> result = NestedSetBuilder.linkOrder();
+ result.addAll(ccLinkingOutputs.getDynamicLibraries());
+ for (CcNativeLibraryProvider dep : AnalysisUtils.getProviders(
+ deps, CcNativeLibraryProvider.class)) {
+ result.addTransitive(dep.getTransitiveCcNativeLibraries());
+ }
+
+ return result.build();
+ }
+
+ private CcExecutionDynamicLibrariesProvider collectExecutionDynamicLibraryArtifacts(
+ List<LibraryToLink> executionDynamicLibraries) {
+ Iterable<Artifact> artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries);
+ if (!Iterables.isEmpty(artifacts)) {
+ return new CcExecutionDynamicLibrariesProvider(
+ NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts));
+ }
+
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (CcExecutionDynamicLibrariesProvider dep :
+ AnalysisUtils.getProviders(deps, CcExecutionDynamicLibrariesProvider.class)) {
+ builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts());
+ }
+ return builder.isEmpty()
+ ? CcExecutionDynamicLibrariesProvider.EMPTY
+ : new CcExecutionDynamicLibrariesProvider(builder.build());
+ }
+
+ private TempsProvider getTemps(CcCompilationOutputs compilationOutputs) {
+ return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()
+ ? new TempsProvider(ImmutableList.<Artifact>of())
+ : new TempsProvider(compilationOutputs.getTemps());
+ }
+
+ private ImmutableList<Artifact> getFilesToCompile(CcCompilationOutputs compilationOutputs) {
+ return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()
+ ? ImmutableList.<Artifact>of()
+ : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java
new file mode 100644
index 0000000..4e4804b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java
@@ -0,0 +1,357 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+
+import java.util.Collection;
+import java.util.Objects;
+
+/**
+ * Parameters to be passed to the linker.
+ *
+ * <p>The parameters concerned are the link options (strings) passed to the linker, linkstamps and a
+ * list of libraries to be linked in.
+ *
+ * <p>Items in the collections are stored in nested sets. Link options and libraries are stored in
+ * link order (preorder) and linkstamps are sorted.
+ */
+public final class CcLinkParams {
+ private final NestedSet<ImmutableList<String>> linkOpts;
+ private final NestedSet<Linkstamp> linkstamps;
+ private final NestedSet<LibraryToLink> libraries;
+
+ private CcLinkParams(NestedSet<ImmutableList<String>> linkOpts,
+ NestedSet<Linkstamp> linkstamps,
+ NestedSet<LibraryToLink> libraries) {
+ this.linkOpts = linkOpts;
+ this.linkstamps = linkstamps;
+ this.libraries = libraries;
+ }
+
+ /**
+ * @return the linkopts
+ */
+ public NestedSet<ImmutableList<String>> getLinkopts() {
+ return linkOpts;
+ }
+
+ public ImmutableList<String> flattenedLinkopts() {
+ return ImmutableList.copyOf(Iterables.concat(linkOpts));
+ }
+
+ /**
+ * @return the linkstamps
+ */
+ public NestedSet<Linkstamp> getLinkstamps() {
+ return linkstamps;
+ }
+
+ /**
+ * @return the libraries
+ */
+ public NestedSet<LibraryToLink> getLibraries() {
+ return libraries;
+ }
+
+ public static final Builder builder(boolean linkingStatically, boolean linkShared) {
+ return new Builder(linkingStatically, linkShared);
+ }
+
+ /**
+ * Builder for {@link CcLinkParams}.
+ *
+ *
+ */
+ public static final class Builder {
+
+ /**
+ * linkingStatically is true when we're linking this target in either FULLY STATIC mode
+ * (linkopts=["-static"]) or MOSTLY STATIC mode (linkstatic=1). When this is true, we want to
+ * use static versions of any libraries that this target depends on (except possibly system
+ * libraries, which are not handled by CcLinkParams). When this is false, we want to use dynamic
+ * versions of any libraries that this target depends on.
+ */
+ private final boolean linkingStatically;
+
+ /**
+ * linkShared is true when we're linking with "-shared" (linkshared=1).
+ */
+ private final boolean linkShared;
+
+ private ImmutableList.Builder<String> localLinkoptsBuilder = ImmutableList.builder();
+
+ private final NestedSetBuilder<ImmutableList<String>> linkOptsBuilder =
+ NestedSetBuilder.linkOrder();
+ private final NestedSetBuilder<Linkstamp> linkstampsBuilder =
+ NestedSetBuilder.compileOrder();
+ private final NestedSetBuilder<LibraryToLink> librariesBuilder =
+ NestedSetBuilder.linkOrder();
+
+ private boolean built = false;
+
+ private Builder(boolean linkingStatically, boolean linkShared) {
+ this.linkingStatically = linkingStatically;
+ this.linkShared = linkShared;
+ }
+
+ /**
+ * Build a {@link CcLinkParams} object.
+ */
+ public CcLinkParams build() {
+ Preconditions.checkState(!built);
+ // Not thread-safe, but builders should not be shared across threads.
+ built = true;
+ ImmutableList<String> localLinkopts = localLinkoptsBuilder.build();
+ if (!localLinkopts.isEmpty()) {
+ linkOptsBuilder.add(localLinkopts);
+ }
+ return new CcLinkParams(linkOptsBuilder.build(), linkstampsBuilder.build(),
+ librariesBuilder.build());
+ }
+
+ private boolean add(CcLinkParamsStore store) {
+ if (store != null) {
+ CcLinkParams args = store.get(linkingStatically, linkShared);
+ addTransitiveArgs(args);
+ }
+ return store != null;
+ }
+
+ /**
+ * Includes link parameters from a collection of dependency targets.
+ */
+ public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> targets) {
+ for (TransitiveInfoCollection target : targets) {
+ addTransitiveTarget(target);
+ }
+ return this;
+ }
+
+ /**
+ * Includes link parameters from a dependency target.
+ *
+ * <p>The target should implement {@link CcLinkParamsProvider}. If it does not,
+ * the method does not do anything.
+ */
+ public Builder addTransitiveTarget(TransitiveInfoCollection target) {
+ return addTransitiveProvider(target.getProvider(CcLinkParamsProvider.class));
+ }
+
+ /**
+ * Includes link parameters from a dependency target. The target is checked for the given
+ * mappings in the order specified, and the first mapping that returns a non-null result is
+ * added.
+ */
+ @SafeVarargs
+ public final Builder addTransitiveTarget(TransitiveInfoCollection target,
+ Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping,
+ @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments.
+ Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) {
+ if (add(firstMapping.apply(target))) {
+ return this;
+ }
+ for (Function<TransitiveInfoCollection, CcLinkParamsStore> mapping : remainingMappings) {
+ if (add(mapping.apply(target))) {
+ return this;
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Includes link parameters from a CcLinkParamsProvider provider.
+ */
+ public Builder addTransitiveProvider(CcLinkParamsProvider provider) {
+ if (provider == null) {
+ return this;
+ }
+
+ CcLinkParams args = provider.getCcLinkParams(linkingStatically, linkShared);
+ addTransitiveArgs(args);
+ return this;
+ }
+
+ /**
+ * Includes link parameters from the given targets. Each target is checked for the given
+ * mappings in the order specified, and the first mapping that returns a non-null result is
+ * added.
+ */
+ @SafeVarargs
+ public final Builder addTransitiveTargets(
+ Iterable<? extends TransitiveInfoCollection> targets,
+ Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping,
+ @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments.
+ Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) {
+ for (TransitiveInfoCollection target : targets) {
+ addTransitiveTarget(target, firstMapping, remainingMappings);
+ }
+ return this;
+ }
+
+ /**
+ * Includes link parameters from the given targets. Each target is checked for the given
+ * mappings in the order specified, and the first mapping that returns a non-null result is
+ * added.
+ *
+ * @deprecated don't add any new uses; all existing uses need to be audited and possibly merged
+ * into a single call - some of them may introduce semantic changes which need to be
+ * carefully vetted
+ */
+ @Deprecated
+ @SafeVarargs
+ public final Builder addTransitiveLangTargets(
+ Iterable<? extends TransitiveInfoCollection> targets,
+ Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping,
+ @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments.
+ Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) {
+ return addTransitiveTargets(targets, firstMapping, remainingMappings);
+ }
+
+ /**
+ * Merges the other {@link CcLinkParams} object into this one.
+ */
+ public Builder addTransitiveArgs(CcLinkParams args) {
+ linkOptsBuilder.addTransitive(args.getLinkopts());
+ linkstampsBuilder.addTransitive(args.getLinkstamps());
+ librariesBuilder.addTransitive(args.getLibraries());
+ return this;
+ }
+
+ /**
+ * Adds a collection of link options.
+ */
+ public Builder addLinkOpts(Collection<String> linkOpts) {
+ localLinkoptsBuilder.addAll(linkOpts);
+ return this;
+ }
+
+ /**
+ * Adds a collection of linkstamps.
+ */
+ public Builder addLinkstamps(Iterable<Artifact> linkstamps, CppCompilationContext context) {
+ ImmutableList<Artifact> declaredIncludeSrcs =
+ ImmutableList.copyOf(context.getDeclaredIncludeSrcs());
+ for (Artifact linkstamp : linkstamps) {
+ linkstampsBuilder.add(new Linkstamp(linkstamp, declaredIncludeSrcs));
+ }
+ return this;
+ }
+
+ /**
+ * Adds a library artifact.
+ */
+ public Builder addLibrary(LibraryToLink library) {
+ librariesBuilder.add(library);
+ return this;
+ }
+
+ /**
+ * Adds a collection of library artifacts.
+ */
+ public Builder addLibraries(Iterable<LibraryToLink> libraries) {
+ librariesBuilder.addAll(libraries);
+ return this;
+ }
+
+ /**
+ * Processes typical dependencies a C/C++ library.
+ *
+ * <p>A helper method that processes getValues() and merges contents of
+ * getPreferredLibraries() and getLinkOpts() into the current link params
+ * object.
+ */
+ public Builder addCcLibrary(RuleContext context, CcCommon common, boolean neverlink,
+ CcLinkingOutputs linkingOutputs) {
+ addTransitiveTargets(
+ context.getPrerequisites("deps", Mode.TARGET),
+ CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS);
+
+ if (!neverlink) {
+ addLibraries(linkingOutputs.getPreferredLibraries(linkingStatically,
+ linkShared || context.getFragment(CppConfiguration.class).forcePic()));
+ addLinkOpts(common.getLinkopts());
+ }
+ return this;
+ }
+ }
+
+ /**
+ * A linkstamp that also knows about its declared includes.
+ *
+ * <p>This object is required because linkstamp files may include other headers which
+ * will have to be provided during compilation.
+ */
+ public static final class Linkstamp {
+ private final Artifact artifact;
+ private final ImmutableList<Artifact> declaredIncludeSrcs;
+
+ private Linkstamp(Artifact artifact, ImmutableList<Artifact> declaredIncludeSrcs) {
+ this.artifact = Preconditions.checkNotNull(artifact);
+ this.declaredIncludeSrcs = Preconditions.checkNotNull(declaredIncludeSrcs);
+ }
+
+ /**
+ * Returns the linkstamp artifact.
+ */
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ /**
+ * Returns the declared includes.
+ */
+ public ImmutableList<Artifact> getDeclaredIncludeSrcs() {
+ return declaredIncludeSrcs;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(artifact, declaredIncludeSrcs);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof Linkstamp)) {
+ return false;
+ }
+ Linkstamp other = (Linkstamp) obj;
+ return artifact.equals(other.artifact)
+ && declaredIncludeSrcs.equals(other.declaredIncludeSrcs);
+ }
+ }
+
+ /**
+ * Empty CcLinkParams.
+ */
+ public static final CcLinkParams EMPTY = new CcLinkParams(
+ NestedSetBuilder.<ImmutableList<String>>emptySet(Order.LINK_ORDER),
+ NestedSetBuilder.<Linkstamp>emptySet(Order.COMPILE_ORDER),
+ NestedSetBuilder.<LibraryToLink>emptySet(Order.LINK_ORDER));
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java
new file mode 100644
index 0000000..11f6011
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl;
+
+/**
+ * A target that provides C linker parameters.
+ */
+@Immutable
+public final class CcLinkParamsProvider implements TransitiveInfoProvider {
+ public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS =
+ new Function<TransitiveInfoCollection, CcLinkParamsStore>() {
+ @Override
+ public CcLinkParamsStore apply(TransitiveInfoCollection input) {
+ CcLinkParamsProvider provider = input.getProvider(
+ CcLinkParamsProvider.class);
+ return provider == null ? null : provider.store;
+ }
+ };
+
+ private final CcLinkParamsStoreImpl store;
+
+ public CcLinkParamsProvider(CcLinkParamsStore store) {
+ this.store = new CcLinkParamsStoreImpl(store);
+ }
+
+ /**
+ * Returns link parameters given static / shared linking settings.
+ */
+ public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) {
+ return store.get(linkingStatically, linkShared);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java
new file mode 100644
index 0000000..a150488
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java
@@ -0,0 +1,136 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder;
+
+/**
+ * A cache of C link parameters.
+ *
+ * <p>The cache holds instances of {@link CcLinkParams} for combinations of
+ * linkingStatically and linkShared. If a requested value is not available in
+ * the cache, it is computed and then stored.
+ *
+ * <p>Typically this class is used on targets that may be linked in as C
+ * libraries as in the following example:
+ *
+ * <pre>
+ * class SomeTarget implements CcLinkParamsProvider {
+ * private final CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() {
+ * @Override
+ * protected void collect(CcLinkParams.Builder builder, boolean linkingStatically,
+ * boolean linkShared) {
+ * builder.add[...]
+ * }
+ * };
+ *
+ * @Override
+ * public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) {
+ * return ccLinkParamsStore.get(linkingStatically, linkShared);
+ * }
+ * }
+ * </pre>
+ */
+public abstract class CcLinkParamsStore {
+
+ private CcLinkParams staticSharedParams;
+ private CcLinkParams staticNoSharedParams;
+ private CcLinkParams noStaticSharedParams;
+ private CcLinkParams noStaticNoSharedParams;
+
+ private CcLinkParams compute(boolean linkingStatically, boolean linkShared) {
+ CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared);
+ collect(builder, linkingStatically, linkShared);
+ return builder.build();
+ }
+
+ /**
+ * Returns {@link CcLinkParams} for a combination of parameters.
+ *
+ * <p>The {@link CcLinkParams} instance is computed lazily and cached.
+ */
+ public synchronized CcLinkParams get(boolean linkingStatically, boolean linkShared) {
+ CcLinkParams result = lookup(linkingStatically, linkShared);
+ if (result == null) {
+ result = compute(linkingStatically, linkShared);
+ put(linkingStatically, linkShared, result);
+ }
+ return result;
+ }
+
+ private CcLinkParams lookup(boolean linkingStatically, boolean linkShared) {
+ if (linkingStatically) {
+ return linkShared ? staticSharedParams : staticNoSharedParams;
+ } else {
+ return linkShared ? noStaticSharedParams : noStaticNoSharedParams;
+ }
+ }
+
+ private void put(boolean linkingStatically, boolean linkShared, CcLinkParams params) {
+ Preconditions.checkNotNull(params);
+ if (linkingStatically) {
+ if (linkShared) {
+ staticSharedParams = params;
+ } else {
+ staticNoSharedParams = params;
+ }
+ } else {
+ if (linkShared) {
+ noStaticSharedParams = params;
+ } else {
+ noStaticNoSharedParams = params;
+ }
+ }
+ }
+
+ /**
+ * Hook for building the actual link params.
+ *
+ * <p>Users should override this method and call methods of the builder to
+ * set up the actual CcLinkParams objects.
+ *
+ * <p>Implementations of this method must not fail or try to report errors on the
+ * configured target.
+ */
+ protected abstract void collect(CcLinkParams.Builder builder, boolean linkingStatically,
+ boolean linkShared);
+
+ /**
+ * An empty CcLinkParamStore.
+ */
+ public static final CcLinkParamsStore EMPTY = new CcLinkParamsStore() {
+
+ @Override
+ protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {}
+ };
+
+ /**
+ * An implementation class for the CcLinkParamsStore.
+ */
+ public static final class CcLinkParamsStoreImpl extends CcLinkParamsStore {
+
+ public CcLinkParamsStoreImpl(CcLinkParamsStore store) {
+ super.staticSharedParams = store.get(true, true);
+ super.staticNoSharedParams = store.get(true, false);
+ super.noStaticSharedParams = store.get(false, true);
+ super.noStaticNoSharedParams = store.get(false, false);
+ }
+
+ @Override
+ protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {}
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java
new file mode 100644
index 0000000..6b45c79
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java
@@ -0,0 +1,243 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A structured representation of the link outputs of a C++ rule.
+ */
+public class CcLinkingOutputs {
+
+ public static final CcLinkingOutputs EMPTY = new Builder().build();
+
+ private final ImmutableList<LibraryToLink> staticLibraries;
+
+ private final ImmutableList<LibraryToLink> picStaticLibraries;
+
+ private final ImmutableList<LibraryToLink> dynamicLibraries;
+
+ private final ImmutableList<LibraryToLink> executionDynamicLibraries;
+
+ private CcLinkingOutputs(ImmutableList<LibraryToLink> staticLibraries,
+ ImmutableList<LibraryToLink> picStaticLibraries,
+ ImmutableList<LibraryToLink> dynamicLibraries,
+ ImmutableList<LibraryToLink> executionDynamicLibraries) {
+ this.staticLibraries = staticLibraries;
+ this.picStaticLibraries = picStaticLibraries;
+ this.dynamicLibraries = dynamicLibraries;
+ this.executionDynamicLibraries = executionDynamicLibraries;
+ }
+
+ public ImmutableList<LibraryToLink> getStaticLibraries() {
+ return staticLibraries;
+ }
+
+ public ImmutableList<LibraryToLink> getPicStaticLibraries() {
+ return picStaticLibraries;
+ }
+
+ public ImmutableList<LibraryToLink> getDynamicLibraries() {
+ return dynamicLibraries;
+ }
+
+ public ImmutableList<LibraryToLink> getExecutionDynamicLibraries() {
+ return executionDynamicLibraries;
+ }
+
+ /**
+ * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of preference depending on the
+ * link preferences.
+ *
+ * <p>This method tries to simulate a search path for adding static and dynamic libraries,
+ * allowing either to be preferred over the other depending on the link {@link LinkStaticness}.
+ *
+ * TODO(bazel-team): (2009) we should preserve the relative ordering of first and second
+ * choice libraries. E.g. if srcs=['foo.a','bar.so','baz.a'] then we should link them in the
+ * same order. Currently we link entries from the first choice list before those from the
+ * second choice list, i.e. in the order {@code ['bar.so', 'foo.a', 'baz.a']}.
+ *
+ * @param linkingStatically whether to prefer static over dynamic libraries. Should be
+ * <code>true</code> for binaries that are linked in fully static or mostly static mode.
+ * @param preferPic whether to prefer pic over non pic libraries (usually used when linking
+ * shared)
+ */
+ public List<LibraryToLink> getPreferredLibraries(
+ boolean linkingStatically, boolean preferPic) {
+ return getPreferredLibraries(linkingStatically, preferPic, false);
+ }
+
+ /**
+ * Returns the shared libraries that are linked against and therefore also need to be in the
+ * runfiles.
+ */
+ public Iterable<Artifact> getLibrariesForRunfiles(boolean linkingStatically) {
+ List<LibraryToLink> libraries =
+ getPreferredLibraries(linkingStatically, /*preferPic*/false, true);
+ return CcCommon.getSharedLibrariesFrom(LinkerInputs.toLibraryArtifacts(libraries));
+ }
+
+ /**
+ * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of
+ * preference depending on the link preferences.
+ */
+ private List<LibraryToLink> getPreferredLibraries(boolean linkingStatically, boolean preferPic,
+ boolean forRunfiles) {
+ List<LibraryToLink> candidates = new ArrayList<>();
+ // It's important that this code keeps the invariant that preferPic has no effect on the output
+ // of .so libraries. That is, the resulting list should contain the same .so files in the same
+ // order.
+ if (linkingStatically) { // Prefer the static libraries.
+ if (preferPic) {
+ // First choice is the PIC static libraries.
+ // Second choice is the other static libraries (may cause link error if they're not PIC,
+ // but I think this is preferable to linking dynamically when you asked for statically).
+ candidates.addAll(picStaticLibraries);
+ candidates.addAll(staticLibraries);
+ } else {
+ // First choice is the non-pic static libraries (best performance);
+ // second choice is the staticPicLibraries (at least they're static;
+ // we can live with the extra overhead of PIC).
+ candidates.addAll(staticLibraries);
+ candidates.addAll(picStaticLibraries);
+ }
+ candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries);
+ } else {
+ // First choice is the dynamicLibraries.
+ candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries);
+ if (preferPic) {
+ // Second choice is the staticPicLibraries (at least they're PIC, so we won't get a
+ // link error).
+ candidates.addAll(picStaticLibraries);
+ candidates.addAll(staticLibraries);
+ } else {
+ candidates.addAll(staticLibraries);
+ candidates.addAll(picStaticLibraries);
+ }
+ }
+ return filterCandidates(candidates);
+ }
+
+ /**
+ * Helper method to filter the candidates by removing equivalent library
+ * entries from the list of candidates.
+ *
+ * @param candidates the library candidates to filter
+ * @return the list of libraries with equivalent duplicate libraries removed.
+ */
+ private List<LibraryToLink> filterCandidates(List<LibraryToLink> candidates) {
+ List<LibraryToLink> libraries = new ArrayList<>();
+ Set<String> identifiers = new HashSet<>();
+ for (LibraryToLink library : candidates) {
+ if (identifiers.add(libraryIdentifierOf(library.getOriginalLibraryArtifact()))) {
+ libraries.add(library);
+ }
+ }
+ return libraries;
+ }
+
+ /**
+ * Returns the library identifier of an artifact: a string that is different for different
+ * libraries, but is the same for the shared, static and pic versions of the same library.
+ */
+ private static String libraryIdentifierOf(Artifact libraryArtifact) {
+ String name = libraryArtifact.getRootRelativePath().getPathString();
+ String basename = FileSystemUtils.removeExtension(name);
+ // Need to special-case file types with double extension.
+ return name.endsWith(".pic.a")
+ ? FileSystemUtils.removeExtension(basename)
+ : name.endsWith(".nopic.a")
+ ? FileSystemUtils.removeExtension(basename)
+ : name.endsWith(".pic.lo")
+ ? FileSystemUtils.removeExtension(basename)
+ : basename;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static final class Builder {
+ private final Set<LibraryToLink> staticLibraries = new LinkedHashSet<>();
+ private final Set<LibraryToLink> picStaticLibraries = new LinkedHashSet<>();
+ private final Set<LibraryToLink> dynamicLibraries = new LinkedHashSet<>();
+ private final Set<LibraryToLink> executionDynamicLibraries = new LinkedHashSet<>();
+
+ public CcLinkingOutputs build() {
+ return new CcLinkingOutputs(ImmutableList.copyOf(staticLibraries),
+ ImmutableList.copyOf(picStaticLibraries), ImmutableList.copyOf(dynamicLibraries),
+ ImmutableList.copyOf(executionDynamicLibraries));
+ }
+
+ public Builder merge(CcLinkingOutputs outputs) {
+ staticLibraries.addAll(outputs.getStaticLibraries());
+ picStaticLibraries.addAll(outputs.getPicStaticLibraries());
+ dynamicLibraries.addAll(outputs.getDynamicLibraries());
+ executionDynamicLibraries.addAll(outputs.getExecutionDynamicLibraries());
+ return this;
+ }
+
+ public Builder addStaticLibrary(LibraryToLink library) {
+ staticLibraries.add(library);
+ return this;
+ }
+
+ public Builder addStaticLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(staticLibraries, libraries);
+ return this;
+ }
+
+ public Builder addPicStaticLibrary(LibraryToLink library) {
+ picStaticLibraries.add(library);
+ return this;
+ }
+
+ public Builder addPicStaticLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(picStaticLibraries, libraries);
+ return this;
+ }
+
+ public Builder addDynamicLibrary(LibraryToLink library) {
+ dynamicLibraries.add(library);
+ return this;
+ }
+
+ public Builder addDynamicLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(dynamicLibraries, libraries);
+ return this;
+ }
+
+ public Builder addExecutionDynamicLibrary(LibraryToLink library) {
+ executionDynamicLibraries.add(library);
+ return this;
+ }
+
+ public Builder addExecutionDynamicLibraries(Iterable<LibraryToLink> libraries) {
+ Iterables.addAll(executionDynamicLibraries, libraries);
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java
new file mode 100644
index 0000000..5e96291
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A target that provides native libraries in the transitive closure of its deps that are needed for
+ * executing C++ code.
+ */
+@Immutable
+public final class CcNativeLibraryProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<LinkerInput> transitiveCcNativeLibraries;
+
+ public CcNativeLibraryProvider(NestedSet<LinkerInput> transitiveCcNativeLibraries) {
+ this.transitiveCcNativeLibraries = transitiveCcNativeLibraries;
+ }
+
+ /**
+ * Collects native libraries in the transitive closure of its deps that are needed for executing
+ * C/C++ code.
+ *
+ * <p>In effect, returns all dynamic library (.so) artifacts provided by the transitive closure.
+ */
+ public NestedSet<LinkerInput> getTransitiveCcNativeLibraries() {
+ return transitiveCcNativeLibraries;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java
new file mode 100644
index 0000000..dfcecc2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl;
+
+/**
+ * A target that provides libraries to be only linked into other C++ targets (and not targets
+ * for other languages)
+ */
+@Immutable
+public final class CcSpecificLinkParamsProvider implements TransitiveInfoProvider {
+ private final CcLinkParamsStoreImpl store;
+
+ public CcSpecificLinkParamsProvider(CcLinkParamsStore store) {
+ this.store = new CcLinkParamsStoreImpl(store);
+ }
+
+ public CcLinkParamsStore getLinkParams() {
+ return store;
+ }
+
+ public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS =
+ new Function<TransitiveInfoCollection, CcLinkParamsStore>() {
+ @Override
+ public CcLinkParamsStore apply(TransitiveInfoCollection input) {
+ CcSpecificLinkParamsProvider provider = input.getProvider(
+ CcSpecificLinkParamsProvider.class);
+ return provider == null ? null : provider.getLinkParams();
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java
new file mode 100644
index 0000000..7827183
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * A configured target class for cc_test rules.
+ */
+public abstract class CcTest implements RuleConfiguredTargetFactory {
+
+ private final CppSemantics semantics;
+
+ protected CcTest(CppSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext context) throws InterruptedException {
+ return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ true);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java
new file mode 100644
index 0000000..bd39d0f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java
@@ -0,0 +1,249 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.CompilationHelper;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.LicensesProvider;
+import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense;
+import com.google.devtools.build.lib.analysis.MiddlemanProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.License;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * Implementation for the cc_toolchain rule.
+ */
+public class CcToolchain implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ final Label label = ruleContext.getLabel();
+ final NestedSet<Artifact> crosstool = ruleContext.getPrerequisite("all_files", Mode.HOST)
+ .getProvider(FileProvider.class).getFilesToBuild();
+ final NestedSet<Artifact> crosstoolMiddleman = getFiles(ruleContext, "all_files");
+ final NestedSet<Artifact> compile = getFiles(ruleContext, "compiler_files");
+ final NestedSet<Artifact> strip = getFiles(ruleContext, "strip_files");
+ final NestedSet<Artifact> objcopy = getFiles(ruleContext, "objcopy_files");
+ final NestedSet<Artifact> link = getFiles(ruleContext, "linker_files");
+ final NestedSet<Artifact> dwp = getFiles(ruleContext, "dwp_files");
+ final NestedSet<Artifact> libcLink = inputsForLibcLink(ruleContext);
+ String purposePrefix = Actions.escapeLabel(label) + "_";
+ String runtimeSolibDirBase = "_solib_" + "_" + Actions.escapeLabel(label);
+ final PathFragment runtimeSolibDir = ruleContext.getConfiguration()
+ .getBinFragment().getRelative(runtimeSolibDirBase);
+
+ CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class);
+ // Static runtime inputs.
+ TransitiveInfoCollection staticRuntimeLibDep = selectDep(ruleContext, "static_runtime_libs",
+ cppConfiguration.getStaticRuntimeLibsLabel());
+ final NestedSet<Artifact> staticRuntimeLinkInputs;
+ final Artifact staticRuntimeLinkMiddleman;
+ if (cppConfiguration.supportsEmbeddedRuntimes()) {
+ staticRuntimeLinkInputs = staticRuntimeLibDep
+ .getProvider(FileProvider.class)
+ .getFilesToBuild();
+ } else {
+ staticRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (!staticRuntimeLinkInputs.isEmpty()) {
+ NestedSet<Artifact> staticRuntimeLinkMiddlemanSet = CompilationHelper.getAggregatingMiddleman(
+ ruleContext,
+ purposePrefix + "static_runtime_link",
+ staticRuntimeLibDep);
+ staticRuntimeLinkMiddleman = staticRuntimeLinkMiddlemanSet.isEmpty()
+ ? null : Iterables.getOnlyElement(staticRuntimeLinkMiddlemanSet);
+ } else {
+ staticRuntimeLinkMiddleman = null;
+ }
+
+ Preconditions.checkState(
+ (staticRuntimeLinkMiddleman == null) == staticRuntimeLinkInputs.isEmpty());
+
+ // Dynamic runtime inputs.
+ TransitiveInfoCollection dynamicRuntimeLibDep = selectDep(ruleContext, "dynamic_runtime_libs",
+ cppConfiguration.getDynamicRuntimeLibsLabel());
+ final NestedSet<Artifact> dynamicRuntimeLinkInputs;
+ final Artifact dynamicRuntimeLinkMiddleman;
+ if (cppConfiguration.supportsEmbeddedRuntimes()) {
+ NestedSetBuilder<Artifact> dynamicRuntimeLinkInputsBuilder = NestedSetBuilder.stableOrder();
+ for (Artifact artifact : dynamicRuntimeLibDep
+ .getProvider(FileProvider.class).getFilesToBuild()) {
+ if (CppHelper.SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) {
+ dynamicRuntimeLinkInputsBuilder.add(SolibSymlinkAction.getCppRuntimeSymlink(
+ ruleContext, artifact, runtimeSolibDirBase,
+ ruleContext.getConfiguration()).getArtifact());
+ } else {
+ dynamicRuntimeLinkInputsBuilder.add(artifact);
+ }
+ }
+ dynamicRuntimeLinkInputs = dynamicRuntimeLinkInputsBuilder.build();
+ } else {
+ dynamicRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (!dynamicRuntimeLinkInputs.isEmpty()) {
+ List<Artifact> dynamicRuntimeLinkMiddlemanSet =
+ CppHelper.getAggregatingMiddlemanForCppRuntimes(
+ ruleContext,
+ purposePrefix + "dynamic_runtime_link",
+ dynamicRuntimeLibDep,
+ runtimeSolibDirBase,
+ ruleContext.getConfiguration());
+ dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddlemanSet.isEmpty()
+ ? null : Iterables.getOnlyElement(dynamicRuntimeLinkMiddlemanSet);
+ } else {
+ dynamicRuntimeLinkMiddleman = null;
+ }
+
+ Preconditions.checkState(
+ (dynamicRuntimeLinkMiddleman == null) == dynamicRuntimeLinkInputs.isEmpty());
+
+ CppCompilationContext.Builder contextBuilder =
+ new CppCompilationContext.Builder(ruleContext);
+ CppModuleMap moduleMap = createCrosstoolModuleMap(ruleContext);
+ if (moduleMap != null) {
+ contextBuilder.setCppModuleMap(moduleMap);
+ }
+ final CppCompilationContext context = contextBuilder.build();
+ boolean supportsParamFiles = ruleContext.attributes().get("supports_param_files", BOOLEAN);
+ boolean supportsHeaderParsing =
+ ruleContext.attributes().get("supports_header_parsing", BOOLEAN);
+
+ CcToolchainProvider provider = new CcToolchainProvider(
+ Preconditions.checkNotNull(ruleContext.getFragment(CppConfiguration.class)),
+ crosstool,
+ fullInputsForCrosstool(ruleContext, crosstoolMiddleman),
+ compile,
+ strip,
+ objcopy,
+ fullInputsForLink(ruleContext, link),
+ dwp,
+ libcLink,
+ staticRuntimeLinkInputs,
+ staticRuntimeLinkMiddleman,
+ dynamicRuntimeLinkInputs,
+ dynamicRuntimeLinkMiddleman,
+ runtimeSolibDir,
+ context,
+ supportsParamFiles,
+ supportsHeaderParsing);
+ RuleConfiguredTargetBuilder builder =
+ new RuleConfiguredTargetBuilder(ruleContext)
+ .add(CcToolchainProvider.class, provider)
+ .setFilesToBuild(new NestedSetBuilder<Artifact>(Order.STABLE_ORDER).build())
+ .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY));
+
+ // If output_license is specified on the cc_toolchain rule, override the transitive licenses
+ // with that one. This is necessary because cc_toolchain is used in the target configuration,
+ // but it is sort-of-kind-of a tool, but various parts of it are linked into the output...
+ // ...so we trust the judgment of the author of the cc_toolchain rule to figure out what
+ // licenses should be propagated to C++ targets.
+ License outputLicense = ruleContext.getRule().getToolOutputLicense(ruleContext.attributes());
+ if (outputLicense != null && outputLicense != License.NO_LICENSE) {
+ final NestedSet<TargetLicense> license = NestedSetBuilder.create(Order.STABLE_ORDER,
+ new TargetLicense(ruleContext.getLabel(), outputLicense));
+ LicensesProvider licensesProvider = new LicensesProvider() {
+ @Override
+ public NestedSet<TargetLicense> getTransitiveLicenses() {
+ return license;
+ }
+ };
+
+ builder.add(LicensesProvider.class, licensesProvider);
+ }
+
+ return builder.build();
+ }
+
+ private NestedSet<Artifact> inputsForLibcLink(RuleContext ruleContext) {
+ TransitiveInfoCollection libcLink = ruleContext.getPrerequisite(":libc_link", Mode.HOST);
+ return libcLink != null
+ ? libcLink.getProvider(FileProvider.class).getFilesToBuild()
+ : NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER);
+ }
+
+ private NestedSet<Artifact> fullInputsForCrosstool(RuleContext ruleContext,
+ NestedSet<Artifact> crosstoolMiddleman) {
+ return NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(crosstoolMiddleman)
+ // Use "libc_link" here, because it is functionally identical to the case
+ // below. If we introduce separate filegroups for compiling and linking, we
+ // need to fix that here.
+ .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link"))
+ .build();
+ }
+
+ private NestedSet<Artifact> fullInputsForLink(RuleContext ruleContext, NestedSet<Artifact> link) {
+ return NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(link)
+ .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link"))
+ .add(ruleContext.getAnalysisEnvironment().getEmbeddedToolArtifact(
+ CppRuleClasses.BUILD_INTERFACE_SO))
+ .build();
+ }
+
+ private CppModuleMap createCrosstoolModuleMap(RuleContext ruleContext) {
+ if (ruleContext.getPrerequisite("module_map", Mode.HOST) == null) {
+ return null;
+ }
+ Artifact moduleMapArtifact = ruleContext.getPrerequisiteArtifact("module_map", Mode.HOST);
+ if (moduleMapArtifact == null) {
+ return null;
+ }
+ return new CppModuleMap(moduleMapArtifact, "crosstool");
+ }
+
+ private TransitiveInfoCollection selectDep(
+ RuleContext ruleContext, String attribute, Label label) {
+ for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attribute, Mode.TARGET)) {
+ if (dep.getLabel().equals(label)) {
+ return dep;
+ }
+ }
+
+ return ruleContext.getPrerequisites(attribute, Mode.TARGET).get(0);
+ }
+
+ private NestedSet<Artifact> getFiles(RuleContext context, String attribute) {
+ TransitiveInfoCollection dep = context.getPrerequisite(attribute, Mode.HOST);
+ MiddlemanProvider middlemanProvider = dep.getProvider(MiddlemanProvider.class);
+ // We use the middleman if we can (if the dep is a filegroup), otherwise, just the regular
+ // filesToBuild (e.g. if it is a simple input file)
+ return middlemanProvider != null
+ ? middlemanProvider.getMiddlemanArtifact()
+ : dep.getProvider(FileProvider.class).getFilesToBuild();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java
new file mode 100644
index 0000000..29ab45c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java
@@ -0,0 +1,802 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+/**
+ * Provides access to features supported by a specific toolchain.
+ *
+ * <p>This class can be generated from the CToolchain protocol buffer.
+ *
+ * <p>TODO(bazel-team): Implement support for specifying the toolchain configuration directly from
+ * the BUILD file.
+ *
+ * <p>TODO(bazel-team): Find a place to put the public-facing documentation and link to it from
+ * here.
+ *
+ * <p>TODO(bazel-team): Split out Feature as CcToolchainFeature, which will modularize the
+ * crosstool configuration into one part that is about handling a set of features (including feature
+ * selection) and one part that is about how to apply a single feature (parsing flags and expanding
+ * them from build variables).
+ */
+@Immutable
+public class CcToolchainFeatures implements Serializable {
+
+ /**
+ * Thrown when a flag value cannot be expanded under a set of build variables.
+ *
+ * <p>This happens for example when a flag references a variable that is not provided by the
+ * action, or when a flag group references multiple variables of sequence type.
+ */
+ public static class ExpansionException extends RuntimeException {
+ ExpansionException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * A piece of a single flag.
+ *
+ * <p>A single flag can contain a combination of text and variables (for example
+ * "-f %{var1}/%{var2}"). We split the flag into chunks, where each chunk represents either a
+ * text snippet, or a variable that is to be replaced.
+ */
+ interface FlagChunk {
+
+ /**
+ * Expands this chunk.
+ *
+ * @param variables variable names mapped to their values for a single flag expansion.
+ * @param flag the flag content to append to.
+ */
+ void expand(Map<String, String> variables, StringBuilder flag);
+ }
+
+ /**
+ * A plain text chunk of a flag.
+ */
+ @Immutable
+ private static class StringChunk implements FlagChunk, Serializable {
+ private final String text;
+
+ private StringChunk(String text) {
+ this.text = text;
+ }
+
+ @Override
+ public void expand(Map<String, String> variables, StringBuilder flag) {
+ flag.append(text);
+ }
+ }
+
+ /**
+ * A chunk of a flag into which a variable should be expanded.
+ */
+ @Immutable
+ private static class VariableChunk implements FlagChunk, Serializable {
+ private final String variableName;
+
+ private VariableChunk(String variableName) {
+ this.variableName = variableName;
+ }
+
+ @Override
+ public void expand(Map<String, String> variables, StringBuilder flag) {
+ String value = variables.get(variableName);
+ if (value == null) {
+ // We check all variables in FlagGroup.expandCommandLine, so if we arrive here with a
+ // null value, the variable map originally handed to the feature selection must have
+ // contained an explicit null value.
+ throw new ExpansionException("Internal blaze error: build variable was set to 'null'.");
+ }
+ flag.append(variables.get(variableName));
+ }
+ }
+
+ /**
+ * Parser for toolchain flags.
+ *
+ * <p>A flag contains a snippet of text supporting variable expansion. For example, a flag value
+ * "-f %{var1}/%{var2}" will expand the values of the variables "var1" and "var2" in the
+ * corresponding places in the string.
+ *
+ * <p>The {@code FlagParser} takes a flag string and parses it into a list of {@code FlagChunk}
+ * objects, where each chunk represents either a snippet of text or a variable to be expanded. In
+ * the above example, the resulting chunks would be ["-f ", var1, "/", var2].
+ *
+ * <p>In addition to the list of chunks, the {@code FlagParser} also provides the set of variables
+ * necessary for the expansion of this flag via {@code getUsedVariables}.
+ *
+ * <p>To get a literal percent character, "%%" can be used in the flag text.
+ */
+ private static class FlagParser {
+
+ /**
+ * The given flag value.
+ */
+ private final String value;
+
+ /**
+ * The current position in {@value} during parsing.
+ */
+ private int current = 0;
+
+ private final ImmutableList.Builder<FlagChunk> chunks = ImmutableList.builder();
+ private final ImmutableSet.Builder<String> usedVariables = ImmutableSet.builder();
+
+ private FlagParser(String value) throws InvalidConfigurationException {
+ this.value = value;
+ parse();
+ }
+
+ /**
+ * @return the parsed chunks for this flag.
+ */
+ private ImmutableList<FlagChunk> getChunks() {
+ return chunks.build();
+ }
+
+ /**
+ * @return all variable names needed to expand this flag.
+ */
+ private ImmutableSet<String> getUsedVariables() {
+ return usedVariables.build();
+ }
+
+ /**
+ * Parses the flag.
+ *
+ * @throws InvalidConfigurationException if there is a parsing error.
+ */
+ private void parse() throws InvalidConfigurationException {
+ while (current < value.length()) {
+ if (atVariableStart()) {
+ parseVariableChunk();
+ } else {
+ parseStringChunk();
+ }
+ }
+ }
+
+ /**
+ * @return whether the current position is the start of a variable.
+ */
+ private boolean atVariableStart() {
+ // We parse a variable when value starts with '%', but not '%%'.
+ return value.charAt(current) == '%'
+ && (current + 1 >= value.length() || value.charAt(current + 1) != '%');
+ }
+
+ /**
+ * Parses a chunk of text until the next '%', which indicates either an escaped literal '%'
+ * or a variable.
+ */
+ private void parseStringChunk() {
+ int start = current;
+ // We only parse string chunks starting with '%' if they also start with '%%'.
+ // In that case, we want to have a single '%' in the string, so we start at the second
+ // character.
+ // Note that for flags like "abc%%def" this will lead to two string chunks, the first
+ // referencing the subtring "abc", and a second referencing the substring "%def".
+ if (value.charAt(current) == '%') {
+ current = current + 1;
+ start = current;
+ }
+ current = value.indexOf('%', current + 1);
+ if (current == -1) {
+ current = value.length();
+ }
+ final String text = value.substring(start, current);
+ chunks.add(new StringChunk(text));
+ }
+
+ /**
+ * Parses a variable to be expanded.
+ *
+ * @throws InvalidConfigurationException if there is a parsing error.
+ */
+ private void parseVariableChunk() throws InvalidConfigurationException {
+ current = current + 1;
+ if (current >= value.length() || value.charAt(current) != '{') {
+ abort("expected '{'");
+ }
+ current = current + 1;
+ if (current >= value.length() || value.charAt(current) == '}') {
+ abort("expected variable name");
+ }
+ int end = value.indexOf('}', current);
+ final String name = value.substring(current, end);
+ usedVariables.add(name);
+ chunks.add(new VariableChunk(name));
+ current = end + 1;
+ }
+
+ /**
+ * @throws InvalidConfigurationException with the given error text, adding information about
+ * the current position in the flag.
+ */
+ private void abort(String error) throws InvalidConfigurationException {
+ throw new InvalidConfigurationException("Invalid toolchain configuration: " + error
+ + " at position " + current + " while parsing a flag containing '" + value + "'");
+ }
+ }
+
+ /**
+ * A single flag to be expanded under a set of variables.
+ *
+ * <p>TODO(bazel-team): Consider specializing Flag for the simple case that a flag is just a bit
+ * of text.
+ */
+ @Immutable
+ private static class Flag implements Serializable {
+ private final ImmutableList<FlagChunk> chunks;
+
+ private Flag(ImmutableList<FlagChunk> chunks) {
+ this.chunks = chunks;
+ }
+
+ /**
+ * Expand this flag into a single new entry in {@code commandLine}.
+ */
+ private void expandCommandLine(Map<String, String> variables, List<String> commandLine) {
+ StringBuilder flag = new StringBuilder();
+ for (FlagChunk chunk : chunks) {
+ chunk.expand(variables, flag);
+ }
+ commandLine.add(flag.toString());
+ }
+ }
+
+ /**
+ * A group of flags.
+ */
+ @Immutable
+ private static class FlagGroup implements Serializable {
+ private final ImmutableList<Flag> flags;
+ private final ImmutableSet<String> usedVariables;
+
+ private FlagGroup(CToolchain.FlagGroup flagGroup) throws InvalidConfigurationException {
+ ImmutableList.Builder<Flag> flags = ImmutableList.builder();
+ ImmutableSet.Builder<String> usedVariables = ImmutableSet.builder();
+ for (String flag : flagGroup.getFlagList()) {
+ FlagParser parser = new FlagParser(flag);
+ flags.add(new Flag(parser.getChunks()));
+ usedVariables.addAll(parser.getUsedVariables());
+ }
+ this.flags = flags.build();
+ this.usedVariables = usedVariables.build();
+ }
+
+ /**
+ * Expands all flags in this group and adds them to {@code commandLine}.
+ *
+ * <p>The flags of the group will be expanded either:
+ * <ul>
+ * <li>once, if there is no variable of sequence type in any of the group's flags, or</li>
+ * <li>for each element in the sequence, if there is one variable of sequence type within
+ * the flags.</li>
+ * </ul>
+ *
+ * <p>Having more than a single variable of sequence type in a single flag group is not
+ * supported.
+ */
+ private void expandCommandLine(Multimap<String, String> variables, List<String> commandLine) {
+ Map<String, String> variableView = new HashMap<>();
+ String sequenceName = null;
+ for (String name : usedVariables) {
+ Collection<String> value = variables.get(name);
+ if (value.isEmpty()) {
+ throw new ExpansionException("Invalid toolchain configuration: unknown variable '" + name
+ + "' can not be expanded.");
+ } else if (value.size() > 1) {
+ if (sequenceName != null) {
+ throw new ExpansionException(
+ "Invalid toolchain configuration: trying to expand two variable list in one "
+ + "flag group: '" + sequenceName + "' and '" + name + "'");
+ }
+ sequenceName = name;
+ } else {
+ variableView.put(name, value.iterator().next());
+ }
+ }
+ if (sequenceName != null) {
+ for (String value : variables.get(sequenceName)) {
+ variableView.put(sequenceName, value);
+ expandOnce(variableView, commandLine);
+ }
+ } else {
+ expandOnce(variableView, commandLine);
+ }
+ }
+
+ /**
+ * Expanding all flags of this group into {@code commandLine}.
+ */
+ private void expandOnce(Map<String, String> variables, List<String> commandLine) {
+ for (Flag flag : flags) {
+ flag.expandCommandLine(variables, commandLine);
+ }
+ }
+ }
+
+ /**
+ * Groups a set of flags to apply for certain actions.
+ */
+ @Immutable
+ private static class FlagSet implements Serializable {
+ private final ImmutableSet<String> actions;
+ private final ImmutableList<FlagGroup> flagGroups;
+
+ private FlagSet(CToolchain.FlagSet flagSet) throws InvalidConfigurationException {
+ this.actions = ImmutableSet.copyOf(flagSet.getActionList());
+ ImmutableList.Builder<FlagGroup> builder = ImmutableList.builder();
+ for (CToolchain.FlagGroup flagGroup : flagSet.getFlagGroupList()) {
+ builder.add(new FlagGroup(flagGroup));
+ }
+ this.flagGroups = builder.build();
+ }
+
+ /**
+ * Adds the flags that apply to the given {@code action} to {@code commandLine}.
+ */
+ private void expandCommandLine(String action, Multimap<String, String> variables,
+ List<String> commandLine) {
+ if (!actions.contains(action)) {
+ return;
+ }
+ for (FlagGroup flagGroup : flagGroups) {
+ flagGroup.expandCommandLine(variables, commandLine);
+ }
+ }
+ }
+
+ /**
+ * Contains flags for a specific feature.
+ */
+ @Immutable
+ private static class Feature implements Serializable {
+ private final String name;
+ private final ImmutableList<FlagSet> flagSets;
+
+ private Feature(CToolchain.Feature feature) throws InvalidConfigurationException {
+ this.name = feature.getName();
+ ImmutableList.Builder<FlagSet> builder = ImmutableList.builder();
+ for (CToolchain.FlagSet flagSet : feature.getFlagSetList()) {
+ builder.add(new FlagSet(flagSet));
+ }
+ this.flagSets = builder.build();
+ }
+
+ /**
+ * @return the features's name.
+ */
+ private String getName() {
+ return name;
+ }
+
+ /**
+ * Adds the flags that apply to the given {@code action} to {@code commandLine}.
+ */
+ private void expandCommandLine(String action, Multimap<String, String> variables,
+ List<String> commandLine) {
+ for (FlagSet flagSet : flagSets) {
+ flagSet.expandCommandLine(action, variables, commandLine);
+ }
+ }
+ }
+
+ /**
+ * Captures the set of enabled features for a rule.
+ */
+ @Immutable
+ public static class FeatureConfiguration {
+ private final ImmutableSet<String> enabledFeatureNames;
+ private final ImmutableList<Feature> enabledFeatures;
+
+ public FeatureConfiguration() {
+ enabledFeatureNames = ImmutableSet.of();
+ enabledFeatures = ImmutableList.of();
+ }
+
+ private FeatureConfiguration(ImmutableList<Feature> enabledFeatures) {
+ this.enabledFeatures = enabledFeatures;
+ ImmutableSet.Builder<String> builder = ImmutableSet.builder();
+ for (Feature feature : enabledFeatures) {
+ builder.add(feature.getName());
+ }
+ this.enabledFeatureNames = builder.build();
+ }
+
+ /**
+ * @return whether the given {@code feature} is enabled.
+ */
+ boolean isEnabled(String feature) {
+ return enabledFeatureNames.contains(feature);
+ }
+
+ /**
+ * @return the command line for the given {@code action}.
+ */
+ List<String> getCommandLine(String action, Multimap<String, String> variables) {
+ List<String> commandLine = new ArrayList<>();
+ for (Feature feature : enabledFeatures) {
+ feature.expandCommandLine(action, variables, commandLine);
+ }
+ return commandLine;
+ }
+ }
+
+ /**
+ * All features in the order in which they were specified in the configuration.
+ *
+ * <p>We guarantee the command line to be in the order in which the flags were specified in the
+ * configuration.
+ */
+ private final ImmutableList<Feature> features;
+
+ /**
+ * Maps from the feature's name to the feature.
+ */
+ private final ImmutableMap<String, Feature> featuresByName;
+
+ /**
+ * Maps from a feature to a set of all the features it has a direct 'implies' edge to.
+ */
+ private final ImmutableMultimap<Feature, Feature> implies;
+
+ /**
+ * Maps from a feature to all features that have an direct 'implies' edge to this feature.
+ */
+ private final ImmutableMultimap<Feature, Feature> impliedBy;
+
+ /**
+ * Maps from a feature to a set of feature sets, where:
+ * <ul>
+ * <li>a feature set satisfies the 'requires' condition, if all features in the feature set are
+ * enabled</li>
+ * <li>the 'requires' condition is satisfied, if at least one of the feature sets satisfies the
+ * 'requires' condition.</li>
+ * </ul>
+ */
+ private final ImmutableMultimap<Feature, ImmutableSet<Feature>> requires;
+
+ /**
+ * Maps from a feature to all features that have a requirement referencing it.
+ *
+ * <p>This will be used to determine which features need to be re-checked after a feature was
+ * disabled.
+ */
+ private final ImmutableMultimap<Feature, Feature> requiredBy;
+
+ /**
+ * A cache of feature selection results, so we do not recalculate the feature selection for
+ * all actions.
+ */
+ private transient LoadingCache<Collection<String>, FeatureConfiguration>
+ configurationCache = buildConfigurationCache();
+
+ /**
+ * Constructs the feature configuration from a {@code CToolchain} protocol buffer.
+ *
+ * @param toolchain the toolchain configuration as specified by the user.
+ * @throws InvalidConfigurationException if the configuration has logical errors.
+ */
+ CcToolchainFeatures(CToolchain toolchain) throws InvalidConfigurationException {
+ // Build up the feature graph.
+ // First, we build up the map of name -> features in one pass, so that earlier features can
+ // reference later features in their configuration.
+ ImmutableList.Builder<Feature> features = ImmutableList.builder();
+ HashMap<String, Feature> featuresByName = new HashMap<>();
+ for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) {
+ Feature feature = new Feature(toolchainFeature);
+ features.add(feature);
+ if (featuresByName.put(feature.getName(), feature) != null) {
+ throw new InvalidConfigurationException("Invalid toolchain configuration: feature '"
+ + feature.getName() + "' was specified multiple times.");
+ }
+ }
+ this.features = features.build();
+ this.featuresByName = ImmutableMap.copyOf(featuresByName);
+
+ // Next, we build up all forward references for 'implies' and 'requires' edges.
+ ImmutableMultimap.Builder<Feature, Feature> implies = ImmutableMultimap.builder();
+ ImmutableMultimap.Builder<Feature, ImmutableSet<Feature>> requires =
+ ImmutableMultimap.builder();
+ // We also store the reverse 'implied by' and 'required by' edges during this pass.
+ ImmutableMultimap.Builder<Feature, Feature> impliedBy = ImmutableMultimap.builder();
+ ImmutableMultimap.Builder<Feature, Feature> requiredBy = ImmutableMultimap.builder();
+ for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) {
+ String name = toolchainFeature.getName();
+ Feature feature = featuresByName.get(name);
+ for (CToolchain.FeatureSet requiredFeatures : toolchainFeature.getRequiresList()) {
+ ImmutableSet.Builder<Feature> allOf = ImmutableSet.builder();
+ for (String requiredName : requiredFeatures.getFeatureList()) {
+ Feature required = getFeatureOrFail(requiredName, name);
+ allOf.add(required);
+ requiredBy.put(required, feature);
+ }
+ requires.put(feature, allOf.build());
+ }
+ for (String impliedName : toolchainFeature.getImpliesList()) {
+ Feature implied = getFeatureOrFail(impliedName, name);
+ impliedBy.put(implied, feature);
+ implies.put(feature, implied);
+ }
+ }
+ this.implies = implies.build();
+ this.requires = requires.build();
+ this.impliedBy = impliedBy.build();
+ this.requiredBy = requiredBy.build();
+ }
+
+ /**
+ * Assign an empty cache after default-deserializing all non-transient members.
+ */
+ private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException {
+ in.defaultReadObject();
+ this.configurationCache = buildConfigurationCache();
+ }
+
+ /**
+ * @return an empty {@code FeatureConfiguration} cache.
+ */
+ private LoadingCache<Collection<String>, FeatureConfiguration> buildConfigurationCache() {
+ return CacheBuilder.newBuilder()
+ // TODO(klimek): Benchmark and tweak once we support a larger configuration.
+ .maximumSize(10000)
+ .build(new CacheLoader<Collection<String>, FeatureConfiguration>() {
+ @Override
+ public FeatureConfiguration load(Collection<String> requestedFeatures) {
+ return computeFeatureConfiguration(requestedFeatures);
+ }
+ });
+ }
+
+ /**
+ * Given a list of {@code requestedFeatures}, returns all features that are enabled by the
+ * toolchain configuration.
+ *
+ * <p>A requested feature will not be enabled if the toolchain does not support it (which may
+ * depend on other requested features).
+ *
+ * <p>Additional features will be enabled if the toolchain supports them and they are implied by
+ * requested features.
+ */
+ FeatureConfiguration getFeatureConfiguration(Collection<String> requestedFeatures) {
+ return configurationCache.getUnchecked(requestedFeatures);
+ }
+
+ private FeatureConfiguration computeFeatureConfiguration(Collection<String> requestedFeatures) {
+ // Command line flags will be output in the order in which they are specified in the toolchain
+ // configuration.
+ return new FeatureSelection(requestedFeatures).run();
+ }
+
+ /**
+ * Convenience method taking a variadic string argument list for testing.
+ */
+ FeatureConfiguration getFeatureConfiguration(String... requestedFeatures) {
+ return getFeatureConfiguration(Arrays.asList(requestedFeatures));
+ }
+
+ /**
+ * @return the feature with the given {@code name}.
+ *
+ * @throws InvalidConfigurationException if no feature with the given name was configured.
+ */
+ private Feature getFeatureOrFail(String name, String reference)
+ throws InvalidConfigurationException {
+ if (!featuresByName.containsKey(name)) {
+ throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + name
+ + "', which is referenced from feature '" + reference + "', is not defined.");
+ }
+ return featuresByName.get(name);
+ }
+
+ @VisibleForTesting
+ Collection<String> getFeatureNames() {
+ Collection<String> featureNames = new HashSet<>();
+ for (Feature feature : features) {
+ featureNames.add(feature.getName());
+ }
+ return featureNames;
+ }
+
+ /**
+ * Implements the feature selection algorithm.
+ *
+ * <p>Feature selection is done by first enabling all features reachable by an 'implies' edge,
+ * and then iteratively pruning features that have unmet requirements.
+ */
+ private class FeatureSelection {
+
+ /**
+ * The features Bazel would like to enable; either because they are supported and generally
+ * useful, or because the user required them (for example through the command line).
+ */
+ private final ImmutableSet<Feature> requestedFeatures;
+
+ /**
+ * The currently enabled feature; during feature selection, we first put all features reachable
+ * via an 'implies' edge into the enabled feature set, and than prune that set from features
+ * that have unmet requirements.
+ */
+ private Set<Feature> enabled = new HashSet<>();
+
+ private FeatureSelection(Collection<String> requestedFeatures) {
+ ImmutableSet.Builder<Feature> builder = ImmutableSet.builder();
+ for (String name : requestedFeatures) {
+ if (featuresByName.containsKey(name)) {
+ builder.add(featuresByName.get(name));
+ }
+ }
+ this.requestedFeatures = builder.build();
+ }
+
+ /**
+ * @return all enabled features in the order in which they were specified in the configuration.
+ */
+ private FeatureConfiguration run() {
+ for (Feature feature : requestedFeatures) {
+ enableAllImpliedBy(feature);
+ }
+ disableUnsupportedFeatures();
+ ImmutableList.Builder<Feature> enabledFeaturesInOrder = ImmutableList.builder();
+ for (Feature feature : features) {
+ if (enabled.contains(feature)) {
+ enabledFeaturesInOrder.add(feature);
+ }
+ }
+ return new FeatureConfiguration(enabledFeaturesInOrder.build());
+ }
+
+ /**
+ * Transitively and unconditionally enable all features implied by the given feature and the
+ * feature itself to the enabled feature set.
+ */
+ private void enableAllImpliedBy(Feature feature) {
+ if (enabled.contains(feature)) {
+ return;
+ }
+ enabled.add(feature);
+ for (Feature implied : implies.get(feature)) {
+ enableAllImpliedBy(implied);
+ }
+ }
+
+ /**
+ * Remove all unsupported features from the enabled feature set.
+ */
+ private void disableUnsupportedFeatures() {
+ Queue<Feature> check = new ArrayDeque<>(enabled);
+ while (!check.isEmpty()) {
+ checkFeature(check.poll());
+ }
+ }
+
+ /**
+ * Check if the given feature is still satisfied within the set of currently enabled features.
+ *
+ * <p>If it is not, remove the feature from the set of enabled features, and re-check all
+ * features that may now also become disabled.
+ */
+ private void checkFeature(Feature feature) {
+ if (!enabled.contains(feature) || isSatisfied(feature)) {
+ return;
+ }
+ enabled.remove(feature);
+
+ // Once we disable a feature, we have to re-check all features that can be affected by
+ // that removal.
+ // 1. A feature that implied the current feature is now going to be disabled.
+ for (Feature impliesCurrent : impliedBy.get(feature)) {
+ checkFeature(impliesCurrent);
+ }
+ // 2. A feature that required the current feature may now be disabled, depending on whether
+ // the requirement was optional.
+ for (Feature requiresCurrent : requiredBy.get(feature)) {
+ checkFeature(requiresCurrent);
+ }
+ // 3. A feature that this feature implied may now be disabled if no other feature also implies
+ // it.
+ for (Feature implied : implies.get(feature)) {
+ checkFeature(implied);
+ }
+ }
+
+ /**
+ * @return whether all requirements of the feature are met in the set of currently enabled
+ * features.
+ */
+ private boolean isSatisfied(Feature feature) {
+ return (requestedFeatures.contains(feature) || isImpliedByEnabledFeature(feature))
+ && allImplicationsEnabled(feature) && allRequirementsMet(feature);
+ }
+
+ /**
+ * @return whether a currently enabled feature implies the given feature.
+ */
+ private boolean isImpliedByEnabledFeature(Feature feature) {
+ for (Feature implies : impliedBy.get(feature)) {
+ if (enabled.contains(implies)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * @return whether all implications of the given feature are enabled.
+ */
+ private boolean allImplicationsEnabled(Feature feature) {
+ for (Feature implied : implies.get(feature)) {
+ if (!enabled.contains(implied)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @return whether all requirements are enabled.
+ *
+ * <p>This implies that for any of the feature sets all of the specified features are enabled.
+ */
+ private boolean allRequirementsMet(Feature feature) {
+ if (!requires.containsKey(feature)) {
+ return true;
+ }
+ for (ImmutableSet<Feature> requiresAllOf : requires.get(feature)) {
+ boolean requirementMet = true;
+ for (Feature required : requiresAllOf) {
+ if (!enabled.contains(required)) {
+ requirementMet = false;
+ break;
+ }
+ }
+ if (requirementMet) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java
new file mode 100644
index 0000000..e1940a5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java
@@ -0,0 +1,226 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import javax.annotation.Nullable;
+
+/**
+ * Information about a C++ compiler used by the <code>cc_*</code> rules.
+ */
+@Immutable
+public final class CcToolchainProvider implements TransitiveInfoProvider {
+ /**
+ * An empty toolchain to be returned in the error case (instead of null).
+ */
+ public static final CcToolchainProvider EMPTY_TOOLCHAIN_IS_ERROR = new CcToolchainProvider(
+ null,
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ null,
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ null,
+ PathFragment.EMPTY_FRAGMENT,
+ CppCompilationContext.EMPTY,
+ false,
+ false);
+
+ @Nullable private final CppConfiguration cppConfiguration;
+ private final NestedSet<Artifact> crosstool;
+ private final NestedSet<Artifact> crosstoolMiddleman;
+ private final NestedSet<Artifact> compile;
+ private final NestedSet<Artifact> strip;
+ private final NestedSet<Artifact> objCopy;
+ private final NestedSet<Artifact> link;
+ private final NestedSet<Artifact> dwp;
+ private final NestedSet<Artifact> libcLink;
+ private final NestedSet<Artifact> staticRuntimeLinkInputs;
+ @Nullable private final Artifact staticRuntimeLinkMiddleman;
+ private final NestedSet<Artifact> dynamicRuntimeLinkInputs;
+ @Nullable private final Artifact dynamicRuntimeLinkMiddleman;
+ private final PathFragment dynamicRuntimeSolibDir;
+ private final CppCompilationContext cppCompilationContext;
+ private final boolean supportsParamFiles;
+ private final boolean supportsHeaderParsing;
+
+ public CcToolchainProvider(
+ @Nullable CppConfiguration cppConfiguration,
+ NestedSet<Artifact> crosstool,
+ NestedSet<Artifact> crosstoolMiddleman,
+ NestedSet<Artifact> compile,
+ NestedSet<Artifact> strip,
+ NestedSet<Artifact> objCopy,
+ NestedSet<Artifact> link,
+ NestedSet<Artifact> dwp,
+ NestedSet<Artifact> libcLink,
+ NestedSet<Artifact> staticRuntimeLinkInputs,
+ @Nullable Artifact staticRuntimeLinkMiddleman,
+ NestedSet<Artifact> dynamicRuntimeLinkInputs,
+ @Nullable Artifact dynamicRuntimeLinkMiddleman,
+ PathFragment dynamicRuntimeSolibDir,
+ CppCompilationContext cppCompilationContext,
+ boolean supportsParamFiles,
+ boolean supportsHeaderParsing) {
+ this.cppConfiguration = cppConfiguration;
+ this.crosstool = Preconditions.checkNotNull(crosstool);
+ this.crosstoolMiddleman = Preconditions.checkNotNull(crosstoolMiddleman);
+ this.compile = Preconditions.checkNotNull(compile);
+ this.strip = Preconditions.checkNotNull(strip);
+ this.objCopy = Preconditions.checkNotNull(objCopy);
+ this.link = Preconditions.checkNotNull(link);
+ this.dwp = Preconditions.checkNotNull(dwp);
+ this.libcLink = Preconditions.checkNotNull(libcLink);
+ this.staticRuntimeLinkInputs = Preconditions.checkNotNull(staticRuntimeLinkInputs);
+ this.staticRuntimeLinkMiddleman = staticRuntimeLinkMiddleman;
+ this.dynamicRuntimeLinkInputs = Preconditions.checkNotNull(dynamicRuntimeLinkInputs);
+ this.dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddleman;
+ this.dynamicRuntimeSolibDir = Preconditions.checkNotNull(dynamicRuntimeSolibDir);
+ this.cppCompilationContext = Preconditions.checkNotNull(cppCompilationContext);
+ this.supportsParamFiles = supportsParamFiles;
+ this.supportsHeaderParsing = supportsHeaderParsing;
+ }
+
+ /**
+ * Returns all the files in Crosstool. Is not a middleman.
+ */
+ public NestedSet<Artifact> getCrosstool() {
+ return crosstool;
+ }
+
+ /**
+ * Returns a middleman for all the files in Crosstool.
+ */
+ public NestedSet<Artifact> getCrosstoolMiddleman() {
+ return crosstoolMiddleman;
+ }
+
+ /**
+ * Returns the files necessary for compilation.
+ */
+ public NestedSet<Artifact> getCompile() {
+ // If include scanning is disabled, we need the entire crosstool filegroup, including header
+ // files. If it is enabled, we use the filegroup without header files - they are found by
+ // include scanning. For go, we also don't need the header files.
+ return cppConfiguration != null && cppConfiguration.shouldScanIncludes() ? compile : crosstool;
+ }
+
+ /**
+ * Returns the files necessary for a 'strip' invocation.
+ */
+ public NestedSet<Artifact> getStrip() {
+ return strip;
+ }
+
+ /**
+ * Returns the files necessary for an 'objcopy' invocation.
+ */
+ public NestedSet<Artifact> getObjcopy() {
+ return objCopy;
+ }
+
+ /**
+ * Returns the files necessary for linking, including the files needed for libc.
+ */
+ public NestedSet<Artifact> getLink() {
+ return link;
+ }
+
+ public NestedSet<Artifact> getDwp() {
+ return dwp;
+ }
+
+ public NestedSet<Artifact> getLibcLink() {
+ return libcLink;
+ }
+
+ /**
+ * Returns the static runtime libraries.
+ */
+ public NestedSet<Artifact> getStaticRuntimeLinkInputs() {
+ return staticRuntimeLinkInputs;
+ }
+
+ /**
+ * Returns an aggregating middleman that represents the static runtime libraries.
+ */
+ @Nullable public Artifact getStaticRuntimeLinkMiddleman() {
+ return staticRuntimeLinkMiddleman;
+ }
+
+ /**
+ * Returns the dynamic runtime libraries.
+ */
+ public NestedSet<Artifact> getDynamicRuntimeLinkInputs() {
+ return dynamicRuntimeLinkInputs;
+ }
+
+ /**
+ * Returns an aggregating middleman that represents the dynamic runtime libraries.
+ */
+ @Nullable public Artifact getDynamicRuntimeLinkMiddleman() {
+ return dynamicRuntimeLinkMiddleman;
+ }
+
+ /**
+ * Returns the name of the directory where the solib symlinks for the dynamic runtime libraries
+ * live. The directory itself will be under the root of the host configuration in the 'bin'
+ * directory.
+ */
+ public PathFragment getDynamicRuntimeSolibDir() {
+ return dynamicRuntimeSolibDir;
+ }
+
+ /**
+ * Returns the C++ compilation context for the toolchain.
+ */
+ public CppCompilationContext getCppCompilationContext() {
+ return cppCompilationContext;
+ }
+
+ /**
+ * Whether the toolchains supports parameter files.
+ */
+ public boolean supportsParamFiles() {
+ return supportsParamFiles;
+ }
+
+ /**
+ * Whether the toolchains supports header parsing.
+ */
+ public boolean supportsHeaderParsing() {
+ return supportsHeaderParsing;
+ }
+
+ /**
+ * Returns the configured features of the toolchain.
+ */
+ public CcToolchainFeatures getFeatures() {
+ return cppConfiguration.getFeatures();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java
new file mode 100644
index 0000000..6c68f00
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.LICENSE;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * Rule definition for compiler definition.
+ */
+@BlazeRule(name = "cc_toolchain",
+ ancestors = { BaseRuleClasses.BaseRule.class },
+ factoryClass = CcToolchain.class)
+public final class CcToolchainRule implements RuleDefinition {
+ private static final LateBoundLabel<BuildConfiguration> LIBC_LINK =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(CppConfiguration.class).getLibcLabel();
+ }
+ };
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setUndocumented()
+ .add(attr("output_licenses", LICENSE))
+ .add(attr("cpu", STRING).mandatory())
+ .add(attr("all_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory())
+ .add(attr("compiler_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory())
+ .add(attr("strip_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory())
+ .add(attr("objcopy_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory())
+ .add(attr("linker_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory())
+ .add(attr("dwp_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory())
+ .add(attr("static_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory())
+ .add(attr("dynamic_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory())
+ .add(attr("module_map", LABEL).legacyAllowAnyFileType().cfg(HOST))
+ .add(attr("supports_param_files", BOOLEAN).value(true))
+ .add(attr("supports_header_parsing", BOOLEAN).value(false))
+ // TODO(bazel-team): Should be using the TARGET configuration.
+ .add(attr(":libc_link", LABEL).cfg(HOST).value(LIBC_LINK))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java
new file mode 100644
index 0000000..78a5f89
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java
@@ -0,0 +1,89 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * C++ build info creation - generates header files that contain the corresponding build-info data.
+ */
+public final class CppBuildInfo implements BuildInfoFactory {
+ public static final BuildInfoKey KEY = new BuildInfoKey("C++");
+
+ private static final PathFragment BUILD_INFO_NONVOLATILE_HEADER_NAME =
+ new PathFragment("build-info-nonvolatile.h");
+ private static final PathFragment BUILD_INFO_VOLATILE_HEADER_NAME =
+ new PathFragment("build-info-volatile.h");
+ // TODO(bazel-team): (2011) Get rid of the redacted build info. We should try to make
+ // the linkstamping process handle the case where those values are undefined.
+ private static final PathFragment BUILD_INFO_REDACTED_HEADER_NAME =
+ new PathFragment("build-info-redacted.h");
+
+ @Override
+ public BuildInfoCollection create(BuildInfoContext buildInfoContext, BuildConfiguration config,
+ Artifact buildInfo, Artifact buildChangelist) {
+ List<Action> actions = new ArrayList<>();
+ WriteBuildInfoHeaderAction redactedInfo = getHeader(buildInfoContext, config,
+ BUILD_INFO_REDACTED_HEADER_NAME,
+ Artifact.NO_ARTIFACTS, true, true);
+ WriteBuildInfoHeaderAction nonvolatileInfo = getHeader(buildInfoContext, config,
+ BUILD_INFO_NONVOLATILE_HEADER_NAME,
+ ImmutableList.of(buildInfo),
+ false, true);
+ WriteBuildInfoHeaderAction volatileInfo = getHeader(buildInfoContext, config,
+ BUILD_INFO_VOLATILE_HEADER_NAME,
+ ImmutableList.of(buildChangelist),
+ true, false);
+ actions.add(redactedInfo);
+ actions.add(nonvolatileInfo);
+ actions.add(volatileInfo);
+ return new BuildInfoCollection(actions,
+ ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()),
+ ImmutableList.of(redactedInfo.getPrimaryOutput()));
+ }
+
+ private WriteBuildInfoHeaderAction getHeader(BuildInfoContext buildInfoContext,
+ BuildConfiguration config, PathFragment headerName,
+ Collection<Artifact> inputs,
+ boolean writeVolatileInfo, boolean writeNonVolatileInfo) {
+ Root outputPath = config.getIncludeDirectory();
+ final Artifact header =
+ buildInfoContext.getBuildInfoArtifact(headerName, outputPath,
+ writeVolatileInfo && !inputs.isEmpty()
+ ? BuildInfoType.NO_REBUILD : BuildInfoType.FORCE_REBUILD_IF_CHANGED);
+ return new WriteBuildInfoHeaderAction(
+ inputs, header, writeVolatileInfo, writeNonVolatileInfo);
+ }
+
+ @Override
+ public BuildInfoKey getKey() {
+ return KEY;
+ }
+
+ @Override
+ public boolean isEnabled(BuildConfiguration config) {
+ return config.hasFragment(CppConfiguration.class);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java
new file mode 100644
index 0000000..cf39ef5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java
@@ -0,0 +1,918 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MiddlemanFactory;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Immutable store of information needed for C++ compilation that is aggregated
+ * across dependencies.
+ */
+@Immutable
+public final class CppCompilationContext implements TransitiveInfoProvider {
+ /** An empty compilation context. */
+ public static final CppCompilationContext EMPTY = new Builder(null).build();
+
+ private final CommandLineContext commandLineContext;
+ private final ImmutableList<DepsContext> depsContexts;
+ private final CppModuleMap cppModuleMap;
+ private final Artifact headerModule;
+ private final Artifact picHeaderModule;
+ private final ImmutableSet<Artifact> compilationPrerequisites;
+
+ private CppCompilationContext(CommandLineContext commandLineContext,
+ List<DepsContext> depsContexts, CppModuleMap cppModuleMap, Artifact headerModule,
+ Artifact picHeaderModule) {
+ Preconditions.checkNotNull(commandLineContext);
+ Preconditions.checkArgument(!depsContexts.isEmpty());
+ this.commandLineContext = commandLineContext;
+ this.depsContexts = ImmutableList.copyOf(depsContexts);
+ this.cppModuleMap = cppModuleMap;
+ this.headerModule = headerModule;
+ this.picHeaderModule = picHeaderModule;
+
+ if (depsContexts.size() == 1) {
+ // Only LIPO targets have more than one DepsContexts. This codepath avoids creating
+ // an ImmutableSet.Builder for the vast majority of the cases.
+ compilationPrerequisites = (depsContexts.get(0).compilationPrerequisiteStampFile != null)
+ ? ImmutableSet.<Artifact>of(depsContexts.get(0).compilationPrerequisiteStampFile)
+ : ImmutableSet.<Artifact>of();
+ } else {
+ ImmutableSet.Builder<Artifact> prerequisites = ImmutableSet.builder();
+ for (DepsContext depsContext : depsContexts) {
+ if (depsContext.compilationPrerequisiteStampFile != null) {
+ prerequisites.add(depsContext.compilationPrerequisiteStampFile);
+ }
+ }
+ compilationPrerequisites = prerequisites.build();
+ }
+ }
+
+ /**
+ * Returns the compilation prerequisites consolidated into middlemen
+ * prerequisites, or an empty set if there are no prerequisites.
+ *
+ * <p>For correct dependency tracking, and to reduce the overhead to establish
+ * dependencies on generated headers, we express the dependency on compilation
+ * prerequisites as a transitive dependency via a middleman. After they have
+ * been accumulated (using
+ * {@link Builder#addCompilationPrerequisites(Iterable)},
+ * {@link Builder#mergeDependentContext(CppCompilationContext)}, and
+ * {@link Builder#mergeDependentContexts(Iterable)}, they are consolidated
+ * into a single middleman Artifact when {@link Builder#build()} is called.
+ *
+ * <p>The returned set can be empty if there are no prerequisites. Usually it
+ * contains a single middleman, but if LIPO is used there can be two.
+ */
+ public ImmutableSet<Artifact> getCompilationPrerequisites() {
+ return compilationPrerequisites;
+ }
+
+ /**
+ * Returns the immutable list of include directories to be added with "-I"
+ * (possibly empty but never null). This includes the include dirs from the
+ * transitive deps closure of the target. This list does not contain
+ * duplicates. All fragments are either absolute or relative to the exec root
+ * (see {@link BuildConfiguration#getExecRoot}).
+ */
+ public ImmutableList<PathFragment> getIncludeDirs() {
+ return commandLineContext.includeDirs;
+ }
+
+ /**
+ * Returns the immutable list of include directories to be added with
+ * "-iquote" (possibly empty but never null). This includes the include dirs
+ * from the transitive deps closure of the target. This list does not contain
+ * duplicates. All fragments are either absolute or relative to the exec root
+ * (see {@link BuildConfiguration#getExecRoot}).
+ */
+ public ImmutableList<PathFragment> getQuoteIncludeDirs() {
+ return commandLineContext.quoteIncludeDirs;
+ }
+
+ /**
+ * Returns the immutable list of include directories to be added with
+ * "-isystem" (possibly empty but never null). This includes the include dirs
+ * from the transitive deps closure of the target. This list does not contain
+ * duplicates. All fragments are either absolute or relative to the exec root
+ * (see {@link BuildConfiguration#getExecRoot}).
+ */
+ public ImmutableList<PathFragment> getSystemIncludeDirs() {
+ return commandLineContext.systemIncludeDirs;
+ }
+
+ /**
+ * Returns the immutable set of declared include directories, relative to a
+ * "-I" or "-iquote" directory" (possibly empty but never null). The returned
+ * collection may contain duplicate elements.
+ *
+ * <p>Note: The iteration order of this list is preserved as ide_build_info
+ * writes these directories and sources out and the ordering will help when
+ * used by consumers.
+ */
+ public NestedSet<PathFragment> getDeclaredIncludeDirs() {
+ if (depsContexts.isEmpty()) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (depsContexts.size() == 1) {
+ return depsContexts.get(0).declaredIncludeDirs;
+ }
+
+ NestedSetBuilder<PathFragment> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.declaredIncludeDirs);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Returns the immutable set of include directories, relative to a "-I" or
+ * "-iquote" directory", from which inclusion will produce a warning (possibly
+ * empty but never null). The returned collection may contain duplicate
+ * elements.
+ *
+ * <p>Note: The iteration order of this list is preserved as ide_build_info
+ * writes these directories and sources out and the ordering will help when
+ * used by consumers.
+ */
+ public NestedSet<PathFragment> getDeclaredIncludeWarnDirs() {
+ if (depsContexts.isEmpty()) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (depsContexts.size() == 1) {
+ return depsContexts.get(0).declaredIncludeWarnDirs;
+ }
+
+ NestedSetBuilder<PathFragment> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.declaredIncludeWarnDirs);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Returns the immutable set of headers that have been declared in the
+ * {@code src} or {@code headers attribute} (possibly empty but never null).
+ * The returned collection may contain duplicate elements.
+ *
+ * <p>Note: The iteration order of this list is preserved as ide_build_info
+ * writes these directories and sources out and the ordering will help when
+ * used by consumers.
+ */
+ public NestedSet<Artifact> getDeclaredIncludeSrcs() {
+ if (depsContexts.isEmpty()) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (depsContexts.size() == 1) {
+ return depsContexts.get(0).declaredIncludeSrcs;
+ }
+
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.declaredIncludeSrcs);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Returns the immutable pairs of (header file, pregrepped header file).
+ */
+ public NestedSet<Pair<Artifact, Artifact>> getPregreppedHeaders() {
+ if (depsContexts.isEmpty()) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (depsContexts.size() == 1) {
+ return depsContexts.get(0).pregreppedHdrs;
+ }
+
+ NestedSetBuilder<Pair<Artifact, Artifact>> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.pregreppedHdrs);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Returns the immutable set of additional transitive inputs needed for
+ * compilation, like C++ module map artifacts.
+ */
+ public NestedSet<Artifact> getAdditionalInputs() {
+ if (depsContexts.isEmpty()) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ if (depsContexts.size() == 1) {
+ return depsContexts.get(0).auxiliaryInputs;
+ }
+
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.auxiliaryInputs);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Returns optional inputs that are needed by any C++ compilations that use header modules.
+ *
+ * <p>For every target that the current target depends on transitively and that is built as header
+ * module, contains:
+ * <ul>
+ * <li>the pic/non-pic header module (pcm file)</li>
+ * <li>the transitive list of module maps.</li>
+ * </ul>
+ */
+ private NestedSet<Artifact> getTransitiveAuxiliaryInputs() {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.transitiveAuxiliaryInputs);
+ }
+ return builder.build();
+ }
+
+ /**
+ * @return all modules maps in the transitive closure.
+ */
+ private NestedSet<Artifact> getTransitiveModuleMaps() {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.transitiveModuleMaps);
+ }
+ return builder.build();
+ }
+
+ /**
+ * @return all headers whose transitive closure of includes needs to be
+ * available when compiling anything in the current target.
+ */
+ protected NestedSet<Artifact> getTransitiveHeaderModuleSrcs() {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.transitiveHeaderModuleSrcs);
+ }
+ return builder.build();
+ }
+
+ /**
+ * @return all declared headers of the current module if the current target
+ * is compiled as a module.
+ */
+ protected NestedSet<Artifact> getHeaderModuleSrcs() {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ for (DepsContext depsContext : depsContexts) {
+ builder.addTransitive(depsContext.headerModuleSrcs);
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns the set of defines needed to compile this target (possibly empty
+ * but never null). This includes definitions from the transitive deps closure
+ * for the target. The order of the returned collection is deterministic.
+ */
+ public ImmutableList<String> getDefines() {
+ return commandLineContext.defines;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof CppCompilationContext)) {
+ return false;
+ }
+ CppCompilationContext other = (CppCompilationContext) obj;
+ return Objects.equals(headerModule, other.headerModule)
+ && Objects.equals(picHeaderModule, other.picHeaderModule)
+ && commandLineContext.equals(other.commandLineContext)
+ && depsContexts.equals(other.depsContexts);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(headerModule, picHeaderModule, commandLineContext, depsContexts);
+ }
+
+ /**
+ * Returns a context that is based on a given context but returns empty sets
+ * for {@link #getDeclaredIncludeDirs()} and {@link #getDeclaredIncludeWarnDirs()}.
+ */
+ public static CppCompilationContext disallowUndeclaredHeaders(CppCompilationContext context) {
+ ImmutableList.Builder<DepsContext> builder = ImmutableList.builder();
+ for (DepsContext depsContext : context.depsContexts) {
+ builder.add(new DepsContext(
+ depsContext.compilationPrerequisiteStampFile,
+ NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER),
+ depsContext.declaredIncludeSrcs,
+ depsContext.pregreppedHdrs,
+ depsContext.auxiliaryInputs,
+ depsContext.headerModuleSrcs,
+ depsContext.transitiveAuxiliaryInputs,
+ depsContext.transitiveHeaderModuleSrcs,
+ depsContext.transitiveModuleMaps));
+ }
+ return new CppCompilationContext(context.commandLineContext, builder.build(),
+ context.cppModuleMap, context.headerModule, context.picHeaderModule);
+ }
+
+ /**
+ * Returns the context for a LIPO compile action. This uses the include dirs
+ * and defines of the library, but the declared inclusion dirs/srcs from both
+ * the library and the owner binary.
+
+ * TODO(bazel-team): this might make every LIPO target have an unnecessary large set of
+ * inclusion dirs/srcs. The correct behavior would be to merge only the contexts
+ * of actual referred targets (as listed in .imports file).
+ *
+ * <p>Undeclared inclusion checking ({@link #getDeclaredIncludeDirs()},
+ * {@link #getDeclaredIncludeWarnDirs()}, and
+ * {@link #getDeclaredIncludeSrcs()}) needs to use the union of the contexts
+ * of the involved source files.
+ *
+ * <p>For include and define command line flags ({@link #getIncludeDirs()}
+ * {@link #getQuoteIncludeDirs()}, {@link #getSystemIncludeDirs()}, and
+ * {@link #getDefines()}) LIPO compilations use the same values as non-LIPO
+ * compilation.
+ *
+ * <p>Include scanning is not handled by this method. See
+ * {@code IncludeScannable#getAuxiliaryScannables()} instead.
+ *
+ * @param ownerContext the compilation context of the owner binary
+ * @param libContext the compilation context of the library
+ */
+ public static CppCompilationContext mergeForLipo(CppCompilationContext ownerContext,
+ CppCompilationContext libContext) {
+ return new CppCompilationContext(libContext.commandLineContext,
+ ImmutableList.copyOf(Iterables.concat(ownerContext.depsContexts, libContext.depsContexts)),
+ libContext.cppModuleMap, libContext.headerModule, libContext.picHeaderModule);
+ }
+
+ /**
+ * @return the C++ module map of the owner.
+ */
+ public CppModuleMap getCppModuleMap() {
+ return cppModuleMap;
+ }
+
+ /**
+ * @return the non-pic C++ header module of the owner.
+ */
+ private Artifact getHeaderModule() {
+ return headerModule;
+ }
+
+ /**
+ * @return the pic C++ header module of the owner.
+ */
+ private Artifact getPicHeaderModule() {
+ return picHeaderModule;
+ }
+
+ /**
+ * The parts of the compilation context that influence the command line of
+ * compilation actions.
+ */
+ @Immutable
+ private static class CommandLineContext {
+ private final ImmutableList<PathFragment> includeDirs;
+ private final ImmutableList<PathFragment> quoteIncludeDirs;
+ private final ImmutableList<PathFragment> systemIncludeDirs;
+ private final ImmutableList<String> defines;
+
+ CommandLineContext(ImmutableList<PathFragment> includeDirs,
+ ImmutableList<PathFragment> quoteIncludeDirs,
+ ImmutableList<PathFragment> systemIncludeDirs,
+ ImmutableList<String> defines) {
+ this.includeDirs = includeDirs;
+ this.quoteIncludeDirs = quoteIncludeDirs;
+ this.systemIncludeDirs = systemIncludeDirs;
+ this.defines = defines;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof CommandLineContext)) {
+ return false;
+ }
+ CommandLineContext other = (CommandLineContext) obj;
+ return Objects.equals(includeDirs, other.includeDirs)
+ && Objects.equals(quoteIncludeDirs, other.quoteIncludeDirs)
+ && Objects.equals(systemIncludeDirs, other.systemIncludeDirs)
+ && Objects.equals(defines, other.defines);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(includeDirs, quoteIncludeDirs, systemIncludeDirs, defines);
+ }
+ }
+
+ /**
+ * The parts of the compilation context that defined the dependencies of
+ * actions of scheduling and inclusion validity checking.
+ */
+ @Immutable
+ private static class DepsContext {
+ private final Artifact compilationPrerequisiteStampFile;
+ private final NestedSet<PathFragment> declaredIncludeDirs;
+ private final NestedSet<PathFragment> declaredIncludeWarnDirs;
+ private final NestedSet<Artifact> declaredIncludeSrcs;
+ private final NestedSet<Pair<Artifact, Artifact>> pregreppedHdrs;
+
+ /**
+ * Optional inputs that are used by some forms of compilation, containing:
+ * <ul>
+ * <li>module map of the current target</li>
+ * <li>module maps of all direct dependencies that are not compiled as header modules</li>
+ * <li>all transitiveAuxiliaryInputs.</li>
+ * </ul>
+ */
+ private final NestedSet<Artifact> auxiliaryInputs;
+
+ /**
+ * All declared headers of the current module, if compiled as a header module.
+ */
+ private final NestedSet<Artifact> headerModuleSrcs;
+
+ private final NestedSet<Artifact> transitiveAuxiliaryInputs;
+
+ /**
+ * Headers whose transitive closure of includes needs to be available when compiling the current
+ * target. For every target that the current target depends on transitively and that is built as
+ * header module, contains all headers that are part of its header module.
+ */
+ private final NestedSet<Artifact> transitiveHeaderModuleSrcs;
+
+ /**
+ * The module maps from all targets the current target depends on transitively.
+ */
+ private final NestedSet<Artifact> transitiveModuleMaps;
+
+ DepsContext(Artifact compilationPrerequisiteStampFile,
+ NestedSet<PathFragment> declaredIncludeDirs,
+ NestedSet<PathFragment> declaredIncludeWarnDirs,
+ NestedSet<Artifact> declaredIncludeSrcs,
+ NestedSet<Pair<Artifact, Artifact>> pregreppedHdrs,
+ NestedSet<Artifact> auxiliaryInputs,
+ NestedSet<Artifact> headerModuleSrcs,
+ NestedSet<Artifact> transitiveAuxiliaryInputs,
+ NestedSet<Artifact> transitiveHeaderModuleSrcs,
+ NestedSet<Artifact> transitiveModuleMaps) {
+ this.compilationPrerequisiteStampFile = compilationPrerequisiteStampFile;
+ this.declaredIncludeDirs = declaredIncludeDirs;
+ this.declaredIncludeWarnDirs = declaredIncludeWarnDirs;
+ this.declaredIncludeSrcs = declaredIncludeSrcs;
+ this.pregreppedHdrs = pregreppedHdrs;
+ this.auxiliaryInputs = auxiliaryInputs;
+ this.headerModuleSrcs = headerModuleSrcs;
+ this.transitiveAuxiliaryInputs = transitiveAuxiliaryInputs;
+ this.transitiveHeaderModuleSrcs = transitiveHeaderModuleSrcs;
+ this.transitiveModuleMaps = transitiveModuleMaps;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof DepsContext)) {
+ return false;
+ }
+ DepsContext other = (DepsContext) obj;
+ return Objects.equals(
+ compilationPrerequisiteStampFile, other.compilationPrerequisiteStampFile)
+ && Objects.equals(declaredIncludeDirs, other.declaredIncludeDirs)
+ && Objects.equals(declaredIncludeWarnDirs, other.declaredIncludeWarnDirs)
+ && Objects.equals(declaredIncludeSrcs, other.declaredIncludeSrcs)
+ && Objects.equals(auxiliaryInputs, other.auxiliaryInputs)
+ && Objects.equals(headerModuleSrcs, other.headerModuleSrcs)
+ // Due to the NestedSet equals being ==, and the code flow only setting them if at least
+ // auxiliaryInputs is set, these checks cannot be executed. We leave them in so the equals
+ // is still correct if that connection ever changes.R
+ && Objects.equals(transitiveAuxiliaryInputs, other.transitiveAuxiliaryInputs)
+ && Objects.equals(transitiveHeaderModuleSrcs, other.transitiveHeaderModuleSrcs)
+ && Objects.equals(transitiveModuleMaps, other.transitiveModuleMaps)
+ ;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(compilationPrerequisiteStampFile,
+ declaredIncludeDirs,
+ declaredIncludeWarnDirs,
+ declaredIncludeSrcs,
+ auxiliaryInputs,
+ headerModuleSrcs,
+ transitiveAuxiliaryInputs,
+ transitiveHeaderModuleSrcs,
+ transitiveModuleMaps);
+ }
+ }
+
+ /**
+ * Builder class for {@link CppCompilationContext}.
+ */
+ public static class Builder {
+ private String purpose = "cpp_compilation_prerequisites";
+ private final Set<Artifact> compilationPrerequisites = new LinkedHashSet<>();
+ private final Set<PathFragment> includeDirs = new LinkedHashSet<>();
+ private final Set<PathFragment> quoteIncludeDirs = new LinkedHashSet<>();
+ private final Set<PathFragment> systemIncludeDirs = new LinkedHashSet<>();
+ private final NestedSetBuilder<PathFragment> declaredIncludeDirs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<PathFragment> declaredIncludeWarnDirs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Artifact> declaredIncludeSrcs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Pair<Artifact, Artifact>> pregreppedHdrs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Artifact> auxiliaryInputs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Artifact> headerModuleSrcs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Artifact> transitiveAuxiliaryInputs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Artifact> transitiveHeaderModuleSrcs =
+ NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<Artifact> transitiveModuleMaps =
+ NestedSetBuilder.stableOrder();
+ private final Set<String> defines = new LinkedHashSet<>();
+ private CppModuleMap cppModuleMap;
+ private Artifact headerModule;
+ private Artifact picHeaderModule;
+
+ /** The rule that owns the context */
+ private final RuleContext ruleContext;
+
+ /**
+ * Creates a new builder for a {@link CppCompilationContext} instance.
+ */
+ public Builder(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ /**
+ * Overrides the purpose of this context. This is useful if a Target
+ * needs more than one CppCompilationContext. (The purpose is used to
+ * construct the name of the prerequisites middleman for the context, and
+ * all artifacts for a given Target must have distinct names.)
+ *
+ * @param purpose must be a string which is suitable for use as a filename.
+ * A single rule may have many middlemen with distinct purposes.
+ *
+ * @see MiddlemanFactory#createErrorPropagatingMiddleman
+ */
+ public Builder setPurpose(String purpose) {
+ this.purpose = purpose;
+ return this;
+ }
+
+ public String getPurpose() {
+ return purpose;
+ }
+
+ /**
+ * Merges the context of a dependency into this one by adding the contents
+ * of all of its attributes.
+ */
+ public Builder mergeDependentContext(CppCompilationContext otherContext) {
+ Preconditions.checkNotNull(otherContext);
+ compilationPrerequisites.addAll(otherContext.getCompilationPrerequisites());
+ includeDirs.addAll(otherContext.getIncludeDirs());
+ quoteIncludeDirs.addAll(otherContext.getQuoteIncludeDirs());
+ systemIncludeDirs.addAll(otherContext.getSystemIncludeDirs());
+ declaredIncludeDirs.addTransitive(otherContext.getDeclaredIncludeDirs());
+ declaredIncludeWarnDirs.addTransitive(otherContext.getDeclaredIncludeWarnDirs());
+ declaredIncludeSrcs.addTransitive(otherContext.getDeclaredIncludeSrcs());
+ pregreppedHdrs.addTransitive(otherContext.getPregreppedHeaders());
+
+ // Forward transitive information.
+ transitiveAuxiliaryInputs.addTransitive(otherContext.getTransitiveAuxiliaryInputs());
+ transitiveModuleMaps.addTransitive(otherContext.getTransitiveModuleMaps());
+ transitiveHeaderModuleSrcs.addTransitive(otherContext.getTransitiveHeaderModuleSrcs());
+
+ // All module maps of direct dependencies are inputs to the current compile independently of
+ // the build type.
+ if (otherContext.getCppModuleMap() != null) {
+ auxiliaryInputs.add(otherContext.getCppModuleMap().getArtifact());
+ }
+ if (otherContext.getHeaderModule() != null || otherContext.getPicHeaderModule() != null) {
+ // If we depend directly on a target that has a compiled header module, all targets
+ // transitively depending on us will need that header module, and all transitive module
+ // maps.
+ if (otherContext.getHeaderModule() != null) {
+ transitiveAuxiliaryInputs.add(otherContext.getHeaderModule());
+ }
+ if (otherContext.getPicHeaderModule() != null) {
+ transitiveAuxiliaryInputs.add(otherContext.getPicHeaderModule());
+ }
+ transitiveAuxiliaryInputs.addAll(otherContext.getTransitiveModuleMaps());
+
+ // All targets transitively depending on us will need to have the full transitive #include
+ // closure of the headers in that module available.
+ transitiveHeaderModuleSrcs.addAll(otherContext.getHeaderModuleSrcs());
+ }
+ // All compile actions in the current target will need the transitive inputs.
+ auxiliaryInputs.addAll(transitiveAuxiliaryInputs.build().toCollection());
+
+ defines.addAll(otherContext.getDefines());
+ return this;
+ }
+
+ /**
+ * Merges the context of some targets into this one by adding the contents
+ * of all of their attributes. Targets that do not implement
+ * {@link CppCompilationContext} are ignored.
+ */
+ public Builder mergeDependentContexts(Iterable<CppCompilationContext> targets) {
+ for (CppCompilationContext target : targets) {
+ mergeDependentContext(target);
+ }
+ return this;
+ }
+
+ /**
+ * Adds multiple compilation prerequisites.
+ */
+ public Builder addCompilationPrerequisites(Iterable<Artifact> prerequisites) {
+ // LIPO collector must not add compilation prerequisites in order to avoid
+ // the creation of a middleman action.
+ Iterables.addAll(compilationPrerequisites, prerequisites);
+ return this;
+ }
+
+ /**
+ * Add a single include directory to be added with "-I". It can be either
+ * relative to the exec root (see {@link BuildConfiguration#getExecRoot}) or
+ * absolute. Before it is stored, the include directory is normalized.
+ */
+ public Builder addIncludeDir(PathFragment includeDir) {
+ includeDirs.add(includeDir.normalize());
+ return this;
+ }
+
+ /**
+ * Add multiple include directories to be added with "-I". These can be
+ * either relative to the exec root (see {@link
+ * BuildConfiguration#getExecRoot}) or absolute. The entries are normalized
+ * before they are stored.
+ */
+ public Builder addIncludeDirs(Iterable<PathFragment> includeDirs) {
+ for (PathFragment includeDir : includeDirs) {
+ addIncludeDir(includeDir);
+ }
+ return this;
+ }
+
+ /**
+ * Add a single include directory to be added with "-iquote". It can be
+ * either relative to the exec root (see {@link
+ * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the
+ * include directory is normalized.
+ */
+ public Builder addQuoteIncludeDir(PathFragment quoteIncludeDir) {
+ quoteIncludeDirs.add(quoteIncludeDir.normalize());
+ return this;
+ }
+
+ /**
+ * Add a single include directory to be added with "-isystem". It can be
+ * either relative to the exec root (see {@link
+ * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the
+ * include directory is normalized.
+ */
+ public Builder addSystemIncludeDir(PathFragment systemIncludeDir) {
+ systemIncludeDirs.add(systemIncludeDir.normalize());
+ return this;
+ }
+
+ /**
+ * Add a single declared include dir, relative to a "-I" or "-iquote"
+ * directory".
+ */
+ public Builder addDeclaredIncludeDir(PathFragment dir) {
+ declaredIncludeDirs.add(dir);
+ return this;
+ }
+
+ /**
+ * Add a single declared include directory, relative to a "-I" or "-iquote"
+ * directory", from which inclusion will produce a warning.
+ */
+ public Builder addDeclaredIncludeWarnDir(PathFragment dir) {
+ declaredIncludeWarnDirs.add(dir);
+ return this;
+ }
+
+ /**
+ * Adds a header that has been declared in the {@code src} or {@code headers attribute}. The
+ * header will also be added to the compilation prerequisites.
+ */
+ public Builder addDeclaredIncludeSrc(Artifact header) {
+ declaredIncludeSrcs.add(header);
+ compilationPrerequisites.add(header);
+ headerModuleSrcs.add(header);
+ return this;
+ }
+
+ /**
+ * Adds multiple headers that have been declared in the {@code src} or {@code headers
+ * attribute}. The headers will also be added to the compilation prerequisites.
+ */
+ public Builder addDeclaredIncludeSrcs(Iterable<Artifact> declaredIncludeSrcs) {
+ this.declaredIncludeSrcs.addAll(declaredIncludeSrcs);
+ this.headerModuleSrcs.addAll(declaredIncludeSrcs);
+ return addCompilationPrerequisites(declaredIncludeSrcs);
+ }
+
+ /**
+ * Add a map of generated source or header Artifact to an output Artifact after grepping
+ * the file for include statements.
+ */
+ public Builder addPregreppedHeaderMap(Map<Artifact, Artifact> pregrepped) {
+ addCompilationPrerequisites(pregrepped.values());
+ for (Map.Entry<Artifact, Artifact> entry : pregrepped.entrySet()) {
+ this.pregreppedHdrs.add(Pair.of(entry.getKey(), entry.getValue()));
+ }
+ return this;
+ }
+
+ /**
+ * Adds a single define.
+ */
+ public Builder addDefine(String define) {
+ defines.add(define);
+ return this;
+ }
+
+ /**
+ * Adds multiple defines.
+ */
+ public Builder addDefines(Iterable<String> defines) {
+ Iterables.addAll(this.defines, defines);
+ return this;
+ }
+
+ /**
+ * Sets the C++ module map.
+ */
+ public Builder setCppModuleMap(CppModuleMap cppModuleMap) {
+ this.cppModuleMap = cppModuleMap;
+ return this;
+ }
+
+ /**
+ * Sets the C++ header module in non-pic mode.
+ */
+ public Builder setHeaderModule(Artifact headerModule) {
+ this.headerModule = headerModule;
+ return this;
+ }
+
+ /**
+ * Sets the C++ header module in pic mode.
+ */
+ public Builder setPicHeaderModule(Artifact picHeaderModule) {
+ this.picHeaderModule = picHeaderModule;
+ return this;
+ }
+
+ /**
+ * Builds the {@link CppCompilationContext}.
+ */
+ public CppCompilationContext build() {
+ return build(
+ ruleContext == null ? null : ruleContext.getActionOwner(),
+ ruleContext == null ? null : ruleContext.getAnalysisEnvironment().getMiddlemanFactory());
+ }
+
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public CppCompilationContext build(ActionOwner owner, MiddlemanFactory middlemanFactory) {
+ if (cppModuleMap != null) {
+ // .cppmap files should also be mandatory inputs for compile actions
+ auxiliaryInputs.add(cppModuleMap.getArtifact());
+ transitiveModuleMaps.add(cppModuleMap.getArtifact());
+ }
+
+ // We don't create middlemen in LIPO collector subtree, because some target CT
+ // will do that instead.
+ Artifact prerequisiteStampFile = (ruleContext != null
+ && ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector())
+ ? getMiddlemanArtifact(middlemanFactory)
+ : createMiddleman(owner, middlemanFactory);
+
+ return new CppCompilationContext(
+ new CommandLineContext(ImmutableList.copyOf(includeDirs),
+ ImmutableList.copyOf(quoteIncludeDirs), ImmutableList.copyOf(systemIncludeDirs),
+ ImmutableList.copyOf(defines)),
+ ImmutableList.of(new DepsContext(prerequisiteStampFile,
+ declaredIncludeDirs.build(),
+ declaredIncludeWarnDirs.build(),
+ declaredIncludeSrcs.build(),
+ pregreppedHdrs.build(),
+ auxiliaryInputs.build(),
+ headerModuleSrcs.build(),
+ transitiveAuxiliaryInputs.build(),
+ transitiveHeaderModuleSrcs.build(),
+ transitiveModuleMaps.build())),
+ cppModuleMap,
+ headerModule,
+ picHeaderModule);
+ }
+
+ /**
+ * Creates a middleman for the compilation prerequisites.
+ *
+ * @return the middleman or null if there are no prerequisites
+ */
+ private Artifact createMiddleman(ActionOwner owner,
+ MiddlemanFactory middlemanFactory) {
+ if (compilationPrerequisites.isEmpty()) {
+ return null;
+ }
+
+ // Compilation prerequisites gathered in the compilationPrerequisites
+ // must be generated prior to executing C++ compilation step that depends
+ // on them (since these prerequisites include all potential header files, etc
+ // that could be referenced during compilation). So there is a definite need
+ // to ensure scheduling edge dependency. However, those prerequisites should
+ // have no effect on the decision whether C++ compilation should happen in
+ // the first place - only CppCompileAction outputs (*.o and *.d files) and
+ // all files referenced by the *.d file should be used to make that decision.
+ // If this action was never executed, then *.d file would be missing, forcing
+ // compilation to occur. If *.d file is present and has not changed then the
+ // only reason that would force us to re-compile would be change in one of
+ // the files referenced by the *.d file, since no other files participated
+ // in the compilation. We also need to propagate errors through this
+ // dependency link. So we use an error propagating middleman.
+ // Such middleman will be ignored by the dependency checker yet will still
+ // represent an edge in the action dependency graph - forcing proper execution
+ // order and error propagation.
+ return middlemanFactory.createErrorPropagatingMiddleman(
+ owner, ruleContext.getLabel().toString(), purpose,
+ ImmutableList.copyOf(compilationPrerequisites),
+ ruleContext.getConfiguration().getMiddlemanDirectory());
+ }
+
+ /**
+ * Returns the same set of artifacts as createMiddleman() would, but without
+ * actually creating middlemen.
+ */
+ private Artifact getMiddlemanArtifact(MiddlemanFactory middlemanFactory) {
+ if (compilationPrerequisites.isEmpty()) {
+ return null;
+ }
+
+ return middlemanFactory.getErrorPropagatingMiddlemanArtifact(ruleContext.getLabel()
+ .toString(), purpose, ruleContext.getConfiguration().getMiddlemanDirectory());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
new file mode 100644
index 0000000..e90f9f7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
@@ -0,0 +1,1356 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander;
+import com.google.devtools.build.lib.actions.ArtifactResolver;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.extra.CppCompileInfo;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.Tool;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.DependencySet;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * Action that represents some kind of C++ compilation step.
+ */
+@ThreadCompatible
+public class CppCompileAction extends AbstractAction implements IncludeScannable {
+ /**
+ * Represents logic that determines which artifacts, if any, should be added to the actual inputs
+ * for each included file (in addition to the included file itself)
+ */
+ public interface IncludeResolver {
+ /**
+ * Returns the set of files to be added for an included file (as returned in the .d file)
+ */
+ Iterable<Artifact> getInputsForIncludedFile(
+ Artifact includedFile, ArtifactResolver artifactResolver);
+ }
+
+ public static final IncludeResolver VOID_INCLUDE_RESOLVER = new IncludeResolver() {
+ @Override
+ public Iterable<Artifact> getInputsForIncludedFile(Artifact includedFile,
+ ArtifactResolver artifactResolver) {
+ return ImmutableList.of();
+ }
+ };
+
+ private static final int VALIDATION_DEBUG = 0; // 0==none, 1==warns/errors, 2==all
+ private static final boolean VALIDATION_DEBUG_WARN = VALIDATION_DEBUG >= 1;
+
+ /**
+ * A string constant for the c compilation action.
+ */
+ public static final String C_COMPILE = "c-compile";
+
+ /**
+ * A string constant for the c++ compilation action.
+ */
+ public static final String CPP_COMPILE = "c++-compile";
+
+ /**
+ * A string constant for the c++ header parsing.
+ */
+ public static final String CPP_HEADER_PARSING = "c++-header-parsing";
+
+ /**
+ * A string constant for the c++ header preprocessing.
+ */
+ public static final String CPP_HEADER_PREPROCESSING = "c++-header-preprocessing";
+
+ /**
+ * A string constant for the c++ module compilation action.
+ * Note: currently we don't support C module compilation.
+ */
+ public static final String CPP_MODULE_COMPILE = "c++-module-compile";
+
+ /**
+ * A string constant for the preprocessing assembler action.
+ */
+ public static final String PREPROCESS_ASSEMBLE = "preprocess-assemble";
+
+
+ private final BuildConfiguration configuration;
+ protected final Artifact outputFile;
+ private final Label sourceLabel;
+ private final Artifact dwoFile;
+ private final Artifact optionalSourceFile;
+ private final NestedSet<Artifact> mandatoryInputs;
+ private final CppCompilationContext context;
+ private final Collection<PathFragment> extraSystemIncludePrefixes;
+ private final Iterable<IncludeScannable> lipoScannables;
+ private final CppCompileCommandLine cppCompileCommandLine;
+ private final boolean enableLayeringCheck;
+ private final boolean compileHeaderModules;
+
+ @VisibleForTesting
+ final CppConfiguration cppConfiguration;
+ private final Class<? extends CppCompileActionContext> actionContext;
+ private final IncludeResolver includeResolver;
+
+ /**
+ * Identifier for the actual execution time behavior of the action.
+ *
+ * <p>Required because the behavior of this class can be modified by injecting code in the
+ * constructor or by inheritance, and we want to have different cache keys for those.
+ */
+ private final UUID actionClassId;
+
+ private boolean inputsKnown = false;
+
+ /**
+ * Set when the action prepares for execution. Used to preserve state between preparation and
+ * execution.
+ */
+ private Collection<? extends ActionInput> additionalInputs = null;
+
+ /**
+ * Creates a new action to compile C/C++ source files.
+ *
+ * @param owner the owner of the action, usually the configured target that
+ * emitted it
+ * @param sourceFile the source file that should be compiled. {@code mandatoryInputs} must
+ * contain this file
+ * @param sourceLabel the label of the rule the source file is generated by
+ * @param mandatoryInputs any additional files that need to be present for the
+ * compilation to succeed, can be empty but not null, for example, extra sources for FDO.
+ * @param outputFile the object file that is written as result of the
+ * compilation, or the fake object for {@link FakeCppCompileAction}s
+ * @param dotdFile the .d file that is generated as a side-effect of
+ * compilation
+ * @param gcnoFile the coverage notes that are written in coverage mode, can
+ * be null
+ * @param dwoFile the .dwo output file where debug information is stored for Fission
+ * builds (null if Fission mode is disabled)
+ * @param optionalSourceFile an additional optional source file (null if unneeded)
+ * @param configuration the build configurations
+ * @param context the compilation context
+ * @param copts options for the compiler
+ * @param coptsFilter regular expression to remove options from {@code copts}
+ * @param compileHeaderModules whether to compile C++ header modules
+ */
+ protected CppCompileAction(ActionOwner owner,
+ // TODO(bazel-team): Eventually we will remove 'features'; all functionality in 'features'
+ // will be provided by 'featureConfiguration'.
+ ImmutableList<String> features,
+ FeatureConfiguration featureConfiguration,
+ Artifact sourceFile,
+ Label sourceLabel,
+ NestedSet<Artifact> mandatoryInputs,
+ Artifact outputFile,
+ DotdFile dotdFile,
+ @Nullable Artifact gcnoFile,
+ @Nullable Artifact dwoFile,
+ Artifact optionalSourceFile,
+ BuildConfiguration configuration,
+ CppConfiguration cppConfiguration,
+ CppCompilationContext context,
+ Class<? extends CppCompileActionContext> actionContext,
+ ImmutableList<String> copts,
+ ImmutableList<String> pluginOpts,
+ Predicate<String> coptsFilter,
+ ImmutableList<PathFragment> extraSystemIncludePrefixes,
+ boolean enableLayeringCheck,
+ @Nullable String fdoBuildStamp,
+ IncludeResolver includeResolver,
+ Iterable<IncludeScannable> lipoScannables,
+ UUID actionClassId,
+ boolean compileHeaderModules) {
+ // getInputs() method is overridden in this class so we pass a dummy empty
+ // list to the AbstractAction constructor in place of a real input collection.
+ super(owner,
+ Artifact.NO_ARTIFACTS,
+ CollectionUtils.asListWithoutNulls(outputFile, dotdFile.artifact(),
+ gcnoFile, dwoFile));
+ this.configuration = configuration;
+ this.sourceLabel = sourceLabel;
+ this.outputFile = Preconditions.checkNotNull(outputFile);
+ this.dwoFile = dwoFile;
+ this.optionalSourceFile = optionalSourceFile;
+ this.context = context;
+ this.extraSystemIncludePrefixes = extraSystemIncludePrefixes;
+ this.enableLayeringCheck = enableLayeringCheck;
+ this.includeResolver = includeResolver;
+ this.cppConfiguration = cppConfiguration;
+ if (cppConfiguration != null && !cppConfiguration.shouldScanIncludes()) {
+ inputsKnown = true;
+ }
+ this.cppCompileCommandLine = new CppCompileCommandLine(sourceFile, dotdFile,
+ context.getCppModuleMap(), copts, coptsFilter, pluginOpts,
+ (gcnoFile != null), features, featureConfiguration, fdoBuildStamp);
+ this.actionContext = actionContext;
+ this.lipoScannables = lipoScannables;
+ this.actionClassId = actionClassId;
+ this.compileHeaderModules = compileHeaderModules;
+
+ // We do not need to include the middleman artifact since it is a generated
+ // artifact and will definitely exist prior to this action execution.
+ this.mandatoryInputs = mandatoryInputs;
+ setInputs(createInputs(mandatoryInputs, context.getCompilationPrerequisites(),
+ optionalSourceFile));
+ }
+
+ private static NestedSet<Artifact> createInputs(
+ NestedSet<Artifact> mandatoryInputs,
+ Set<Artifact> prerequisites, Artifact optionalSourceFile) {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ if (optionalSourceFile != null) {
+ builder.add(optionalSourceFile);
+ }
+ builder.addAll(prerequisites);
+ builder.addTransitive(mandatoryInputs);
+ return builder.build();
+ }
+
+ public boolean shouldScanIncludes() {
+ return cppConfiguration.shouldScanIncludes();
+ }
+
+ @Override
+ public List<PathFragment> getBuiltInIncludeDirectories() {
+ return cppConfiguration.getBuiltInIncludeDirectories();
+ }
+
+ public String getHostSystemName() {
+ return cppConfiguration.getHostSystemName();
+ }
+
+ @Override
+ public NestedSet<Artifact> getMandatoryInputs() {
+ return mandatoryInputs;
+ }
+
+ @Override
+ public boolean inputsKnown() {
+ return inputsKnown;
+ }
+
+ /**
+ * Returns the list of additional inputs found by dependency discovery, during action preparation,
+ * and clears the stored list. {@link #prepare} must be called before this method is called, on
+ * each action execution.
+ */
+ public Collection<? extends ActionInput> getAdditionalInputs() {
+ Collection<? extends ActionInput> result = Preconditions.checkNotNull(additionalInputs);
+ additionalInputs = null;
+ return result;
+ }
+
+ @Override
+ public boolean discoversInputs() {
+ return true;
+ }
+
+ @Override
+ public void discoverInputs(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ try {
+ this.additionalInputs = executor.getContext(CppCompileActionContext.class)
+ .findAdditionalInputs(this, actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException("Include scanning of rule '" + getOwner().getLabel() + "'",
+ executor.getVerboseFailures(), this);
+ }
+ }
+
+ @Override
+ public Artifact getPrimaryInput() {
+ return getSourceFile();
+ }
+
+ @Override
+ public Artifact getPrimaryOutput() {
+ return getOutputFile();
+ }
+
+ /**
+ * Returns the path of the c/cc source for gcc.
+ */
+ public final Artifact getSourceFile() {
+ return cppCompileCommandLine.sourceFile;
+ }
+
+ /**
+ * Returns the path where gcc should put its result.
+ */
+ public Artifact getOutputFile() {
+ return outputFile;
+ }
+
+ /**
+ * Returns the path of the debug info output file (when debug info is
+ * spliced out of the .o file via fission).
+ */
+ @Nullable
+ Artifact getDwoFile() {
+ return dwoFile;
+ }
+
+ protected PathFragment getInternalOutputFile() {
+ return outputFile.getExecPath();
+ }
+
+ @VisibleForTesting
+ public List<String> getPluginOpts() {
+ return cppCompileCommandLine.pluginOpts;
+ }
+
+ Collection<PathFragment> getExtraSystemIncludePrefixes() {
+ return extraSystemIncludePrefixes;
+ }
+
+ @Override
+ public Map<Artifact, Path> getLegalGeneratedScannerFileMap() {
+ Map<Artifact, Path> legalOuts = new HashMap<>();
+
+ for (Artifact a : context.getDeclaredIncludeSrcs()) {
+ if (!a.isSourceArtifact()) {
+ legalOuts.put(a, null);
+ }
+ }
+ for (Pair<Artifact, Artifact> pregreppedSrcs : context.getPregreppedHeaders()) {
+ Artifact hdr = pregreppedSrcs.getFirst();
+ Preconditions.checkState(!hdr.isSourceArtifact(), hdr);
+ legalOuts.put(hdr, pregreppedSrcs.getSecond().getPath());
+ }
+ return Collections.unmodifiableMap(legalOuts);
+ }
+
+ /**
+ * Returns the path where gcc should put the discovered dependency
+ * information.
+ */
+ public DotdFile getDotdFile() {
+ return cppCompileCommandLine.dotdFile;
+ }
+
+ protected boolean needsIncludeScanning(Executor executor) {
+ return executor.getContext(actionContext).needsIncludeScanning();
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return executor.getContext(actionContext).strategyLocality();
+ }
+
+ @VisibleForTesting
+ public CppCompilationContext getContext() {
+ return context;
+ }
+
+ @Override
+ public List<PathFragment> getQuoteIncludeDirs() {
+ return context.getQuoteIncludeDirs();
+ }
+
+ @Override
+ public List<PathFragment> getIncludeDirs() {
+ ImmutableList.Builder<PathFragment> result = ImmutableList.builder();
+ result.addAll(context.getIncludeDirs());
+ for (String opt : cppCompileCommandLine.copts) {
+ if (opt.startsWith("-I") && opt.length() > 2) {
+ // We insist on the combined form "-Idir".
+ result.add(new PathFragment(opt.substring(2)));
+ }
+ }
+ return result.build();
+ }
+
+ @Override
+ public List<PathFragment> getSystemIncludeDirs() {
+ ImmutableList.Builder<PathFragment> result = ImmutableList.builder();
+ result.addAll(context.getSystemIncludeDirs());
+ for (String opt : cppCompileCommandLine.copts) {
+ if (opt.startsWith("-isystem") && opt.length() > 8) {
+ // We insist on the combined form "-isystemdir".
+ result.add(new PathFragment(opt.substring(8)));
+ }
+ }
+ return result.build();
+ }
+
+ @Override
+ public List<String> getCmdlineIncludes() {
+ ImmutableList.Builder<String> cmdlineIncludes = ImmutableList.builder();
+ List<String> args = getArgv();
+ for (Iterator<String> argi = args.iterator(); argi.hasNext();) {
+ String arg = argi.next();
+ if (arg.equals("-include") && argi.hasNext()) {
+ cmdlineIncludes.add(argi.next());
+ }
+ }
+ return cmdlineIncludes.build();
+ }
+
+ @Override
+ public Collection<Artifact> getIncludeScannerSources() {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ // For every header module we use for the build we need the set of sources that it can
+ // reference.
+ builder.addAll(context.getTransitiveHeaderModuleSrcs());
+ if (CppFileTypes.CPP_MODULE_MAP.matches(getSourceFile().getPath())) {
+ // If this is an action that compiles the header module itself, the source we build is the
+ // module map, and we need to include-scan all headers that are referenced in the module map.
+ // We need to do include scanning as long as we want to support building code bases that are
+ // not fully strict layering clean.
+ builder.addAll(context.getHeaderModuleSrcs());
+ } else {
+ builder.add(getSourceFile());
+ }
+ return builder.build().toCollection();
+ }
+
+ @Override
+ public Iterable<IncludeScannable> getAuxiliaryScannables() {
+ return lipoScannables;
+ }
+
+ /**
+ * Returns the list of "-D" arguments that should be used by this gcc
+ * invocation. Only used for testing.
+ */
+ @VisibleForTesting
+ public ImmutableCollection<String> getDefines() {
+ return context.getDefines();
+ }
+
+ /**
+ * Returns an (immutable) map of environment key, value pairs to be
+ * provided to the C++ compiler.
+ */
+ public ImmutableMap<String, String> getEnvironment() {
+ Map<String, String> environment =
+ new LinkedHashMap<>(configuration.getDefaultShellEnvironment());
+ if (configuration.isCodeCoverageEnabled()) {
+ environment.put("PWD", "/proc/self/cwd");
+ }
+ if (OS.getCurrent() == OS.WINDOWS) {
+ // TODO(bazel-team): Both GCC and clang rely on their execution directories being on
+ // PATH, otherwise they fail to find dependent DLLs (and they fail silently...). On
+ // the other hand, Windows documentation says that the directory of the executable
+ // is always searched for DLLs first. Not sure what to make of it.
+ // Other options are to forward the system path (brittle), or to add a PATH field to
+ // the crosstool file.
+ environment.put("PATH", cppConfiguration.getToolPathFragment(Tool.GCC).getParentDirectory()
+ .getPathString());
+ }
+ return ImmutableMap.copyOf(environment);
+ }
+
+ /**
+ * Returns a new, mutable list of command and arguments (argv) to be passed
+ * to the gcc subprocess.
+ */
+ public final List<String> getArgv() {
+ return getArgv(getInternalOutputFile());
+ }
+
+ protected final List<String> getArgv(PathFragment outputFile) {
+ return cppCompileCommandLine.getArgv(outputFile);
+ }
+
+ @Override
+ public ExtraActionInfo.Builder getExtraActionInfo() {
+ CppCompileInfo.Builder info = CppCompileInfo.newBuilder();
+ info.setTool(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString());
+ for (String option : getCompilerOptions()) {
+ info.addCompilerOption(option);
+ }
+ info.setOutputFile(outputFile.getExecPathString());
+ info.setSourceFile(getSourceFile().getExecPathString());
+ if (inputsKnown()) {
+ info.addAllSourcesAndHeaders(Artifact.toExecPaths(getInputs()));
+ } else {
+ info.addSourcesAndHeaders(getSourceFile().getExecPathString());
+ info.addAllSourcesAndHeaders(
+ Artifact.toExecPaths(context.getDeclaredIncludeSrcs()));
+ }
+
+ return super.getExtraActionInfo()
+ .setExtension(CppCompileInfo.cppCompileInfo, info.build());
+ }
+
+ /**
+ * Returns the compiler options.
+ */
+ @VisibleForTesting
+ public List<String> getCompilerOptions() {
+ return cppCompileCommandLine.getCompilerOptions();
+ }
+
+ /**
+ * Enforce that the includes actually visited during the compile were properly
+ * declared in the rules.
+ *
+ * <p>The technique is to walk through all of the reported includes that gcc
+ * emits into the .d file, and verify that they came from acceptable
+ * relative include directories. This is done in two steps:
+ *
+ * <p>First, each included file is stripped of any include path prefix from
+ * {@code quoteIncludeDirs} to produce an effective relative include dir+name.
+ *
+ * <p>Second, the remaining directory is looked up in {@code declaredIncludeDirs},
+ * a list of acceptable dirs. This list contains a set of dir fragments that
+ * have been calculated by the configured target to be allowable for inclusion
+ * by this source. If no match is found, an error is reported and an exception
+ * is thrown.
+ *
+ * @throws ActionExecutionException iff there was an undeclared dependency
+ */
+ @VisibleForTesting
+ public void validateInclusions(
+ MiddlemanExpander middlemanExpander, EventHandler eventHandler)
+ throws ActionExecutionException {
+ if (!cppConfiguration.shouldScanIncludes() || !inputsKnown()) {
+ return;
+ }
+
+ IncludeProblems errors = new IncludeProblems();
+ IncludeProblems warnings = new IncludeProblems();
+ Set<Artifact> allowedIncludes = new HashSet<>();
+ for (Artifact input : mandatoryInputs) {
+ if (input.isMiddlemanArtifact()) {
+ middlemanExpander.expand(input, allowedIncludes);
+ }
+ allowedIncludes.add(input);
+ }
+
+ if (optionalSourceFile != null) {
+ allowedIncludes.add(optionalSourceFile);
+ }
+ List<PathFragment> cxxSystemIncludeDirs =
+ cppConfiguration.getBuiltInIncludeDirectories();
+ Iterable<PathFragment> ignoreDirs = Iterables.concat(cxxSystemIncludeDirs,
+ extraSystemIncludePrefixes, context.getSystemIncludeDirs());
+
+ // Copy the sets to hash sets for fast contains checking.
+ // Avoid immutable sets here to limit memory churn.
+ Set<PathFragment> declaredIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeDirs());
+ Set<PathFragment> warnIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeWarnDirs());
+ Set<Artifact> declaredIncludeSrcs = Sets.newHashSet(context.getDeclaredIncludeSrcs());
+ for (Artifact input : getInputs()) {
+ if (context.getCompilationPrerequisites().contains(input)
+ || allowedIncludes.contains(input)) {
+ continue; // ignore our fixed source in mandatoryInput: we just want includes
+ }
+ // Ignore headers from built-in include directories.
+ if (FileSystemUtils.startsWithAny(input.getExecPath(), ignoreDirs)) {
+ continue;
+ }
+ if (!isDeclaredIn(input, declaredIncludeDirs, declaredIncludeSrcs)) {
+ // This call can never match the declared include sources (they would be matched above).
+ // There are no declared include sources we need to warn about, so use an empty set here.
+ if (isDeclaredIn(input, warnIncludeDirs, ImmutableSet.<Artifact>of())) {
+ warnings.add(input.getPath().toString());
+ } else {
+ errors.add(input.getPath().toString());
+ }
+ }
+ }
+ if (VALIDATION_DEBUG_WARN) {
+ synchronized (System.err) {
+ if (VALIDATION_DEBUG >= 2 || errors.hasProblems() || warnings.hasProblems()) {
+ if (errors.hasProblems()) {
+ System.err.println("ERROR: Include(s) were not in declared srcs:");
+ } else if (warnings.hasProblems()) {
+ System.err.println("WARN: Include(s) were not in declared srcs:");
+ } else {
+ System.err.println("INFO: Include(s) were OK for '" + getSourceFile()
+ + "', declared srcs:");
+ }
+ for (Artifact a : context.getDeclaredIncludeSrcs()) {
+ System.err.println(" '" + a.toDetailString() + "'");
+ }
+ System.err.println(" or under declared dirs:");
+ for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeDirs())) {
+ System.err.println(" '" + f + "'");
+ }
+ System.err.println(" or under declared warn dirs:");
+ for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeWarnDirs())) {
+ System.err.println(" '" + f + "'");
+ }
+ System.err.println(" with prefixes:");
+ for (PathFragment dirpath : context.getQuoteIncludeDirs()) {
+ System.err.println(" '" + dirpath + "'");
+ }
+ }
+ }
+ }
+
+ if (warnings.hasProblems()) {
+ eventHandler.handle(
+ new Event(EventKind.WARNING,
+ getOwner().getLocation(), warnings.getMessage(this, getSourceFile()),
+ Label.print(getOwner().getLabel())));
+ }
+ errors.assertProblemFree(this, getSourceFile());
+ }
+
+ /**
+ * Returns true if an included artifact is declared in a set of allowed
+ * include directories. The simple case is that the artifact's parent
+ * directory is contained in the set, or is empty.
+ *
+ * <p>This check also supports a wildcard suffix of '**' for the cases where the
+ * calculations are inexact.
+ *
+ * <p>It also handles unseen non-nested-package subdirs by walking up the path looking
+ * for matches.
+ */
+ private static boolean isDeclaredIn(Artifact input, Set<PathFragment> declaredIncludeDirs,
+ Set<Artifact> declaredIncludeSrcs) {
+ // First check if it's listed in "srcs". If so, then its declared & OK.
+ if (declaredIncludeSrcs.contains(input)) {
+ return true;
+ }
+ // If it's a derived artifact, then it MUST be listed in "srcs" as checked above.
+ // We define derived here as being not source and not under the include link tree.
+ if (!input.isSourceArtifact()
+ && !input.getRoot().getExecPath().getBaseName().equals("include")) {
+ return false;
+ }
+ // Need to do dir/package matching: first try a quick exact lookup.
+ PathFragment includeDir = input.getRootRelativePath().getParentDirectory();
+ if (includeDir.segmentCount() == 0 || declaredIncludeDirs.contains(includeDir)) {
+ return true; // OK: quick exact match.
+ }
+ // Not found in the quick lookup: try the wildcards.
+ for (PathFragment declared : declaredIncludeDirs) {
+ if (declared.getBaseName().equals("**")) {
+ if (includeDir.startsWith(declared.getParentDirectory())) {
+ return true; // OK: under a wildcard dir.
+ }
+ }
+ }
+ // Still not found: see if it is in a subdir of a declared package.
+ Path root = input.getRoot().getPath();
+ for (Path dir = input.getPath().getParentDirectory();;) {
+ if (dir.getRelative("BUILD").exists()) {
+ return false; // Bad: this is a sub-package, not a subdir of a declared package.
+ }
+ dir = dir.getParentDirectory();
+ if (dir.equals(root)) {
+ return false; // Bad: at the top, give up.
+ }
+ if (declaredIncludeDirs.contains(dir.relativeTo(root))) {
+ return true; // OK: found under a declared dir.
+ }
+ }
+ }
+
+ /**
+ * Recalculates this action's live input collection, including sources, middlemen.
+ *
+ * @throws ActionExecutionException iff any errors happen during update.
+ */
+ @VisibleForTesting
+ @ThreadCompatible
+ public final void updateActionInputs(Path execRoot,
+ ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply)
+ throws ActionExecutionException {
+ if (!cppConfiguration.shouldScanIncludes()) {
+ return;
+ }
+ inputsKnown = false;
+ NestedSetBuilder<Artifact> inputs = NestedSetBuilder.stableOrder();
+ Profiler.instance().startTask(ProfilerTask.ACTION_UPDATE, this);
+ try {
+ inputs.addTransitive(mandatoryInputs);
+ if (optionalSourceFile != null) {
+ inputs.add(optionalSourceFile);
+ }
+ inputs.addAll(context.getCompilationPrerequisites());
+ populateActionInputs(execRoot, artifactResolver, reply, inputs);
+ inputsKnown = true;
+ } finally {
+ Profiler.instance().completeTask(ProfilerTask.ACTION_UPDATE);
+ synchronized (this) {
+ setInputs(inputs.build());
+ }
+ }
+ }
+
+ private DependencySet processDepset(Path execRoot, CppCompileActionContext.Reply reply)
+ throws IOException {
+ DependencySet depSet = new DependencySet(execRoot);
+
+ // artifact() is null if we are not using in-memory .d files. We also want to prepare for the
+ // case where we expected an in-memory .d file, but we did not get an appropriate response.
+ // Perhaps we produced the file locally.
+ if (getDotdFile().artifact() != null || reply == null) {
+ return depSet.read(getDotdFile().getPath());
+ } else {
+ // This is an in-memory .d file.
+ return depSet.process(reply.getContents());
+ }
+ }
+
+ /**
+ * Populates the given ordered collection with additional input artifacts
+ * relevant to the specific action implementation.
+ *
+ * <p>The default implementation updates this Action's input set by reading
+ * dynamically-discovered dependency information out of the .d file.
+ *
+ * <p>Artifacts are considered inputs but not "mandatory" inputs.
+ *
+ *
+ * @param reply the reply from the compilation.
+ * @param inputs the ordered collection of inputs to append to
+ * @throws ActionExecutionException iff the .d is missing, malformed or has
+ * unresolvable included artifacts.
+ */
+ @ThreadCompatible
+ private void populateActionInputs(Path execRoot,
+ ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply,
+ NestedSetBuilder<Artifact> inputs)
+ throws ActionExecutionException {
+ try {
+ // Read .d file.
+ DependencySet depSet = processDepset(execRoot, reply);
+
+ // Determine prefixes of allowed absolute inclusions.
+ CppConfiguration toolchain = cppConfiguration;
+ List<PathFragment> systemIncludePrefixes = new ArrayList<>();
+ for (PathFragment includePath : toolchain.getBuiltInIncludeDirectories()) {
+ if (includePath.isAbsolute()) {
+ systemIncludePrefixes.add(includePath);
+ }
+ }
+ systemIncludePrefixes.addAll(extraSystemIncludePrefixes);
+
+ // Check inclusions.
+ IncludeProblems problems = new IncludeProblems();
+ Map<PathFragment, Artifact> allowedDerivedInputsMap = getAllowedDerivedInputsMap();
+ for (PathFragment execPath : depSet.getDependencies()) {
+ if (execPath.isAbsolute()) {
+ // Absolute includes from system paths are ignored.
+ if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) {
+ continue;
+ }
+ // Since gcc is given only relative paths on the command line,
+ // non-system include paths here should never be absolute. If they
+ // are, it's probably due to a non-hermetic #include, & we should stop
+ // the build with an error.
+ if (execPath.startsWith(execRoot.asFragment())) {
+ execPath = execPath.relativeTo(execRoot.asFragment()); // funky but tolerable path
+ } else {
+ problems.add(execPath.getPathString());
+ continue;
+ }
+ }
+ Artifact artifact = allowedDerivedInputsMap.get(execPath);
+ if (artifact == null) {
+ artifact = artifactResolver.resolveSourceArtifact(execPath);
+ }
+ if (artifact != null) {
+ inputs.add(artifact);
+ // In some cases, execution backends need extra files for each included file. Add them
+ // to the set of actual inputs.
+ inputs.addAll(includeResolver.getInputsForIncludedFile(artifact, artifactResolver));
+ } else {
+ // Abort if we see files that we can't resolve, likely caused by
+ // undeclared includes or illegal include constructs.
+ problems.add(execPath.getPathString());
+ }
+ }
+ problems.assertProblemFree(this, getSourceFile());
+ } catch (IOException e) {
+ // Some kind of IO or parse exception--wrap & rethrow it to stop the build.
+ throw new ActionExecutionException("error while parsing .d file", e, this, false);
+ }
+ }
+
+ @Override
+ public void updateInputsFromCache(
+ ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths) {
+ // Note that this method may trigger a violation of the desirable invariant that getInputs()
+ // is a superset of getMandatoryInputs(). See bug about an "action not in canonical form"
+ // error message and the integration test test_crosstool_change_and_failure().
+
+ Map<PathFragment, Artifact> allowedDerivedInputsMap = getAllowedDerivedInputsMap();
+ List<Artifact> inputs = new ArrayList<>();
+ for (PathFragment execPath : inputPaths) {
+ // The artifact may be a derived artifact, and if it has been created already, then we still
+ // want to keep it to preserve incrementality.
+ Artifact artifact = allowedDerivedInputsMap.get(execPath);
+ if (artifact == null) {
+ artifact = artifactResolver.resolveSourceArtifact(execPath);
+ }
+ // If PathFragment cannot be resolved into the artifact - ignore it. This could happen if
+ // rule definition has changed and action no longer depends on, e.g., additional source file
+ // in the separate package and that package is no longer referenced anywhere else.
+ // It is safe to ignore such paths because dependency checker would identify change in inputs
+ // (ignored path was used before) and will force action execution.
+ if (artifact != null) {
+ inputs.add(artifact);
+ }
+ }
+ inputsKnown = true;
+ synchronized (this) {
+ setInputs(inputs);
+ }
+ }
+
+ private Map<PathFragment, Artifact> getAllowedDerivedInputsMap() {
+ Map<PathFragment, Artifact> allowedDerivedInputMap = new HashMap<>();
+ addToMap(allowedDerivedInputMap, mandatoryInputs);
+ addToMap(allowedDerivedInputMap, context.getDeclaredIncludeSrcs());
+ addToMap(allowedDerivedInputMap, context.getCompilationPrerequisites());
+ Artifact artifact = getSourceFile();
+ if (!artifact.isSourceArtifact()) {
+ allowedDerivedInputMap.put(artifact.getExecPath(), artifact);
+ }
+ return allowedDerivedInputMap;
+ }
+
+ private void addToMap(Map<PathFragment, Artifact> map, Iterable<Artifact> artifacts) {
+ for (Artifact artifact : artifacts) {
+ if (!artifact.isSourceArtifact()) {
+ map.put(artifact.getExecPath(), artifact);
+ }
+ }
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Compiling " + getSourceFile().prettyPrint();
+ }
+
+ /**
+ * Return the directories in which to look for headers (pertains to headers
+ * not specifically listed in {@code declaredIncludeSrcs}). The return value
+ * may contain duplicate elements.
+ */
+ public NestedSet<PathFragment> getDeclaredIncludeDirs() {
+ return context.getDeclaredIncludeDirs();
+ }
+
+ /**
+ * Return the directories in which to look for headers and issue a warning.
+ * (pertains to headers not specifically listed in {@code
+ * declaredIncludeSrcs}). The return value may contain duplicate elements.
+ */
+ public NestedSet<PathFragment> getDeclaredIncludeWarnDirs() {
+ return context.getDeclaredIncludeWarnDirs();
+ }
+
+ /**
+ * Return explicit header files (i.e., header files explicitly listed). The
+ * return value may contain duplicate elements.
+ */
+ public NestedSet<Artifact> getDeclaredIncludeSrcs() {
+ return context.getDeclaredIncludeSrcs();
+ }
+
+ /**
+ * Return explicit header files (i.e., header files explicitly listed) in an order
+ * that is stable between builds.
+ */
+ protected final List<PathFragment> getDeclaredIncludeSrcsInStableOrder() {
+ List<PathFragment> paths = new ArrayList<>();
+ for (Artifact declaredIncludeSrc : context.getDeclaredIncludeSrcs()) {
+ paths.add(declaredIncludeSrc.getExecPath());
+ }
+ Collections.sort(paths); // Order is not important, but stability is.
+ return paths;
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return executor.getContext(actionContext).estimateResourceConsumption(this);
+ }
+
+ @VisibleForTesting
+ public Class<? extends CppCompileActionContext> getActionContext() {
+ return actionContext;
+ }
+
+ /**
+ * Estimate resource consumption when this action is executed locally.
+ */
+ public ResourceSet estimateResourceConsumptionLocal() {
+ // We use a local compile, so much of the time is spent waiting for IO,
+ // but there is still significant CPU; hence we estimate 50% cpu usage.
+ return new ResourceSet(/*memoryMb=*/200, /*cpuUsage=*/0.5, /*ioUsage=*/0.0);
+ }
+
+ @Override
+ public String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addUUID(actionClassId);
+ f.addStrings(getArgv());
+
+ /*
+ * getArgv() above captures all changes which affect the compilation
+ * command and hence the contents of the object file. But we need to
+ * also make sure that we reexecute the action if any of the fields
+ * that affect whether validateIncludes() will report an error or warning
+ * have changed, otherwise we might miss some errors.
+ */
+ f.addPaths(context.getDeclaredIncludeDirs());
+ f.addPaths(context.getDeclaredIncludeWarnDirs());
+ f.addPaths(getDeclaredIncludeSrcsInStableOrder());
+ f.addPaths(getExtraSystemIncludePrefixes());
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ @ThreadCompatible
+ public void execute(
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ CppCompileActionContext.Reply reply;
+ try {
+ reply = executor.getContext(actionContext).execWithReply(this, actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException("C++ compilation of rule '" + getOwner().getLabel() + "'",
+ executor.getVerboseFailures(), this);
+ }
+ ensureCoverageNotesFilesExist();
+ IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class);
+ updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply);
+ reply = null; // Clear in-memory .d files early.
+ validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler());
+ }
+
+ /**
+ * Gcc only creates ".gcno" files if the compilation unit is non-empty.
+ * To ensure that the set of outputs for a CppCompileAction remains consistent
+ * and doesn't vary dynamically depending on the _contents_ of the input files,
+ * we create empty ".gcno" files if gcc didn't create them.
+ */
+ private void ensureCoverageNotesFilesExist() throws ActionExecutionException {
+ for (Artifact output : getOutputs()) {
+ if (CppFileTypes.COVERAGE_NOTES.matches(output.getFilename()) // ".gcno"
+ && !output.getPath().exists()) {
+ try {
+ FileSystemUtils.createEmptyFile(output.getPath());
+ } catch (IOException e) {
+ throw new ActionExecutionException(
+ "Error creating file '" + output.getPath() + "': " + e.getMessage(), e, this, false);
+ }
+ }
+ }
+ }
+
+ /**
+ * Provides list of include files needed for performing extra actions on this action when run
+ * remotely. The list of include files is created by performing a header scan on the known input
+ * files.
+ */
+ @Override
+ public Iterable<Artifact> getInputFilesForExtraAction(
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Collection<Artifact> scannedIncludes =
+ actionExecutionContext.getExecutor().getContext(actionContext)
+ .getScannedIncludeFiles(this, actionExecutionContext);
+ // Use a set to eliminate duplicates.
+ ImmutableSet.Builder<Artifact> result = ImmutableSet.builder();
+ return result.addAll(getInputs()).addAll(scannedIncludes).build();
+ }
+
+ @Override
+ public String getMnemonic() { return "CppCompile"; }
+
+ @Override
+ public String describeKey() {
+ StringBuilder message = new StringBuilder();
+ message.append(getProgressMessage());
+ message.append('\n');
+ message.append(" Command: ");
+ message.append(
+ ShellEscaper.escapeString(cppConfiguration.getLdExecutable().getPathString()));
+ message.append('\n');
+ // Outputting one argument per line makes it easier to diff the results.
+ for (String argument : ShellEscaper.escapeAll(getArgv())) {
+ message.append(" Argument: ");
+ message.append(argument);
+ message.append('\n');
+ }
+
+ for (PathFragment path : context.getDeclaredIncludeDirs()) {
+ message.append(" Declared include directory: ");
+ message.append(ShellEscaper.escapeString(path.getPathString()));
+ message.append('\n');
+ }
+
+ for (PathFragment path : getDeclaredIncludeSrcsInStableOrder()) {
+ message.append(" Declared include source: ");
+ message.append(ShellEscaper.escapeString(path.getPathString()));
+ message.append('\n');
+ }
+
+ for (PathFragment path : getExtraSystemIncludePrefixes()) {
+ message.append(" Extra system include prefix: ");
+ message.append(ShellEscaper.escapeString(path.getPathString()));
+ message.append('\n');
+ }
+ return message.toString();
+ }
+
+ /**
+ * The compile command line for the enclosing C++ compile action.
+ */
+ public final class CppCompileCommandLine {
+ private final Artifact sourceFile;
+ private final DotdFile dotdFile;
+ private final CppModuleMap cppModuleMap;
+ private final List<String> copts;
+ private final Predicate<String> coptsFilter;
+ private final List<String> pluginOpts;
+ private final boolean isInstrumented;
+ private final Collection<String> features;
+ private final FeatureConfiguration featureConfiguration;
+
+ // The value of the BUILD_FDO_TYPE macro to be defined on command line
+ @Nullable private final String fdoBuildStamp;
+
+ public CppCompileCommandLine(Artifact sourceFile, DotdFile dotdFile, CppModuleMap cppModuleMap,
+ ImmutableList<String> copts, Predicate<String> coptsFilter,
+ ImmutableList<String> pluginOpts, boolean isInstrumented,
+ Collection<String> features, FeatureConfiguration featureConfiguration,
+ @Nullable String fdoBuildStamp) {
+ this.sourceFile = Preconditions.checkNotNull(sourceFile);
+ this.dotdFile = Preconditions.checkNotNull(dotdFile);
+ this.cppModuleMap = cppModuleMap;
+ this.copts = Preconditions.checkNotNull(copts);
+ this.coptsFilter = coptsFilter;
+ this.pluginOpts = Preconditions.checkNotNull(pluginOpts);
+ this.isInstrumented = isInstrumented;
+ this.features = Preconditions.checkNotNull(features);
+ this.featureConfiguration = featureConfiguration;
+ this.fdoBuildStamp = fdoBuildStamp;
+ }
+
+ protected List<String> getArgv(PathFragment outputFile) {
+ List<String> commandLine = new ArrayList<>();
+
+ // first: The command name.
+ commandLine.add(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString());
+
+ // second: The compiler options.
+ commandLine.addAll(getCompilerOptions());
+
+ // third: The file to compile!
+ commandLine.add("-c");
+ commandLine.add(sourceFile.getExecPathString());
+
+ // finally: The output file. (Prefixed with -o).
+ commandLine.add("-o");
+ commandLine.add(outputFile.getPathString());
+
+ return commandLine;
+ }
+
+ private String getActionName() {
+ PathFragment sourcePath = sourceFile.getExecPath();
+ if (CppFileTypes.CPP_MODULE_MAP.matches(sourcePath)) {
+ return CPP_MODULE_COMPILE;
+ } else if (CppFileTypes.CPP_HEADER.matches(sourcePath)) {
+ // TODO(bazel-team): Handle C headers that probably don't work in C++ mode.
+ // TODO(bazel-team): Replace use of features.contains with featureConfiguration.isEnabled
+ // here (the other instances will need to stay with the current feature selection process
+ // until all crosstool configurations have been converted).
+ if (features.contains(CppRuleClasses.PARSE_HEADERS)) {
+ return CPP_HEADER_PARSING;
+ } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) {
+ return CPP_HEADER_PREPROCESSING;
+ } else {
+ // CcCommon.collectCAndCppSources() ensures we do not add headers to
+ // the compilation artifacts unless either 'parse_headers' or
+ // 'preprocess_headers' is set.
+ throw new IllegalStateException();
+ }
+ } else if (CppFileTypes.C_SOURCE.matches(sourcePath)) {
+ return C_COMPILE;
+ } else if (CppFileTypes.CPP_SOURCE.matches(sourcePath)) {
+ return CPP_COMPILE;
+ } else if (CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR.matches(sourcePath)) {
+ return PREPROCESS_ASSEMBLE;
+ }
+ // CcLibraryHelper ensures CppCompileAction only gets instantiated for supported file types.
+ throw new IllegalStateException();
+ }
+
+ public List<String> getCompilerOptions() {
+ List<String> options = new ArrayList<>();
+
+ // TODO(bazel-team): Extract combinations of options into sections in the CROSSTOOL file.
+ if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFile.getExecPath())) {
+ options.add("-x");
+ options.add("c++");
+ } else if (CppFileTypes.CPP_HEADER.matches(sourceFile.getExecPath())) {
+ // TODO(bazel-team): Read the compiler flag settings out of the CROSSTOOL file.
+ // TODO(bazel-team): Handle C headers that probably don't work in C++ mode.
+ if (features.contains(CppRuleClasses.PARSE_HEADERS)) {
+ options.add("-x");
+ options.add("c++-header");
+ // Specifying -x c++-header will make clang/gcc create precompiled
+ // headers, which we suppress with -fsyntax-only.
+ options.add("-fsyntax-only");
+ } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) {
+ options.add("-E");
+ options.add("-x");
+ options.add("c++");
+ } else {
+ // CcCommon.collectCAndCppSources() ensures we do not add headers to
+ // the compilation artifacts unless either 'parse_headers' or
+ // 'preprocess_headers' is set.
+ throw new IllegalStateException();
+ }
+ }
+
+ for (PathFragment quoteIncludePath : context.getQuoteIncludeDirs()) {
+ // "-iquote" is a gcc-specific option. For C compilers that don't support "-iquote",
+ // we should instead use "-I".
+ options.add("-iquote");
+ options.add(quoteIncludePath.getSafePathString());
+ }
+ for (PathFragment includePath : context.getIncludeDirs()) {
+ options.add("-I" + includePath.getSafePathString());
+ }
+ for (PathFragment systemIncludePath : context.getSystemIncludeDirs()) {
+ options.add("-isystem");
+ options.add(systemIncludePath.getSafePathString());
+ }
+
+ CppConfiguration toolchain = cppConfiguration;
+
+ // pluginOpts has to be added before defaultCopts because -fplugin must precede -plugin-arg.
+ options.addAll(pluginOpts);
+ addFilteredOptions(options, toolchain.getCompilerOptions(features));
+
+ // Enable instrumentation if requested.
+ if (isInstrumented) {
+ addFilteredOptions(options, ImmutableList.of("-fprofile-arcs", "-ftest-coverage"));
+ }
+
+ String sourceFilename = sourceFile.getExecPathString();
+ if (CppFileTypes.C_SOURCE.matches(sourceFilename)) {
+ addFilteredOptions(options, toolchain.getCOptions());
+ }
+ if (CppFileTypes.CPP_SOURCE.matches(sourceFilename)
+ || CppFileTypes.CPP_HEADER.matches(sourceFilename)
+ || CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) {
+ addFilteredOptions(options, toolchain.getCxxOptions(features));
+ }
+
+ // Users don't expect the explicit copts to be filtered by coptsFilter, add them verbatim.
+ options.addAll(copts);
+
+ for (String warn : cppConfiguration.getCWarns()) {
+ options.add("-W" + warn);
+ }
+ for (String define : context.getDefines()) {
+ options.add("-D" + define);
+ }
+
+ // Stamp FDO builds with FDO subtype string
+ if (fdoBuildStamp != null) {
+ options.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\"");
+ }
+
+ options.addAll(toolchain.getUnfilteredCompilerOptions(features));
+
+ // GCC gives randomized names to symbols which are defined in
+ // an anonymous namespace but have external linkage. To make
+ // computation of these deterministic, we want to override the
+ // default seed for the random number generator. It's safe to use
+ // any value which differs for all translation units; we use the
+ // path to the object file.
+ options.add("-frandom-seed=" + outputFile.getExecPathString());
+
+ // Add the options of --per_file_copt, if the label or the base name of the source file
+ // matches the specified regular expression filter.
+ for (PerLabelOptions perLabelOptions : cppConfiguration.getPerFileCopts()) {
+ if ((sourceLabel != null && perLabelOptions.isIncluded(sourceLabel))
+ || perLabelOptions.isIncluded(sourceFile)) {
+ options.addAll(perLabelOptions.getOptions());
+ }
+ }
+
+ // Enable <object>.d file generation.
+ if (dotdFile != null) {
+ // Gcc options:
+ // -MD turns on .d file output as a side-effect (doesn't imply -E)
+ // -MM[D] enables user includes only, not system includes
+ // -MF <name> specifies the dotd file name
+ // Issues:
+ // -M[M] alone subverts actual .o output (implies -E)
+ // -M[M]D alone breaks some of the .d naming assumptions
+ // This combination gets user and system includes with specified name:
+ // -MD -MF <name>
+ options.add("-MD");
+ options.add("-MF");
+ options.add(dotdFile.getSafeExecPath().getPathString());
+ }
+
+ if (cppModuleMap != null && (compileHeaderModules || enableLayeringCheck)) {
+ options.add("-Xclang-only=-fmodule-maps");
+ options.add("-Xclang-only=-fmodule-name=" + cppModuleMap.getName());
+ options.add("-Xclang-only=-fmodule-map-file="
+ + cppModuleMap.getArtifact().getExecPathString());
+ options.add("-Xclang=-fno-modules-implicit-maps");
+
+ if (compileHeaderModules) {
+ options.add("-Xclang-only=-fmodules");
+ if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) {
+ options.add("-Xclang=-emit-module");
+ options.add("-Xcrosstool-module-compilation");
+ }
+ // Select .pcm inputs to pass on the command line depending on whether
+ // we are in pic or non-pic mode.
+ // TODO(bazel-team): We want to add these to the compile even if the
+ // current target is not built as a module; currently that implies
+ // passing -fmodules to the compiler, which is experimental; thus, we
+ // do not use the header modules files for now if the current
+ // compilation is not modules enabled on its own.
+ boolean pic = copts.contains("-fPIC");
+ for (Artifact source : context.getAdditionalInputs()) {
+ if ((pic && source.getFilename().endsWith(".pic.pcm")) || (!pic
+ && !source.getFilename().endsWith(".pic.pcm")
+ && source.getFilename().endsWith(".pcm"))) {
+ options.add("-Xclang=-fmodule-file=" + source.getExecPathString());
+ }
+ }
+ }
+ if (enableLayeringCheck) {
+ options.add("-Xclang-only=-fmodules-strict-decluse");
+ }
+ }
+
+ if (FileType.contains(outputFile, CppFileTypes.ASSEMBLER, CppFileTypes.PIC_ASSEMBLER)) {
+ options.add("-S");
+ } else if (FileType.contains(outputFile, CppFileTypes.PREPROCESSED_C,
+ CppFileTypes.PREPROCESSED_CPP, CppFileTypes.PIC_PREPROCESSED_C,
+ CppFileTypes.PIC_PREPROCESSED_CPP)) {
+ options.add("-E");
+ }
+
+ if (cppConfiguration.useFission()) {
+ options.add("-gsplit-dwarf");
+ }
+
+ options.addAll(featureConfiguration.getCommandLine(getActionName(),
+ ImmutableMultimap.<String, String>of()));
+ return options;
+ }
+
+ // For each option in 'in', add it to 'out' unless it is matched by the 'coptsFilter' regexp.
+ private void addFilteredOptions(List<String> out, List<String> in) {
+ Iterables.addAll(out, Iterables.filter(in, coptsFilter));
+ }
+ }
+
+ /**
+ * A reference to a .d file. There are two modes:
+ * <ol>
+ * <li>an Artifact that represents a real on-disk file
+ * <li>just an execPath that refers to a virtual .d file that is not written to disk
+ * </ol>
+ */
+ public static class DotdFile {
+ private final Artifact artifact;
+ private final PathFragment execPath;
+
+ public DotdFile(Artifact artifact) {
+ this.artifact = artifact;
+ this.execPath = null;
+ }
+
+ public DotdFile(PathFragment execPath) {
+ this.artifact = null;
+ this.execPath = execPath;
+ }
+
+ /**
+ * @return the Artifact or null
+ */
+ public Artifact artifact() {
+ return artifact;
+ }
+
+ /**
+ * @return Gets the execPath regardless of whether this is a real Artifact
+ */
+ public PathFragment getSafeExecPath() {
+ return execPath == null ? artifact.getExecPath() : execPath;
+ }
+
+ /**
+ * @return the on-disk location of the .d file or null
+ */
+ public Path getPath() {
+ return artifact.getPath();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java
new file mode 100644
index 0000000..0d8da3e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java
@@ -0,0 +1,439 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
+import com.google.devtools.build.lib.rules.cpp.CppCompileAction.DotdFile;
+import com.google.devtools.build.lib.rules.cpp.CppCompileAction.IncludeResolver;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+/**
+ * Builder class to construct C++ compile actions.
+ */
+public class CppCompileActionBuilder {
+ public static final UUID GUID = UUID.fromString("cee5db0a-d2ad-4c69-9b81-97c936a29075");
+
+ private final ActionOwner owner;
+ private final List<String> features = new ArrayList<>();
+ private CcToolchainFeatures.FeatureConfiguration featureConfiguration;
+ private final Artifact sourceFile;
+ private final Label sourceLabel;
+ private final NestedSetBuilder<Artifact> mandatoryInputsBuilder;
+ private NestedSetBuilder<Artifact> pluginInputsBuilder;
+ private Artifact optionalSourceFile;
+ private Artifact outputFile;
+ private PathFragment tempOutputFile;
+ private DotdFile dotdFile;
+ private Artifact gcnoFile;
+ private final BuildConfiguration configuration;
+ private CppCompilationContext context = CppCompilationContext.EMPTY;
+ private final List<String> copts = new ArrayList<>();
+ private final List<String> pluginOpts = new ArrayList<>();
+ private final List<Pattern> nocopts = new ArrayList<>();
+ private AnalysisEnvironment analysisEnvironment;
+ private ImmutableList<PathFragment> extraSystemIncludePrefixes = ImmutableList.of();
+ private boolean enableLayeringCheck;
+ private boolean compileHeaderModules;
+ private String fdoBuildStamp;
+ private IncludeResolver includeResolver = CppCompileAction.VOID_INCLUDE_RESOLVER;
+ private UUID actionClassId = GUID;
+ private Class<? extends CppCompileActionContext> actionContext;
+ private CppConfiguration cppConfiguration;
+ private ImmutableMap<Artifact, IncludeScannable> lipoScannableMap;
+
+ /**
+ * Creates a builder from a rule. This also uses the configuration and
+ * artifact factory from the rule.
+ */
+ public CppCompileActionBuilder(RuleContext ruleContext, Artifact sourceFile, Label sourceLabel) {
+ this.owner = ruleContext.getActionOwner();
+ this.actionContext = CppCompileActionContext.class;
+ this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class);
+ this.analysisEnvironment = ruleContext.getAnalysisEnvironment();
+ this.sourceFile = sourceFile;
+ this.sourceLabel = sourceLabel;
+ this.configuration = ruleContext.getConfiguration();
+ this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder();
+ this.pluginInputsBuilder = NestedSetBuilder.stableOrder();
+ this.lipoScannableMap = getLipoScannableMap(ruleContext);
+
+ features.addAll(ruleContext.getFeatures());
+ }
+
+ private static ImmutableMap<Artifact, IncludeScannable> getLipoScannableMap(
+ RuleContext ruleContext) {
+ if (!ruleContext.getFragment(CppConfiguration.class).isLipoOptimization()) {
+ return null;
+ }
+
+ LipoContextProvider provider = ruleContext.getPrerequisite(
+ ":lipo_context_collector", Mode.DONT_CHECK, LipoContextProvider.class);
+ return provider.getIncludeScannables();
+ }
+
+ /**
+ * Creates a builder for an owner that is not required to be rule.
+ */
+ public CppCompileActionBuilder(
+ ActionOwner owner, AnalysisEnvironment analysisEnvironment, Artifact sourceFile,
+ Label sourceLabel, BuildConfiguration configuration) {
+ this.owner = owner;
+ this.actionContext = CppCompileActionContext.class;
+ this.cppConfiguration = configuration.getFragment(CppConfiguration.class);
+ this.analysisEnvironment = analysisEnvironment;
+ this.sourceFile = sourceFile;
+ this.sourceLabel = sourceLabel;
+ this.configuration = configuration;
+ this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder();
+ this.pluginInputsBuilder = NestedSetBuilder.stableOrder();
+ this.lipoScannableMap = ImmutableMap.of();
+ }
+
+ /**
+ * Creates a builder that is a copy of another builder.
+ */
+ public CppCompileActionBuilder(CppCompileActionBuilder other) {
+ this.owner = other.owner;
+ this.features.addAll(other.features);
+ this.featureConfiguration = other.featureConfiguration;
+ this.sourceFile = other.sourceFile;
+ this.sourceLabel = other.sourceLabel;
+ this.mandatoryInputsBuilder = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(other.mandatoryInputsBuilder.build());
+ this.pluginInputsBuilder = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(other.pluginInputsBuilder.build());
+ this.optionalSourceFile = other.optionalSourceFile;
+ this.outputFile = other.outputFile;
+ this.tempOutputFile = other.tempOutputFile;
+ this.dotdFile = other.dotdFile;
+ this.gcnoFile = other.gcnoFile;
+ this.configuration = other.configuration;
+ this.context = other.context;
+ this.copts.addAll(other.copts);
+ this.pluginOpts.addAll(other.pluginOpts);
+ this.nocopts.addAll(other.nocopts);
+ this.analysisEnvironment = other.analysisEnvironment;
+ this.extraSystemIncludePrefixes = ImmutableList.copyOf(other.extraSystemIncludePrefixes);
+ this.enableLayeringCheck = other.enableLayeringCheck;
+ this.compileHeaderModules = other.compileHeaderModules;
+ this.includeResolver = other.includeResolver;
+ this.actionClassId = other.actionClassId;
+ this.actionContext = other.actionContext;
+ this.cppConfiguration = other.cppConfiguration;
+ this.lipoScannableMap = other.lipoScannableMap;
+ }
+
+ public PathFragment getTempOutputFile() {
+ return tempOutputFile;
+ }
+
+ public Artifact getSourceFile() {
+ return sourceFile;
+ }
+
+ public CppCompilationContext getContext() {
+ return context;
+ }
+
+ public NestedSet<Artifact> getMandatoryInputs() {
+ return mandatoryInputsBuilder.build();
+ }
+
+ /**
+ * Returns the .dwo output file that matches the specified .o output file. If Fission mode
+ * isn't enabled for this build, this is null (we don't produce .dwo files in that case).
+ */
+ private static Artifact getDwoFile(Artifact outputFile, AnalysisEnvironment artifactFactory,
+ CppConfiguration cppConfiguration) {
+
+ // Only create .dwo's for .o compilations (i.e. not .ii or .S).
+ boolean isObjectOutput = CppFileTypes.OBJECT_FILE.matches(outputFile.getExecPath())
+ || CppFileTypes.PIC_OBJECT_FILE.matches(outputFile.getExecPath());
+
+ // Note configurations can be null for tests.
+ if (cppConfiguration != null && cppConfiguration.useFission() && isObjectOutput) {
+ return artifactFactory.getDerivedArtifact(
+ FileSystemUtils.replaceExtension(outputFile.getRootRelativePath(), ".dwo"),
+ outputFile.getRoot());
+ } else {
+ return null;
+ }
+ }
+
+ private static Predicate<String> getNocoptPredicate(Collection<Pattern> patterns) {
+ final ImmutableList<Pattern> finalPatterns = ImmutableList.copyOf(patterns);
+ if (finalPatterns.isEmpty()) {
+ return Predicates.alwaysTrue();
+ } else {
+ return new Predicate<String>() {
+ @Override
+ public boolean apply(String option) {
+ for (Pattern pattern : finalPatterns) {
+ if (pattern.matcher(option).matches()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ };
+ }
+ }
+
+ private Iterable<IncludeScannable> getLipoScannables(NestedSet<Artifact> realMandatoryInputs) {
+ return lipoScannableMap == null ? ImmutableList.<IncludeScannable>of() : Iterables.filter(
+ Iterables.transform(
+ Iterables.filter(
+ FileType.filter(
+ realMandatoryInputs,
+ CppFileTypes.C_SOURCE, CppFileTypes.CPP_SOURCE,
+ CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR),
+ Predicates.not(Predicates.equalTo(getSourceFile()))),
+ Functions.forMap(lipoScannableMap, null)),
+ Predicates.notNull());
+ }
+
+ /**
+ * Builds the Action as configured and returns the to be generated Artifact.
+ *
+ * <p>This method may be called multiple times to create multiple compile
+ * actions (usually after calling some setters to modify the generated
+ * action).
+ */
+ public CppCompileAction build() {
+ // Configuration can be null in tests.
+ NestedSetBuilder<Artifact> realMandatoryInputsBuilder = NestedSetBuilder.compileOrder();
+ realMandatoryInputsBuilder.addTransitive(mandatoryInputsBuilder.build());
+ if (tempOutputFile == null && configuration != null
+ && !configuration.getFragment(CppConfiguration.class).shouldScanIncludes()) {
+ realMandatoryInputsBuilder.addTransitive(context.getDeclaredIncludeSrcs());
+ }
+ realMandatoryInputsBuilder.addTransitive(context.getAdditionalInputs());
+ realMandatoryInputsBuilder.addTransitive(pluginInputsBuilder.build());
+ realMandatoryInputsBuilder.add(sourceFile);
+ boolean fake = tempOutputFile != null;
+
+ // Copying the collections is needed to make the builder reusable.
+ if (fake) {
+ return new FakeCppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration,
+ sourceFile, sourceLabel, realMandatoryInputsBuilder.build(), outputFile, tempOutputFile,
+ dotdFile, configuration, cppConfiguration, context, ImmutableList.copyOf(copts),
+ ImmutableList.copyOf(pluginOpts), getNocoptPredicate(nocopts),
+ extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp);
+ } else {
+ NestedSet<Artifact> realMandatoryInputs = realMandatoryInputsBuilder.build();
+
+ return new CppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration,
+ sourceFile, sourceLabel, realMandatoryInputs, outputFile, dotdFile,
+ gcnoFile, getDwoFile(outputFile, analysisEnvironment, cppConfiguration),
+ optionalSourceFile, configuration, cppConfiguration, context,
+ actionContext, ImmutableList.copyOf(copts),
+ ImmutableList.copyOf(pluginOpts),
+ getNocoptPredicate(nocopts),
+ extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp,
+ includeResolver, getLipoScannables(realMandatoryInputs), actionClassId,
+ compileHeaderModules);
+ }
+ }
+
+ /**
+ * Sets the feature configuration to be used for the action.
+ */
+ public CppCompileActionBuilder setFeatureConfiguration(
+ FeatureConfiguration featureConfiguration) {
+ this.featureConfiguration = featureConfiguration;
+ return this;
+ }
+
+ public CppCompileActionBuilder setIncludeResolver(IncludeResolver includeResolver) {
+ this.includeResolver = includeResolver;
+ return this;
+ }
+
+ public CppCompileActionBuilder setCppConfiguration(CppConfiguration cppConfiguration) {
+ this.cppConfiguration = cppConfiguration;
+ return this;
+ }
+
+ public CppCompileActionBuilder setActionContext(
+ Class<? extends CppCompileActionContext> actionContext) {
+ this.actionContext = actionContext;
+ return this;
+ }
+
+ public CppCompileActionBuilder setActionClassId(UUID uuid) {
+ this.actionClassId = uuid;
+ return this;
+ }
+
+ public CppCompileActionBuilder setExtraSystemIncludePrefixes(
+ Collection<PathFragment> extraSystemIncludePrefixes) {
+ this.extraSystemIncludePrefixes = ImmutableList.copyOf(extraSystemIncludePrefixes);
+ return this;
+ }
+
+ public CppCompileActionBuilder addPluginInput(Artifact artifact) {
+ pluginInputsBuilder.add(artifact);
+ return this;
+ }
+
+ public CppCompileActionBuilder clearPluginInputs() {
+ pluginInputsBuilder = NestedSetBuilder.stableOrder();
+ return this;
+ }
+
+ /**
+ * Set an optional source file (usually with metadata of the main source file). The optional
+ * source file can only be set once, whether via this method or through the constructor
+ * {@link #CppCompileActionBuilder(CppCompileActionBuilder)}.
+ */
+ public CppCompileActionBuilder addOptionalSourceFile(Artifact artifact) {
+ Preconditions.checkState(optionalSourceFile == null, "%s %s", optionalSourceFile, artifact);
+ optionalSourceFile = artifact;
+ return this;
+ }
+
+ public CppCompileActionBuilder addMandatoryInputs(Iterable<Artifact> artifacts) {
+ mandatoryInputsBuilder.addAll(artifacts);
+ return this;
+ }
+
+ public CppCompileActionBuilder addTransitiveMandatoryInputs(NestedSet<Artifact> artifacts) {
+ mandatoryInputsBuilder.addTransitive(artifacts);
+ return this;
+ }
+
+ public CppCompileActionBuilder setOutputFile(Artifact outputFile) {
+ this.outputFile = outputFile;
+ return this;
+ }
+
+ /**
+ * The temp output file is not an artifact, since it does not appear in the outputs of the
+ * action.
+ *
+ * <p>This is theoretically a problem if that file already existed before, since then Blaze
+ * does not delete it before executing the rule, but 1. that only applies for local
+ * execution which does not happen very often and 2. it is only a problem if the compiler is
+ * affected by the presence of this file, which it should not be.
+ */
+ public CppCompileActionBuilder setTempOutputFile(PathFragment tempOutputFile) {
+ this.tempOutputFile = tempOutputFile;
+ return this;
+ }
+
+ @VisibleForTesting
+ public CppCompileActionBuilder setDotdFileForTesting(Artifact dotdFile) {
+ this.dotdFile = new DotdFile(dotdFile);
+ return this;
+ }
+
+ public CppCompileActionBuilder setDotdFile(PathFragment outputName, String extension,
+ RuleContext ruleContext) {
+ if (configuration.getFragment(CppConfiguration.class).getInmemoryDotdFiles()) {
+ // Just set the path, no artifact is constructed
+ PathFragment file = FileSystemUtils.replaceExtension(outputName, extension);
+ Root root = configuration.getBinDirectory();
+ dotdFile = new DotdFile(root.getExecPath().getRelative(file));
+ } else {
+ dotdFile = new DotdFile(ruleContext.getRelatedArtifact(outputName, extension));
+ }
+ return this;
+ }
+
+ public CppCompileActionBuilder setGcnoFile(Artifact gcnoFile) {
+ this.gcnoFile = gcnoFile;
+ return this;
+ }
+
+ public CppCompileActionBuilder addCopt(String copt) {
+ copts.add(copt);
+ return this;
+ }
+
+ public CppCompileActionBuilder addPluginOpt(String opt) {
+ pluginOpts.add(opt);
+ return this;
+ }
+
+ public CppCompileActionBuilder clearPluginOpts() {
+ pluginOpts.clear();
+ return this;
+ }
+
+ public CppCompileActionBuilder addCopts(Iterable<? extends String> copts) {
+ Iterables.addAll(this.copts, copts);
+ return this;
+ }
+
+ public CppCompileActionBuilder addCopts(int position, Iterable<? extends String> copts) {
+ this.copts.addAll(position, ImmutableList.copyOf(copts));
+ return this;
+ }
+
+ public CppCompileActionBuilder addNocopts(Pattern nocopts) {
+ this.nocopts.add(nocopts);
+ return this;
+ }
+
+ public CppCompileActionBuilder setContext(CppCompilationContext context) {
+ this.context = context;
+ return this;
+ }
+
+ public CppCompileActionBuilder setEnableLayeringCheck(boolean enableLayeringCheck) {
+ this.enableLayeringCheck = enableLayeringCheck;
+ return this;
+ }
+
+ /**
+ * Sets whether the CompileAction should use header modules for its compilation.
+ */
+ public CppCompileActionBuilder setCompileHeaderModules(boolean compileHeaderModules) {
+ this.compileHeaderModules = compileHeaderModules;
+ return this;
+ }
+
+ public CppCompileActionBuilder setFdoBuildStamp(String fdoBuildStamp) {
+ this.fdoBuildStamp = fdoBuildStamp;
+ return this;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java
new file mode 100644
index 0000000..4bbfa44
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java
@@ -0,0 +1,84 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.ActionContextMarker;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ResourceSet;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+
+/**
+ * Context for compiling plain C++.
+ */
+@ActionContextMarker(name = "C++")
+public interface CppCompileActionContext extends ActionContext {
+ /**
+ * Reply for the execution of a C++ compilation.
+ */
+ public interface Reply {
+ /**
+ * Returns the contents of the .d file.
+ */
+ byte[] getContents() throws IOException;
+ }
+
+ /** Does include scanning to find the list of files needed to execute the action. */
+ public Collection<? extends ActionInput> findAdditionalInputs(CppCompileAction action,
+ ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException, ActionExecutionException;
+
+ /**
+ * Executes the given action and return the reply of the executor.
+ */
+ Reply execWithReply(CppCompileAction action,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException;
+
+ /**
+ * Returns the executor reply from an exec exception, if available.
+ */
+ @Nullable Reply getReplyFromException(
+ ExecException e, CppCompileAction action);
+
+ /**
+ * Returns the estimated resource consumption of the action.
+ */
+ ResourceSet estimateResourceConsumption(CppCompileAction action);
+
+ /**
+ * Returns where the action actually runs.
+ */
+ String strategyLocality();
+
+ /**
+ * Returns whether include scanning needs to be run.
+ */
+ boolean needsIncludeScanning();
+
+ /**
+ * Returns the include files that should be shipped to the executor in addition the ones that
+ * were declared.
+ */
+ Collection<Artifact> getScannedIncludeFiles(
+ CppCompileAction action, ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java
new file mode 100644
index 0000000..c5cc9a5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java
@@ -0,0 +1,1691 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.PackageRootResolver;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.CompilationMode;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.rules.cpp.CppConfigurationLoader.CppConfigurationParameters;
+import com.google.devtools.build.lib.rules.cpp.FdoSupport.FdoException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.util.IncludeScanningUtil;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LinkingModeFlags;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipException;
+
+/**
+ * This class represents the C/C++ parts of the {@link BuildConfiguration},
+ * including the host architecture, target architecture, compiler version, and
+ * a standard library version. It has information about the tools locations and
+ * the flags required for compiling.
+ */
+@SkylarkModule(name = "cpp", doc = "A configuration fragment for C++")
+@Immutable
+public class CppConfiguration extends BuildConfiguration.Fragment {
+ /**
+ * An enumeration of all the tools that comprise a toolchain.
+ */
+ public enum Tool {
+ AR("ar"),
+ CPP("cpp"),
+ GCC("gcc"),
+ GCOV("gcov"),
+ GCOVTOOL("gcov-tool"),
+ LD("ld"),
+ NM("nm"),
+ OBJCOPY("objcopy"),
+ OBJDUMP("objdump"),
+ STRIP("strip"),
+ DWP("dwp");
+
+ private final String namePart;
+
+ private Tool(String namePart) {
+ this.namePart = namePart;
+ }
+
+ public String getNamePart() {
+ return namePart;
+ }
+ }
+
+ /**
+ * Values for the --hdrs_check option.
+ */
+ public static enum HeadersCheckingMode {
+ /** Legacy behavior: Silently allow undeclared headers. */
+ LOOSE,
+ /** Warn about undeclared headers. */
+ WARN,
+ /** Disallow undeclared headers. */
+ STRICT
+ }
+
+ /**
+ * --dynamic_mode parses to DynamicModeFlag, but AUTO will be translated based on platform,
+ * resulting in a DynamicMode value.
+ */
+ public enum DynamicMode { OFF, DEFAULT, FULLY }
+
+ /**
+ * This enumeration is used for the --strip option.
+ */
+ public static enum StripMode {
+
+ ALWAYS("always"), // Always strip.
+ SOMETIMES("sometimes"), // Strip iff compilationMode == FASTBUILD.
+ NEVER("never"); // Never strip.
+
+ private final String mode;
+
+ private StripMode(String mode) {
+ this.mode = mode;
+ }
+
+ @Override
+ public String toString() {
+ return mode;
+ }
+ }
+
+ /** Storage for the libc label, if given. */
+ public static class LibcTop implements Serializable {
+ private final Label label;
+
+ LibcTop(Label label) {
+ Preconditions.checkArgument(label != null);
+ this.label = label;
+ }
+
+ public Label getLabel() {
+ return label;
+ }
+
+ public PathFragment getSysroot() {
+ return label.getPackageFragment();
+ }
+
+ @Override
+ public String toString() {
+ return label.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ } else if (other instanceof LibcTop) {
+ return label.equals(((LibcTop) other).label);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return label.hashCode();
+ }
+ }
+
+ /**
+ * This macro will be passed as a command-line parameter (eg. -DBUILD_FDO_TYPE="LIPO").
+ * For possible values see {@code CppModel.getFdoBuildStamp()}.
+ */
+ public static final String FDO_STAMP_MACRO = "BUILD_FDO_TYPE";
+
+ /**
+ * Represents an optional flag that can be toggled using the package features mechanism.
+ */
+ @VisibleForTesting
+ static class OptionalFlag implements Serializable {
+ private final String name;
+ private final List<String> flags;
+
+ @VisibleForTesting
+ OptionalFlag(String name, List<String> flags) {
+ this.name = name;
+ this.flags = flags;
+ }
+
+ private List<String> getFlags() {
+ return flags;
+ }
+
+ private String getName() {
+ return name;
+ }
+ }
+
+ @VisibleForTesting
+ static class FlagList implements Serializable {
+ private List<String> prefixFlags;
+ private List<OptionalFlag> optionalFlags;
+ private List<String> suffixFlags;
+
+ @VisibleForTesting
+ FlagList(List<String> prefixFlags,
+ List<OptionalFlag> optionalFlags,
+ List<String> suffixFlags) {
+ this.prefixFlags = prefixFlags;
+ this.optionalFlags = optionalFlags;
+ this.suffixFlags = suffixFlags;
+ }
+
+ @VisibleForTesting
+ List<String> evaluate(Collection<String> features) {
+ ImmutableList.Builder<String> result = ImmutableList.builder();
+ result.addAll(prefixFlags);
+ for (OptionalFlag optionalFlag : optionalFlags) {
+ // The flag is added if the default is true and the flag is not specified,
+ // or if the default is false and the flag is specified.
+ if (features.contains(optionalFlag.getName())) {
+ result.addAll(optionalFlag.getFlags());
+ }
+ }
+
+ result.addAll(suffixFlags);
+ return result.build();
+ }
+ }
+
+ private final Label crosstoolTop;
+ private final String hostSystemName;
+ private final String compiler;
+ private final String targetCpu;
+ private final String targetSystemName;
+ private final String targetLibc;
+ private final LipoMode lipoMode;
+ private final PathFragment crosstoolTopPathFragment;
+
+ private final String abi;
+ private final String abiGlibcVersion;
+
+ private final String toolchainIdentifier;
+ private final String cacheKey;
+
+ private final CcToolchainFeatures toolchainFeatures;
+ private final boolean supportsGoldLinker;
+ private final boolean supportsThinArchives;
+ private final boolean supportsStartEndLib;
+ private final boolean supportsInterfaceSharedObjects;
+ private final boolean supportsEmbeddedRuntimes;
+ private final boolean supportsFission;
+
+ // We encode three states with two booleans:
+ // (1) (false false) -> no pic code
+ // (2) (true false) -> shared libraries as pic, but not binaries
+ // (3) (true true) -> both shared libraries and binaries as pic
+ private final boolean toolchainNeedsPic;
+ private final boolean usePicForBinaries;
+
+ private final FdoSupport fdoSupport;
+
+ // TODO(bazel-team): All these labels (except for ccCompilerRuleLabel) can be removed once the
+ // transition to the cc_compiler rule is complete.
+ private final Label libcLabel;
+ private final Label staticRuntimeLibsLabel;
+ private final Label dynamicRuntimeLibsLabel;
+ private final Label ccToolchainLabel;
+
+ private final PathFragment sysroot;
+ private final PathFragment runtimeSysroot;
+ private final List<PathFragment> builtInIncludeDirectories;
+
+ private final Map<String, PathFragment> toolPaths;
+ private final PathFragment ldExecutable;
+
+ // Only used during construction.
+ private final List<String> commonLinkOptions;
+ private final ListMultimap<CompilationMode, String> linkOptionsFromCompilationMode;
+ private final ListMultimap<LipoMode, String> linkOptionsFromLipoMode;
+ private final ListMultimap<LinkingMode, String> linkOptionsFromLinkingMode;
+
+ private final FlagList compilerFlags;
+ private final FlagList cxxFlags;
+ private final FlagList unfilteredCompilerFlags;
+ private final List<String> cOptions;
+
+ private FlagList fullyStaticLinkFlags;
+ private FlagList mostlyStaticLinkFlags;
+ private FlagList mostlyStaticSharedLinkFlags;
+ private FlagList dynamicLinkFlags;
+ private FlagList dynamicLibraryLinkFlags;
+ private final List<String> testOnlyLinkFlags;
+
+ private final List<String> linkOptions;
+
+ private final List<String> objcopyOptions;
+ private final List<String> ldOptions;
+ private final List<String> arOptions;
+ private final List<String> arThinArchivesOptions;
+
+ private final Map<String, String> additionalMakeVariables;
+
+ private final CppOptions cppOptions;
+
+ // The dynamic mode for linking.
+ private final DynamicMode dynamicMode;
+ private final boolean stripBinaries;
+ private final ImmutableMap<String, String> commandLineDefines;
+ private final String solibDirectory;
+ private final CompilationMode compilationMode;
+ private final Path execRoot;
+ /**
+ * If true, the ConfiguredTarget is only used to get the necessary cross-referenced
+ * CppCompilationContexts, but registering build actions is disabled.
+ */
+ private final boolean lipoContextCollector;
+ private final Root greppedIncludesDirectory;
+
+ protected CppConfiguration(CppConfigurationParameters params)
+ throws InvalidConfigurationException {
+ CrosstoolConfig.CToolchain toolchain = params.toolchain;
+ cppOptions = params.buildOptions.get(CppOptions.class);
+ this.hostSystemName = toolchain.getHostSystemName();
+ this.compiler = toolchain.getCompiler();
+ this.targetCpu = toolchain.getTargetCpu();
+ this.lipoMode = cppOptions.getLipoMode();
+ this.targetSystemName = toolchain.getTargetSystemName();
+ this.targetLibc = toolchain.getTargetLibc();
+ this.crosstoolTop = params.crosstoolTop;
+ this.ccToolchainLabel = params.ccToolchainLabel;
+ this.compilationMode =
+ params.buildOptions.get(BuildConfiguration.Options.class).compilationMode;
+ this.lipoContextCollector = cppOptions.lipoCollector;
+ this.execRoot = params.execRoot;
+
+ // Note that the grepped includes directory is not configuration-specific; the paths of the
+ // files within that directory, however, are configuration-specific.
+ this.greppedIncludesDirectory = Root.asDerivedRoot(execRoot,
+ execRoot.getRelative(IncludeScanningUtil.GREPPED_INCLUDES));
+
+ this.crosstoolTopPathFragment = crosstoolTop.getPackageFragment();
+
+ try {
+ this.staticRuntimeLibsLabel =
+ crosstoolTop.getRelative(toolchain.hasStaticRuntimesFilegroup() ?
+ toolchain.getStaticRuntimesFilegroup() : "static-runtime-libs-" + targetCpu);
+ this.dynamicRuntimeLibsLabel =
+ crosstoolTop.getRelative(toolchain.hasDynamicRuntimesFilegroup() ?
+ toolchain.getDynamicRuntimesFilegroup() : "dynamic-runtime-libs-" + targetCpu);
+ } catch (SyntaxException e) {
+ // All of the above label.getRelative() calls are valid labels, and the crosstool_top
+ // was already checked earlier in the process.
+ throw new AssertionError(e);
+ }
+
+ if (cppOptions.lipoMode == LipoMode.BINARY) {
+ // TODO(bazel-team): implement dynamic linking with LIPO
+ this.dynamicMode = DynamicMode.OFF;
+ } else {
+ switch (cppOptions.dynamicMode) {
+ case DEFAULT:
+ this.dynamicMode = DynamicMode.DEFAULT; break;
+ case OFF: this.dynamicMode = DynamicMode.OFF; break;
+ case FULLY: this.dynamicMode = DynamicMode.FULLY; break;
+ default: throw new IllegalStateException("Invalid dynamicMode.");
+ }
+ }
+
+ this.fdoSupport = new FdoSupport(
+ params.buildOptions.get(CppOptions.class).fdoInstrument, params.fdoZip,
+ cppOptions.lipoMode, execRoot);
+
+ this.stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS ||
+ (cppOptions.stripBinaries == StripMode.SOMETIMES &&
+ compilationMode == CompilationMode.FASTBUILD));
+
+ CrosstoolConfigurationIdentifier crosstoolConfig =
+ CrosstoolConfigurationIdentifier.fromToolchain(toolchain);
+ Preconditions.checkState(crosstoolConfig.getCpu().equals(targetCpu));
+ Preconditions.checkState(crosstoolConfig.getCompiler().equals(compiler));
+ Preconditions.checkState(crosstoolConfig.getLibc().equals(targetLibc));
+
+ this.solibDirectory = "_solib_" + targetCpu;
+
+ this.toolchainIdentifier = toolchain.getToolchainIdentifier();
+ this.cacheKey = this + ":" + crosstoolTop + ":" + params.cacheKeySuffix + ":"
+ + lipoContextCollector;
+
+ this.toolchainFeatures = new CcToolchainFeatures(toolchain);
+ this.supportsGoldLinker = toolchain.getSupportsGoldLinker();
+ this.supportsThinArchives = toolchain.getSupportsThinArchives();
+ this.supportsStartEndLib = toolchain.getSupportsStartEndLib();
+ this.supportsInterfaceSharedObjects = toolchain.getSupportsInterfaceSharedObjects();
+ this.supportsEmbeddedRuntimes = toolchain.getSupportsEmbeddedRuntimes();
+ this.supportsFission = toolchain.getSupportsFission();
+ this.toolchainNeedsPic = toolchain.getNeedsPic();
+ this.usePicForBinaries =
+ toolchain.getNeedsPic() && compilationMode != CompilationMode.OPT;
+
+ this.toolPaths = Maps.newHashMap();
+ for (CrosstoolConfig.ToolPath tool : toolchain.getToolPathList()) {
+ PathFragment path = new PathFragment(tool.getPath());
+ if (!path.isNormalized()) {
+ throw new IllegalArgumentException("The include path '" + tool.getPath()
+ + "' is not normalized.");
+ }
+ toolPaths.put(tool.getName(), crosstoolTopPathFragment.getRelative(path));
+ }
+
+ if (toolPaths.isEmpty()) {
+ // If no paths are specified, we just use the names of the tools as the path.
+ for (Tool tool : Tool.values()) {
+ toolPaths.put(tool.getNamePart(),
+ crosstoolTopPathFragment.getRelative(tool.getNamePart()));
+ }
+ } else {
+ Iterable<Tool> neededTools = Iterables.filter(EnumSet.allOf(Tool.class),
+ new Predicate<Tool>() {
+ @Override
+ public boolean apply(Tool tool) {
+ if (tool == Tool.DWP) {
+ // When fission is unsupported, don't check for the dwp tool.
+ return supportsFission();
+ } else if (tool == Tool.GCOVTOOL) {
+ // gcov-tool is optional, don't check whether it's present
+ return false;
+ } else {
+ return true;
+ }
+ }
+ });
+ for (Tool tool : neededTools) {
+ if (!toolPaths.containsKey(tool.getNamePart())) {
+ throw new IllegalArgumentException("Tool path for '" + tool.getNamePart()
+ + "' is missing");
+ }
+ }
+ }
+
+ // We can't use an ImmutableMap.Builder here; we need the ability (at least
+ // in tests) to add entries with keys that are already in the map, and only
+ // HashMap supports this (by replacing the existing entry under the key).
+ Map<String, String> commandLineDefinesBuilder = new HashMap<>();
+ for (Map.Entry<String, String> define : cppOptions.commandLineDefinedVariables) {
+ commandLineDefinesBuilder.put(define.getKey(), define.getValue());
+ }
+ commandLineDefines = ImmutableMap.copyOf(commandLineDefinesBuilder);
+
+ ListMultimap<CompilationMode, String> cFlags = ArrayListMultimap.create();
+ ListMultimap<CompilationMode, String> cxxFlags = ArrayListMultimap.create();
+ linkOptionsFromCompilationMode = ArrayListMultimap.create();
+ for (CrosstoolConfig.CompilationModeFlags flags : toolchain.getCompilationModeFlagsList()) {
+ // Remove this when CROSSTOOL files no longer contain 'coverage'.
+ if (flags.getMode() == CrosstoolConfig.CompilationMode.COVERAGE) {
+ continue;
+ }
+ CompilationMode realmode = importCompilationMode(flags.getMode());
+ cFlags.putAll(realmode, flags.getCompilerFlagList());
+ cxxFlags.putAll(realmode, flags.getCxxFlagList());
+ linkOptionsFromCompilationMode.putAll(realmode, flags.getLinkerFlagList());
+ }
+
+ ListMultimap<LipoMode, String> lipoCFlags = ArrayListMultimap.create();
+ ListMultimap<LipoMode, String> lipoCxxFlags = ArrayListMultimap.create();
+ linkOptionsFromLipoMode = ArrayListMultimap.create();
+ for (CrosstoolConfig.LipoModeFlags flags : toolchain.getLipoModeFlagsList()) {
+ LipoMode realmode = flags.getMode();
+ lipoCFlags.putAll(realmode, flags.getCompilerFlagList());
+ lipoCxxFlags.putAll(realmode, flags.getCxxFlagList());
+ linkOptionsFromLipoMode.putAll(realmode, flags.getLinkerFlagList());
+ }
+
+ linkOptionsFromLinkingMode = ArrayListMultimap.create();
+ for (LinkingModeFlags flags : toolchain.getLinkingModeFlagsList()) {
+ LinkingMode realmode = importLinkingMode(flags.getMode());
+ linkOptionsFromLinkingMode.putAll(realmode, flags.getLinkerFlagList());
+ }
+
+ this.commonLinkOptions = ImmutableList.copyOf(toolchain.getLinkerFlagList());
+ dynamicLibraryLinkFlags = new FlagList(
+ ImmutableList.copyOf(toolchain.getDynamicLibraryLinkerFlagList()),
+ convertOptionalOptions(toolchain.getOptionalDynamicLibraryLinkerFlagList()),
+ Collections.<String>emptyList());
+ this.objcopyOptions = ImmutableList.copyOf(toolchain.getObjcopyEmbedFlagList());
+ this.ldOptions = ImmutableList.copyOf(toolchain.getLdEmbedFlagList());
+ this.arOptions = copyOrDefaultIfEmpty(toolchain.getArFlagList(), "rcsD");
+ this.arThinArchivesOptions = copyOrDefaultIfEmpty(
+ toolchain.getArThinArchivesFlagList(), "rcsDT");
+
+ this.abi = toolchain.getAbiVersion();
+ this.abiGlibcVersion = toolchain.getAbiLibcVersion();
+
+ // The default value for optional string attributes is the empty string.
+ PathFragment defaultSysroot = toolchain.getBuiltinSysroot().length() == 0
+ ? null
+ : new PathFragment(toolchain.getBuiltinSysroot());
+ if ((defaultSysroot != null) && !defaultSysroot.isNormalized()) {
+ throw new IllegalArgumentException("The built-in sysroot '" + defaultSysroot
+ + "' is not normalized.");
+ }
+
+ if ((cppOptions.libcTop != null) && (defaultSysroot == null)) {
+ throw new InvalidConfigurationException("The selected toolchain " + toolchainIdentifier
+ + " does not support setting --grte_top.");
+ }
+ LibcTop libcTop = cppOptions.libcTop;
+ if ((libcTop == null) && !toolchain.getDefaultGrteTop().isEmpty()) {
+ try {
+ libcTop = new CppOptions.LibcTopConverter().convert(toolchain.getDefaultGrteTop());
+ } catch (OptionsParsingException e) {
+ throw new InvalidConfigurationException(e.getMessage(), e);
+ }
+ }
+ if ((libcTop != null) && (libcTop.getLabel() != null)) {
+ libcLabel = libcTop.getLabel();
+ } else {
+ libcLabel = null;
+ }
+
+ ImmutableList.Builder<PathFragment> builtInIncludeDirectoriesBuilder
+ = ImmutableList.builder();
+ sysroot = libcTop == null ? defaultSysroot : libcTop.getSysroot();
+ for (String s : toolchain.getCxxBuiltinIncludeDirectoryList()) {
+ builtInIncludeDirectoriesBuilder.add(
+ resolveIncludeDir(s, sysroot, crosstoolTopPathFragment));
+ }
+ builtInIncludeDirectories = builtInIncludeDirectoriesBuilder.build();
+
+ // The runtime sysroot should really be set from --grte_top. However, currently libc has no
+ // way to set the sysroot. The CROSSTOOL file does set the runtime sysroot, in the
+ // builtin_sysroot field. This implies that you can not arbitrarily mix and match Crosstool
+ // and libc versions, you must always choose compatible ones.
+ runtimeSysroot = defaultSysroot;
+
+ String sysrootFlag;
+ if (sysroot != null && !sysroot.equals(defaultSysroot)) {
+ // Only specify the --sysroot option if it is different from the built-in one.
+ sysrootFlag = "--sysroot=" + sysroot;
+ } else {
+ sysrootFlag = null;
+ }
+
+ ImmutableList.Builder<String> unfilteredCoptsBuilder = ImmutableList.builder();
+ if (sysrootFlag != null) {
+ unfilteredCoptsBuilder.add(sysrootFlag);
+ }
+ unfilteredCoptsBuilder.addAll(toolchain.getUnfilteredCxxFlagList());
+ unfilteredCompilerFlags = new FlagList(
+ unfilteredCoptsBuilder.build(),
+ convertOptionalOptions(toolchain.getOptionalUnfilteredCxxFlagList()),
+ Collections.<String>emptyList());
+
+ ImmutableList.Builder<String> linkoptsBuilder = ImmutableList.builder();
+ linkoptsBuilder.addAll(cppOptions.linkoptList);
+ if (cppOptions.experimentalOmitfp) {
+ linkoptsBuilder.add("-Wl,--eh-frame-hdr");
+ }
+ if (sysrootFlag != null) {
+ linkoptsBuilder.add(sysrootFlag);
+ }
+ this.linkOptions = linkoptsBuilder.build();
+
+ ImmutableList.Builder<String> coptsBuilder = ImmutableList.<String>builder()
+ .addAll(toolchain.getCompilerFlagList())
+ .addAll(cFlags.get(compilationMode))
+ .addAll(lipoCFlags.get(cppOptions.getLipoMode()));
+ if (cppOptions.experimentalOmitfp) {
+ coptsBuilder.add("-fomit-frame-pointer");
+ coptsBuilder.add("-fasynchronous-unwind-tables");
+ coptsBuilder.add("-DNO_FRAME_POINTER");
+ }
+ this.compilerFlags = new FlagList(
+ coptsBuilder.build(),
+ convertOptionalOptions(toolchain.getOptionalCompilerFlagList()),
+ cppOptions.coptList);
+
+ this.cOptions = ImmutableList.copyOf(cppOptions.conlyoptList);
+
+ ImmutableList.Builder<String> cxxOptsBuilder = ImmutableList.<String>builder()
+ .addAll(toolchain.getCxxFlagList())
+ .addAll(cxxFlags.get(compilationMode))
+ .addAll(lipoCxxFlags.get(cppOptions.getLipoMode()));
+
+ this.cxxFlags = new FlagList(
+ cxxOptsBuilder.build(),
+ convertOptionalOptions(toolchain.getOptionalCxxFlagList()),
+ cppOptions.cxxoptList);
+
+ this.ldExecutable = getToolPathFragment(CppConfiguration.Tool.LD);
+
+ boolean stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS) ||
+ ((cppOptions.stripBinaries == StripMode.SOMETIMES) &&
+ (compilationMode == CompilationMode.FASTBUILD));
+
+ fullyStaticLinkFlags = new FlagList(
+ configureLinkerOptions(compilationMode, lipoMode, LinkingMode.FULLY_STATIC,
+ ldExecutable, stripBinaries),
+ convertOptionalOptions(toolchain.getOptionalLinkerFlagList()),
+ Collections.<String>emptyList());
+ mostlyStaticLinkFlags = new FlagList(
+ configureLinkerOptions(compilationMode, lipoMode, LinkingMode.MOSTLY_STATIC,
+ ldExecutable, stripBinaries),
+ convertOptionalOptions(toolchain.getOptionalLinkerFlagList()),
+ Collections.<String>emptyList());
+ mostlyStaticSharedLinkFlags = new FlagList(
+ configureLinkerOptions(compilationMode, lipoMode,
+ LinkingMode.MOSTLY_STATIC_LIBRARIES, ldExecutable, stripBinaries),
+ convertOptionalOptions(toolchain.getOptionalLinkerFlagList()),
+ Collections.<String>emptyList());
+ dynamicLinkFlags = new FlagList(
+ configureLinkerOptions(compilationMode, lipoMode, LinkingMode.DYNAMIC,
+ ldExecutable, stripBinaries),
+ convertOptionalOptions(toolchain.getOptionalLinkerFlagList()),
+ Collections.<String>emptyList());
+ testOnlyLinkFlags = ImmutableList.copyOf(toolchain.getTestOnlyLinkerFlagList());
+
+ Map<String, String> makeVariablesBuilder = new HashMap<>();
+ // The following are to be used to allow some build rules to avoid the limits on stack frame
+ // sizes and variable-length arrays. Ensure that these are always set.
+ makeVariablesBuilder.put("STACK_FRAME_UNLIMITED", "");
+ makeVariablesBuilder.put("CC_FLAGS", "");
+ for (CrosstoolConfig.MakeVariable variable : toolchain.getMakeVariableList()) {
+ makeVariablesBuilder.put(variable.getName(), variable.getValue());
+ }
+ if (sysrootFlag != null) {
+ String ccFlags = makeVariablesBuilder.get("CC_FLAGS");
+ ccFlags = ccFlags.isEmpty() ? sysrootFlag : ccFlags + " " + sysrootFlag;
+ makeVariablesBuilder.put("CC_FLAGS", ccFlags);
+ }
+ this.additionalMakeVariables = ImmutableMap.copyOf(makeVariablesBuilder);
+ }
+
+ private List<OptionalFlag> convertOptionalOptions(
+ List<CrosstoolConfig.CToolchain.OptionalFlag> optionalFlagList)
+ throws IllegalArgumentException {
+ List<OptionalFlag> result = new ArrayList<>();
+
+ for (CrosstoolConfig.CToolchain.OptionalFlag crosstoolOptionalFlag : optionalFlagList) {
+ String name = crosstoolOptionalFlag.getDefaultSettingName();
+ result.add(new OptionalFlag(
+ name,
+ ImmutableList.copyOf(crosstoolOptionalFlag.getFlagList())));
+ }
+
+ return result;
+ }
+
+ private static ImmutableList<String> copyOrDefaultIfEmpty(List<String> list,
+ String defaultValue) {
+ return list.isEmpty() ? ImmutableList.of(defaultValue) : ImmutableList.copyOf(list);
+ }
+
+ @VisibleForTesting
+ static CompilationMode importCompilationMode(CrosstoolConfig.CompilationMode mode) {
+ return CompilationMode.valueOf(mode.name());
+ }
+
+ @VisibleForTesting
+ static LinkingMode importLinkingMode(CrosstoolConfig.LinkingMode mode) {
+ return LinkingMode.valueOf(mode.name());
+ }
+
+ private static final PathFragment SYSROOT_FRAGMENT = new PathFragment("%sysroot%");
+
+ /**
+ * Resolve the given include directory. If it is not absolute, it is
+ * interpreted relative to the crosstool top. If it starts with %sysroot%/,
+ * that part is replaced with the actual sysroot.
+ */
+ static PathFragment resolveIncludeDir(String s, PathFragment sysroot,
+ PathFragment crosstoolTopPathFragment) {
+ PathFragment path = new PathFragment(s);
+ if (!path.isNormalized()) {
+ throw new IllegalArgumentException("The include path '" + s + "' is not normalized.");
+ }
+ if (path.startsWith(SYSROOT_FRAGMENT)) {
+ if (sysroot == null) {
+ throw new IllegalArgumentException("A %sysroot% prefix is only allowed if the "
+ + "default_sysroot option is set");
+ }
+ return sysroot.getRelative(path.relativeTo(SYSROOT_FRAGMENT));
+ } else {
+ return crosstoolTopPathFragment.getRelative(path);
+ }
+ }
+
+ /**
+ * Returns the configuration-independent grepped-includes directory.
+ */
+ public Root getGreppedIncludesDirectory() {
+ return greppedIncludesDirectory;
+ }
+
+ @VisibleForTesting
+ List<String> configureLinkerOptions(
+ CompilationMode compilationMode, LipoMode lipoMode, LinkingMode linkingMode,
+ PathFragment ldExecutable, boolean stripBinaries) {
+ List<String> result = new ArrayList<>();
+ result.addAll(commonLinkOptions);
+
+ result.add("-B" + ldExecutable.getParentDirectory().getPathString());
+ if (stripBinaries) {
+ result.add("-Wl,-S");
+ }
+
+ result.addAll(linkOptionsFromCompilationMode.get(compilationMode));
+ result.addAll(linkOptionsFromLipoMode.get(lipoMode));
+ result.addAll(linkOptionsFromLinkingMode.get(linkingMode));
+ return ImmutableList.copyOf(result);
+ }
+
+ /**
+ * Returns the toolchain identifier, which uniquely identifies the compiler
+ * version, target libc version, target cpu, and LIPO linkage.
+ */
+ public String getToolchainIdentifier() {
+ return toolchainIdentifier;
+ }
+
+ /**
+ * Returns the system name which is required by the toolchain to run.
+ */
+ public String getHostSystemName() {
+ return hostSystemName;
+ }
+
+ @Override
+ public String toString() {
+ return toolchainIdentifier;
+ }
+
+ /**
+ * Returns the compiler version string (e.g. "gcc-4.1.1").
+ */
+ @SkylarkCallable(name = "compiler", structField = true, doc = "C++ compiler.")
+ public String getCompiler() {
+ return compiler;
+ }
+
+ /**
+ * Returns the libc version string (e.g. "glibc-2.2.2").
+ */
+ public String getTargetLibc() {
+ return targetLibc;
+ }
+
+ /**
+ * Returns the target architecture using blaze-specific constants (e.g. "piii").
+ */
+ @SkylarkCallable(name = "cpu", structField = true, doc = "Target CPU of the C++ toolchain.")
+ public String getTargetCpu() {
+ return targetCpu;
+ }
+
+ /**
+ * Returns the path fragment that is either absolute or relative to the
+ * execution root that can be used to execute the given tool.
+ *
+ * <p>Note that you must not use this method to get the linker location, but
+ * use {@link #getLdExecutable} instead!
+ */
+ public PathFragment getToolPathFragment(CppConfiguration.Tool tool) {
+ return toolPaths.get(tool.getNamePart());
+ }
+
+ /**
+ * Returns a label that forms a dependency to the files required for the
+ * sysroot that is used.
+ */
+ public Label getLibcLabel() {
+ return libcLabel;
+ }
+
+ /**
+ * Returns a label that references the library files needed to statically
+ * link the C++ runtime (i.e. libgcc.a, libgcc_eh.a, libstdc++.a) for the
+ * target architecture.
+ */
+ public Label getStaticRuntimeLibsLabel() {
+ return supportsEmbeddedRuntimes() ? staticRuntimeLibsLabel : null;
+ }
+
+ /**
+ * Returns a label that references the library files needed to dynamically
+ * link the C++ runtime (i.e. libgcc_s.so, libstdc++.so) for the target
+ * architecture.
+ */
+ public Label getDynamicRuntimeLibsLabel() {
+ return supportsEmbeddedRuntimes() ? dynamicRuntimeLibsLabel : null;
+ }
+
+ /**
+ * Returns the label of the <code>cc_compiler</code> rule for the C++ configuration.
+ */
+ public Label getCcToolchainRuleLabel() {
+ return ccToolchainLabel;
+ }
+
+ /**
+ * Returns the abi we're using, which is a gcc version. E.g.: "gcc-3.4".
+ * Note that in practice we might be using gcc-3.4 as ABI even when compiling
+ * with gcc-4.1.0, because ABIs are backwards compatible.
+ */
+ // TODO(bazel-team): The javadoc should clarify how this is used in Blaze.
+ public String getAbi() {
+ return abi;
+ }
+
+ /**
+ * Returns the glibc version used by the abi we're using. This is a
+ * glibc version number (e.g., "2.2.2"). Note that in practice we
+ * might be using glibc 2.2.2 as ABI even when compiling with
+ * gcc-4.2.2, gcc-4.3.1, or gcc-4.4.0 (which use glibc 2.3.6),
+ * because ABIs are backwards compatible.
+ */
+ // TODO(bazel-team): The javadoc should clarify how this is used in Blaze.
+ public String getAbiGlibcVersion() {
+ return abiGlibcVersion;
+ }
+
+ /**
+ * Returns the configured features of the toolchain. Rules should not call this directly, but
+ * instead use {@code CcToolchainProvider.getFeatures}.
+ */
+ public CcToolchainFeatures getFeatures() {
+ return toolchainFeatures;
+ }
+
+ /**
+ * Returns whether the toolchain supports the gold linker.
+ */
+ public boolean supportsGoldLinker() {
+ return supportsGoldLinker;
+ }
+
+ /**
+ * Returns whether the toolchain supports thin archives.
+ */
+ public boolean supportsThinArchives() {
+ return supportsThinArchives;
+ }
+
+ /**
+ * Returns whether the toolchain supports the --start-lib/--end-lib options.
+ */
+ public boolean supportsStartEndLib() {
+ return supportsStartEndLib;
+ }
+
+ /**
+ * Returns whether build_interface_so can build interface shared objects for this toolchain.
+ * Should be true if this toolchain generates ELF objects.
+ */
+ public boolean supportsInterfaceSharedObjects() {
+ return supportsInterfaceSharedObjects;
+ }
+
+ /**
+ * Returns whether the toolchain supports linking C/C++ runtime libraries
+ * supplied inside the toolchain distribution.
+ */
+ public boolean supportsEmbeddedRuntimes() {
+ return supportsEmbeddedRuntimes;
+ }
+
+ /**
+ * Returns whether the toolchain supports EXEC_ORIGIN libraries resolution.
+ */
+ public boolean supportsExecOrigin() {
+ // We're rolling out support for this in the same release that also supports embedded runtimes.
+ return supportsEmbeddedRuntimes;
+ }
+
+ /**
+ * Returns whether the toolchain supports "Fission" C++ builds, i.e. builds
+ * where compilation partitions object code and debug symbols into separate
+ * output files.
+ */
+ public boolean supportsFission() {
+ return supportsFission;
+ }
+
+ /**
+ * Returns whether shared libraries must be compiled with position
+ * independent code on this platform.
+ */
+ public boolean toolchainNeedsPic() {
+ return toolchainNeedsPic;
+ }
+
+ /**
+ * Returns whether binaries must be compiled with position independent code.
+ */
+ public boolean usePicForBinaries() {
+ return usePicForBinaries;
+ }
+
+ /**
+ * Returns the type of archives being used.
+ */
+ public Link.ArchiveType archiveType() {
+ if (useStartEndLib()) {
+ return Link.ArchiveType.START_END_LIB;
+ }
+ if (useThinArchives()) {
+ return Link.ArchiveType.THIN;
+ }
+ return Link.ArchiveType.FAT;
+ }
+
+ /**
+ * Returns the ar flags to be used.
+ */
+ public List<String> getArFlags(boolean thinArchives) {
+ return thinArchives ? arThinArchivesOptions : arOptions;
+ }
+
+ /**
+ * Returns a string that uniquely identifies the toolchain.
+ */
+ @Override
+ public String cacheKey() {
+ return cacheKey;
+ }
+
+ /**
+ * Returns the built-in list of system include paths for the toolchain
+ * compiler. All paths in this list should be relative to the exec directory.
+ * They may be absolute if they are also installed on the remote build nodes or
+ * for local compilation.
+ */
+ public List<PathFragment> getBuiltInIncludeDirectories() {
+ return builtInIncludeDirectories;
+ }
+
+ /**
+ * Returns the sysroot to be used. If the toolchain compiler does not support
+ * different sysroots, or the sysroot is the same as the default sysroot, then
+ * this method returns <code>null</code>.
+ */
+ public PathFragment getSysroot() {
+ return sysroot;
+ }
+
+ /**
+ * Returns the run time sysroot, which is where the dynamic linker
+ * and system libraries are found at runtime. This is usually an absolute path. If the
+ * toolchain compiler does not support sysroots, then this method returns <code>null</code>.
+ */
+ public PathFragment getRuntimeSysroot() {
+ return runtimeSysroot;
+ }
+
+ /**
+ * Returns the default options to use for compiling C, C++, and assembler.
+ * This is just the options that should be used for all three languages.
+ * There may be additional C-specific or C++-specific options that should be used,
+ * in addition to the ones returned by this method;
+ */
+ public List<String> getCompilerOptions(Collection<String> features) {
+ return compilerFlags.evaluate(features);
+ }
+
+ /**
+ * Returns the list of additional C-specific options to use for compiling
+ * C. These should be go on the command line after the common options
+ * returned by {@link #getCompilerOptions}.
+ */
+ public List<String> getCOptions() {
+ return cOptions;
+ }
+
+ /**
+ * Returns the list of additional C++-specific options to use for compiling
+ * C++. These should be go on the command line after the common options
+ * returned by {@link #getCompilerOptions}.
+ */
+ public List<String> getCxxOptions(Collection<String> features) {
+ return cxxFlags.evaluate(features);
+ }
+
+ /**
+ * Returns the default list of options which cannot be filtered by BUILD
+ * rules. These should be appended to the command line after filtering.
+ */
+ public List<String> getUnfilteredCompilerOptions(Collection<String> features) {
+ return unfilteredCompilerFlags.evaluate(features);
+ }
+
+ /**
+ * Returns the set of command-line linker options, including any flags
+ * inferred from the command-line options.
+ *
+ * @see Link
+ */
+ // TODO(bazel-team): Clean up the linker options computation!
+ public List<String> getLinkOptions() {
+ return linkOptions;
+ }
+
+ /**
+ * Returns the immutable list of linker options for fully statically linked
+ * outputs. Does not include command-line options passed via --linkopt or
+ * --linkopts.
+ *
+ * @param features default settings affecting this link
+ * @param sharedLib true if the output is a shared lib, false if it's an executable
+ */
+ public List<String> getFullyStaticLinkOptions(Collection<String> features,
+ boolean sharedLib) {
+ if (sharedLib) {
+ return getSharedLibraryLinkOptions(mostlyStaticLinkFlags, features);
+ } else {
+ return fullyStaticLinkFlags.evaluate(features);
+ }
+ }
+
+ /**
+ * Returns the immutable list of linker options for mostly statically linked
+ * outputs. Does not include command-line options passed via --linkopt or
+ * --linkopts.
+ *
+ * @param features default settings affecting this link
+ * @param sharedLib true if the output is a shared lib, false if it's an executable
+ */
+ public List<String> getMostlyStaticLinkOptions(Collection<String> features,
+ boolean sharedLib) {
+ if (sharedLib) {
+ return getSharedLibraryLinkOptions(
+ supportsEmbeddedRuntimes ? mostlyStaticSharedLinkFlags : dynamicLinkFlags,
+ features);
+ } else {
+ return mostlyStaticLinkFlags.evaluate(features);
+ }
+ }
+
+ /**
+ * Returns the immutable list of linker options for artifacts that are not
+ * fully or mostly statically linked. Does not include command-line options
+ * passed via --linkopt or --linkopts.
+ *
+ * @param features default settings affecting this link
+ * @param sharedLib true if the output is a shared lib, false if it's an executable
+ */
+ public List<String> getDynamicLinkOptions(Collection<String> features,
+ boolean sharedLib) {
+ if (sharedLib) {
+ return getSharedLibraryLinkOptions(dynamicLinkFlags, features);
+ } else {
+ return dynamicLinkFlags.evaluate(features);
+ }
+ }
+
+ /**
+ * Returns link options for the specified flag list, combined with universal options
+ * for all shared libraries (regardless of link staticness).
+ */
+ private List<String> getSharedLibraryLinkOptions(FlagList flags,
+ Collection<String> features) {
+ return ImmutableList.<String>builder()
+ .addAll(flags.evaluate(features))
+ .addAll(dynamicLibraryLinkFlags.evaluate(features))
+ .build();
+ }
+
+ /**
+ * Returns test-only link options such that certain test-specific features can be configured
+ * separately (e.g. lazy binding).
+ */
+ public List<String> getTestOnlyLinkOptions() {
+ return testOnlyLinkFlags;
+ }
+
+
+ /**
+ * Returns the list of options to be used with 'objcopy' when converting
+ * binary files to object files, or {@code null} if this operation is not
+ * supported.
+ */
+ public List<String> getObjCopyOptionsForEmbedding() {
+ return objcopyOptions;
+ }
+
+ /**
+ * Returns the list of options to be used with 'ld' when converting
+ * binary files to object files, or {@code null} if this operation is not
+ * supported.
+ */
+ public List<String> getLdOptionsForEmbedding() {
+ return ldOptions;
+ }
+
+ /**
+ * Returns a map of additional make variables for use by {@link
+ * BuildConfiguration}. These are to used to allow some build rules to
+ * avoid the limits on stack frame sizes and variable-length arrays.
+ *
+ * <p>The returned map must contain an entry for {@code STACK_FRAME_UNLIMITED},
+ * though the entry may be an empty string.
+ */
+ @VisibleForTesting
+ public Map<String, String> getAdditionalMakeVariables() {
+ return additionalMakeVariables;
+ }
+
+ /**
+ * Returns the execution path to the linker binary to use for this build.
+ * Relative paths are relative to the execution root.
+ */
+ public PathFragment getLdExecutable() {
+ return ldExecutable;
+ }
+
+ /**
+ * Returns the dynamic linking mode (full, off, or default).
+ */
+ public DynamicMode getDynamicMode() {
+ return dynamicMode;
+ }
+
+ /*
+ * If true then the directory name for non-LIPO targets will have a '-lipodata' suffix in
+ * AutoFDO mode.
+ */
+ public boolean getAutoFdoLipoData() {
+ return cppOptions.autoFdoLipoData;
+ }
+
+ /**
+ * Returns the STL label if given on the command line. {@code null}
+ * otherwise.
+ */
+ public Label getStl() {
+ return cppOptions.stl;
+ }
+
+ /*
+ * Returns the command-line "Make" variable overrides.
+ */
+ @Override
+ public ImmutableMap<String, String> getCommandLineDefines() {
+ return commandLineDefines;
+ }
+
+ /**
+ * Returns the command-line override value for the specified "Make" variable
+ * for this configuration, or null if none.
+ */
+ public String getMakeVariableOverride(String var) {
+ return commandLineDefines.get(var);
+ }
+
+ public boolean shouldScanIncludes() {
+ return cppOptions.scanIncludes;
+ }
+
+ /**
+ * Returns the currently active LIPO compilation mode.
+ */
+ public LipoMode getLipoMode() {
+ return cppOptions.lipoMode;
+ }
+
+ public boolean isFdo() {
+ return cppOptions.isFdo();
+ }
+
+ public boolean isLipoOptimization() {
+ // The LIPO optimization bits are set in the LIPO context collector configuration, too.
+ return cppOptions.isLipoOptimization() && !isLipoContextCollector();
+ }
+
+ public boolean isLipoOptimizationOrInstrumentation() {
+ return cppOptions.isLipoOptimizationOrInstrumentation();
+ }
+
+ /**
+ * Returns true if it is AutoFDO LIPO build.
+ */
+ public boolean isAutoFdoLipo() {
+ return cppOptions.fdoOptimize != null && FdoSupport.isAutoFdo(cppOptions.fdoOptimize)
+ && getLipoMode() != LipoMode.OFF;
+ }
+
+ /**
+ * Returns the default header check mode.
+ */
+ public HeadersCheckingMode getHeadersCheckingMode() {
+ return cppOptions.headersCheckingMode;
+ }
+
+ /**
+ * Returns whether or not to strip the binaries.
+ */
+ public boolean shouldStripBinaries() {
+ return stripBinaries;
+ }
+
+ /**
+ * Returns the additional options to pass to strip when generating a
+ * {@code <name>.stripped} binary by this build.
+ */
+ public List<String> getStripOpts() {
+ return cppOptions.stripoptList;
+ }
+
+ /**
+ * Returns whether temporary outputs from gcc will be saved.
+ */
+ public boolean getSaveTemps() {
+ return cppOptions.saveTemps;
+ }
+
+ /**
+ * Returns the {@link PerLabelOptions} to apply to the gcc command line, if
+ * the label of the compiled file matches the regular expression.
+ */
+ public List<PerLabelOptions> getPerFileCopts() {
+ return cppOptions.perFileCopts;
+ }
+
+ public Label getLipoContextLabel() {
+ return cppOptions.getLipoContextLabel();
+ }
+
+ /**
+ * Returns the custom malloc library label.
+ */
+ public Label customMalloc() {
+ return cppOptions.customMalloc;
+ }
+
+ /**
+ * Returns the extra warnings enabled for C compilation.
+ */
+ public List<String> getCWarns() {
+ return cppOptions.cWarns;
+ }
+
+ /**
+ * Returns true if mostly-static C++ binaries should be skipped.
+ */
+ public boolean skipStaticOutputs() {
+ return cppOptions.skipStaticOutputs;
+ }
+
+ /**
+ * Returns true if Fission is specified for this build and supported by the crosstool.
+ */
+ public boolean useFission() {
+ return cppOptions.fissionModes.contains(compilationMode) && supportsFission();
+ }
+
+ /**
+ * Returns true if all C++ compilations should produce position-independent code, links should
+ * produce position-independent executables, and dependencies with equivalent pre-built pic and
+ * nopic versions should apply the pic versions. Returns false if default settings should be
+ * applied (i.e. make no special provisions for pic code).
+ */
+ public boolean forcePic() {
+ return cppOptions.forcePic;
+ }
+
+ public boolean useStartEndLib() {
+ return cppOptions.useStartEndLib && supportsStartEndLib();
+ }
+
+ public boolean useThinArchives() {
+ return cppOptions.useThinArchives && supportsThinArchives();
+ }
+
+ /**
+ * Returns true if interface shared objects should be used.
+ */
+ public boolean useInterfaceSharedObjects() {
+ return supportsInterfaceSharedObjects() && cppOptions.useInterfaceSharedObjects;
+ }
+
+ public boolean forceIgnoreDashStatic() {
+ return cppOptions.forceIgnoreDashStatic;
+ }
+
+ /**
+ * Returns true iff this build configuration requires inclusion extraction
+ * (for include scanning) in the action graph.
+ */
+ public boolean needsIncludeScanning() {
+ return cppOptions.extractInclusions;
+ }
+
+ public boolean createCppModuleMaps() {
+ return cppOptions.cppModuleMaps;
+ }
+
+ /**
+ * Returns true if shared libraries must be compiled with position independent code
+ * on this platform or in this configuration.
+ */
+ public boolean needsPic() {
+ return forcePic() || toolchainNeedsPic();
+ }
+
+ /**
+ * Returns true iff we should use ".pic.o" files when linking executables.
+ */
+ public boolean usePicObjectsForBinaries() {
+ return forcePic() || usePicForBinaries();
+ }
+
+ public boolean legacyWholeArchive() {
+ return cppOptions.legacyWholeArchive;
+ }
+
+ public boolean getSymbolCounts() {
+ return cppOptions.symbolCounts;
+ }
+
+ public boolean getInmemoryDotdFiles() {
+ return cppOptions.inmemoryDotdFiles;
+ }
+
+ public boolean useIsystemForIncludes() {
+ return cppOptions.useIsystemForIncludes;
+ }
+
+ public LibcTop getLibcTop() {
+ return cppOptions.libcTop;
+ }
+
+ public boolean getUseInterfaceSharedObjects() {
+ return cppOptions.useInterfaceSharedObjects;
+ }
+
+ /**
+ * Returns the FDO support object.
+ */
+ public FdoSupport getFdoSupport() {
+ return fdoSupport;
+ }
+
+ /**
+ * Return the name of the directory (relative to the bin directory) that
+ * holds mangled links to shared libraries. This name is always set to
+ * the '{@code _solib_<cpu_archictecture_name>}.
+ */
+ public String getSolibDirectory() {
+ return solibDirectory;
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'objcopy' binary to use for this
+ * build. (Corresponds to $(OBJCOPY) in make-dbg.) Relative paths are
+ * relative to the execution root.
+ */
+ public PathFragment getObjCopyExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.OBJCOPY);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'gcc' binary that should be used
+ * by this build. This binary should support compilation of both C (*.c)
+ * and C++ (*.cc) files. Relative paths are relative to the execution root.
+ */
+ public PathFragment getCppExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.GCC);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'g++' binary that should be used
+ * by this build. This binary should support linking of both C (*.c)
+ * and C++ (*.cc) files. Relative paths are relative to the execution root.
+ */
+ public PathFragment getCppLinkExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.GCC);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'cpp' binary that should be used
+ * by this build. Relative paths are relative to the execution root.
+ */
+ public PathFragment getCpreprocessorExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.CPP);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'gcov' binary that should be used
+ * by this build to analyze C++ coverage data. Relative paths are relative to
+ * the execution root.
+ */
+ public PathFragment getGcovExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.GCOV);
+ }
+
+ /**
+ * Returns the path to the 'gcov-tool' executable that should be used
+ * by this build. Relative paths are relative to the execution root.
+ */
+ public PathFragment getGcovToolExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.GCOVTOOL);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'nm' executable that should be used
+ * by this build. Used only for testing. Relative paths are relative to the
+ * execution root.
+ */
+ public PathFragment getNmExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.NM);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'objdump' executable that should be
+ * used by this build. Used only for testing. Relative paths are relative to
+ * the execution root.
+ */
+ public PathFragment getObjdumpExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.OBJDUMP);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'ar' binary to use for this build.
+ * Relative paths are relative to the execution root.
+ */
+ public PathFragment getArExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.AR);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'strip' executable that should be used
+ * by this build. Relative paths are relative to the execution root.
+ */
+ public PathFragment getStripExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.STRIP);
+ }
+
+ /**
+ * Returns the path to the GNU binutils 'dwp' binary that should be used by this
+ * build to combine debug info output from individual C++ compilations (i.e. .dwo
+ * files) into aggregate target-level debug packages. Relative paths are relative to the
+ * execution root. See https://gcc.gnu.org/wiki/DebugFission .
+ */
+ public PathFragment getDwpExecutable() {
+ return getToolPathFragment(CppConfiguration.Tool.DWP);
+ }
+
+ /**
+ * Returns the GNU System Name
+ */
+ public String getTargetGnuSystemName() {
+ return targetSystemName;
+ }
+
+ /**
+ * Returns the architecture component of the GNU System Name
+ */
+ public String getGnuSystemArch() {
+ if (targetSystemName.indexOf('-') == -1) {
+ return targetSystemName;
+ }
+ return targetSystemName.substring(0, targetSystemName.indexOf('-'));
+ }
+
+ /**
+ * Returns whether the configuration's purpose is only to collect LIPO-related data.
+ */
+ public boolean isLipoContextCollector() {
+ return lipoContextCollector;
+ }
+
+ @Override
+ public String getName() {
+ return "cpp";
+ }
+
+ @Override
+ public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
+ CppOptions cppOptions = buildOptions.get(CppOptions.class);
+ if (stripBinaries) {
+ boolean warn = cppOptions.coptList.contains("-g");
+ for (PerLabelOptions opt : cppOptions.perFileCopts) {
+ warn |= opt.getOptions().contains("-g");
+ }
+ if (warn) {
+ reporter.handle(Event.warn("Stripping enabled, but '--copt=-g' (or --per_file_copt=...@-g) "
+ + "specified. Debug information will be generated and then stripped away. This is "
+ + "probably not what you want! Use '-c dbg' for debug mode, or use '--strip=never' "
+ + "to disable stripping"));
+ }
+ }
+
+ if (cppOptions.fdoInstrument != null && cppOptions.fdoOptimize != null) {
+ reporter.handle(Event.error("Cannot instrument and optimize for FDO at the same time. "
+ + "Remove one of the '--fdo_instrument' and '--fdo_optimize' options"));
+ }
+
+ if (cppOptions.lipoContext != null) {
+ if (cppOptions.lipoMode != LipoMode.BINARY || cppOptions.fdoOptimize == null) {
+ reporter.handle(Event.warn("The --lipo_context option can only be used together with "
+ + "--fdo_optimize=<profile zip> and --lipo=binary. LIPO context will be ignored."));
+ }
+ } else {
+ if (cppOptions.lipoMode == LipoMode.BINARY && cppOptions.fdoOptimize != null) {
+ reporter.handle(Event.error("The --lipo_context option must be specified when using "
+ + "--fdo_optimize=<profile zip> and --lipo=binary"));
+ }
+ }
+ if (cppOptions.lipoMode == LipoMode.BINARY &&
+ compilationMode != CompilationMode.OPT) {
+ reporter.handle(Event.error(
+ "'--lipo=binary' can only be used with '--compilation_mode=opt' (or '-c opt')"));
+ }
+
+ if (cppOptions.fissionModes.contains(compilationMode) && !supportsFission()) {
+ reporter.handle(
+ Event.warn("Fission is not supported by this crosstool. Please use a supporting " +
+ "crosstool to enable fission"));
+ }
+ }
+
+ @Override
+ public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) {
+ // hardcoded CC->gcc setting for unit tests
+ globalMakeEnvBuilder.put("CC", getCppExecutable().getPathString());
+
+ // Make variables provided by crosstool/gcc compiler suite.
+ globalMakeEnvBuilder.put("AR", getArExecutable().getPathString());
+ globalMakeEnvBuilder.put("NM", getNmExecutable().getPathString());
+ globalMakeEnvBuilder.put("OBJCOPY", getObjCopyExecutable().getPathString());
+ globalMakeEnvBuilder.put("STRIP", getStripExecutable().getPathString());
+
+ PathFragment gcovtool = getGcovToolExecutable();
+ if (gcovtool != null) {
+ // gcov-tool is optional in Crosstool
+ globalMakeEnvBuilder.put("GCOVTOOL", gcovtool.getPathString());
+ }
+
+ if (getTargetLibc().startsWith("glibc-")) {
+ globalMakeEnvBuilder.put("GLIBC_VERSION",
+ getTargetLibc().substring("glibc-".length()));
+ } else {
+ globalMakeEnvBuilder.put("GLIBC_VERSION", getTargetLibc());
+ }
+
+ globalMakeEnvBuilder.put("C_COMPILER", getCompiler());
+ globalMakeEnvBuilder.put("TARGET_CPU", getTargetCpu());
+
+ // Deprecated variables
+
+ // TODO(bazel-team): (2009) These variables are so rarely used we should try to eliminate
+ // them entirely. see: "cs -f=BUILD -noi GNU_TARGET" and "cs -f=build_defs -noi
+ // GNU_TARGET"
+ globalMakeEnvBuilder.put("CROSSTOOLTOP", crosstoolTopPathFragment.getPathString());
+ globalMakeEnvBuilder.put("GLIBC", getTargetLibc());
+ globalMakeEnvBuilder.put("GNU_TARGET", targetSystemName);
+
+ globalMakeEnvBuilder.putAll(getAdditionalMakeVariables());
+
+ globalMakeEnvBuilder.put("ABI_GLIBC_VERSION", getAbiGlibcVersion());
+ globalMakeEnvBuilder.put("ABI", abi);
+ }
+
+ @Override
+ public void addImplicitLabels(Multimap<String, Label> implicitLabels) {
+ if (getLibcLabel() != null) {
+ implicitLabels.put("crosstool", getLibcLabel());
+ }
+
+ implicitLabels.put("crosstool", crosstoolTop);
+ }
+
+ @Override
+ public void prepareHook(Path execRoot, ArtifactFactory artifactFactory, PathFragment genfilesPath,
+ PackageRootResolver resolver) throws ViewCreationFailedException {
+ try {
+ getFdoSupport().prepareToBuild(execRoot, genfilesPath, artifactFactory, resolver);
+ } catch (ZipException e) {
+ throw new ViewCreationFailedException("Error reading provided FDO zip file", e);
+ } catch (FdoException | IOException e) {
+ throw new ViewCreationFailedException("Error while initializing FDO support", e);
+ }
+ }
+
+ @Override
+ public void declareSkyframeDependencies(Environment env) {
+ getFdoSupport().declareSkyframeDependencies(env, execRoot);
+ }
+
+ @Override
+ public void addRoots(List<Root> roots) {
+ // Fdo root can only exist for the target configuration.
+ FdoSupport fdoSupport = getFdoSupport();
+ if (fdoSupport.getFdoRoot() != null) {
+ roots.add(fdoSupport.getFdoRoot());
+ }
+
+ // Grepped header includes; this root is not configuration specific.
+ roots.add(getGreppedIncludesDirectory());
+ }
+
+ @Override
+ public Map<String, String> getCoverageEnvironment() {
+ ImmutableMap.Builder<String, String> env = ImmutableMap.builder();
+ env.put("COVERAGE_GCOV_PATH", getGcovExecutable().getPathString());
+ PathFragment fdoInstrument = getFdoSupport().getFdoInstrument();
+ if (fdoInstrument != null) {
+ env.put("FDO_DIR", fdoInstrument.getPathString());
+ }
+ return env.build();
+ }
+
+ @Override
+ public ImmutableList<Label> getCoverageLabels() {
+ // TODO(bazel-team): Using a gcov-specific crosstool filegroup here could reduce the number of
+ // inputs significantly. We'd also need to add logic in tools/coverage/collect_coverage.sh to
+ // drop crosstool dependency if metadataFiles does not contain *.gcno artifacts.
+ return ImmutableList.of(crosstoolTop);
+ }
+
+ @Override
+ public String getOutputDirectoryName() {
+ String lipoSuffix;
+ if (getLipoMode() != LipoMode.OFF && !isAutoFdoLipo()) {
+ lipoSuffix = "-lipo";
+ } else if (getAutoFdoLipoData()) {
+ lipoSuffix = "-lipodata";
+ } else {
+ lipoSuffix = "";
+ }
+ return toolchainIdentifier + lipoSuffix;
+ }
+
+ @Override
+ public String getConfigurationNameSuffix() {
+ return isLipoContextCollector() ? "collector" : null;
+ }
+
+ @Override
+ public String getPlatformName() {
+ return getToolchainIdentifier();
+ }
+
+ @Override
+ public boolean supportsIncrementalBuild() {
+ return !isLipoOptimization();
+ }
+
+ @Override
+ public boolean performsStaticLink() {
+ return getLinkOptions().contains("-static");
+ }
+
+ /**
+ * Returns true if we should share identical native libraries between different targets.
+ */
+ public boolean shareNativeDeps() {
+ return cppOptions.shareNativeDeps;
+ }
+
+ @Override
+ public void prepareForExecutionPhase() throws IOException {
+ // _fdo has a prefix of "_", but it should nevertheless be deleted. Detailed description
+ // of the structure of the symlinks / directories can be found at FdoSupport.extractFdoZip().
+ // We actually create a directory named "blaze-fdo" under the exec root, the previous version
+ // of which is deleted in FdoSupport.prepareToBuildExec(). We cannot do that just before the
+ // execution phase because that needs to happen before the analysis phase (in order to create
+ // the artifacts corresponding to the .gcda files).
+ Path tempPath = execRoot.getRelative("_fdo");
+ if (tempPath.exists()) {
+ FileSystemUtils.deleteTree(tempPath);
+ }
+ }
+
+ @Override
+ public Map<String, Object> lateBoundOptionDefaults() {
+ // --cpu defaults to null. With that default, the actual target cpu string gets picked up
+ // by the "default_target_cpu" crosstool parameter.
+ return ImmutableMap.<String, Object>of("cpu", getTargetCpu());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java
new file mode 100644
index 0000000..de20283
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java
@@ -0,0 +1,174 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.RedirectChaser;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig;
+
+import javax.annotation.Nullable;
+
+/**
+ * Loader for C++ configurations.
+ */
+public class CppConfigurationLoader implements ConfigurationFragmentFactory {
+ @Override
+ public Class<? extends Fragment> creates() {
+ return CppConfiguration.class;
+ }
+
+ private final Function<String, String> cpuTransformer;
+
+ /**
+ * Creates a new CrosstoolConfigurationLoader instance with the given
+ * configuration provider. The configuration provider is used to perform
+ * caller-specific configuration file lookup.
+ */
+ public CppConfigurationLoader(Function<String, String> cpuTransformer) {
+ this.cpuTransformer = cpuTransformer;
+ }
+
+ @Override
+ public CppConfiguration create(ConfigurationEnvironment env, BuildOptions options)
+ throws InvalidConfigurationException {
+ CppConfigurationParameters params = createParameters(env, options);
+ if (params == null) {
+ return null;
+ }
+ return new CppConfiguration(params);
+ }
+
+ /**
+ * Value class for all the data needed to create a {@link CppConfiguration}.
+ */
+ public static class CppConfigurationParameters {
+ protected final CrosstoolConfig.CToolchain toolchain;
+ protected final String cacheKeySuffix;
+ protected final BuildOptions buildOptions;
+ protected final Label crosstoolTop;
+ protected final Label ccToolchainLabel;
+ protected final Path fdoZip;
+ protected final Path execRoot;
+
+ CppConfigurationParameters(CrosstoolConfig.CToolchain toolchain,
+ String cacheKeySuffix,
+ BuildOptions buildOptions,
+ Path fdoZip,
+ Path execRoot,
+ Label crosstoolTop,
+ Label ccToolchainLabel) {
+ this.toolchain = toolchain;
+ this.cacheKeySuffix = cacheKeySuffix;
+ this.buildOptions = buildOptions;
+ this.fdoZip = fdoZip;
+ this.execRoot = execRoot;
+ this.crosstoolTop = crosstoolTop;
+ this.ccToolchainLabel = ccToolchainLabel;
+ }
+ }
+
+ @Nullable
+ protected CppConfigurationParameters createParameters(
+ ConfigurationEnvironment env, BuildOptions options) throws InvalidConfigurationException {
+ BlazeDirectories directories = env.getBlazeDirectories();
+ if (directories == null) {
+ return null;
+ }
+ Label crosstoolTop = RedirectChaser.followRedirects(env,
+ options.get(CppOptions.class).crosstoolTop, "crosstool_top");
+ if (crosstoolTop == null) {
+ return null;
+ }
+ CrosstoolConfigurationLoader.CrosstoolFile file =
+ CrosstoolConfigurationLoader.readCrosstool(env, crosstoolTop);
+ if (file == null) {
+ return null;
+ }
+ CrosstoolConfig.CToolchain toolchain =
+ CrosstoolConfigurationLoader.selectToolchain(file.getProto(), options, cpuTransformer);
+
+ // FDO
+ // TODO(bazel-team): move this to CppConfiguration.prepareHook
+ CppOptions cppOptions = options.get(CppOptions.class);
+ Path fdoZip;
+ if (cppOptions.fdoOptimize == null) {
+ fdoZip = null;
+ } else if (cppOptions.fdoOptimize.startsWith("//")) {
+ try {
+ Target target = env.getTarget(Label.parseAbsolute(cppOptions.fdoOptimize));
+ if (target == null) {
+ return null;
+ }
+ if (!(target instanceof InputFile)) {
+ throw new InvalidConfigurationException(
+ "--fdo_optimize cannot accept targets that do not refer to input files");
+ }
+ fdoZip = env.getPath(target.getPackage(), target.getName());
+ if (fdoZip == null) {
+ throw new InvalidConfigurationException(
+ "The --fdo_optimize parameter you specified resolves to a file that does not exist");
+ }
+ } catch (NoSuchPackageException | NoSuchTargetException | SyntaxException e) {
+ throw new InvalidConfigurationException(e);
+ }
+ } else {
+ fdoZip = directories.getWorkspace().getRelative(cppOptions.fdoOptimize);
+ }
+
+ Label ccToolchainLabel;
+ try {
+ ccToolchainLabel = crosstoolTop.getRelative("cc-compiler-" + toolchain.getTargetCpu());
+ } catch (Label.SyntaxException e) {
+ throw new InvalidConfigurationException(String.format(
+ "'%s' is not a valid CPU. It should only consist of characters valid in labels",
+ toolchain.getTargetCpu()));
+ }
+
+ Target ccToolchain;
+ try {
+ ccToolchain = env.getTarget(ccToolchainLabel);
+ if (ccToolchain == null) {
+ return null;
+ }
+ } catch (NoSuchThingException e) {
+ throw new InvalidConfigurationException(String.format(
+ "The toolchain rule '%s' does not exist", ccToolchainLabel));
+ }
+
+ if (!(ccToolchain instanceof Rule)
+ || !((Rule) ccToolchain).getRuleClass().equals("cc_toolchain")) {
+ throw new InvalidConfigurationException(String.format(
+ "The label '%s' is not a cc_toolchain rule", ccToolchainLabel));
+ }
+
+ return new CppConfigurationParameters(toolchain, file.getMd5(), options,
+ fdoZip, directories.getExecRoot(), crosstoolTop, ccToolchainLabel);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java
new file mode 100644
index 0000000..c0fcb11
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A target that provides .dwo files which can be combined into a .dwp packaging step. See
+ * https://gcc.gnu.org/wiki/DebugFission for details.
+ */
+@Immutable
+public final class CppDebugFileProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> transitiveDwoFiles;
+ private final NestedSet<Artifact> transitivePicDwoFiles;
+
+ public CppDebugFileProvider(NestedSet<Artifact> transitiveDwoFiles,
+ NestedSet<Artifact> transitivePicDwoFiles) {
+ this.transitiveDwoFiles = transitiveDwoFiles;
+ this.transitivePicDwoFiles = transitivePicDwoFiles;
+ }
+
+ /**
+ * Returns the .dwo files that should be included in this target's .dwp packaging (if this
+ * target is linked) or passed through to a dependant's .dwp packaging (e.g. if this is a
+ * cc_library depended on by a statically linked cc_binary).
+ *
+ * Assumes the corresponding link consumes .o files (vs. .pic.o files).
+ */
+ public NestedSet<Artifact> getTransitiveDwoFiles() {
+ return transitiveDwoFiles;
+ }
+
+ /**
+ * Same as above, but assumes the corresponding link consumes pic.o files.
+ */
+ public NestedSet<Artifact> getTransitivePicDwoFiles() {
+ return transitivePicDwoFiles;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java
new file mode 100644
index 0000000..864a4d5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import javax.annotation.Nullable;
+
+/**
+ * Provides the binary artifact and its associated .dwp files, if fission is enabled.
+ * If Fission ({@link https://gcc.gnu.org/wiki/DebugFission}) is not enabled, the
+ * dwp file will be null.
+ */
+@Immutable
+public final class CppDebugPackageProvider implements TransitiveInfoProvider {
+
+ private final Artifact strippedArtifact;
+ private final Artifact unstrippedArtifact;
+ @Nullable private final Artifact dwpArtifact;
+
+ public CppDebugPackageProvider(
+ Artifact strippedArtifact,
+ Artifact unstrippedArtifact,
+ @Nullable Artifact dwpArtifact) {
+ Preconditions.checkNotNull(strippedArtifact);
+ Preconditions.checkNotNull(unstrippedArtifact);
+ this.strippedArtifact = strippedArtifact;
+ this.unstrippedArtifact = unstrippedArtifact;
+ this.dwpArtifact = dwpArtifact;
+ }
+
+ /**
+ * Returns the stripped file (the explicit ".stripped" target).
+ */
+ public final Artifact getStrippedArtifact() {
+ return strippedArtifact;
+ }
+
+ /**
+ * Returns the unstripped file (the default executable target).
+ */
+ public final Artifact getUnstrippedArtifact() {
+ return unstrippedArtifact;
+ }
+
+ /**
+ * Returns the .dwp file (for fission builds) or null if --fission=no.
+ */
+ @Nullable
+ public final Artifact getDwpArtifact() {
+ return dwpArtifact;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java
new file mode 100644
index 0000000..d9bb7b6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.util.FileType;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+/**
+ * C++-related file type definitions.
+ */
+public final class CppFileTypes {
+ public static final FileType CPP_SOURCE = FileType.of(".cc", ".cpp", ".cxx", ".C");
+ public static final FileType C_SOURCE = FileType.of(".c");
+ public static final FileType CPP_HEADER = FileType.of(".h", ".hh", ".hpp", ".hxx", ".inc");
+ public static final FileType CPP_TEXTUAL_INCLUDE = FileType.of(".inc");
+
+ public static final FileType PIC_PREPROCESSED_C = FileType.of(".pic.i");
+ public static final FileType PREPROCESSED_C = new FileType() {
+ final String ext = ".i";
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext) && !PIC_PREPROCESSED_C.matches(filename);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+ public static final FileType PIC_PREPROCESSED_CPP = FileType.of(".pic.ii");
+ public static final FileType PREPROCESSED_CPP = new FileType() {
+ final String ext = ".ii";
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext) && !PIC_PREPROCESSED_CPP.matches(filename);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+
+ public static final FileType ASSEMBLER_WITH_C_PREPROCESSOR = FileType.of(".S");
+ public static final FileType PIC_ASSEMBLER = FileType.of(".pic.s");
+ public static final FileType ASSEMBLER = new FileType() {
+ final String ext = ".s";
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext) && !PIC_ASSEMBLER.matches(filename);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+
+ public static final FileType PIC_ARCHIVE = FileType.of(".pic.a");
+ public static final FileType ARCHIVE = new FileType() {
+ final String ext = ".a";
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext) && !PIC_ARCHIVE.matches(filename);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+
+ public static final FileType ALWAYS_LINK_PIC_LIBRARY = FileType.of(".pic.lo");
+ public static final FileType ALWAYS_LINK_LIBRARY = new FileType() {
+ final String ext = ".lo";
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext) && !ALWAYS_LINK_PIC_LIBRARY.matches(filename);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+
+ public static final FileType PIC_OBJECT_FILE = FileType.of(".pic.o");
+ public static final FileType OBJECT_FILE = new FileType() {
+ final String ext = ".o";
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext) && !PIC_OBJECT_FILE.matches(filename);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+
+
+ public static final FileType SHARED_LIBRARY = FileType.of(".so");
+ public static final FileType INTERFACE_SHARED_LIBRARY = FileType.of(".ifso");
+ public static final FileType LINKER_SCRIPT = FileType.of(".lds");
+ // Matches shared libraries with version names in the extension, i.e.
+ // libmylib.so.2 or libmylib.so.2.10.
+ private static final Pattern VERSIONED_SHARED_LIBRARY_PATTERN =
+ Pattern.compile("^.+\\.so(\\.\\d+)+$");
+ public static final FileType VERSIONED_SHARED_LIBRARY = new FileType() {
+ @Override
+ public boolean apply(String filename) {
+ // Because regex matching can be slow, we first do a quick digit check on the final
+ // character before risking the full-on regex match. This should eliminate the performance
+ // hit on practically every non-qualifying file type.
+ if (Character.isDigit(filename.charAt(filename.length() - 1))) {
+ return VERSIONED_SHARED_LIBRARY_PATTERN.matcher(filename).matches();
+ } else {
+ return false;
+ }
+ }
+ };
+
+ public static final FileType COVERAGE_NOTES = FileType.of(".gcno");
+ public static final FileType COVERAGE_DATA = FileType.of(".gcda");
+ public static final FileType COVERAGE_DATA_IMPORTS = FileType.of(".gcda.imports");
+ public static final FileType GCC_AUTO_PROFILE = FileType.of(".afdo");
+
+ public static final FileType CPP_MODULE_MAP = FileType.of(".cppmap");
+ public static final FileType CPP_MODULE = FileType.of(".pcm");
+
+ // Output of the dwp tool
+ public static final FileType DEBUG_INFO_PACKAGE = FileType.of(".dwp");
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java
new file mode 100644
index 0000000..5bc1363
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java
@@ -0,0 +1,529 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MiddlemanFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.Util;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Linkstamp;
+import com.google.devtools.build.lib.rules.cpp.CppCompilationContext.Builder;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.util.IncludeScanningUtil;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class for functionality shared by cpp related rules.
+ *
+ * <p>This class can be used only after the loading phase.
+ */
+public class CppHelper {
+ // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES?
+ public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of(
+ CppFileTypes.SHARED_LIBRARY,
+ CppFileTypes.VERSIONED_SHARED_LIBRARY);
+
+ private static final FileTypeSet CPP_FILETYPES = FileTypeSet.of(
+ CppFileTypes.CPP_HEADER,
+ CppFileTypes.CPP_SOURCE);
+
+ private CppHelper() {
+ // prevents construction
+ }
+
+ /**
+ * Merges the STL and toolchain contexts into context builder. The STL is automatically determined
+ * using the ":stl" attribute.
+ */
+ public static void mergeToolchainDependentContext(RuleContext ruleContext,
+ Builder contextBuilder) {
+ TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET);
+ if (stl != null) {
+ // TODO(bazel-team): Clean this up.
+ contextBuilder.addSystemIncludeDir(stl.getLabel().getPackageFragment().getRelative("gcc3"));
+ contextBuilder.mergeDependentContext(stl.getProvider(CppCompilationContext.class));
+ }
+ CcToolchainProvider toolchain = getToolchain(ruleContext);
+ if (toolchain != null) {
+ contextBuilder.mergeDependentContext(toolchain.getCppCompilationContext());
+ }
+ }
+
+ /**
+ * Returns the malloc implementation for the given target.
+ */
+ public static TransitiveInfoCollection mallocForTarget(RuleContext ruleContext) {
+ if (ruleContext.getFragment(CppConfiguration.class).customMalloc() != null) {
+ return ruleContext.getPrerequisite(":default_malloc", Mode.TARGET);
+ } else {
+ return ruleContext.getPrerequisite("malloc", Mode.TARGET);
+ }
+ }
+
+ /**
+ * Expands Make variables in a list of string and tokenizes the result. If the package feature
+ * no_copts_tokenization is set, tokenize only items consisting of a single make variable.
+ *
+ * @param ruleContext the ruleContext to be used as the context of Make variable expansion
+ * @param attributeName the name of the attribute to use in error reporting
+ * @param input the list of strings to expand
+ * @return a list of strings containing the expanded and tokenized values for the
+ * attribute
+ */
+ // TODO(bazel-team): Move to CcCommon; refactor CcPlugin to use either CcLibraryHelper or
+ // CcCommon.
+ static List<String> expandMakeVariables(
+ RuleContext ruleContext, String attributeName, List<String> input) {
+ boolean tokenization =
+ !ruleContext.getFeatures().contains("no_copts_tokenization");
+
+ List<String> tokens = new ArrayList<>();
+ for (String token : input) {
+ try {
+ // Legacy behavior: tokenize all items.
+ if (tokenization) {
+ ruleContext.tokenizeAndExpandMakeVars(tokens, attributeName, token);
+ } else {
+ String exp = ruleContext.expandSingleMakeVariable(attributeName, token);
+ if (exp != null) {
+ ShellUtils.tokenize(tokens, exp);
+ } else {
+ tokens.add(ruleContext.expandMakeVariables(attributeName, token));
+ }
+ }
+ } catch (ShellUtils.TokenizationException e) {
+ ruleContext.attributeError(attributeName, e.getMessage());
+ }
+ }
+ return ImmutableList.copyOf(tokens);
+ }
+
+ /**
+ * Appends the tokenized values of the copts attribute to copts.
+ */
+ public static ImmutableList<String> getAttributeCopts(RuleContext ruleContext, String attr) {
+ Preconditions.checkArgument(ruleContext.getRule().isAttrDefined(attr, Type.STRING_LIST));
+ List<String> unexpanded = ruleContext.attributes().get(attr, Type.STRING_LIST);
+
+ return ImmutableList.copyOf(expandMakeVariables(ruleContext, attr, unexpanded));
+ }
+
+ /**
+ * Expands attribute value either using label expansion
+ * (if attemptLabelExpansion == {@code true} and it does not look like make
+ * variable or flag) or tokenizes and expands make variables.
+ */
+ public static void expandAttribute(RuleContext ruleContext,
+ List<String> values, String attrName, String attrValue, boolean attemptLabelExpansion) {
+ if (attemptLabelExpansion && CppHelper.isLinkoptLabel(attrValue)) {
+ if (!CppHelper.expandLabel(ruleContext, values, attrValue)) {
+ ruleContext.attributeError(attrName, "could not resolve label '" + attrValue + "'");
+ }
+ } else {
+ ruleContext.tokenizeAndExpandMakeVars(values, attrName, attrValue);
+ }
+ }
+
+ /**
+ * Determines if a linkopt can be a label. Linkopts come in 2 varieties:
+ * literals -- flags like -Xl and makefile vars like $(LD) -- and labels,
+ * which we should expand into filenames.
+ *
+ * @param linkopt the link option to test.
+ * @return true if the linkopt is not a flag (starting with "-") or a makefile
+ * variable (starting with "$");
+ */
+ private static boolean isLinkoptLabel(String linkopt) {
+ return !linkopt.startsWith("$") && !linkopt.startsWith("-");
+ }
+
+ /**
+ * Expands a label against the target's deps, adding the expanded path strings
+ * to the linkopts.
+ *
+ * @param linkopts the linkopts to add the expanded label to
+ * @param labelName the name of the label to expand
+ * @return true if the label was expanded successfully, false otherwise
+ */
+ private static boolean expandLabel(RuleContext ruleContext, List<String> linkopts,
+ String labelName) {
+ try {
+ Label label = ruleContext.getLabel().getRelative(labelName);
+ for (FileProvider target : ruleContext
+ .getPrerequisites("deps", Mode.TARGET, FileProvider.class)) {
+ if (target.getLabel().equals(label)) {
+ for (Artifact artifact : target.getFilesToBuild()) {
+ linkopts.add(artifact.getExecPathString());
+ }
+ return true;
+ }
+ }
+ } catch (SyntaxException e) {
+ // Quietly ignore and fall through.
+ }
+ linkopts.add(labelName);
+ return false;
+ }
+
+ /**
+ * This almost trivial method looks up the :cc_toolchain attribute on the rule context, makes sure
+ * that it refers to a rule that has a {@link CcToolchainProvider} (gives an error otherwise), and
+ * returns a reference to that {@link CcToolchainProvider}. The method only returns {@code null}
+ * if there is no such attribute (this is currently not an error).
+ */
+ @Nullable public static CcToolchainProvider getToolchain(RuleContext ruleContext) {
+ if (ruleContext.attributes().getAttributeDefinition(":cc_toolchain") == null) {
+ // TODO(bazel-team): Report an error or throw an exception in this case.
+ return null;
+ }
+ TransitiveInfoCollection dep = ruleContext.getPrerequisite(":cc_toolchain", Mode.TARGET);
+ return getToolchain(ruleContext, dep);
+ }
+
+ /**
+ * This almost trivial method makes sure that the given info collection has a {@link
+ * CcToolchainProvider} (gives an error otherwise), and returns a reference to that {@link
+ * CcToolchainProvider}. The method never returns {@code null}, even if there is no toolchain.
+ */
+ public static CcToolchainProvider getToolchain(RuleContext ruleContext,
+ TransitiveInfoCollection dep) {
+ // TODO(bazel-team): Consider checking this generally at the attribute level.
+ if ((dep == null) || (dep.getProvider(CcToolchainProvider.class) == null)) {
+ ruleContext.ruleError("The selected C++ toolchain is not a cc_toolchain rule");
+ return CcToolchainProvider.EMPTY_TOOLCHAIN_IS_ERROR;
+ }
+ return dep.getProvider(CcToolchainProvider.class);
+ }
+
+ /**
+ * Returns the directory where object files are created.
+ */
+ public static PathFragment getObjDirectory(Label ruleLabel) {
+ return AnalysisUtils.getUniqueDirectory(ruleLabel, new PathFragment("_objs"));
+ }
+
+ /**
+ * Creates a grep-includes ExtractInclusions action for generated sources/headers in the
+ * needsIncludeScanning() BuildConfiguration case. Returns a map from original header
+ * Artifact to the output Artifact of grepping over it. The return value only includes
+ * entries for generated sources or headers when --extract_generated_inclusions is enabled.
+ *
+ * <p>Previously, incremental rebuilds redid all include scanning work
+ * for a given .cc source in serial. For high-latency file systems, this could cause
+ * performance problems if many headers are generated.
+ */
+ @Nullable
+ public static final Map<Artifact, Artifact> createExtractInclusions(RuleContext ruleContext,
+ Iterable<Artifact> prerequisites) {
+ Map<Artifact, Artifact> extractions = new HashMap<>();
+ for (Artifact prerequisite : prerequisites) {
+ Artifact scanned = createExtractInclusions(ruleContext, prerequisite);
+ if (scanned != null) {
+ extractions.put(prerequisite, scanned);
+ }
+ }
+ return extractions;
+ }
+
+ /**
+ * Creates a grep-includes ExtractInclusions action for generated sources/headers in the
+ * needsIncludeScanning() BuildConfiguration case.
+ *
+ * <p>Previously, incremental rebuilds redid all include scanning work for a given
+ * .cc source in serial. For high-latency file systems, this could cause
+ * performance problems if many headers are generated.
+ */
+ private static final Artifact createExtractInclusions(RuleContext ruleContext,
+ Artifact prerequisite) {
+ if (ruleContext != null &&
+ ruleContext.getFragment(CppConfiguration.class).needsIncludeScanning() &&
+ !prerequisite.isSourceArtifact() &&
+ CPP_FILETYPES.matches(prerequisite.getFilename())) {
+ Artifact scanned = getIncludesOutput(ruleContext, prerequisite);
+ ruleContext.registerAction(
+ new ExtractInclusionAction(ruleContext.getActionOwner(), prerequisite, scanned));
+ return scanned;
+ }
+ return null;
+ }
+
+ private static Artifact getIncludesOutput(RuleContext ruleContext, Artifact src) {
+ Root root = ruleContext.getFragment(CppConfiguration.class).getGreppedIncludesDirectory();
+ PathFragment relOut = IncludeScanningUtil.getRootRelativeOutputPath(src.getExecPath());
+ return ruleContext.getAnalysisEnvironment().getDerivedArtifact(relOut, root);
+ }
+
+ /**
+ * Returns the workspace-relative filename for the linked artifact.
+ */
+ public static PathFragment getLinkedFilename(RuleContext ruleContext,
+ LinkTargetType linkType) {
+ PathFragment relativePath = Util.getWorkspaceRelativePath(ruleContext.getTarget());
+ PathFragment linkedFileName = (linkType == LinkTargetType.EXECUTABLE) ?
+ relativePath :
+ relativePath.replaceName("lib" + relativePath.getBaseName() + linkType.getExtension());
+ return linkedFileName;
+ }
+
+ /**
+ * Resolves the linkstamp collection from the {@code CcLinkParams} into a map.
+ *
+ * <p>Emits a warning on the rule if there are identical linkstamp artifacts with different
+ * compilation contexts.
+ */
+ public static Map<Artifact, ImmutableList<Artifact>> resolveLinkstamps(RuleContext ruleContext,
+ CcLinkParams linkParams) {
+ Map<Artifact, ImmutableList<Artifact>> result = new LinkedHashMap<>();
+ for (Linkstamp pair : linkParams.getLinkstamps()) {
+ Artifact artifact = pair.getArtifact();
+ if (result.containsKey(artifact)) {
+ ruleContext.ruleWarning("rule inherits the '" + artifact.toDetailString()
+ + "' linkstamp file from more than one cc_library rule");
+ }
+ result.put(artifact, pair.getDeclaredIncludeSrcs());
+ }
+ return result;
+ }
+
+ public static void addTransitiveLipoInfoForCommonAttributes(
+ RuleContext ruleContext,
+ CcCompilationOutputs outputs,
+ NestedSetBuilder<IncludeScannable> scannableBuilder) {
+
+ TransitiveLipoInfoProvider stl = null;
+ if (ruleContext.getRule().getAttributeDefinition(":stl") != null &&
+ ruleContext.getPrerequisite(":stl", Mode.TARGET) != null) {
+ // If the attribute is defined, it is never null.
+ stl = ruleContext.getPrerequisite(":stl", Mode.TARGET)
+ .getProvider(TransitiveLipoInfoProvider.class);
+ }
+ if (stl != null) {
+ scannableBuilder.addTransitive(stl.getTransitiveIncludeScannables());
+ }
+
+ for (TransitiveLipoInfoProvider dep :
+ ruleContext.getPrerequisites("deps", Mode.TARGET, TransitiveLipoInfoProvider.class)) {
+ scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables());
+ }
+
+ if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) {
+ TransitiveInfoCollection malloc = mallocForTarget(ruleContext);
+ TransitiveLipoInfoProvider provider = malloc.getProvider(TransitiveLipoInfoProvider.class);
+ if (provider != null) {
+ scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables());
+ }
+ }
+
+ for (IncludeScannable scannable : outputs.getLipoScannables()) {
+ Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1);
+ scannableBuilder.add(scannable);
+ }
+ }
+
+ // TODO(bazel-team): figure out a way to merge these 2 methods. See the Todo in
+ // CcCommonConfiguredTarget.noCoptsMatches().
+ /**
+ * Determines if we should apply -fPIC for this rule's C++ compilations. This determination
+ * is generally made by the global C++ configuration settings "needsPic" and
+ * and "usePicForBinaries". However, an individual rule may override these settings by applying
+ * -fPIC" to its "nocopts" attribute. This allows incompatible rules to "opt out" of global PIC
+ * settings (see bug: "Provide a way to turn off -fPIC for targets that can't be built that way").
+ *
+ * @param ruleContext the context of the rule to check
+ * @param forBinary true if compiling for a binary, false if for a shared library
+ * @return true if this rule's compilations should apply -fPIC, false otherwise
+ */
+ public static boolean usePic(RuleContext ruleContext, boolean forBinary) {
+ if (CcCommon.noCoptsMatches("-fPIC", ruleContext)) {
+ return false;
+ }
+ CppConfiguration config = ruleContext.getFragment(CppConfiguration.class);
+ return forBinary ? config.usePicObjectsForBinaries() : config.needsPic();
+ }
+
+ /**
+ * Returns the LIPO context provider for configured target,
+ * or null if such a provider doesn't exist.
+ */
+ public static LipoContextProvider getLipoContextProvider(RuleContext ruleContext) {
+ if (ruleContext.getRule().getAttributeDefinition(":lipo_context_collector") == null) {
+ return null;
+ }
+
+ TransitiveInfoCollection dep =
+ ruleContext.getPrerequisite(":lipo_context_collector", Mode.DONT_CHECK);
+ return (dep != null) ? dep.getProvider(LipoContextProvider.class) : null;
+ }
+
+ // Creates CppModuleMap object, and adds it to C++ compilation context.
+ public static CppModuleMap addCppModuleMapToContext(RuleContext ruleContext,
+ CppCompilationContext.Builder contextBuilder) {
+ if (!ruleContext.getFragment(CppConfiguration.class).createCppModuleMaps()) {
+ return null;
+ }
+ if (getToolchain(ruleContext).getCppCompilationContext().getCppModuleMap() == null) {
+ return null;
+ }
+ // Create the module map artifact as a genfile.
+ PathFragment mapPath = FileSystemUtils.appendExtension(ruleContext.getLabel().toPathFragment(),
+ Iterables.getOnlyElement(CppFileTypes.CPP_MODULE_MAP.getExtensions()));
+ Artifact mapFile = ruleContext.getAnalysisEnvironment().getDerivedArtifact(mapPath,
+ ruleContext.getConfiguration().getGenfilesDirectory());
+ CppModuleMap moduleMap =
+ new CppModuleMap(mapFile, ruleContext.getLabel().toString());
+ contextBuilder.setCppModuleMap(moduleMap);
+ return moduleMap;
+ }
+
+ /**
+ * Returns a middleman for all files to build for the given configured target,
+ * substituting shared library artifacts with corresponding solib symlinks. If
+ * multiple calls are made, then it returns the same artifact for configurations
+ * with the same internal directory.
+ *
+ * <p>The resulting middleman only aggregates the inputs and must be expanded
+ * before populating the set of files necessary to execute an action.
+ */
+ static List<Artifact> getAggregatingMiddlemanForCppRuntimes(RuleContext ruleContext,
+ String purpose, TransitiveInfoCollection dep, String solibDirOverride,
+ BuildConfiguration configuration) {
+ return getMiddlemanInternal(
+ ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose,
+ dep, true, true, solibDirOverride, configuration);
+ }
+
+ @VisibleForTesting
+ public static List<Artifact> getAggregatingMiddlemanForTesting(AnalysisEnvironment env,
+ RuleContext ruleContext, ActionOwner owner, String purpose, TransitiveInfoCollection dep,
+ boolean useSolibSymlinks, BuildConfiguration configuration) {
+ return getMiddlemanInternal(
+ env, ruleContext, owner, purpose, dep, useSolibSymlinks, false, null, configuration);
+ }
+
+ /**
+ * Internal implementation for getAggregatingMiddlemanForCppRuntimes.
+ */
+ private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env,
+ RuleContext ruleContext, ActionOwner actionOwner, String purpose,
+ TransitiveInfoCollection dep, boolean useSolibSymlinks, boolean isCppRuntime,
+ String solibDirOverride, BuildConfiguration configuration) {
+ if (dep == null) {
+ return ImmutableList.of();
+ }
+ MiddlemanFactory factory = env.getMiddlemanFactory();
+ Iterable<Artifact> artifacts = dep.getProvider(FileProvider.class).getFilesToBuild();
+ if (useSolibSymlinks) {
+ List<Artifact> symlinkedArtifacts = new ArrayList<>();
+ for (Artifact artifact : artifacts) {
+ symlinkedArtifacts.add(solibArtifactMaybe(
+ ruleContext, artifact, isCppRuntime, solibDirOverride, configuration));
+ }
+ artifacts = symlinkedArtifacts;
+ purpose += "_with_solib";
+ }
+ return ImmutableList.of(factory.createMiddlemanAllowMultiple(
+ env, actionOwner, purpose, artifacts, configuration.getMiddlemanDirectory()));
+ }
+
+ /**
+ * If the artifact is a shared library, returns the solib symlink artifact associated with it.
+ *
+ * @param ruleContext the context of the rule that creates the symlink
+ * @param artifact the library the solib symlink should point to
+ * @param isCppRuntime whether the library is a C++ runtime
+ * @param solibDirOverride if not null, forces the solib symlink to be in this directory
+ */
+ private static Artifact solibArtifactMaybe(RuleContext ruleContext, Artifact artifact,
+ boolean isCppRuntime, String solibDirOverride, BuildConfiguration configuration) {
+ if (SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) {
+ return isCppRuntime
+ ? SolibSymlinkAction.getCppRuntimeSymlink(
+ ruleContext, artifact, solibDirOverride, configuration)
+ .getArtifact()
+ : SolibSymlinkAction.getDynamicLibrarySymlink(
+ ruleContext, artifact, false, true, configuration)
+ .getArtifact();
+ } else {
+ return artifact;
+ }
+ }
+
+ /**
+ * Returns the type of archives being used.
+ */
+ public static Link.ArchiveType archiveType(BuildConfiguration config) {
+ CppConfiguration cppConfig = config.getFragment(CppConfiguration.class);
+ return cppConfig.archiveType();
+ }
+
+ /**
+ * Returns the FDO build subtype.
+ */
+ public static String getFdoBuildStamp(CppConfiguration cppConfiguration) {
+ if (cppConfiguration.getFdoSupport().isAutoFdoEnabled()) {
+ return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "ALIPO" : "AFDO";
+ }
+ if (cppConfiguration.isFdo()) {
+ return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "LIPO" : "FDO";
+ }
+ return null;
+ }
+
+ /**
+ * Returns a relative path to the bin directory for data in AutoFDO LIPO mode.
+ */
+ public static PathFragment getLipoDataBinFragment(BuildConfiguration configuration) {
+ PathFragment parent = configuration.getBinFragment().getParentDirectory();
+ return parent.replaceName(parent.getBaseName() + "-lipodata")
+ .getChild(configuration.getBinFragment().getBaseName());
+ }
+
+ /**
+ * Returns a relative path to the genfiles directory for data in AutoFDO LIPO mode.
+ */
+ public static PathFragment getLipoDataGenfilesFragment(BuildConfiguration configuration) {
+ PathFragment parent = configuration.getGenfilesFragment().getParentDirectory();
+ return parent.replaceName(parent.getBaseName() + "-lipodata")
+ .getChild(configuration.getGenfilesFragment().getBaseName());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java
new file mode 100644
index 0000000..ecf3431
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java
@@ -0,0 +1,1074 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ParameterFile;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.extra.CppLinkInfo;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.ImmutableIterable;
+import com.google.devtools.build.lib.collect.IterablesChain;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Action that represents an ELF linking step.
+ */
+@ThreadCompatible
+public final class CppLinkAction extends AbstractAction {
+ private static final String LINK_GUID = "58ec78bd-1176-4e36-8143-439f656b181d";
+ private static final String FAKE_LINK_GUID = "da36f819-5a15-43a9-8a45-e01b60e10c8b";
+
+ private final CppConfiguration cppConfiguration;
+ private final LibraryToLink outputLibrary;
+ private final LibraryToLink interfaceOutputLibrary;
+
+ private final LinkCommandLine linkCommandLine;
+
+ /** True for cc_fake_binary targets. */
+ private final boolean fake;
+
+ private final Iterable<Artifact> mandatoryInputs;
+
+ // Linking uses a lot of memory; estimate 1 MB per input file, min 1.5 Gib.
+ // It is vital to not underestimate too much here,
+ // because running too many concurrent links can
+ // thrash the machine to the point where it stops
+ // responding to keystrokes or mouse clicks.
+ // CPU and IO do not scale similarly and still use the static minimum estimate.
+ public static final ResourceSet LINK_RESOURCES_PER_INPUT = new ResourceSet(1, 0, 0);
+
+ // This defines the minimum of each resource that will be reserved.
+ public static final ResourceSet MIN_STATIC_LINK_RESOURCES = new ResourceSet(1536, 1, 0.3);
+
+ // Dynamic linking should be cheaper than static linking.
+ public static final ResourceSet MIN_DYNAMIC_LINK_RESOURCES = new ResourceSet(1024, 0.3, 0.2);
+
+ /**
+ * Use {@link Builder} to create instances of this class. Also see there for
+ * the documentation of all parameters.
+ *
+ * <p>This constructor is intentionally private and is only to be called from
+ * {@link Builder#build()}.
+ */
+ private CppLinkAction(ActionOwner owner,
+ Iterable<Artifact> inputs,
+ ImmutableList<Artifact> outputs,
+ CppConfiguration cppConfiguration,
+ LibraryToLink outputLibrary,
+ LibraryToLink interfaceOutputLibrary,
+ boolean fake,
+ LinkCommandLine linkCommandLine) {
+ super(owner, inputs, outputs);
+ this.mandatoryInputs = inputs;
+ this.cppConfiguration = cppConfiguration;
+ this.outputLibrary = outputLibrary;
+ this.interfaceOutputLibrary = interfaceOutputLibrary;
+ this.fake = fake;
+
+ this.linkCommandLine = linkCommandLine;
+ }
+
+ private static Iterable<LinkerInput> filterLinkerInputs(Iterable<LinkerInput> inputs) {
+ return Iterables.filter(inputs, new Predicate<LinkerInput>() {
+ @Override
+ public boolean apply(LinkerInput input) {
+ return Link.VALID_LINKER_INPUTS.matches(input.getArtifact().getFilename());
+ }
+ });
+ }
+
+ private static Iterable<Artifact> filterLinkerInputArtifacts(Iterable<Artifact> inputs) {
+ return Iterables.filter(inputs, new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact input) {
+ return Link.VALID_LINKER_INPUTS.matches(input.getFilename());
+ }
+ });
+ }
+
+ private CppConfiguration getCppConfiguration() {
+ return cppConfiguration;
+ }
+
+ @VisibleForTesting
+ public String getTargetCpu() {
+ return getCppConfiguration().getTargetCpu();
+ }
+
+ public String getHostSystemName() {
+ return getCppConfiguration().getHostSystemName();
+ }
+
+ /**
+ * Returns the link configuration; for correctness you should not call this method during
+ * execution - only the argv is part of the action cache key, and we therefore don't guarantee
+ * that the action will be re-executed if the contents change in a way that does not affect the
+ * argv.
+ */
+ @VisibleForTesting
+ public LinkCommandLine getLinkCommandLine() {
+ return linkCommandLine;
+ }
+
+ public LibraryToLink getOutputLibrary() {
+ return outputLibrary;
+ }
+
+ public LibraryToLink getInterfaceOutputLibrary() {
+ return interfaceOutputLibrary;
+ }
+
+ /**
+ * Returns the path to the output artifact produced by the linker.
+ */
+ public Path getOutputFile() {
+ return outputLibrary.getArtifact().getPath();
+ }
+
+ @VisibleForTesting
+ public List<String> getRawLinkArgv() {
+ return linkCommandLine.getRawLinkArgv();
+ }
+
+ @VisibleForTesting
+ public List<String> getArgv() {
+ return linkCommandLine.arguments();
+ }
+
+ /**
+ * Prepares and returns the command line specification for this link.
+ * Splits appropriate parts into a .params file and adds any required
+ * linkstamp compilation steps.
+ *
+ * @return a finalized command line suitable for execution
+ */
+ public final List<String> prepareCommandLine(Path execRoot, List<String> inputFiles)
+ throws ExecException {
+ List<String> commandlineArgs;
+ // Try to shorten the command line by use of a parameter file.
+ // This makes the output with --subcommands (et al) more readable.
+ if (linkCommandLine.canBeSplit()) {
+ PathFragment paramExecPath = ParameterFile.derivePath(
+ outputLibrary.getArtifact().getExecPath());
+ Pair<List<String>, List<String>> split = linkCommandLine.splitCommandline(paramExecPath);
+ commandlineArgs = split.first;
+ writeToParamFile(execRoot, paramExecPath, split.second);
+ if (inputFiles != null) {
+ inputFiles.add(paramExecPath.getPathString());
+ }
+ } else {
+ commandlineArgs = linkCommandLine.getRawLinkArgv();
+ }
+ return linkCommandLine.finalizeWithLinkstampCommands(commandlineArgs);
+ }
+
+ private static void writeToParamFile(Path workingDir, PathFragment paramExecPath,
+ List<String> paramFileArgs) throws ExecException {
+ // Create parameter file.
+ ParameterFile paramFile = new ParameterFile(workingDir, paramExecPath, ISO_8859_1,
+ ParameterFileType.UNQUOTED);
+ Path paramFilePath = paramFile.getPath();
+ try {
+ // writeContent() fails for existing files that are marked readonly.
+ paramFilePath.delete();
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("could not delete file '" + paramFilePath + "'", e);
+ }
+ paramFile.writeContent(paramFileArgs);
+
+ // Normally Blaze chmods all output files automatically (see
+ // SkyframeActionExecutor#setOutputsReadOnlyAndExecutable), but this params file is created
+ // out-of-band and is not declared as an output. By chmodding the file, other processes
+ // can observe this file being created.
+ try {
+ paramFilePath.setWritable(false);
+ paramFilePath.setExecutable(true); // for consistency with other action outputs
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("could not chmod param file '" + paramFilePath + "'", e);
+ }
+ }
+
+ @Override
+ @ThreadCompatible
+ public void execute(
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ if (fake) {
+ executeFake();
+ } else {
+ Executor executor = actionExecutionContext.getExecutor();
+
+ try {
+ executor.getContext(CppLinkActionContext.class).exec(
+ this, actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException("Linking of rule '" + getOwner().getLabel() + "'",
+ executor.getVerboseFailures(), this);
+ }
+ }
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return fake
+ ? "fake,local"
+ : executor.getContext(CppLinkActionContext.class).strategyLocality(this);
+ }
+
+ // Don't forget to update FAKE_LINK_GUID if you modify this method.
+ @ThreadCompatible
+ private void executeFake()
+ throws ActionExecutionException {
+ // The uses of getLinkConfiguration in this method may not be consistent with the computed key.
+ // I.e., this may be incrementally incorrect.
+ final Collection<Artifact> linkstampOutputs = getLinkCommandLine().getLinkstamps().values();
+
+ // Prefix all fake output files in the command line with $TEST_TMPDIR/.
+ final String outputPrefix = "$TEST_TMPDIR/";
+ List<String> escapedLinkArgv = escapeLinkArgv(linkCommandLine.getRawLinkArgv(),
+ linkstampOutputs, outputPrefix);
+ // Write the commands needed to build the real target to the fake target
+ // file.
+ StringBuilder s = new StringBuilder();
+ Joiner.on('\n').appendTo(s,
+ "# This is a fake target file, automatically generated.",
+ "# Do not edit by hand!",
+ "echo $0 is a fake target file and not meant to be executed.",
+ "exit 0",
+ "EOS",
+ "",
+ "makefile_dir=.",
+ "");
+
+ try {
+ // Concatenate all the (fake) .o files into the result.
+ for (LinkerInput linkerInput : getLinkCommandLine().getLinkerInputs()) {
+ Artifact objectFile = linkerInput.getArtifact();
+ if (CppFileTypes.OBJECT_FILE.matches(objectFile.getFilename())
+ && linkerInput.isFake()) {
+ s.append(FileSystemUtils.readContentAsLatin1(objectFile.getPath())); // (IOException)
+ }
+ }
+
+ s.append(getOutputFile().getBaseName()).append(": ");
+ for (Artifact linkstamp : linkstampOutputs) {
+ s.append("mkdir -p " + outputPrefix +
+ linkstamp.getExecPath().getParentDirectory() + " && ");
+ }
+ Joiner.on(' ').appendTo(s,
+ ShellEscaper.escapeAll(linkCommandLine.finalizeAlreadyEscapedWithLinkstampCommands(
+ escapedLinkArgv, outputPrefix)));
+ s.append('\n');
+ if (getOutputFile().exists()) {
+ getOutputFile().setWritable(true); // (IOException)
+ }
+ FileSystemUtils.writeContent(getOutputFile(), ISO_8859_1, s.toString());
+ getOutputFile().setExecutable(true); // (IOException)
+ for (Artifact linkstamp : linkstampOutputs) {
+ FileSystemUtils.touchFile(linkstamp.getPath());
+ }
+ } catch (IOException e) {
+ throw new ActionExecutionException("failed to create fake link command for rule '" +
+ getOwner().getLabel() + ": " + e.getMessage(),
+ this, false);
+ }
+ }
+
+ /**
+ * Shell-escapes the raw link command line.
+ *
+ * @param rawLinkArgv raw link command line
+ * @param linkstampOutputs linkstamp artifacts
+ * @param outputPrefix to be prepended to any outputs
+ * @return escaped link command line
+ */
+ private List<String> escapeLinkArgv(List<String> rawLinkArgv,
+ final Collection<Artifact> linkstampOutputs, final String outputPrefix) {
+ final List<String> linkstampExecPaths = Artifact.asExecPaths(linkstampOutputs);
+ ImmutableList.Builder<String> escapedArgs = ImmutableList.builder();
+ for (String rawArg : rawLinkArgv) {
+ String escapedArg;
+ if (rawArg.equals(getPrimaryOutput().getExecPathString())
+ || linkstampExecPaths.contains(rawArg)) {
+ escapedArg = outputPrefix + ShellEscaper.escapeString(rawArg);
+ } else if (rawArg.startsWith(Link.FAKE_OBJECT_PREFIX)) {
+ escapedArg = outputPrefix + ShellEscaper.escapeString(
+ rawArg.substring(Link.FAKE_OBJECT_PREFIX.length()));
+ } else {
+ escapedArg = ShellEscaper.escapeString(rawArg);
+ }
+ escapedArgs.add(escapedArg);
+ }
+ return escapedArgs.build();
+ }
+
+ @Override
+ public ExtraActionInfo.Builder getExtraActionInfo() {
+ // The uses of getLinkConfiguration in this method may not be consistent with the computed key.
+ // I.e., this may be incrementally incorrect.
+ CppLinkInfo.Builder info = CppLinkInfo.newBuilder();
+ info.addAllInputFile(Artifact.toExecPaths(
+ LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getLinkerInputs())));
+ info.addAllInputFile(Artifact.toExecPaths(
+ LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getRuntimeInputs())));
+ info.setOutputFile(getPrimaryOutput().getExecPathString());
+ if (interfaceOutputLibrary != null) {
+ info.setInterfaceOutputFile(interfaceOutputLibrary.getArtifact().getExecPathString());
+ }
+ info.setLinkTargetType(getLinkCommandLine().getLinkTargetType().name());
+ info.setLinkStaticness(getLinkCommandLine().getLinkStaticness().name());
+ info.addAllLinkStamp(Artifact.toExecPaths(getLinkCommandLine().getLinkstamps().values()));
+ info.addAllBuildInfoHeaderArtifact(
+ Artifact.toExecPaths(getLinkCommandLine().getBuildInfoHeaderArtifacts()));
+ info.addAllLinkOpt(getLinkCommandLine().getLinkopts());
+
+ return super.getExtraActionInfo()
+ .setExtension(CppLinkInfo.cppLinkInfo, info.build());
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(fake ? FAKE_LINK_GUID : LINK_GUID);
+ f.addString(getCppConfiguration().getLdExecutable().getPathString());
+ f.addStrings(linkCommandLine.arguments());
+ // TODO(bazel-team): For correctness, we need to ensure the invariant that all values accessed
+ // during the execution phase are also covered by the key. Above, we add the argv to the key,
+ // which covers most cases. Unfortunately, the extra action and fake support methods above also
+ // sometimes directly access settings from the link configuration that may or may not affect the
+ // key. We either need to change the code to cover them in the key computation, or change the
+ // LinkConfiguration to disallow the combinations where the value of a setting does not affect
+ // the argv.
+ f.addBoolean(linkCommandLine.isNativeDeps());
+ f.addBoolean(linkCommandLine.useTestOnlyFlags());
+ if (linkCommandLine.getRuntimeSolibDir() != null) {
+ f.addPath(linkCommandLine.getRuntimeSolibDir());
+ }
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public String describeKey() {
+ StringBuilder message = new StringBuilder();
+ if (fake) {
+ message.append("Fake ");
+ }
+ message.append(getProgressMessage());
+ message.append('\n');
+ message.append(" Command: ");
+ message.append(ShellEscaper.escapeString(
+ getCppConfiguration().getLdExecutable().getPathString()));
+ message.append('\n');
+ // Outputting one argument per line makes it easier to diff the results.
+ for (String argument : ShellEscaper.escapeAll(linkCommandLine.arguments())) {
+ message.append(" Argument: ");
+ message.append(argument);
+ message.append('\n');
+ }
+ return message.toString();
+ }
+
+ @Override
+ public String getMnemonic() { return "CppLink"; }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Linking " + outputLibrary.getArtifact().prettyPrint();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return executor.getContext(CppLinkActionContext.class).estimateResourceConsumption(this);
+ }
+
+ /**
+ * Estimate the resources consumed when this action is run locally.
+ */
+ public ResourceSet estimateResourceConsumptionLocal() {
+ // It's ok if this behaves differently even if the key is identical.
+ ResourceSet minLinkResources =
+ getLinkCommandLine().getLinkStaticness() == Link.LinkStaticness.DYNAMIC
+ ? MIN_DYNAMIC_LINK_RESOURCES
+ : MIN_STATIC_LINK_RESOURCES;
+
+ final int inputSize = Iterables.size(getLinkCommandLine().getLinkerInputs())
+ + Iterables.size(getLinkCommandLine().getRuntimeInputs());
+
+ return new ResourceSet(
+ Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getMemoryMb(),
+ minLinkResources.getMemoryMb()),
+ Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getCpuUsage(),
+ minLinkResources.getCpuUsage()),
+ Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getIoUsage(),
+ minLinkResources.getIoUsage())
+ );
+ }
+
+ @Override
+ public Iterable<Artifact> getMandatoryInputs() {
+ return mandatoryInputs;
+ }
+
+ /**
+ * Determines whether or not this link should output a symbol counts file.
+ */
+ private static boolean enableSymbolsCounts(CppConfiguration cppConfiguration, boolean fake,
+ LinkTargetType linkType) {
+ return cppConfiguration.getSymbolCounts()
+ && cppConfiguration.supportsGoldLinker()
+ && linkType == LinkTargetType.EXECUTABLE
+ && !fake;
+ }
+
+ /**
+ * Builder class to construct {@link CppLinkAction}s.
+ */
+ public static class Builder {
+ // Builder-only
+ private final RuleContext ruleContext;
+ private final AnalysisEnvironment analysisEnvironment;
+ private final PathFragment outputPath;
+ private final CcToolchainProvider toolchain;
+ private PathFragment interfaceOutputPath;
+ private PathFragment runtimeSolibDir;
+ protected final BuildConfiguration configuration;
+ private final CppConfiguration cppConfiguration;
+
+ // Morally equivalent with {@link Context}, except these are mutable.
+ // Keep these in sync with {@link Context}.
+ private final Set<LinkerInput> nonLibraries = new LinkedHashSet<>();
+ private final NestedSetBuilder<LibraryToLink> libraries = NestedSetBuilder.linkOrder();
+ private NestedSet<Artifact> crosstoolInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ private Artifact runtimeMiddleman;
+ private NestedSet<Artifact> runtimeInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ private final NestedSetBuilder<Artifact> compilationInputs = NestedSetBuilder.stableOrder();
+ private final Set<Artifact> linkstamps = new LinkedHashSet<>();
+ private List<String> linkstampOptions = new ArrayList<>();
+ private final List<String> linkopts = new ArrayList<>();
+ private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY;
+ private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC;
+ private boolean fake;
+ private boolean isNativeDeps;
+ private boolean useTestOnlyFlags;
+ private boolean wholeArchive;
+ private boolean supportsParamFiles = true;
+
+ /**
+ * Creates a builder that builds {@link CppLinkAction} instances.
+ *
+ * @param ruleContext the rule that owns the action
+ * @param outputPath the path of the ELF file to be created, relative to the
+ * 'bin' directory
+ */
+ public Builder(RuleContext ruleContext, PathFragment outputPath) {
+ this(ruleContext, outputPath, ruleContext.getConfiguration(),
+ ruleContext.getAnalysisEnvironment(), CppHelper.getToolchain(ruleContext));
+ }
+
+ /**
+ * Creates a builder that builds {@link CppLinkAction} instances.
+ *
+ * @param ruleContext the rule that owns the action
+ * @param outputPath the path of the ELF file to be created, relative to the
+ * 'bin' directory
+ */
+ public Builder(RuleContext ruleContext, PathFragment outputPath,
+ BuildConfiguration configuration, CcToolchainProvider toolchain) {
+ this(ruleContext, outputPath, configuration,
+ ruleContext.getAnalysisEnvironment(), toolchain);
+ }
+
+ /**
+ * Creates a builder that builds {@link CppLinkAction}s.
+ *
+ * @param ruleContext the rule that owns the action
+ * @param outputPath the path of the ELF file to be created, relative to the
+ * 'bin' directory
+ * @param configuration the configuration used to determine the tool chain
+ * and the default link options
+ */
+ private Builder(RuleContext ruleContext, PathFragment outputPath,
+ BuildConfiguration configuration, AnalysisEnvironment analysisEnvironment,
+ CcToolchainProvider toolchain) {
+ this.ruleContext = ruleContext;
+ this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment);
+ this.outputPath = Preconditions.checkNotNull(outputPath);
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.cppConfiguration = configuration.getFragment(CppConfiguration.class);
+ this.toolchain = toolchain;
+
+ // The toolchain != null is here for CppLinkAction.createTestBuilder(). Meh.
+ if (cppConfiguration.supportsEmbeddedRuntimes() && toolchain != null) {
+ runtimeSolibDir = toolchain.getDynamicRuntimeSolibDir();
+ }
+ if (toolchain != null) {
+ supportsParamFiles = toolchain.supportsParamFiles();
+ }
+ }
+
+ /**
+ * Given a Context, creates a Builder that builds {@link CppLinkAction}s.
+ * Note well: Keep the Builder->Context and Context->Builder transforms consistent!
+ * @param ruleContext the rule that owns the action
+ * @param outputPath the path of the ELF file to be created, relative to the
+ * 'bin' directory
+ * @param linkContext an immutable CppLinkAction.Context from the original builder
+ */
+ public Builder(RuleContext ruleContext, PathFragment outputPath, Context linkContext,
+ BuildConfiguration configuration) {
+ // These Builder-only fields get set in the constructor:
+ // ruleContext, analysisEnvironment, outputPath, configuration, runtimeSolibDir
+ this(ruleContext, outputPath, configuration, ruleContext.getAnalysisEnvironment(),
+ CppHelper.getToolchain(ruleContext));
+ Preconditions.checkNotNull(linkContext);
+
+ // All linkContext fields should be transferred to this Builder.
+ this.nonLibraries.addAll(linkContext.nonLibraries);
+ this.libraries.addTransitive(linkContext.libraries);
+ this.crosstoolInputs = linkContext.crosstoolInputs;
+ this.runtimeMiddleman = linkContext.runtimeMiddleman;
+ this.runtimeInputs = linkContext.runtimeInputs;
+ this.compilationInputs.addTransitive(linkContext.compilationInputs);
+ this.linkstamps.addAll(linkContext.linkstamps);
+ this.linkopts.addAll(linkContext.linkopts);
+ this.linkType = linkContext.linkType;
+ this.linkStaticness = linkContext.linkStaticness;
+ this.fake = linkContext.fake;
+ this.isNativeDeps = linkContext.isNativeDeps;
+ this.useTestOnlyFlags = linkContext.useTestOnlyFlags;
+ }
+
+ /**
+ * Builds the Action as configured and returns it.
+ *
+ * <p>This method may only be called once.
+ */
+ public CppLinkAction build() {
+ if (interfaceOutputPath != null && (fake || linkType != LinkTargetType.DYNAMIC_LIBRARY)) {
+ throw new RuntimeException("Interface output can only be used "
+ + "with non-fake DYNAMIC_LIBRARY targets");
+ }
+
+ final Artifact output = createArtifact(outputPath);
+ final Artifact interfaceOutput = (interfaceOutputPath != null)
+ ? createArtifact(interfaceOutputPath)
+ : null;
+
+ final ImmutableList<Artifact> buildInfoHeaderArtifacts = !linkstamps.isEmpty()
+ ? ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, CppBuildInfo.KEY)
+ : ImmutableList.<Artifact>of();
+
+ final Artifact symbolCountOutput = enableSymbolsCounts(cppConfiguration, fake, linkType)
+ ? createArtifact(output.getRootRelativePath().replaceName(
+ output.getExecPath().getBaseName() + ".sc"))
+ : null;
+
+ boolean needWholeArchive = wholeArchive || needWholeArchive(
+ linkStaticness, linkType, linkopts, isNativeDeps, cppConfiguration);
+
+ NestedSet<LibraryToLink> uniqueLibraries = libraries.build();
+ final Iterable<Artifact> filteredNonLibraryArtifacts = filterLinkerInputArtifacts(
+ LinkerInputs.toLibraryArtifacts(nonLibraries));
+ final Iterable<LinkerInput> linkerInputs = IterablesChain.<LinkerInput>builder()
+ .add(ImmutableList.copyOf(filterLinkerInputs(nonLibraries)))
+ .add(ImmutableIterable.from(Link.mergeInputsCmdLine(
+ uniqueLibraries, needWholeArchive, cppConfiguration.archiveType())))
+ .build();
+
+ // ruleContext can only be null during testing. This is kind of ugly.
+ final ImmutableSet<String> features = (ruleContext == null)
+ ? ImmutableSet.<String>of()
+ : ruleContext.getFeatures();
+
+ final LibraryToLink outputLibrary =
+ LinkerInputs.newInputLibrary(output, filteredNonLibraryArtifacts);
+ final LibraryToLink interfaceOutputLibrary = interfaceOutput == null ? null :
+ LinkerInputs.newInputLibrary(interfaceOutput, filteredNonLibraryArtifacts);
+
+ final ImmutableMap<Artifact, Artifact> linkstampMap =
+ mapLinkstampsToOutputs(linkstamps, ruleContext, output);
+
+ final ImmutableList<Artifact> actionOutputs = constructOutputs(
+ outputLibrary.getArtifact(),
+ linkstampMap.values(),
+ interfaceOutputLibrary == null ? null : interfaceOutputLibrary.getArtifact(),
+ symbolCountOutput);
+
+ LinkCommandLine linkCommandLine = new LinkCommandLine.Builder(configuration, getOwner())
+ .setOutput(outputLibrary.getArtifact())
+ .setInterfaceOutput(interfaceOutput)
+ .setSymbolCountsOutput(symbolCountOutput)
+ .setBuildInfoHeaderArtifacts(buildInfoHeaderArtifacts)
+ .setLinkerInputs(linkerInputs)
+ .setRuntimeInputs(ImmutableList.copyOf(LinkerInputs.simpleLinkerInputs(runtimeInputs)))
+ .setLinkTargetType(linkType)
+ .setLinkStaticness(linkStaticness)
+ .setLinkopts(ImmutableList.copyOf(linkopts))
+ .setFeatures(features)
+ .setLinkstamps(linkstampMap)
+ .addLinkstampCompileOptions(linkstampOptions)
+ .setRuntimeSolibDir(linkType.isStaticLibraryLink() ? null : runtimeSolibDir)
+ .setNativeDeps(isNativeDeps)
+ .setUseTestOnlyFlags(useTestOnlyFlags)
+ .setNeedWholeArchive(needWholeArchive)
+ .setInterfaceSoBuilder(getInterfaceSoBuilder())
+ .setSupportsParamFiles(supportsParamFiles)
+ .build();
+
+ // Compute the set of inputs - we only need stable order here.
+ NestedSetBuilder<Artifact> dependencyInputsBuilder = NestedSetBuilder.stableOrder();
+ dependencyInputsBuilder.addAll(buildInfoHeaderArtifacts);
+ dependencyInputsBuilder.addAll(linkstamps);
+ dependencyInputsBuilder.addTransitive(crosstoolInputs);
+ if (runtimeMiddleman != null) {
+ dependencyInputsBuilder.add(runtimeMiddleman);
+ }
+ dependencyInputsBuilder.addTransitive(compilationInputs.build());
+
+ Iterable<Artifact> expandedInputs =
+ LinkerInputs.toLibraryArtifacts(Link.mergeInputsDependencies(uniqueLibraries,
+ needWholeArchive, cppConfiguration.archiveType()));
+ // getPrimaryInput returns the first element, and that is a public interface - therefore the
+ // order here is important.
+ Iterable<Artifact> inputs = IterablesChain.<Artifact>builder()
+ .add(ImmutableList.copyOf(LinkerInputs.toLibraryArtifacts(nonLibraries)))
+ .add(dependencyInputsBuilder.build())
+ .add(ImmutableIterable.from(expandedInputs))
+ .deduplicate()
+ .build();
+
+ return new CppLinkAction(
+ getOwner(),
+ inputs,
+ actionOutputs,
+ cppConfiguration,
+ outputLibrary,
+ interfaceOutputLibrary,
+ fake,
+ linkCommandLine);
+ }
+
+ /**
+ * The default heuristic on whether we need to use whole-archive for the link.
+ */
+ private static boolean needWholeArchive(LinkStaticness staticness,
+ LinkTargetType type, Collection<String> linkopts, boolean isNativeDeps,
+ CppConfiguration cppConfig) {
+ boolean fullyStatic = (staticness == LinkStaticness.FULLY_STATIC);
+ boolean mostlyStatic = (staticness == LinkStaticness.MOSTLY_STATIC);
+ boolean sharedLinkopts = type == LinkTargetType.DYNAMIC_LIBRARY
+ || linkopts.contains("-shared")
+ || cppConfig.getLinkOptions().contains("-shared");
+ return (isNativeDeps || cppConfig.legacyWholeArchive())
+ && (fullyStatic || mostlyStatic)
+ && sharedLinkopts;
+ }
+
+ private static ImmutableList<Artifact> constructOutputs(Artifact primaryOutput,
+ Collection<Artifact> outputList, Artifact... outputs) {
+ return new ImmutableList.Builder<Artifact>()
+ .add(primaryOutput)
+ .addAll(outputList)
+ .addAll(CollectionUtils.asListWithoutNulls(outputs))
+ .build();
+ }
+
+ /**
+ * Translates a collection of linkstamp source files to an immutable
+ * mapping from source files to object files. In other words, given a
+ * set of source files, this method determines the output path to which
+ * each file should be compiled.
+ *
+ * @param linkstamps collection of linkstamp source files
+ * @param ruleContext the rule for which this link is being performed
+ * @param outputBinary the binary output path for this link
+ * @return an immutable map that pairs each source file with the
+ * corresponding object file that should be fed into the link
+ */
+ public static ImmutableMap<Artifact, Artifact> mapLinkstampsToOutputs(
+ Collection<Artifact> linkstamps, RuleContext ruleContext, Artifact outputBinary) {
+ ImmutableMap.Builder<Artifact, Artifact> mapBuilder = ImmutableMap.builder();
+
+ PathFragment outputBinaryPath = outputBinary.getRootRelativePath();
+ PathFragment stampOutputDirectory = outputBinaryPath.getParentDirectory().
+ getRelative("_objs").getRelative(outputBinaryPath.getBaseName());
+
+ for (Artifact linkstamp : linkstamps) {
+ PathFragment stampOutputPath = stampOutputDirectory.getRelative(
+ FileSystemUtils.replaceExtension(linkstamp.getRootRelativePath(), ".o"));
+ mapBuilder.put(linkstamp,
+ ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ stampOutputPath, outputBinary.getRoot()));
+ }
+ return mapBuilder.build();
+ }
+
+ protected ActionOwner getOwner() {
+ return ruleContext.getActionOwner();
+ }
+
+ protected Artifact createArtifact(PathFragment path) {
+ return analysisEnvironment.getDerivedArtifact(path, configuration.getBinDirectory());
+ }
+
+ protected Artifact getInterfaceSoBuilder() {
+ return analysisEnvironment.getEmbeddedToolArtifact(CppRuleClasses.BUILD_INTERFACE_SO);
+ }
+
+ /**
+ * Set the crosstool inputs required for the action.
+ */
+ public Builder setCrosstoolInputs(NestedSet<Artifact> inputs) {
+ this.crosstoolInputs = inputs;
+ return this;
+ }
+
+ /**
+ * Sets the C++ runtime library inputs for the action.
+ */
+ public Builder setRuntimeInputs(Artifact middleman, NestedSet<Artifact> inputs) {
+ Preconditions.checkArgument((middleman == null) == inputs.isEmpty());
+ this.runtimeMiddleman = middleman;
+ this.runtimeInputs = inputs;
+ return this;
+ }
+
+ /**
+ * Sets the interface output of the link. A non-null argument can
+ * only be provided if the link type is {@code DYNAMIC_LIBRARY}
+ * and fake is false.
+ */
+ public Builder setInterfaceOutputPath(PathFragment path) {
+ this.interfaceOutputPath = path;
+ return this;
+ }
+
+ /**
+ * Add additional inputs needed for the linkstamp compilation that is being done as part of the
+ * link.
+ */
+ public Builder addCompilationInputs(Iterable<Artifact> inputs) {
+ this.compilationInputs.addAll(inputs);
+ return this;
+ }
+
+ public Builder addTransitiveCompilationInputs(NestedSet<Artifact> inputs) {
+ this.compilationInputs.addTransitive(inputs);
+ return this;
+ }
+
+ private void addNonLibraryInput(LinkerInput input) {
+ String name = input.getArtifact().getFilename();
+ Preconditions.checkArgument(
+ !Link.ARCHIVE_LIBRARY_FILETYPES.matches(name)
+ && !Link.SHARED_LIBRARY_FILETYPES.matches(name),
+ "'%s' is a library file", input);
+ this.nonLibraries.add(input);
+ }
+ /**
+ * Adds a single artifact to the set of inputs (C++ source files, header files, etc). Artifacts
+ * that are not of recognized types will be used for dependency checking but will not be passed
+ * to the linker. The artifact must not be an archive or a shared library.
+ */
+ public Builder addNonLibraryInput(Artifact input) {
+ addNonLibraryInput(LinkerInputs.simpleLinkerInput(input));
+ return this;
+ }
+
+ /**
+ * Adds multiple artifacts to the set of inputs (C++ source files, header files, etc).
+ * Artifacts that are not of recognized types will be used for dependency checking but will
+ * not be passed to the linker. The artifacts must not be archives or shared libraries.
+ */
+ public Builder addNonLibraryInputs(Iterable<Artifact> inputs) {
+ for (Artifact input : inputs) {
+ addNonLibraryInput(LinkerInputs.simpleLinkerInput(input));
+ }
+ return this;
+ }
+
+ public Builder addFakeNonLibraryInputs(Iterable<Artifact> inputs) {
+ for (Artifact input : inputs) {
+ addNonLibraryInput(LinkerInputs.fakeLinkerInput(input));
+ }
+ return this;
+ }
+
+ private void checkLibrary(LibraryToLink input) {
+ String name = input.getArtifact().getFilename();
+ Preconditions.checkArgument(
+ Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) ||
+ Link.SHARED_LIBRARY_FILETYPES.matches(name),
+ "'%s' is not a library file", input);
+ }
+
+ /**
+ * Adds a single artifact to the set of inputs. The artifact must be an archive or a shared
+ * library. Note that all directly added libraries are implicitly ordered before all nested
+ * sets added with {@link #addLibraries}, even if added in the opposite order.
+ */
+ public Builder addLibrary(LibraryToLink input) {
+ checkLibrary(input);
+ libraries.add(input);
+ return this;
+ }
+
+ /**
+ * Adds multiple artifact to the set of inputs. The artifacts must be archives or shared
+ * libraries.
+ */
+ public Builder addLibraries(NestedSet<LibraryToLink> inputs) {
+ for (LibraryToLink input : inputs) {
+ checkLibrary(input);
+ }
+ this.libraries.addTransitive(inputs);
+ return this;
+ }
+
+ /**
+ * Sets the type of ELF file to be created (.a, .so, .lo, executable). The
+ * default is {@link LinkTargetType#STATIC_LIBRARY}.
+ */
+ public Builder setLinkType(LinkTargetType linkType) {
+ this.linkType = linkType;
+ return this;
+ }
+
+ /**
+ * Sets the degree of "staticness" of the link: fully static (static binding
+ * of all symbols), mostly static (use dynamic binding only for symbols from
+ * glibc), dynamic (use dynamic binding wherever possible). The default is
+ * {@link LinkStaticness#FULLY_STATIC}.
+ */
+ public Builder setLinkStaticness(LinkStaticness linkStaticness) {
+ this.linkStaticness = linkStaticness;
+ return this;
+ }
+
+ /**
+ * Adds a C++ source file which will be compiled at link time. This is used
+ * to embed various values from the build system into binaries to identify
+ * their provenance.
+ *
+ * <p>Link stamps are also automatically added to the inputs.
+ */
+ public Builder addLinkstamps(Map<Artifact, ImmutableList<Artifact>> linkstamps) {
+ this.linkstamps.addAll(linkstamps.keySet());
+ // Add inputs for linkstamping.
+ if (!linkstamps.isEmpty()) {
+ // This will just be the compiler unless include scanning is disabled, in which case it will
+ // include all header files. Since we insist that linkstamps declare all their headers, all
+ // header files would be overkill, but that only happens when include scanning is disabled.
+ addTransitiveCompilationInputs(toolchain.getCompile());
+ for (Map.Entry<Artifact, ImmutableList<Artifact>> entry : linkstamps.entrySet()) {
+ addCompilationInputs(entry.getValue());
+ }
+ }
+ return this;
+ }
+
+ public Builder addLinkstampCompilerOptions(ImmutableList<String> linkstampOptions) {
+ this.linkstampOptions = linkstampOptions;
+ return this;
+ }
+
+ /**
+ * Adds an additional linker option.
+ */
+ public Builder addLinkopt(String linkopt) {
+ this.linkopts.add(linkopt);
+ return this;
+ }
+
+ /**
+ * Adds multiple linker options at once.
+ *
+ * @see #addLinkopt(String)
+ */
+ public Builder addLinkopts(Collection<String> linkopts) {
+ this.linkopts.addAll(linkopts);
+ return this;
+ }
+
+ /**
+ * Sets whether this link action will be used for a cc_fake_binary; false by
+ * default.
+ */
+ public Builder setFake(boolean fake) {
+ this.fake = fake;
+ return this;
+ }
+
+ /**
+ * Sets whether this link action is used for a native dependency library.
+ */
+ public Builder setNativeDeps(boolean isNativeDeps) {
+ this.isNativeDeps = isNativeDeps;
+ return this;
+ }
+
+ /**
+ * Setting this to true overrides the default whole-archive computation and force-enables
+ * whole archives for every archive in the link. This is only necessary for linking executable
+ * binaries that are supposed to export symbols.
+ *
+ * <p>Usually, the link action while use whole archives for dynamic libraries that are native
+ * deps (or the legacy whole archive flag is enabled), and that are not dynamically linked.
+ *
+ * <p>(Note that it is possible to build dynamic libraries with cc_binary rules by specifying
+ * linkshared = 1, and giving the rule a name that matches the pattern {@code
+ * lib<name>.so}.)
+ */
+ public Builder setWholeArchive(boolean wholeArchive) {
+ this.wholeArchive = wholeArchive;
+ return this;
+ }
+
+ /**
+ * Sets whether this link action should use test-specific flags (e.g. $EXEC_ORIGIN instead of
+ * $ORIGIN for the solib search path or lazy binding); false by default.
+ */
+ public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) {
+ this.useTestOnlyFlags = useTestOnlyFlags;
+ return this;
+ }
+
+ /**
+ * Sets the name of the directory where the solib symlinks for the dynamic runtime libraries
+ * live. This is usually automatically set from the cc_toolchain.
+ */
+ public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) {
+ this.runtimeSolibDir = runtimeSolibDir;
+ return this;
+ }
+
+ /**
+ * Creates a builder without the need for a {@link RuleContext}.
+ * This is to be used exclusively for testing purposes.
+ *
+ * <p>Link stamping is not supported if using this method.
+ */
+ @VisibleForTesting
+ public static Builder createTestBuilder(
+ final ActionOwner owner, final AnalysisEnvironment analysisEnvironment,
+ final PathFragment outputPath, BuildConfiguration config) {
+ return new Builder(null, outputPath, config, analysisEnvironment, null) {
+ @Override
+ protected Artifact createArtifact(PathFragment path) {
+ return new Artifact(configuration.getBinDirectory().getPath().getRelative(path),
+ configuration.getBinDirectory(), configuration.getBinFragment().getRelative(path),
+ analysisEnvironment.getOwner());
+ }
+ @Override
+ protected ActionOwner getOwner() {
+ return owner;
+ }
+ };
+ }
+ }
+
+ /**
+ * Immutable ELF linker context, suitable for serialization.
+ */
+ @Immutable @ThreadSafe
+ public static final class Context implements TransitiveInfoProvider {
+ // Morally equivalent with {@link Builder}, except these are immutable.
+ // Keep these in sync with {@link Builder}.
+ private final ImmutableSet<LinkerInput> nonLibraries;
+ private final NestedSet<LibraryToLink> libraries;
+ private final NestedSet<Artifact> crosstoolInputs;
+ private final Artifact runtimeMiddleman;
+ private final NestedSet<Artifact> runtimeInputs;
+ private final NestedSet<Artifact> compilationInputs;
+ private final ImmutableSet<Artifact> linkstamps;
+ private final ImmutableList<String> linkopts;
+ private final LinkTargetType linkType;
+ private final LinkStaticness linkStaticness;
+ private final boolean fake;
+ private final boolean isNativeDeps;
+ private final boolean useTestOnlyFlags;
+
+ /**
+ * Given a {@link Builder}, creates a {@code Context} to pass to another target.
+ * Note well: Keep the Builder->Context and Context->Builder transforms consistent!
+ * @param builder a mutable {@link CppLinkAction.Builder} to clone from
+ */
+ public Context(Builder builder) {
+ this.nonLibraries = ImmutableSet.copyOf(builder.nonLibraries);
+ this.libraries = NestedSetBuilder.<LibraryToLink>linkOrder()
+ .addTransitive(builder.libraries.build()).build();
+ this.crosstoolInputs =
+ NestedSetBuilder.<Artifact>stableOrder().addTransitive(builder.crosstoolInputs).build();
+ this.runtimeMiddleman = builder.runtimeMiddleman;
+ this.runtimeInputs =
+ NestedSetBuilder.<Artifact>stableOrder().addTransitive(builder.runtimeInputs).build();
+ this.compilationInputs = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(builder.compilationInputs.build()).build();
+ this.linkstamps = ImmutableSet.copyOf(builder.linkstamps);
+ this.linkopts = ImmutableList.copyOf(builder.linkopts);
+ this.linkType = builder.linkType;
+ this.linkStaticness = builder.linkStaticness;
+ this.fake = builder.fake;
+ this.isNativeDeps = builder.isNativeDeps;
+ this.useTestOnlyFlags = builder.useTestOnlyFlags;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java
new file mode 100644
index 0000000..24a936b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.ActionContextMarker;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ResourceSet;
+
+/**
+ * Context for executing {@link CppLinkAction}s.
+ */
+@ActionContextMarker(name = "C++ link")
+public interface CppLinkActionContext extends ActionContext {
+ /**
+ * Returns where the action actually runs.
+ */
+ String strategyLocality(CppLinkAction action);
+
+ /**
+ * Returns the estimated resource consumption of the action.
+ */
+ ResourceSet estimateResourceConsumption(CppLinkAction action);
+
+ /**
+ * Executes the specified action.
+ */
+ void exec(CppLinkAction action,
+ ActionExecutionContext actionExecutionContext)
+ throws ExecException, ActionExecutionException, InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java
new file mode 100644
index 0000000..44258a5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java
@@ -0,0 +1,707 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CcCompilationOutputs.Builder;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * Representation of a C/C++ compilation. Its purpose is to share the code that creates compilation
+ * actions between all classes that need to do so. It follows the builder pattern - load up the
+ * necessary settings and then call {@link #createCcCompileActions}.
+ *
+ * <p>This class is not thread-safe, and it should only be used once for each set of source files,
+ * i.e. calling {@link #createCcCompileActions} will throw an Exception if called twice.
+ */
+public final class CppModel {
+ private final CppSemantics semantics;
+ private final RuleContext ruleContext;
+ private final BuildConfiguration configuration;
+ private final CppConfiguration cppConfiguration;
+
+ // compile model
+ private CppCompilationContext context;
+ private final List<Pair<Artifact, Label>> sourceFiles = new ArrayList<>();
+ private final List<String> copts = new ArrayList<>();
+ private final List<PathFragment> additionalIncludes = new ArrayList<>();
+ @Nullable private Pattern nocopts;
+ private boolean fake;
+ private boolean maySaveTemps;
+ private boolean onlySingleOutput;
+ private CcCompilationOutputs compilationOutputs;
+ private boolean enableLayeringCheck;
+ private boolean compileHeaderModules;
+
+ // link model
+ private final List<String> linkopts = new ArrayList<>();
+ private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY;
+ private boolean neverLink;
+ private boolean allowInterfaceSharedObjects;
+ private boolean createDynamicLibrary = true;
+ private PathFragment soImplFilename;
+ private FeatureConfiguration featureConfiguration;
+
+ public CppModel(RuleContext ruleContext, CppSemantics semantics) {
+ this.ruleContext = ruleContext;
+ this.semantics = semantics;
+ configuration = ruleContext.getConfiguration();
+ cppConfiguration = configuration.getFragment(CppConfiguration.class);
+ }
+
+ /**
+ * If the cpp compilation is a fake, then it creates only a single compile action without PIC.
+ * Defaults to false.
+ */
+ public CppModel setFake(boolean fake) {
+ this.fake = fake;
+ return this;
+ }
+
+ /**
+ * If set, the CppModel only creates a single .o output that can be linked into a dynamic library,
+ * i.e., it never generates both PIC and non-PIC outputs. Otherwise it creates outputs that can be
+ * linked into both static binaries and dynamic libraries (if both require PIC or both require
+ * non-PIC, then it still only creates a single output). Defaults to false.
+ */
+ public CppModel setOnlySingleOutput(boolean onlySingleOutput) {
+ this.onlySingleOutput = onlySingleOutput;
+ return this;
+ }
+
+ /**
+ * If set, use compiler flags to enable compiler based layering checks.
+ */
+ public CppModel setEnableLayeringCheck(boolean enableLayeringCheck) {
+ this.enableLayeringCheck = enableLayeringCheck;
+ return this;
+ }
+
+ /**
+ * If set, add actions that compile header modules to the build.
+ * See http://clang.llvm.org/docs/Modules.html for more information.
+ */
+ public CppModel setCompileHeaderModules(boolean compileHeaderModules) {
+ this.compileHeaderModules = compileHeaderModules;
+ return this;
+ }
+
+ /**
+ * Whether to create actions for temps. This defaults to false.
+ */
+ public CppModel setSaveTemps(boolean maySaveTemps) {
+ this.maySaveTemps = maySaveTemps;
+ return this;
+ }
+
+ /**
+ * Sets the compilation context, i.e. include directories and allowed header files inclusions.
+ */
+ public CppModel setContext(CppCompilationContext context) {
+ this.context = context;
+ return this;
+ }
+
+ /**
+ * Adds a single source file to be compiled. Note that this should only be called for primary
+ * compilation units, not for header files or files that are otherwise included.
+ */
+ public CppModel addSources(Iterable<Artifact> sourceFiles, Label sourceLabel) {
+ for (Artifact sourceFile : sourceFiles) {
+ this.sourceFiles.add(Pair.of(sourceFile, sourceLabel));
+ }
+ return this;
+ }
+
+ /**
+ * Adds all the source files. Note that this should only be called for primary compilation units,
+ * not for header files or files that are otherwise included.
+ */
+ public CppModel addSources(Iterable<Pair<Artifact, Label>> sources) {
+ Iterables.addAll(this.sourceFiles, sources);
+ return this;
+ }
+
+ /**
+ * Adds the given copts.
+ */
+ public CppModel addCopts(Collection<String> copts) {
+ this.copts.addAll(copts);
+ return this;
+ }
+
+ /**
+ * Sets the nocopts pattern. This is used to filter out flags from the system defined set of
+ * flags. By default no filter is applied.
+ */
+ public CppModel setNoCopts(@Nullable Pattern nocopts) {
+ this.nocopts = nocopts;
+ return this;
+ }
+
+ /**
+ * This can be used to specify additional include directories, without modifying the compilation
+ * context.
+ */
+ public CppModel addAdditionalIncludes(Collection<PathFragment> additionalIncludes) {
+ // TODO(bazel-team): Maybe this could be handled by the compilation context instead?
+ this.additionalIncludes.addAll(additionalIncludes);
+ return this;
+ }
+
+ /**
+ * Adds the given linkopts to the optional dynamic library link command.
+ */
+ public CppModel addLinkopts(Collection<String> linkopts) {
+ this.linkopts.addAll(linkopts);
+ return this;
+ }
+
+ /**
+ * Sets the link type used for the link actions. Note that only static links are supported at this
+ * time.
+ */
+ public CppModel setLinkTargetType(LinkTargetType linkType) {
+ this.linkType = linkType;
+ return this;
+ }
+
+ public CppModel setNeverLink(boolean neverLink) {
+ this.neverLink = neverLink;
+ return this;
+ }
+
+ /**
+ * Whether to allow interface dynamic libraries. Note that setting this to true only has an effect
+ * if the configuration allows it. Defaults to false.
+ */
+ public CppModel setAllowInterfaceSharedObjects(boolean allowInterfaceSharedObjects) {
+ // TODO(bazel-team): Set the default to true, and require explicit action to disable it.
+ this.allowInterfaceSharedObjects = allowInterfaceSharedObjects;
+ return this;
+ }
+
+ public CppModel setCreateDynamicLibrary(boolean createDynamicLibrary) {
+ this.createDynamicLibrary = createDynamicLibrary;
+ return this;
+ }
+
+ public CppModel setDynamicLibraryPath(PathFragment soImplFilename) {
+ this.soImplFilename = soImplFilename;
+ return this;
+ }
+
+ /**
+ * Sets the feature configuration to be used for C/C++ actions.
+ */
+ public CppModel setFeatureConfiguration(FeatureConfiguration featureConfiguration) {
+ this.featureConfiguration = featureConfiguration;
+ return this;
+ }
+
+ /**
+ * @return the non-pic header module artifact for the current target.
+ */
+ public Artifact getHeaderModule(Artifact moduleMapArtifact) {
+ PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel());
+ PathFragment outputName = objectDir.getRelative(
+ semantics.getEffectiveSourcePath(moduleMapArtifact));
+ return ruleContext.getRelatedArtifact(outputName, ".pcm");
+ }
+
+ /**
+ * @return the pic header module artifact for the current target.
+ */
+ public Artifact getPicHeaderModule(Artifact moduleMapArtifact) {
+ PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel());
+ PathFragment outputName = objectDir.getRelative(
+ semantics.getEffectiveSourcePath(moduleMapArtifact));
+ return ruleContext.getRelatedArtifact(outputName, ".pic.pcm");
+ }
+
+ /**
+ * @return whether this target needs to generate pic actions.
+ */
+ public boolean getGeneratePicActions() {
+ return CppHelper.usePic(ruleContext, false);
+ }
+
+ /**
+ * @return whether this target needs to generate non-pic actions.
+ */
+ public boolean getGenerateNoPicActions() {
+ return
+ // If we always need pic for everything, then don't bother to create a no-pic action.
+ (!CppHelper.usePic(ruleContext, true) || !CppHelper.usePic(ruleContext, false))
+ // onlySingleOutput guarantees that the code is only ever linked into a dynamic library - so
+ // we don't need a no-pic action even if linking into a binary would require it.
+ && !((onlySingleOutput && getGeneratePicActions()));
+ }
+
+ /**
+ * @return whether this target needs to generate a pic header module.
+ */
+ public boolean getGeneratesPicHeaderModule() {
+ // TODO(bazel-team): Make sure cc_fake_binary works with header module support.
+ return compileHeaderModules && !fake && getGeneratePicActions();
+ }
+
+ /**
+ * @return whether this target needs to generate a non-pic header module.
+ */
+ public boolean getGeratesNoPicHeaderModule() {
+ return compileHeaderModules && !fake && getGenerateNoPicActions();
+ }
+
+ /**
+ * Returns a {@code CppCompileActionBuilder} with the common fields for a C++ compile action
+ * being initialized.
+ */
+ private CppCompileActionBuilder initializeCompileAction(Artifact sourceArtifact,
+ Label sourceLabel) {
+ CppCompileActionBuilder builder = createCompileActionBuilder(sourceArtifact, sourceLabel);
+ if (nocopts != null) {
+ builder.addNocopts(nocopts);
+ }
+
+ builder.setEnableLayeringCheck(enableLayeringCheck);
+ builder.setCompileHeaderModules(compileHeaderModules);
+ builder.setExtraSystemIncludePrefixes(additionalIncludes);
+ builder.setFdoBuildStamp(CppHelper.getFdoBuildStamp(cppConfiguration));
+ builder.setFeatureConfiguration(featureConfiguration);
+ return builder;
+ }
+
+ /**
+ * Constructs the C++ compiler actions. It generally creates one action for every specified source
+ * file. It takes into account LIPO, fake-ness, coverage, and PIC, in addition to using the
+ * settings specified on the current object. This method should only be called once.
+ */
+ public CcCompilationOutputs createCcCompileActions() {
+ CcCompilationOutputs.Builder result = new CcCompilationOutputs.Builder();
+ Preconditions.checkNotNull(context);
+ AnalysisEnvironment env = ruleContext.getAnalysisEnvironment();
+ PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel());
+
+ if (compileHeaderModules) {
+ Artifact moduleMapArtifact = context.getCppModuleMap().getArtifact();
+ Label moduleMapLabel = Label.parseAbsoluteUnchecked(context.getCppModuleMap().getName());
+ PathFragment outputName = getObjectOutputPath(moduleMapArtifact, objectDir);
+ CppCompileActionBuilder builder = initializeCompileAction(moduleMapArtifact, moduleMapLabel);
+
+ // A header module compile action is just like a normal compile action, but:
+ // - the compiled source file is the module map
+ // - it creates a header module (.pcm file).
+ createSourceAction(outputName, result, env, moduleMapArtifact, builder, ".pcm");
+ }
+
+ for (Pair<Artifact, Label> source : sourceFiles) {
+ Artifact sourceArtifact = source.getFirst();
+ Label sourceLabel = source.getSecond();
+ PathFragment outputName = getObjectOutputPath(sourceArtifact, objectDir);
+ CppCompileActionBuilder builder = initializeCompileAction(sourceArtifact, sourceLabel);
+
+ if (CppFileTypes.CPP_HEADER.matches(source.first.getExecPath())) {
+ createHeaderAction(outputName, result, env, builder);
+ } else {
+ createSourceAction(outputName, result, env, sourceArtifact, builder, ".o");
+ }
+ }
+
+ compilationOutputs = result.build();
+ return compilationOutputs;
+ }
+
+ private void createHeaderAction(PathFragment outputName, Builder result, AnalysisEnvironment env,
+ CppCompileActionBuilder builder) {
+ builder.setOutputFile(ruleContext.getRelatedArtifact(outputName, ".h.processed")).setDotdFile(
+ outputName, ".h.d", ruleContext);
+ semantics.finalizeCompileActionBuilder(ruleContext, builder);
+ CppCompileAction compileAction = builder.build();
+ env.registerAction(compileAction);
+ Artifact tokenFile = compileAction.getOutputFile();
+ result.addHeaderTokenFile(tokenFile);
+ }
+
+ private void createSourceAction(PathFragment outputName,
+ CcCompilationOutputs.Builder result,
+ AnalysisEnvironment env,
+ Artifact sourceArtifact,
+ CppCompileActionBuilder builder,
+ String outputExtension) {
+ PathFragment ccRelativeName = semantics.getEffectiveSourcePath(sourceArtifact);
+ LipoContextProvider lipoProvider = null;
+ if (cppConfiguration.isLipoOptimization()) {
+ // TODO(bazel-team): we shouldn't be needing this, merging context with the binary
+ // is a superset of necessary information.
+ lipoProvider = Preconditions.checkNotNull(CppHelper.getLipoContextProvider(ruleContext),
+ outputName);
+ builder.setContext(CppCompilationContext.mergeForLipo(lipoProvider.getLipoContext(),
+ context));
+ }
+ if (fake) {
+ // For cc_fake_binary, we only create a single fake compile action. It's
+ // not necessary to use -fPIC for negative compilation tests, and using
+ // .pic.o files in cc_fake_binary would break existing uses of
+ // cc_fake_binary.
+ Artifact outputFile = ruleContext.getRelatedArtifact(outputName, outputExtension);
+ PathFragment tempOutputName =
+ FileSystemUtils.replaceExtension(outputFile.getExecPath(), ".temp" + outputExtension);
+ builder
+ .setOutputFile(outputFile)
+ .setDotdFile(outputName, ".d", ruleContext)
+ .setTempOutputFile(tempOutputName);
+ semantics.finalizeCompileActionBuilder(ruleContext, builder);
+ CppCompileAction action = builder.build();
+ env.registerAction(action);
+ result.addObjectFile(action.getOutputFile());
+ } else {
+ boolean generatePicAction = getGeneratePicActions();
+ // If we always need pic for everything, then don't bother to create a no-pic action.
+ boolean generateNoPicAction = getGenerateNoPicActions();
+ Preconditions.checkState(generatePicAction || generateNoPicAction);
+
+ // Create PIC compile actions (same as non-PIC, but use -fPIC and
+ // generate .pic.o, .pic.d, .pic.gcno instead of .o, .d, .gcno.)
+ if (generatePicAction) {
+ CppCompileActionBuilder picBuilder = copyAsPicBuilder(builder, outputName, outputExtension);
+ cppConfiguration.getFdoSupport().configureCompilation(picBuilder, ruleContext, env,
+ ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/true,
+ lipoProvider);
+
+ if (maySaveTemps) {
+ result.addTemps(
+ createTempsActions(sourceArtifact, outputName, picBuilder, /*usePic=*/true));
+ }
+
+ if (isCodeCoverageEnabled()) {
+ picBuilder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".pic.gcno"));
+ }
+
+ semantics.finalizeCompileActionBuilder(ruleContext, picBuilder);
+ CppCompileAction picAction = picBuilder.build();
+ env.registerAction(picAction);
+ result.addPicObjectFile(picAction.getOutputFile());
+ if (picAction.getDwoFile() != null) {
+ // Host targets don't produce .dwo files.
+ result.addPicDwoFile(picAction.getDwoFile());
+ }
+ if (cppConfiguration.isLipoContextCollector() && !generateNoPicAction) {
+ result.addLipoScannable(picAction);
+ }
+ }
+
+ if (generateNoPicAction) {
+ builder
+ .setOutputFile(ruleContext.getRelatedArtifact(outputName, outputExtension))
+ .setDotdFile(outputName, ".d", ruleContext);
+ // Create non-PIC compile actions
+ cppConfiguration.getFdoSupport().configureCompilation(builder, ruleContext, env,
+ ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/false,
+ lipoProvider);
+
+ if (maySaveTemps) {
+ result.addTemps(
+ createTempsActions(sourceArtifact, outputName, builder, /*usePic=*/false));
+ }
+
+ if (!cppConfiguration.isLipoOptimization() && isCodeCoverageEnabled()) {
+ builder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".gcno"));
+ }
+
+ semantics.finalizeCompileActionBuilder(ruleContext, builder);
+ CppCompileAction compileAction = builder.build();
+ env.registerAction(compileAction);
+ Artifact objectFile = compileAction.getOutputFile();
+ result.addObjectFile(objectFile);
+ if (compileAction.getDwoFile() != null) {
+ // Host targets don't produce .dwo files.
+ result.addDwoFile(compileAction.getDwoFile());
+ }
+ if (cppConfiguration.isLipoContextCollector()) {
+ result.addLipoScannable(compileAction);
+ }
+ }
+ }
+ }
+
+ /**
+ * Constructs the C++ linker actions. It generally generates two actions, one for a static library
+ * and one for a dynamic library. If PIC is required for shared libraries, but not for binaries,
+ * it additionally creates a third action to generate a PIC static library.
+ *
+ * <p>For dynamic libraries, this method can additionally create an interface shared library that
+ * can be used for linking, but doesn't contain any executable code. This increases the number of
+ * cache hits for link actions. Call {@link #setAllowInterfaceSharedObjects(boolean)} to enable
+ * this behavior.
+ */
+ public CcLinkingOutputs createCcLinkActions(CcCompilationOutputs ccOutputs) {
+ // For now only handle static links. Note that the dynamic library link below ignores linkType.
+ // TODO(bazel-team): Either support non-static links or move this check to setLinkType().
+ Preconditions.checkState(linkType.isStaticLibraryLink(), "can only handle static links");
+
+ CcLinkingOutputs.Builder result = new CcLinkingOutputs.Builder();
+ if (cppConfiguration.isLipoContextCollector()) {
+ // Don't try to create LIPO link actions in collector mode,
+ // because it needs some data that's not available at this point.
+ return result.build();
+ }
+
+ AnalysisEnvironment env = ruleContext.getAnalysisEnvironment();
+ boolean usePicForBinaries = CppHelper.usePic(ruleContext, true);
+ boolean usePicForSharedLibs = CppHelper.usePic(ruleContext, false);
+
+ // Create static library (.a). The linkType only reflects whether the library is alwayslink or
+ // not. The PIC-ness is determined by whether we need to use PIC or not. There are three cases
+ // for (usePicForSharedLibs usePicForBinaries):
+ //
+ // (1) (false false) -> no pic code
+ // (2) (true false) -> shared libraries as pic, but not binaries
+ // (3) (true true) -> both shared libraries and binaries as pic
+ //
+ // In case (3), we always need PIC, so only create one static library containing the PIC object
+ // files. The name therefore does not match the content.
+ //
+ // Presumably, it is done this way because the .a file is an implicit output of every cc_library
+ // rule, so we can't use ".pic.a" that in the always-PIC case.
+ PathFragment linkedFileName = CppHelper.getLinkedFilename(ruleContext, linkType);
+ CppLinkAction maybePicAction = newLinkActionBuilder(linkedFileName)
+ .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForBinaries))
+ .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles())
+ .setLinkType(linkType)
+ .setLinkStaticness(LinkStaticness.FULLY_STATIC)
+ .build();
+ env.registerAction(maybePicAction);
+ result.addStaticLibrary(maybePicAction.getOutputLibrary());
+
+ // Create a second static library (.pic.a). Only in case (2) do we need both PIC and non-PIC
+ // static libraries. In that case, the first static library contains the non-PIC code, and this
+ // one contains the PIC code, so the names match the content.
+ if (!usePicForBinaries && usePicForSharedLibs) {
+ LinkTargetType picLinkType = (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY)
+ ? LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY
+ : LinkTargetType.PIC_STATIC_LIBRARY;
+
+ PathFragment picFileName = CppHelper.getLinkedFilename(ruleContext, picLinkType);
+ CppLinkAction picAction = newLinkActionBuilder(picFileName)
+ .addNonLibraryInputs(ccOutputs.getObjectFiles(true))
+ .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles())
+ .setLinkType(picLinkType)
+ .setLinkStaticness(LinkStaticness.FULLY_STATIC)
+ .build();
+ env.registerAction(picAction);
+ result.addPicStaticLibrary(picAction.getOutputLibrary());
+ }
+
+ if (!createDynamicLibrary) {
+ return result.build();
+ }
+
+ // Create dynamic library.
+ if (soImplFilename == null) {
+ soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY);
+ }
+ List<String> sonameLinkopts = ImmutableList.of();
+ PathFragment soInterfaceFilename = null;
+ if (cppConfiguration.useInterfaceSharedObjects() && allowInterfaceSharedObjects) {
+ soInterfaceFilename =
+ CppHelper.getLinkedFilename(ruleContext, LinkTargetType.INTERFACE_DYNAMIC_LIBRARY);
+ Artifact dynamicLibrary = env.getDerivedArtifact(
+ soImplFilename, configuration.getBinDirectory());
+ sonameLinkopts = ImmutableList.of("-Wl,-soname=" +
+ SolibSymlinkAction.getDynamicLibrarySoname(dynamicLibrary.getRootRelativePath(), false));
+ }
+
+ // Should we also link in any libraries that this library depends on?
+ // That is required on some systems...
+ CppLinkAction action = newLinkActionBuilder(soImplFilename)
+ .setInterfaceOutputPath(soInterfaceFilename)
+ .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForSharedLibs))
+ .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles())
+ .setLinkType(LinkTargetType.DYNAMIC_LIBRARY)
+ .setLinkStaticness(LinkStaticness.DYNAMIC)
+ .addLinkopts(linkopts)
+ .addLinkopts(sonameLinkopts)
+ .setRuntimeInputs(
+ CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkMiddleman(),
+ CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs())
+ .build();
+ env.registerAction(action);
+
+ LibraryToLink dynamicLibrary = action.getOutputLibrary();
+ LibraryToLink interfaceLibrary = action.getInterfaceOutputLibrary();
+ if (interfaceLibrary == null) {
+ interfaceLibrary = dynamicLibrary;
+ }
+
+ // If shared library has neverlink=1, then leave it untouched. Otherwise,
+ // create a mangled symlink for it and from now on reference it through
+ // mangled name only.
+ if (neverLink) {
+ result.addDynamicLibrary(interfaceLibrary);
+ result.addExecutionDynamicLibrary(dynamicLibrary);
+ } else {
+ LibraryToLink libraryLink = SolibSymlinkAction.getDynamicLibrarySymlink(
+ ruleContext, interfaceLibrary.getArtifact(), false, false,
+ ruleContext.getConfiguration());
+ result.addDynamicLibrary(libraryLink);
+ LibraryToLink implLibraryLink = SolibSymlinkAction.getDynamicLibrarySymlink(
+ ruleContext, dynamicLibrary.getArtifact(), false, false,
+ ruleContext.getConfiguration());
+ result.addExecutionDynamicLibrary(implLibraryLink);
+ }
+ return result.build();
+ }
+
+ private CppLinkAction.Builder newLinkActionBuilder(PathFragment outputPath) {
+ return new CppLinkAction.Builder(ruleContext, outputPath)
+ .setCrosstoolInputs(CppHelper.getToolchain(ruleContext).getLink())
+ .addNonLibraryInputs(context.getCompilationPrerequisites());
+ }
+
+ /**
+ * Returns the output artifact path relative to the object directory.
+ */
+ private PathFragment getObjectOutputPath(Artifact source, PathFragment objectDirectory) {
+ return objectDirectory.getRelative(semantics.getEffectiveSourcePath(source));
+ }
+
+ /**
+ * Creates a basic cpp compile action builder for source file. Configures options,
+ * crosstool inputs, output and dotd file names, compilation context and copts.
+ */
+ private CppCompileActionBuilder createCompileActionBuilder(
+ Artifact source, Label label) {
+ CppCompileActionBuilder builder = new CppCompileActionBuilder(
+ ruleContext, source, label);
+
+ builder
+ .setContext(context)
+ .addCopts(copts);
+ return builder;
+ }
+
+ /**
+ * Creates cpp PIC compile action builder from the given builder by adding necessary copt and
+ * changing output and dotd file names.
+ */
+ private CppCompileActionBuilder copyAsPicBuilder(CppCompileActionBuilder builder,
+ PathFragment outputName, String outputExtension) {
+ CppCompileActionBuilder picBuilder = new CppCompileActionBuilder(builder);
+ picBuilder.addCopt("-fPIC")
+ .setOutputFile(ruleContext.getRelatedArtifact(outputName, ".pic" + outputExtension))
+ .setDotdFile(outputName, ".pic.d", ruleContext);
+ return picBuilder;
+ }
+
+ /**
+ * Create the actions for "--save_temps".
+ */
+ private ImmutableList<Artifact> createTempsActions(Artifact source, PathFragment outputName,
+ CppCompileActionBuilder builder, boolean usePic) {
+ if (!cppConfiguration.getSaveTemps()) {
+ return ImmutableList.of();
+ }
+
+ String path = source.getFilename();
+ boolean isCFile = CppFileTypes.C_SOURCE.matches(path);
+ boolean isCppFile = CppFileTypes.CPP_SOURCE.matches(path);
+
+ if (!isCFile && !isCppFile) {
+ return ImmutableList.of();
+ }
+
+ String iExt = isCFile ? ".i" : ".ii";
+ String picExt = usePic ? ".pic" : "";
+ CppCompileActionBuilder dBuilder = new CppCompileActionBuilder(builder);
+ CppCompileActionBuilder sdBuilder = new CppCompileActionBuilder(builder);
+
+ dBuilder
+ .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + iExt))
+ .setDotdFile(outputName, picExt + iExt + ".d", ruleContext);
+ semantics.finalizeCompileActionBuilder(ruleContext, dBuilder);
+ CppCompileAction dAction = dBuilder.build();
+ ruleContext.registerAction(dAction);
+
+ sdBuilder
+ .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + ".s"))
+ .setDotdFile(outputName, picExt + ".s.d", ruleContext);
+ semantics.finalizeCompileActionBuilder(ruleContext, sdBuilder);
+ CppCompileAction sdAction = sdBuilder.build();
+ ruleContext.registerAction(sdAction);
+ return ImmutableList.of(
+ dAction.getOutputFile(),
+ sdAction.getOutputFile());
+ }
+
+ /**
+ * Returns true iff code coverage is enabled for the given target.
+ */
+ private boolean isCodeCoverageEnabled() {
+ if (configuration.isCodeCoverageEnabled()) {
+ final RegexFilter filter = configuration.getInstrumentationFilter();
+ // If rule is matched by the instrumentation filter, enable instrumentation
+ if (filter.isIncluded(ruleContext.getLabel().toString())) {
+ return true;
+ }
+ // At this point the rule itself is not matched by the instrumentation filter. However, we
+ // might still want to instrument C++ rules if one of the targets listed in "deps" is
+ // instrumented and, therefore, can supply header files that we would want to collect code
+ // coverage for. For example, think about cc_test rule that tests functionality defined in a
+ // header file that is supplied by the cc_library.
+ //
+ // Note that we only check direct prerequisites and not the transitive closure. This is done
+ // for two reasons:
+ // a) It is a good practice to declare libraries which you directly rely on. Including headers
+ // from a library hidden deep inside the transitive closure makes build dependencies less
+ // readable and can lead to unexpected breakage.
+ // b) Traversing the transitive closure for each C++ compile action would require more complex
+ // implementation (with caching results of this method) to avoid O(N^2) slowdown.
+ if (ruleContext.getRule().isAttrDefined("deps", Type.LABEL_LIST)) {
+ for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
+ if (dep.getProvider(CppCompilationContext.class) != null
+ && filter.isIncluded(dep.getLabel().toString())) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java
new file mode 100644
index 0000000..bb27209
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Structure for C++ module maps. Stores the name of the module and a .cppmap artifact.
+ */
+@Immutable
+public class CppModuleMap {
+ private final Artifact artifact;
+ private final String name;
+
+ public CppModuleMap(Artifact artifact, String name) {
+ this.artifact = artifact;
+ this.name = name;
+ }
+
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name + "@" + artifact;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java
new file mode 100644
index 0000000..a350fc4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java
@@ -0,0 +1,185 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Creates C++ module map artifact genfiles. These are then passed to Clang to
+ * do dependency checking.
+ */
+public class CppModuleMapAction extends AbstractFileWriteAction {
+
+ private static final String GUID = "4f407081-1951-40c1-befc-d6b4daff5de3";
+
+ // C++ module map of the current target
+ private final CppModuleMap cppModuleMap;
+
+ /**
+ * If set, the paths in the module map are relative to the current working directory instead
+ * of relative to the module map file's location.
+ */
+ private final boolean moduleMapHomeIsCwd;
+
+ // Headers and dependencies list
+ private final ImmutableList<Artifact> privateHeaders;
+ private final ImmutableList<Artifact> publicHeaders;
+ private final ImmutableList<CppModuleMap> dependencies;
+ private final ImmutableList<PathFragment> additionalExportedHeaders;
+ private final boolean compiledModule;
+
+ public CppModuleMapAction(ActionOwner owner, CppModuleMap cppModuleMap,
+ Iterable<Artifact> privateHeaders, Iterable<Artifact> publicHeaders,
+ Iterable<CppModuleMap> dependencies, Iterable<PathFragment> additionalExportedHeaders,
+ boolean compiledModule, boolean moduleMapHomeIsCwd) {
+ super(owner, ImmutableList.<Artifact>of(), cppModuleMap.getArtifact(),
+ /*makeExecutable=*/false);
+ this.cppModuleMap = cppModuleMap;
+ this.moduleMapHomeIsCwd = moduleMapHomeIsCwd;
+ this.privateHeaders = ImmutableList.copyOf(privateHeaders);
+ this.publicHeaders = ImmutableList.copyOf(publicHeaders);
+ this.dependencies = ImmutableList.copyOf(dependencies);
+ this.additionalExportedHeaders = ImmutableList.copyOf(additionalExportedHeaders);
+ this.compiledModule = compiledModule;
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) {
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ StringBuilder content = new StringBuilder();
+ PathFragment fragment = cppModuleMap.getArtifact().getExecPath();
+ int segmentsToExecPath = fragment.segmentCount() - 1;
+
+ // For details about the different header types, see:
+ // http://clang.llvm.org/docs/Modules.html#header-declaration
+ String leadingPeriods = moduleMapHomeIsCwd ? "" : Strings.repeat("../", segmentsToExecPath);
+ content.append("module \"").append(cppModuleMap.getName()).append("\" {\n");
+ content.append(" export *\n");
+ for (Artifact artifact : privateHeaders) {
+ appendHeader(content, "private", artifact.getExecPath(), leadingPeriods,
+ /*canCompile=*/true);
+ }
+ for (Artifact artifact : publicHeaders) {
+ appendHeader(content, "", artifact.getExecPath(), leadingPeriods, /*canCompile=*/true);
+ }
+ for (PathFragment additionalExportedHeader : additionalExportedHeaders) {
+ appendHeader(content, "", additionalExportedHeader, leadingPeriods, /*canCompile*/false);
+ }
+ for (CppModuleMap dep : dependencies) {
+ content.append(" use \"").append(dep.getName()).append("\"\n");
+ }
+ content.append("}");
+ for (CppModuleMap dep : dependencies) {
+ content.append("\nextern module \"")
+ .append(dep.getName())
+ .append("\" \"")
+ .append(leadingPeriods)
+ .append(dep.getArtifact().getExecPath())
+ .append("\"");
+ }
+ out.write(content.toString().getBytes(StandardCharsets.ISO_8859_1));
+ }
+ };
+ }
+
+ private void appendHeader(StringBuilder content, String visibilitySpecifier, PathFragment path,
+ String leadingPeriods, boolean canCompile) {
+ content.append(" ");
+ if (!visibilitySpecifier.isEmpty()) {
+ content.append(visibilitySpecifier).append(" ");
+ }
+ if (!canCompile || !shouldCompileHeader(path)) {
+ content.append("textual ");
+ }
+ content.append("header \"").append(leadingPeriods).append(path).append("\"\n");
+ }
+
+ private boolean shouldCompileHeader(PathFragment path) {
+ return compiledModule && !CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(path);
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "CppModuleMap";
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addInt(privateHeaders.size());
+ for (Artifact artifact : privateHeaders) {
+ f.addPath(artifact.getRootRelativePath());
+ }
+ f.addInt(publicHeaders.size());
+ for (Artifact artifact : publicHeaders) {
+ f.addPath(artifact.getRootRelativePath());
+ }
+ f.addInt(dependencies.size());
+ for (CppModuleMap dep : dependencies) {
+ f.addPath(dep.getArtifact().getExecPath());
+ }
+ f.addPath(cppModuleMap.getArtifact().getExecPath());
+ f.addString(cppModuleMap.getName());
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumptionLocal() {
+ return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.02);
+ }
+
+ @VisibleForTesting
+ public Collection<Artifact> getPublicHeaders() {
+ return publicHeaders;
+ }
+
+ @VisibleForTesting
+ public Collection<Artifact> getPrivateHeaders() {
+ return privateHeaders;
+ }
+
+ @VisibleForTesting
+ public ImmutableList<PathFragment> getAdditionalExportedHeaders() {
+ return additionalExportedHeaders;
+ }
+
+ @VisibleForTesting
+ public Collection<Artifact> getDependencyArtifacts() {
+ List<Artifact> artifacts = new ArrayList<>();
+ for (CppModuleMap map : dependencies) {
+ artifacts.add(map.getArtifact());
+ }
+ return artifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java
new file mode 100644
index 0000000..b0f2e82
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java
@@ -0,0 +1,646 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter;
+import com.google.devtools.build.lib.analysis.config.CompilationMode;
+import com.google.devtools.build.lib.analysis.config.FragmentOptions;
+import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.LibcTop;
+import com.google.devtools.build.lib.rules.cpp.CppConfiguration.StripMode;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Command-line options for C++.
+ */
+public class CppOptions extends FragmentOptions {
+ /**
+ * Label of a filegroup that contains all crosstool files for all configurations.
+ */
+ @VisibleForTesting
+ public static final String DEFAULT_CROSSTOOL_TARGET = "//tools/cpp:toolchain";
+
+
+ /**
+ * Converter for --cwarn flag
+ */
+ public static class GccWarnConverter implements Converter<String> {
+ @Override
+ public String convert(String input) throws OptionsParsingException {
+ if (input.startsWith("no-") || input.startsWith("-W")) {
+ throw new OptionsParsingException("Not a valid gcc warning to enable");
+ }
+ return input;
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "A gcc warning to enable";
+ }
+ }
+
+ /**
+ * Converts a comma-separated list of compilation mode settings to a properly typed List.
+ */
+ public static class FissionOptionConverter implements Converter<List<CompilationMode>> {
+ @Override
+ public List<CompilationMode> convert(String input) throws OptionsParsingException {
+ ImmutableSet.Builder<CompilationMode> modes = ImmutableSet.builder();
+ if (input.equals("yes")) { // Special case: enable all modes.
+ modes.add(CompilationMode.values());
+ } else if (!input.equals("no")) { // "no" is another special case that disables all modes.
+ CompilationMode.Converter modeConverter = new CompilationMode.Converter();
+ for (String mode : Splitter.on(',').split(input)) {
+ modes.add(modeConverter.convert(mode));
+ }
+ }
+ return modes.build().asList();
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a set of compilation modes";
+ }
+ }
+
+ /**
+ * The same as DynamicMode, but on command-line we also allow AUTO.
+ */
+ public enum DynamicModeFlag { OFF, DEFAULT, FULLY, AUTO }
+
+ /**
+ * Converter for DynamicModeFlag
+ */
+ public static class DynamicModeConverter extends EnumConverter<DynamicModeFlag> {
+ public DynamicModeConverter() {
+ super(DynamicModeFlag.class, "dynamic mode");
+ }
+ }
+
+ /**
+ * Converter for the --strip option.
+ */
+ public static class StripModeConverter extends EnumConverter<StripMode> {
+ public StripModeConverter() {
+ super(StripMode.class, "strip mode");
+ }
+ }
+
+ private static final String LIBC_RELATIVE_LABEL = ":everything";
+
+ /**
+ * Converts a String, which is an absolute path or label into a LibcTop
+ * object.
+ */
+ public static class LibcTopConverter implements Converter<LibcTop> {
+ @Override
+ public LibcTop convert(String input) throws OptionsParsingException {
+ if (!input.startsWith("//")) {
+ throw new OptionsParsingException("Not a label");
+ }
+ try {
+ Label label = Label.parseAbsolute(input).getRelative(LIBC_RELATIVE_LABEL);
+ return new LibcTop(label);
+ } catch (SyntaxException e) {
+ throw new OptionsParsingException(e.getMessage());
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a label";
+ }
+ }
+
+ /**
+ * Converter for the --hdrs_check option.
+ */
+ public static class HdrsCheckConverter extends EnumConverter<HeadersCheckingMode> {
+ public HdrsCheckConverter() {
+ super(HeadersCheckingMode.class, "Headers check mode");
+ }
+ }
+
+ /**
+ * Checks whether a string is a valid regex pattern and compiles it.
+ */
+ public static class NullableRegexPatternConverter implements Converter<Pattern> {
+
+ @Override
+ public Pattern convert(String input) throws OptionsParsingException {
+ if (input.isEmpty()) {
+ return null;
+ }
+ try {
+ return Pattern.compile(input);
+ } catch (PatternSyntaxException e) {
+ throw new OptionsParsingException("Not a valid regular expression: " + e.getMessage());
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a valid Java regular expression";
+ }
+ }
+
+ /**
+ * Converter for the --lipo option.
+ */
+ public static class LipoModeConverter extends EnumConverter<LipoMode> {
+ public LipoModeConverter() {
+ super(LipoMode.class, "LIPO mode");
+ }
+ }
+
+ @Option(name = "lipo input collector",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Internal flag, only used to create configurations with the LIPO-collector flag set.")
+ public boolean lipoCollector;
+
+ @Option(name = "crosstool_top",
+ defaultValue = CppOptions.DEFAULT_CROSSTOOL_TARGET,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "The label of the crosstool package to be used for compiling C++ code.")
+ public Label crosstoolTop;
+
+ @Option(name = "compiler",
+ defaultValue = "null",
+ category = "version",
+ help = "The C++ compiler to use for compiling the target.")
+ public String cppCompiler;
+
+ @Option(name = "glibc",
+ defaultValue = "null",
+ category = "version",
+ help = "The version of glibc the target should be linked against. "
+ + "By default, a suitable version is chosen based on --cpu.")
+ public String glibc;
+
+ @Option(name = "thin_archives",
+ defaultValue = "false",
+ category = "strategy", // but also adds edges to the action graph
+ help = "Pass the 'T' flag to ar if supported by the toolchain. " +
+ "All supported toolchains support this setting.")
+ public boolean useThinArchives;
+
+ // O intrepid reaper of unused options: Be warned that the [no]start_end_lib
+ // option, however tempting to remove, has a use case. Look in our telemetry data.
+ @Option(name = "start_end_lib",
+ defaultValue = "true",
+ category = "strategy", // but also adds edges to the action graph
+ help = "Use the --start-lib/--end-lib ld options if supported by the toolchain.")
+ public boolean useStartEndLib;
+
+ @Option(name = "interface_shared_objects",
+ defaultValue = "true",
+ category = "strategy", // but also adds edges to the action graph
+ help = "Use interface shared objects if supported by the toolchain. " +
+ "All ELF toolchains currently support this setting.")
+ public boolean useInterfaceSharedObjects;
+
+ @Option(name = "cc_include_scanning",
+ defaultValue = "true",
+ category = "strategy",
+ help = "Whether to perform include scanning. Without it, your build will most likely "
+ + "fail.")
+ public boolean scanIncludes;
+
+ @Option(name = "extract_generated_inclusions",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Run grep-includes actions (used for include scanning) over " +
+ "generated headers and sources.")
+ public boolean extractInclusions;
+
+ @Option(name = "fission",
+ defaultValue = "no",
+ converter = FissionOptionConverter.class,
+ category = "semantics",
+ help = "Specifies which compilation modes use fission for C++ compilations and links. "
+ + " May be any combination of {'fastbuild', 'dbg', 'opt'} or the special values 'yes' "
+ + " to enable all modes and 'no' to disable all modes.")
+ public List<CompilationMode> fissionModes;
+
+ @Option(name = "dynamic_mode",
+ defaultValue = "default",
+ converter = DynamicModeConverter.class,
+ category = "semantics",
+ help = "Determines whether C++ binaries will be linked dynamically. 'default' means "
+ + "blaze will choose whether to link dynamically. 'fully' means all libraries "
+ + "will be linked dynamically. 'off' means that all libraries will be linked "
+ + "in mostly static mode.")
+ public DynamicModeFlag dynamicMode;
+
+ @Option(name = "force_pic",
+ defaultValue = "false",
+ category = "semantics",
+ help = "If enabled, all C++ compilations produce position-independent code (\"-fPIC\"),"
+ + " links prefer PIC pre-built libraries over non-PIC libraries, and links produce"
+ + " position-independent executables (\"-pie\").")
+ public boolean forcePic;
+
+ @Option(name = "force_ignore_dash_static",
+ defaultValue = "false",
+ category = "semantics",
+ help = "If set, '-static' options in the linkopts of cc_* rules will be ignored.")
+ public boolean forceIgnoreDashStatic;
+
+ @Option(name = "experimental_skip_static_outputs",
+ defaultValue = "false",
+ category = "semantics",
+ help = "This flag is experimental and may go away at any time. "
+ + "If true, linker output for mostly-static C++ executables is a tiny amount of "
+ + "dummy dependency information, and NOT a usable binary. Kludge, but can reduce "
+ + "network and disk I/O load (and thus, continuous build cycle times) by a lot. "
+ + "NOTE: use of this flag REQUIRES --distinct_host_configuration.")
+ public boolean skipStaticOutputs;
+
+ @Option(name = "hdrs_check",
+ allowMultiple = false,
+ defaultValue = "loose",
+ converter = HdrsCheckConverter.class,
+ category = "semantics",
+ help = "Headers check mode for rules that don't specify it explicitly using a "
+ + "hdrs_check attribute. Allowed values: 'loose' allows undeclared headers, 'warn' "
+ + "warns about undeclared headers, and 'strict' disallows them.")
+ public HeadersCheckingMode headersCheckingMode;
+
+ @Option(name = "copt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional options to pass to gcc.")
+ public List<String> coptList;
+
+ @Option(name = "cwarn",
+ converter = GccWarnConverter.class,
+ defaultValue = "",
+ category = "flags",
+ allowMultiple = true,
+ help = "Additional warnings to enable when compiling C or C++ source files.")
+ public List<String> cWarns;
+
+ @Option(name = "cxxopt",
+ defaultValue = "",
+ category = "flags",
+ allowMultiple = true,
+ help = "Additional option to pass to gcc when compiling C++ source files.")
+ public List<String> cxxoptList;
+
+ @Option(name = "conlyopt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional option to pass to gcc when compiling C source files.")
+ public List<String> conlyoptList;
+
+ @Option(name = "linkopt",
+ defaultValue = "",
+ category = "flags",
+ allowMultiple = true,
+ help = "Additional option to pass to gcc when linking.")
+ public List<String> linkoptList;
+
+ @Option(name = "stripopt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional options to pass to strip when generating a '<name>.stripped' binary.")
+ public List<String> stripoptList;
+
+ @Option(name = "custom_malloc",
+ defaultValue = "null",
+ category = "semantics",
+ help = "Specifies a custom malloc implementation. This setting overrides malloc " +
+ "attributes in build rules.",
+ converter = LabelConverter.class)
+ public Label customMalloc;
+
+ @Option(name = "cpp_module_maps",
+ defaultValue = "true",
+ category = "flags",
+ help = "If true then C++ targets create a module map based on BUILD files, and "
+ + "pass them to the compiler.")
+ public boolean cppModuleMaps;
+
+ @Option(name = "legacy_whole_archive",
+ defaultValue = "true",
+ category = "semantics",
+ help = "When on, use --whole-archive for cc_binary rules that have "
+ + "linkshared=1 and either linkstatic=1 or '-static' in linkopts. "
+ + "This is for backwards compatibility only. "
+ + "A better alternative is to use alwayslink=1 where required.")
+ public boolean legacyWholeArchive;
+
+ @Option(name = "strip",
+ defaultValue = "sometimes",
+ category = "flags",
+ help = "Specifies whether to strip binaries and shared libraries "
+ + " (using \"-Wl,--strip-debug\"). The default value of 'sometimes'"
+ + " means strip iff --compilation_mode=fastbuild.",
+ converter = StripModeConverter.class)
+ public StripMode stripBinaries;
+
+ @Option(name = "fdo_instrument",
+ defaultValue = "null",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ category = "flags",
+ implicitRequirements = {"--copt=-Wno-error"},
+ help = "Generate binaries with FDO instrumentation. Specify the relative " +
+ "directory name for the .gcda files at runtime.")
+ public PathFragment fdoInstrument;
+
+ @Option(name = "fdo_optimize",
+ defaultValue = "null",
+ category = "flags",
+ help = "Use FDO profile information to optimize compilation. Specify the name " +
+ "of the zip file containing the .gcda file tree or an afdo file containing " +
+ "an auto profile. This flag also accepts files specified as labels, for " +
+ "example //foo/bar:file.afdo. Such labels must refer to input files; you may " +
+ "need to add an exports_files directive to the corresponding package to make " +
+ "the file visible to Blaze.")
+ public String fdoOptimize;
+
+ @Option(name = "autofdo_lipo_data",
+ defaultValue = "false",
+ category = "flags",
+ help = "If true then the directory name for non-LIPO targets will have a " +
+ "'-lipodata' suffix in AutoFDO mode.")
+ public boolean autoFdoLipoData;
+
+ @Option(name = "lipo",
+ defaultValue = "off",
+ converter = LipoModeConverter.class,
+ category = "flags",
+ help = "Enable LIPO optimization (lightweight inter-procedural optimization, The allowed "
+ + "values for this option are 'off' and 'binary', which enables LIPO. This option only "
+ + "has an effect when FDO is also enabled. Currently LIPO is only supported when "
+ + "building a single cc_binary rule.")
+ public LipoMode lipoMode;
+
+ @Option(name = "lipo_context",
+ defaultValue = "null",
+ category = "flags",
+ converter = LabelConverter.class,
+ implicitRequirements = {"--linkopt=-Wl,--warn-unresolved-symbols"},
+ help = "Specifies the binary from which the LIPO profile information comes.")
+ public Label lipoContext;
+
+ @Option(name = "experimental_stl",
+ converter = LabelConverter.class,
+ defaultValue = "null",
+ category = "version",
+ help = "If set, use this label instead of the default STL implementation. "
+ + "This option is EXPERIMENTAL and may go away in a future release.")
+ public Label stl;
+
+ @Option(name = "save_temps",
+ defaultValue = "false",
+ category = "what",
+ help = "If set, temporary outputs from gcc will be saved. "
+ + "These include .s files (assembler code), .i files (preprocessed C) and "
+ + ".ii files (preprocessed C++).")
+ public boolean saveTemps;
+
+ @Option(name = "per_file_copt",
+ allowMultiple = true,
+ converter = PerLabelOptions.PerLabelOptionsConverter.class,
+ defaultValue = "",
+ category = "semantics",
+ help = "Additional options to selectively pass to gcc when compiling certain files. "
+ + "This option can be passed multiple times. "
+ + "Syntax: regex_filter@option_1,option_2,...,option_n. Where regex_filter stands "
+ + "for a list of include and exclude regular expression patterns (Also see "
+ + "--instrumentation_filter). option_1 to option_n stand for "
+ + "arbitrary command line options. If an option contains a comma it has to be "
+ + "quoted with a backslash. Options can contain @. Only the first @ is used to "
+ + "split the string. Example: "
+ + "--per_file_copt=//foo/.*\\.cc,-//foo/bar\\.cc@-O0 adds the -O0 "
+ + "command line option to the gcc command line of all cc files in //foo/ "
+ + "except bar.cc.")
+ public List<PerLabelOptions> perFileCopts;
+
+ @Option(name = "host_crosstool_top",
+ defaultValue = "null",
+ converter = LabelConverter.class,
+ category = "semantics",
+ help = "By default, the --crosstool_top, --glibc, and --compiler options are also used " +
+ "for the host configuration. If this flag is provided, Blaze uses the default glibc " +
+ "and compiler for the given crosstool_top.")
+ public Label hostCrosstoolTop;
+
+ @Option(name = "host_copt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional options to pass to gcc for host tools.")
+ public List<String> hostCoptList;
+
+ @Option(name = "define",
+ converter = Converters.AssignmentConverter.class,
+ defaultValue = "",
+ category = "semantics",
+ allowMultiple = true,
+ help = "Each --define option specifies an assignment for a build variable.")
+ public List<Map.Entry<String, String>> commandLineDefinedVariables;
+
+ @Option(name = "grte_top",
+ defaultValue = "null", // The default value is chosen by the toolchain.
+ category = "version",
+ converter = LibcTopConverter.class,
+ help = "A label to a checked-in libc library. The default value is selected by the crosstool "
+ + "toolchain, and you almost never need to override it.")
+ public LibcTop libcTop;
+
+ @Option(name = "host_grte_top",
+ defaultValue = "null", // The default value is chosen by the toolchain.
+ category = "version",
+ converter = LibcTopConverter.class,
+ help = "If specified, this setting overrides the libc top-level directory (--grte_top) "
+ + "for the host configuration.")
+ public LibcTop hostLibcTop;
+
+ @Option(name = "output_symbol_counts",
+ defaultValue = "false",
+ category = "flags",
+ help = "If enabled, every C++ binary linked with gold will store the number of used "
+ + "symbols per object file in a .sc file.")
+ public boolean symbolCounts;
+
+ @Option(name = "experimental_inmemory_dotd_files",
+ defaultValue = "false",
+ category = "experimental",
+ help = "If enabled, C++ .d files will be passed through in memory directly from the remote "
+ + "build nodes instead of being written to disk.")
+ public boolean inmemoryDotdFiles;
+
+ @Option(name = "use_isystem_for_includes",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Instruct C and C++ compilations to treat 'includes' paths as system header " +
+ "paths, by translating it into -isystem instead of -I.")
+ public boolean useIsystemForIncludes;
+
+ @Option(name = "experimental_omitfp",
+ defaultValue = "false",
+ category = "semantics",
+ help = "If true, use libunwind for stack unwinding, and compile with " +
+ "-fomit-frame-pointer and -fasynchronous-unwind-tables.")
+ public boolean experimentalOmitfp;
+
+ @Option(name = "share_native_deps",
+ defaultValue = "true",
+ category = "strategy",
+ help = "If true, native libraries that contain identical functionality "
+ + "will be shared among different targets")
+ public boolean shareNativeDeps;
+
+ @Override
+ public FragmentOptions getHost(boolean fallback) {
+ CppOptions host = (CppOptions) getDefault();
+
+ host.commandLineDefinedVariables = commandLineDefinedVariables;
+
+ // The crosstool options are partially copied from the target configuration.
+ if (!fallback) {
+ if (hostCrosstoolTop == null) {
+ host.cppCompiler = cppCompiler;
+ host.crosstoolTop = crosstoolTop;
+ host.glibc = glibc;
+ } else {
+ host.crosstoolTop = hostCrosstoolTop;
+ }
+ }
+
+ if (hostLibcTop != null) {
+ host.libcTop = hostLibcTop;
+ } else if (hostCrosstoolTop == null) {
+ // Track libc in the host configuration if no host crosstool is set.
+ host.libcTop = libcTop;
+ }
+
+ // -g0 is the default, but allowMultiple options cannot have default values so we just pass
+ // -g0 first and let the user options override it.
+ host.coptList = ImmutableList.<String>builder().add("-g0").addAll(hostCoptList).build();
+
+ host.useThinArchives = useThinArchives;
+ host.useStartEndLib = useStartEndLib;
+ host.extractInclusions = extractInclusions;
+ host.stripBinaries = StripMode.ALWAYS;
+ host.fdoOptimize = null;
+ host.lipoMode = LipoMode.OFF;
+ host.scanIncludes = scanIncludes;
+ host.inmemoryDotdFiles = inmemoryDotdFiles;
+ host.cppModuleMaps = cppModuleMaps;
+
+ return host;
+ }
+
+ @Override
+ public void addAllLabels(Multimap<String, Label> labelMap) {
+ labelMap.put("crosstool", crosstoolTop);
+ if (hostCrosstoolTop != null) {
+ labelMap.put("crosstool", hostCrosstoolTop);
+ }
+
+ if (libcTop != null) {
+ Label libcLabel = libcTop.getLabel();
+ if (libcLabel != null) {
+ labelMap.put("crosstool", libcLabel);
+ }
+ }
+ addOptionalLabel(labelMap, "fdo", fdoOptimize);
+
+ if (stl != null) {
+ labelMap.put("STL", stl);
+ }
+
+ if (customMalloc != null) {
+ labelMap.put("custom_malloc", customMalloc);
+ }
+
+ if (getLipoContextLabel() != null) {
+ labelMap.put("lipo", getLipoContextLabel());
+ }
+ }
+
+ @Override
+ public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) {
+ Set<Label> crosstoolLabels = new LinkedHashSet<>();
+ crosstoolLabels.add(crosstoolTop);
+ if (hostCrosstoolTop != null) {
+ crosstoolLabels.add(hostCrosstoolTop);
+ }
+
+ if (libcTop != null) {
+ Label libcLabel = libcTop.getLabel();
+ if (libcLabel != null) {
+ crosstoolLabels.add(libcLabel);
+ }
+ }
+
+ return ImmutableMap.of(
+ "CROSSTOOL", crosstoolLabels,
+ "COVERAGE", ImmutableSet.<Label>of());
+ }
+
+ public boolean isFdo() {
+ return fdoOptimize != null || fdoInstrument != null;
+ }
+
+ public boolean isLipoOptimization() {
+ return lipoMode == LipoMode.BINARY && fdoOptimize != null && lipoContext != null;
+ }
+
+ public boolean isLipoOptimizationOrInstrumentation() {
+ return lipoMode == LipoMode.BINARY &&
+ ((fdoOptimize != null && lipoContext != null) || fdoInstrument != null);
+ }
+
+ public Label getLipoContextLabel() {
+ return (lipoMode == LipoMode.BINARY && fdoOptimize != null)
+ ? lipoContext : null;
+ }
+
+ public LipoMode getLipoMode() {
+ return lipoMode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java
new file mode 100644
index 0000000..de7c95d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java
@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_LIBRARY;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_PIC_LIBRARY;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ARCHIVE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_HEADER;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_SOURCE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.C_SOURCE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.OBJECT_FILE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_ARCHIVE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_OBJECT_FILE;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.SHARED_LIBRARY;
+import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.VERSIONED_SHARED_LIBRARY;
+
+import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+/**
+ * Rule class definitions for C++ rules.
+ */
+public class CppRuleClasses {
+ // Artifacts of these types are discarded from the 'hdrs' attribute in cc rules
+ static final FileTypeSet DISALLOWED_HDRS_FILES = FileTypeSet.of(
+ ARCHIVE,
+ PIC_ARCHIVE,
+ ALWAYS_LINK_LIBRARY,
+ ALWAYS_LINK_PIC_LIBRARY,
+ SHARED_LIBRARY,
+ VERSIONED_SHARED_LIBRARY,
+ OBJECT_FILE,
+ PIC_OBJECT_FILE);
+
+ /**
+ * The set of instrumented source file types; keep this in sync with the list above. Note that
+ * extension-less header files cannot currently be declared, so we cannot collect coverage for
+ * those.
+ */
+ static final InstrumentationSpec INSTRUMENTATION_SPEC = new InstrumentationSpec(
+ FileTypeSet.of(CPP_SOURCE, C_SOURCE, CPP_HEADER, ASSEMBLER_WITH_C_PREPROCESSOR),
+ "srcs", "deps", "data", "hdrs", "implements", "implementation");
+
+ public static final LibraryLanguage LANGUAGE = new LibraryLanguage("C++");
+
+ /**
+ * Implicit outputs for cc_binary rules.
+ */
+ public static final SafeImplicitOutputsFunction CC_BINARY_STRIPPED =
+ fromTemplates("%{name}.stripped");
+
+
+ // Used for requesting dwp "debug packages".
+ public static final SafeImplicitOutputsFunction CC_BINARY_DEBUG_PACKAGE =
+ fromTemplates("%{name}.dwp");
+
+
+ /**
+ * Path of the build_interface_so script in the Blaze binary.
+ */
+ public static final String BUILD_INTERFACE_SO = "build_interface_so";
+
+ /**
+ * A string constant for the layering_check feature.
+ */
+ public static final String LAYERING_CHECK = "layering_check";
+
+ /**
+ * A string constant for the parse_headers feature.
+ */
+ public static final String PARSE_HEADERS = "parse_headers";
+
+ /**
+ * A string constant for the preprocess_headers feature.
+ */
+ public static final String PREPROCESS_HEADERS = "preprocess_headers";
+
+ /**
+ * A string constant for the header_modules feature.
+ */
+ public static final String HEADER_MODULES = "header_modules";
+
+ /**
+ * A string constant for the module_map_home_cwd feature.
+ */
+ public static final String MODULE_MAP_HOME_CWD = "module_map_home_cwd";
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java
new file mode 100644
index 0000000..f4aa38c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Runfiles provider for C++ targets.
+ *
+ * <p>Contains two {@link Runfiles} objects: one for the eventual statically linked binary and
+ * one for the one that uses shared libraries. Data dependencies are present in both.
+ */
+@Immutable
+public final class CppRunfilesProvider implements TransitiveInfoProvider {
+ private final Runfiles staticRunfiles;
+ private final Runfiles sharedRunfiles;
+
+ public CppRunfilesProvider(Runfiles staticRunfiles, Runfiles sharedRunfiles) {
+ this.staticRunfiles = staticRunfiles;
+ this.sharedRunfiles = sharedRunfiles;
+ }
+
+ public Runfiles getStaticRunfiles() {
+ return staticRunfiles;
+ }
+
+ public Runfiles getSharedRunfiles() {
+ return sharedRunfiles;
+ }
+
+ /**
+ * Returns a function that gets the static C++ runfiles from a {@link TransitiveInfoCollection}
+ * or the empty runfiles instance if it does not contain that provider.
+ */
+ public static final Function<TransitiveInfoCollection, Runfiles> STATIC_RUNFILES =
+ new Function<TransitiveInfoCollection, Runfiles>() {
+ @Override
+ public Runfiles apply(TransitiveInfoCollection input) {
+ CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class);
+ return provider == null
+ ? Runfiles.EMPTY
+ : provider.getStaticRunfiles();
+ }
+ };
+
+ /**
+ * Returns a function that gets the shared C++ runfiles from a {@link TransitiveInfoCollection}
+ * or the empty runfiles instance if it does not contain that provider.
+ */
+ public static final Function<TransitiveInfoCollection, Runfiles> SHARED_RUNFILES =
+ new Function<TransitiveInfoCollection, Runfiles>() {
+ @Override
+ public Runfiles apply(TransitiveInfoCollection input) {
+ CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class);
+ return provider == null
+ ? Runfiles.EMPTY
+ : provider.getSharedRunfiles();
+ }
+ };
+
+ /**
+ * Returns a function that gets the C++ runfiles from a {@link TransitiveInfoCollection} or
+ * the empty runfiles instance if it does not contain that provider.
+ */
+ public static final Function<TransitiveInfoCollection, Runfiles> runfilesFunction(
+ boolean linkingStatically) {
+ return linkingStatically ? STATIC_RUNFILES : SHARED_RUNFILES;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java
new file mode 100644
index 0000000..600b2fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Pluggable C++ compilation semantics.
+ */
+public interface CppSemantics {
+ /**
+ * Returns the "effective source path" of a source file.
+ *
+ * <p>It is used, among other things, for computing the output path.
+ */
+ PathFragment getEffectiveSourcePath(Artifact source);
+
+ /**
+ * Called before a C++ compile action is built.
+ *
+ * <p>Gives the semantics implementation the opportunity to change compile actions at the last
+ * minute.
+ */
+ void finalizeCompileActionBuilder(
+ RuleContext ruleContext, CppCompileActionBuilder actionBuilder);
+
+ /**
+ * Called before {@link CppCompilationContext}s are finalized.
+ *
+ * <p>Gives the semantics implementation the opportunity to change what the C++ rule propagates
+ * to dependent rules.
+ */
+ void setupCompilationContext(
+ RuleContext ruleContext, CppCompilationContext.Builder contextBuilder);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java
new file mode 100644
index 0000000..1111189
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain;
+
+import java.util.Objects;
+
+/**
+ * Contains parameters which uniquely describe a crosstool configuration
+ * and methods for comparing two crosstools against each other.
+ *
+ * <p>Two crosstools which contain equivalent values of these parameters are
+ * considered equal.
+ */
+public final class CrosstoolConfigurationIdentifier implements CrosstoolConfigurationOptions {
+
+ /** The CPU associated with this crosstool configuration. */
+ private final String cpu;
+
+ /** The compiler (e.g. gcc) associated with this crosstool configuration. */
+ private final String compiler;
+
+ /** The version of libc (e.g. glibc-2.11) associated with this crosstool configuration. */
+ private final String libc;
+
+ private CrosstoolConfigurationIdentifier(String cpu, String compiler, String libc) {
+ this.cpu = cpu;
+ this.compiler = compiler;
+ this.libc = libc;
+ }
+
+ /**
+ * Creates a new crosstool configuration from the given crosstool release and
+ * configuration options.
+ */
+ public static CrosstoolConfigurationIdentifier fromReleaseAndCrosstoolConfiguration(
+ CrosstoolConfig.CrosstoolRelease release, BuildOptions buildOptions) {
+ String cpu = buildOptions.get(BuildConfiguration.Options.class).getCpu();
+ if (cpu == null) {
+ cpu = release.getDefaultTargetCpu();
+ }
+ CppOptions cppOptions = buildOptions.get(CppOptions.class);
+ return new CrosstoolConfigurationIdentifier(cpu, cppOptions.cppCompiler, cppOptions.glibc);
+ }
+
+ public static CrosstoolConfigurationIdentifier fromToolchain(CToolchain toolchain) {
+ return new CrosstoolConfigurationIdentifier(
+ toolchain.getTargetCpu(), toolchain.getCompiler(), toolchain.getTargetLibc());
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof CrosstoolConfigurationIdentifier)) {
+ return false;
+ }
+ CrosstoolConfigurationIdentifier otherCrosstool = (CrosstoolConfigurationIdentifier) other;
+ return Objects.equals(cpu, otherCrosstool.cpu)
+ && Objects.equals(compiler, otherCrosstool.compiler)
+ && Objects.equals(libc, otherCrosstool.libc);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(cpu, compiler, libc);
+ }
+
+
+ /**
+ * Returns a series of command line flags which specify the configuration options.
+ * Any of these options may be null, in which case its flag is omitted.
+ *
+ * <p>The appended string will be along the lines of
+ * " --cpu='cpu' --compiler='compiler' --glibc='libc'".
+ */
+ public String describeFlags() {
+ StringBuilder message = new StringBuilder();
+ if (getCpu() != null) {
+ message.append(" --cpu='").append(getCpu()).append("'");
+ }
+ if (getCompiler() != null) {
+ message.append(" --compiler='").append(getCompiler()).append("'");
+ }
+ if (getLibc() != null) {
+ message.append(" --glibc='").append(getLibc()).append("'");
+ }
+ return message.toString();
+ }
+
+ /** Returns true if the specified toolchain is a candidate for use with this crosstool. */
+ public boolean isCandidateToolchain(CToolchain toolchain) {
+ return (toolchain.getTargetCpu().equals(getCpu())
+ && (getLibc() == null || toolchain.getTargetLibc().equals(getLibc()))
+ && (getCompiler() == null || toolchain.getCompiler().equals(
+ getCompiler())));
+ }
+
+ @Override
+ public String toString() {
+ return describeFlags();
+ }
+
+ @Override
+ public String getCpu() {
+ return cpu;
+ }
+
+ @Override
+ public String getCompiler() {
+ return compiler;
+ }
+
+ @Override
+ public String getLibc() {
+ return libc;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java
new file mode 100644
index 0000000..a113f5f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java
@@ -0,0 +1,327 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.analysis.RedirectChaser;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig;
+import com.google.protobuf.TextFormat;
+import com.google.protobuf.TextFormat.ParseException;
+import com.google.protobuf.UninitializedMessageException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.ExecutionException;
+
+import javax.annotation.Nullable;
+
+/**
+ * A loader that reads Crosstool configuration files and creates CToolchain
+ * instances from them.
+ */
+public class CrosstoolConfigurationLoader {
+ private static final String CROSSTOOL_CONFIGURATION_FILENAME = "CROSSTOOL";
+
+ /**
+ * Cache for storing result of toReleaseConfiguration function based on path and md5 sum of
+ * input file. We can use md5 because result of this function depends only on the file content.
+ */
+ private static final LoadingCache<Pair<Path, String>, CrosstoolConfig.CrosstoolRelease>
+ crosstoolReleaseCache = CacheBuilder.newBuilder().concurrencyLevel(4).maximumSize(100).build(
+ new CacheLoader<Pair<Path, String>, CrosstoolConfig.CrosstoolRelease>() {
+ @Override
+ public CrosstoolConfig.CrosstoolRelease load(Pair<Path, String> key) throws IOException {
+ char[] data = FileSystemUtils.readContentAsLatin1(key.first);
+ return toReleaseConfiguration(key.first.getPathString(), new String(data));
+ }
+ });
+
+ /**
+ * A class that holds the results of reading a CROSSTOOL file.
+ */
+ public static class CrosstoolFile {
+ private final Label crosstoolTop;
+ private Path crosstoolPath;
+ private CrosstoolConfig.CrosstoolRelease crosstool;
+ private String md5;
+
+ CrosstoolFile(Label crosstoolTop) {
+ this.crosstoolTop = crosstoolTop;
+ }
+
+ void setCrosstoolPath(Path crosstoolPath) {
+ this.crosstoolPath = crosstoolPath;
+ }
+
+ void setCrosstool(CrosstoolConfig.CrosstoolRelease crosstool) {
+ this.crosstool = crosstool;
+ }
+
+ void setMd5(String md5) {
+ this.md5 = md5;
+ }
+
+ /**
+ * Returns the crosstool top as resolved.
+ */
+ public Label getCrosstoolTop() {
+ return crosstoolTop;
+ }
+
+ /**
+ * Returns the absolute path from which the CROSSTOOL file was read.
+ */
+ public Path getCrosstoolPath() {
+ return crosstoolPath;
+ }
+
+ /**
+ * Returns the parsed contents of the CROSSTOOL file.
+ */
+ public CrosstoolConfig.CrosstoolRelease getProto() {
+ return crosstool;
+ }
+
+ /**
+ * Returns an MD5 hash of the CROSSTOOL file contents.
+ */
+ public String getMd5() {
+ return md5;
+ }
+ }
+
+ private CrosstoolConfigurationLoader() {
+ }
+
+ /**
+ * Reads the given <code>data</code> String, which must be in ascii format,
+ * into a protocol buffer. It uses the <code>name</code> parameter for error
+ * messages.
+ *
+ * @throws IOException if the parsing failed
+ */
+ @VisibleForTesting
+ static CrosstoolConfig.CrosstoolRelease toReleaseConfiguration(String name, String data)
+ throws IOException {
+ CrosstoolConfig.CrosstoolRelease.Builder builder =
+ CrosstoolConfig.CrosstoolRelease.newBuilder();
+ try {
+ TextFormat.merge(data, builder);
+ return builder.build();
+ } catch (ParseException e) {
+ throw new IOException("Could not read the crosstool configuration file '" + name + "', "
+ + "because of a parser error (" + e.getMessage() + ")");
+ } catch (UninitializedMessageException e) {
+ throw new IOException("Could not read the crosstool configuration file '" + name + "', "
+ + "because of an incomplete protocol buffer (" + e.getMessage() + ")");
+ }
+ }
+
+ private static boolean findCrosstoolConfiguration(
+ ConfigurationEnvironment env,
+ CrosstoolConfigurationLoader.CrosstoolFile file)
+ throws IOException, InvalidConfigurationException {
+ Label crosstoolTop = file.getCrosstoolTop();
+ Path path = null;
+ try {
+ Package containingPackage = env.getTarget(crosstoolTop.getLocalTargetLabel("BUILD"))
+ .getPackage();
+ if (containingPackage == null) {
+ return false;
+ }
+ path = env.getPath(containingPackage, CROSSTOOL_CONFIGURATION_FILENAME);
+ } catch (SyntaxException e) {
+ throw new InvalidConfigurationException(e);
+ } catch (NoSuchThingException e) {
+ // Handled later
+ }
+
+ // If we can't find a file, fall back to the provided alternative.
+ if (path == null || !path.exists()) {
+ throw new InvalidConfigurationException("The crosstool_top you specified was resolved to '" +
+ crosstoolTop + "', which does not contain a CROSSTOOL file. " +
+ "You can use a crosstool from the depot by specifying its label.");
+ } else {
+ // Do this before we read the data, so if it changes, we get a different MD5 the next time.
+ // Alternatively, we could calculate the MD5 of the contents, which we also read, but this
+ // is faster if the file comes from a file system with md5 support.
+ file.setCrosstoolPath(path);
+ String md5 = BaseEncoding.base16().lowerCase().encode(path.getMD5Digest());
+ CrosstoolConfig.CrosstoolRelease release;
+ try {
+ release = crosstoolReleaseCache.get(new Pair<Path, String>(path, md5));
+ file.setCrosstool(release);
+ file.setMd5(md5);
+ } catch (ExecutionException e) {
+ throw new InvalidConfigurationException(e);
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Reads a crosstool file.
+ */
+ @Nullable
+ public static CrosstoolConfigurationLoader.CrosstoolFile readCrosstool(
+ ConfigurationEnvironment env, Label crosstoolTop) throws InvalidConfigurationException {
+ crosstoolTop = RedirectChaser.followRedirects(env, crosstoolTop, "crosstool_top");
+ if (crosstoolTop == null) {
+ return null;
+ }
+ CrosstoolConfigurationLoader.CrosstoolFile file =
+ new CrosstoolConfigurationLoader.CrosstoolFile(crosstoolTop);
+ try {
+ boolean allDependenciesPresent = findCrosstoolConfiguration(env, file);
+ return allDependenciesPresent ? file : null;
+ } catch (IOException e) {
+ throw new InvalidConfigurationException(e);
+ }
+ }
+
+ /**
+ * Selects a crosstool toolchain corresponding to the given crosstool
+ * configuration options. If all of these options are null, it returns the default
+ * toolchain specified in the crosstool release. If only cpu is non-null, it
+ * returns the default toolchain for that cpu, as specified in the crosstool
+ * release. Otherwise, all values must be non-null, and this method
+ * returns the toolchain which matches all of the values.
+ *
+ * @throws NullPointerException if {@code release} is null
+ * @throws InvalidConfigurationException if no matching toolchain can be found, or
+ * if the input parameters do not obey the constraints described above
+ */
+ public static CrosstoolConfig.CToolchain selectToolchain(
+ CrosstoolConfig.CrosstoolRelease release, BuildOptions options,
+ Function<String, String> cpuTransformer)
+ throws InvalidConfigurationException {
+ CrosstoolConfigurationIdentifier config =
+ CrosstoolConfigurationIdentifier.fromReleaseAndCrosstoolConfiguration(release, options);
+ if ((config.getCompiler() != null) || (config.getLibc() != null)) {
+ ArrayList<CrosstoolConfig.CToolchain> candidateToolchains = new ArrayList<>();
+ for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) {
+ if (config.isCandidateToolchain(toolchain)) {
+ candidateToolchains.add(toolchain);
+ }
+ }
+ switch (candidateToolchains.size()) {
+ case 0: {
+ StringBuilder message = new StringBuilder();
+ message.append("No toolchain found for");
+ message.append(config.describeFlags());
+ message.append(". Valid toolchains are: ");
+ describeToolchainList(message, release.getToolchainList());
+ throw new InvalidConfigurationException(message.toString());
+ }
+ case 1:
+ return candidateToolchains.get(0);
+ default: {
+ StringBuilder message = new StringBuilder();
+ message.append("Multiple toolchains found for");
+ message.append(config.describeFlags());
+ message.append(": ");
+ describeToolchainList(message, candidateToolchains);
+ throw new InvalidConfigurationException(message.toString());
+ }
+ }
+ }
+ String selectedIdentifier = null;
+ // We use fake CPU values to allow cross-platform builds for other languages that use the
+ // C++ toolchain. Translate to the actual target architecture.
+ String desiredCpu = cpuTransformer.apply(config.getCpu());
+ for (CrosstoolConfig.DefaultCpuToolchain selector : release.getDefaultToolchainList()) {
+ if (selector.getCpu().equals(desiredCpu)) {
+ selectedIdentifier = selector.getToolchainIdentifier();
+ break;
+ }
+ }
+ checkToolChain(selectedIdentifier, desiredCpu);
+ for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) {
+ if (toolchain.getToolchainIdentifier().equals(selectedIdentifier)) {
+ return toolchain;
+ }
+ }
+ throw new InvalidConfigurationException("Inconsistent crosstool configuration; no toolchain "
+ + "corresponding to '" + selectedIdentifier + "' found for cpu '" + config.getCpu() + "'");
+ }
+
+ private static String describeToolchainFlags(CrosstoolConfig.CToolchain toolchain) {
+ return CrosstoolConfigurationIdentifier.fromToolchain(toolchain).describeFlags();
+ }
+
+ /**
+ * Appends a series of toolchain descriptions (as the blaze command line flags
+ * that would specify that toolchain) to 'message'.
+ */
+ private static void describeToolchainList(StringBuilder message,
+ Collection<CrosstoolConfig.CToolchain> toolchains) {
+ message.append("[");
+ for (CrosstoolConfig.CToolchain toolchain : toolchains) {
+ message.append(describeToolchainFlags(toolchain));
+ message.append(",");
+ }
+ message.append("]");
+ }
+
+ /**
+ * Makes sure that {@code selectedIdentifier} is a valid identifier for a toolchain,
+ * i.e. it starts with a letter or an underscore and continues with only dots, dashes,
+ * spaces, letters, digits or underscores (i.e. matches the following regular expression:
+ * "[a-zA-Z_][\.\- \w]*").
+ *
+ * @throws InvalidConfigurationException if selectedIdentifier is null or does not match the
+ * aforementioned regular expression.
+ */
+ private static void checkToolChain(String selectedIdentifier, String cpu)
+ throws InvalidConfigurationException {
+ if (selectedIdentifier == null) {
+ throw new InvalidConfigurationException("No toolchain found for cpu '" + cpu + "'");
+ }
+ // If you update this regex, please do so in the javadoc comment too, and also in the
+ // crosstool_config.proto file.
+ String rx = "[a-zA-Z_][\\.\\- \\w]*";
+ if (!selectedIdentifier.matches(rx)) {
+ throw new InvalidConfigurationException("Toolchain identifier for cpu '" + cpu + "' " +
+ "is illegal (does not match '" + rx + "')");
+ }
+ }
+
+ public static CrosstoolConfig.CrosstoolRelease getCrosstoolReleaseProto(
+ ConfigurationEnvironment env, BuildOptions options,
+ Label crosstoolTop, Function<String, String> cpuTransformer)
+ throws InvalidConfigurationException {
+ CrosstoolConfigurationLoader.CrosstoolFile file =
+ readCrosstool(env, crosstoolTop);
+ // Make sure that we have the requested toolchain in the result. Throw an exception if not.
+ selectToolchain(file.getProto(), options, cpuTransformer);
+ return file.getProto();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java
new file mode 100644
index 0000000..e311ab6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+/**
+ * A container object which provides crosstool configuration options to the build.
+ */
+public interface CrosstoolConfigurationOptions {
+ /** Returns the CPU associated with this crosstool configuration. */
+ public String getCpu();
+
+ /** Returns the compiler associated with this crosstool configuration. */
+ public String getCompiler();
+
+ /** Returns the libc version associated with this crosstool configuration. */
+ public String getLibc();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java
new file mode 100644
index 0000000..a446125
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java
@@ -0,0 +1,139 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactResolver;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Helper for actions that do include scanning. Currently only deals with source files, so is only
+ * appropriate for actions that do not discover generated files. Currently does not do .d file
+ * parsing, so the set of artifacts returned may be an overapproximation to the ones actually used
+ * during execution.
+ */
+public class DiscoveredSourceInputsHelper {
+
+ private DiscoveredSourceInputsHelper() {
+ }
+
+ /**
+ * Converts PathFragments into source Artifacts using an ArtifactResolver, ignoring any that are
+ * already in mandatoryInputs. Silently drops any PathFragments that cannot be resolved into
+ * Artifacts.
+ */
+ public static ImmutableList<Artifact> getDiscoveredInputsFromPaths(
+ Iterable<Artifact> mandatoryInputs, ArtifactResolver artifactResolver,
+ Collection<PathFragment> inputPaths) {
+ Set<PathFragment> knownPathFragments = new HashSet<>();
+ for (Artifact input : mandatoryInputs) {
+ knownPathFragments.add(input.getExecPath());
+ }
+ ImmutableList.Builder<Artifact> foundInputs = ImmutableList.builder();
+ for (PathFragment execPath : inputPaths) {
+ if (!knownPathFragments.add(execPath)) {
+ // Don't add any inputs that we already added, or original inputs, which we probably
+ // couldn't convert into artifacts anyway.
+ continue;
+ }
+ Artifact artifact = artifactResolver.resolveSourceArtifact(execPath);
+ // It is unlikely that this artifact is null, but tolerate the situation just in case.
+ // It is safe to ignore such paths because dependency checker would identify change in inputs
+ // (ignored path was used before) and will force action execution.
+ if (artifact != null) {
+ foundInputs.add(artifact);
+ }
+ }
+ return foundInputs.build();
+ }
+
+ /**
+ * Converts ActionInputs discovered as inputs during execution into source Artifacts, ignoring any
+ * that are already in mandatoryInputs or that live in builtInIncludeDirectories. If any
+ * ActionInputs cannot be resolved, an ActionExecutionException will be thrown.
+ *
+ * <p>This method duplicates the functionality of CppCompileAction#populateActionInputs, though it
+ * is simpler because it need not deal with derived artifacts and doesn't parse the .d file.
+ */
+ public static ImmutableList<Artifact> getDiscoveredInputsFromActionInputs(
+ Iterable<Artifact> mandatoryInputs,
+ ArtifactResolver artifactResolver,
+ Iterable<? extends ActionInput> discoveredInputs,
+ Iterable<PathFragment> builtInIncludeDirectories,
+ Action action,
+ Artifact primaryInput) throws ActionExecutionException {
+ List<PathFragment> systemIncludePrefixes = new ArrayList<>();
+ for (PathFragment includePath : builtInIncludeDirectories) {
+ if (includePath.isAbsolute()) {
+ systemIncludePrefixes.add(includePath);
+ }
+ }
+
+ // Avoid duplicates by keeping track of the ones we've seen so far, even though duplicates are
+ // unlikely, since they would have to be inputs to this (non-CppCompile) action and also
+ // #included by a C++ source file.
+ Set<Artifact> knownInputs = new HashSet<>();
+ Iterables.addAll(knownInputs, mandatoryInputs);
+ ImmutableList.Builder<Artifact> foundInputs = ImmutableList.builder();
+ // Check inclusions.
+ IncludeProblems problems = new IncludeProblems();
+ for (ActionInput input : discoveredInputs) {
+ if (input instanceof Artifact) {
+ Artifact artifact = (Artifact) input;
+ if (knownInputs.add(artifact)) {
+ foundInputs.add(artifact);
+ }
+ continue;
+ }
+ PathFragment execPath = new PathFragment(input.getExecPathString());
+ if (execPath.isAbsolute()) {
+ // Absolute includes from system paths are ignored.
+ if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) {
+ continue;
+ }
+ // Theoretically, the more sophisticated logic of CppCompileAction#populateActioInputs could
+ // be used here, to allow absolute includes that started with the execRoot. However, since
+ // we don't hit this codepath for local execution, that should be unnecessary. If and when
+ // we examine the results of local execution for scanned includes, that case may need to be
+ // dealt with.
+ problems.add(execPath.getPathString());
+ }
+ Artifact artifact = artifactResolver.resolveSourceArtifact(execPath);
+ if (artifact != null) {
+ if (knownInputs.add(artifact)) {
+ foundInputs.add(artifact);
+ }
+ } else {
+ // Abort if we see files that we can't resolve, likely caused by
+ // undeclared includes or illegal include constructs.
+ problems.add(execPath.getPathString());
+ }
+ }
+ problems.assertProblemFree(action, primaryInput);
+ return foundInputs.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java
new file mode 100644
index 0000000..142a67a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java
@@ -0,0 +1,120 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+
+/**
+ * Provides generic functionality for collecting the .dwo artifacts produced by any target
+ * that compiles C++ files. Supports both transitive and "only direct outputs" collection.
+ * Provides accessors for both PIC and non-PIC compilation modes.
+ */
+public class DwoArtifactsCollector {
+
+ /**
+ * The .dwo files collected by this target in non-PIC compilation mode (i.e. myobject.dwo).
+ */
+ private final NestedSet<Artifact> dwoArtifacts;
+
+ /**
+ * The .dwo files collected by this target in PIC compilation mode (i.e. myobject.pic.dwo).
+ */
+ private final NestedSet<Artifact> picDwoArtifacts;
+
+ /**
+ * Instantiates a "real" collector on meaningful data.
+ */
+ private DwoArtifactsCollector(CcCompilationOutputs compilationOutputs,
+ Iterable<TransitiveInfoCollection> deps) {
+
+ Preconditions.checkNotNull(compilationOutputs);
+ Preconditions.checkNotNull(deps);
+
+ // Note: .dwo collection works fine with any order, but tests may assume a
+ // specific order for readability / simplicity purposes. See
+ // DebugInfoPackagingTest for details.
+ NestedSetBuilder<Artifact> dwoBuilder = NestedSetBuilder.compileOrder();
+ NestedSetBuilder<Artifact> picDwoBuilder = NestedSetBuilder.compileOrder();
+
+ dwoBuilder.addAll(compilationOutputs.getDwoFiles());
+ picDwoBuilder.addAll(compilationOutputs.getPicDwoFiles());
+
+ for (TransitiveInfoCollection info : deps) {
+ CppDebugFileProvider provider = info.getProvider(CppDebugFileProvider.class);
+ if (provider != null) {
+ dwoBuilder.addTransitive(provider.getTransitiveDwoFiles());
+ picDwoBuilder.addTransitive(provider.getTransitivePicDwoFiles());
+ }
+ }
+
+ dwoArtifacts = dwoBuilder.build();
+ picDwoArtifacts = picDwoBuilder.build();
+ }
+
+ /**
+ * Instantiates an empty collector.
+ */
+ private DwoArtifactsCollector() {
+ dwoArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.COMPILE_ORDER);
+ picDwoArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.COMPILE_ORDER);
+ }
+
+ /**
+ * Returns a new instance that collects direct outputs and transitive dependencies.
+ *
+ * @param compilationOutputs the output compilation context for the owning target
+ * @param deps which of the target's transitive info collections should be visited
+ */
+ public static DwoArtifactsCollector transitiveCollector(CcCompilationOutputs compilationOutputs,
+ Iterable<TransitiveInfoCollection> deps) {
+ return new DwoArtifactsCollector(compilationOutputs, deps);
+ }
+
+ /**
+ * Returns a new instance that collects direct outputs only.
+ *
+ * @param compilationOutputs the output compilation context for the owning target
+ */
+ public static DwoArtifactsCollector directCollector(CcCompilationOutputs compilationOutputs) {
+ return new DwoArtifactsCollector(
+ compilationOutputs, ImmutableList.<TransitiveInfoCollection>of());
+ }
+
+ /**
+ * Returns a new instance that doesn't collect anything (its artifact sets are empty).
+ */
+ public static DwoArtifactsCollector emptyCollector() {
+ return new DwoArtifactsCollector();
+ }
+
+ /**
+ * Returns the .dwo files applicable to non-PIC compilation mode (i.e. myobject.dwo).
+ */
+ public NestedSet<Artifact> getDwoArtifacts() {
+ return dwoArtifacts;
+ }
+
+ /**
+ * Returns the .dwo files applicable to PIC compilation mode (i.e. myobject.pic.dwo).
+ */
+ public NestedSet<Artifact> getPicDwoArtifacts() {
+ return picDwoArtifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java
new file mode 100644
index 0000000..15d7010
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+
+import java.io.IOException;
+
+/**
+ * An action which greps for includes over a given .cc or .h file.
+ * This is a part of the work required for C++ include scanning.
+ *
+ * <p>Note that this may run grep-includes over-optimistically, where we previously
+ * had not. For example, consider a cc_library of generated headers. If another
+ * library depends on it, and only references one of the headers, the other
+ * grep-includes will have been wasted.
+ */
+final class ExtractInclusionAction extends AbstractAction {
+
+ private static final String GUID = "45b43e5a-4734-43bb-a05e-012313808142";
+
+ /**
+ * Constructs a new action.
+ */
+ public ExtractInclusionAction(ActionOwner owner, Artifact input, Artifact output) {
+ super(owner, ImmutableList.of(input), ImmutableList.of(output));
+ }
+
+ @Override
+ protected String computeKey() {
+ return GUID;
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return executor.getContext(CppCompileActionContext.class).strategyLocality();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "GrepIncludes";
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Extracting include lines from " + getPrimaryInput().prettyPrint();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ IncludeScanningContext context = executor.getContext(IncludeScanningContext.class);
+ try {
+ context.extractIncludes(actionExecutionContext, this, getPrimaryInput(),
+ getPrimaryOutput());
+ } catch (IOException e) {
+ throw new ActionExecutionException(e, this, false);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java
new file mode 100644
index 0000000..bd15455
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java
@@ -0,0 +1,212 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * Action that represents a fake C++ compilation step.
+ */
+@ThreadCompatible
+public class FakeCppCompileAction extends CppCompileAction {
+
+ private static final Logger LOG = Logger.getLogger(FakeCppCompileAction.class.getName());
+
+ public static final UUID GUID = UUID.fromString("b2d95c91-1434-47ae-a786-816017de8494");
+
+ private final PathFragment tempOutputFile;
+
+ FakeCppCompileAction(ActionOwner owner,
+ ImmutableList<String> features,
+ FeatureConfiguration featureConfiguration,
+ Artifact sourceFile,
+ Label sourceLabel,
+ NestedSet<Artifact> mandatoryInputs,
+ Artifact outputFile,
+ PathFragment tempOutputFile,
+ DotdFile dotdFile,
+ BuildConfiguration configuration,
+ CppConfiguration cppConfiguration,
+ CppCompilationContext context,
+ ImmutableList<String> copts,
+ ImmutableList<String> pluginOpts,
+ Predicate<String> nocopts,
+ ImmutableList<PathFragment> extraSystemIncludePrefixes,
+ boolean enableLayeringCheck,
+ @Nullable String fdoBuildStamp) {
+ super(owner, features, featureConfiguration, sourceFile, sourceLabel, mandatoryInputs,
+ outputFile, dotdFile, null, null, null,
+ configuration, cppConfiguration,
+ // We only allow inclusion of header files explicitly declared in
+ // "srcs", so we only use declaredIncludeSrcs, not declaredIncludeDirs.
+ // (Disallowing use of undeclared headers for cc_fake_binary is needed
+ // because the header files get included in the runfiles for the
+ // cc_fake_binary and for the negative compilation tests that depend on
+ // the cc_fake_binary, and the runfiles must be determined at analysis
+ // time, so they can't depend on the contents of the ".d" file.)
+ CppCompilationContext.disallowUndeclaredHeaders(context), null, copts, pluginOpts, nocopts,
+ extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, VOID_INCLUDE_RESOLVER,
+ ImmutableList.<IncludeScannable>of(),
+ GUID, /*compileHeaderModules=*/false);
+ this.tempOutputFile = Preconditions.checkNotNull(tempOutputFile);
+ }
+
+ @Override
+ @ThreadCompatible
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+
+ // First, do an normal compilation, to generate the ".d" file. The generated
+ // object file is built to a temporary location (tempOutputFile) and ignored
+ // afterwards.
+ LOG.info("Generating " + getDotdFile());
+ CppCompileActionContext context = executor.getContext(CppCompileActionContext.class);
+ CppCompileActionContext.Reply reply = null;
+ try {
+ // We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null.
+ reply = context.execWithReply(
+ this, actionExecutionContext.withFileOutErr(new FileOutErr()));
+ } catch (ExecException e) {
+ // We ignore failures here (other than capturing the Distributor reply).
+ // The compilation may well fail (that's the whole point of negative compilation tests).
+ // We execute it here just for the side effect of generating the ".d" file.
+ reply = context.getReplyFromException(e, this);
+ if (reply == null) {
+ // This can only happen if the ExecException does not come from remote execution.
+ throw e.toActionExecutionException("", executor.getVerboseFailures(), this);
+ }
+ }
+ IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class);
+ updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply);
+
+ // Even cc_fake_binary rules need to properly declare their dependencies...
+ // In fact, they need to declare their dependencies even more than cc_binary rules do.
+ // CcCommonConfiguredTarget passes in an empty set of declaredIncludeDirs,
+ // so this check below will only allow inclusion of header files that are explicitly
+ // listed in the "srcs" of the cc_fake_binary or in the "srcs" of a cc_library that it
+ // depends on.
+ try {
+ validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler());
+ } catch (ActionExecutionException e) {
+ // TODO(bazel-team): (2009) make this into an error, once most of the current warnings
+ // are fixed.
+ executor.getEventHandler().handle(Event.warn(
+ getOwner().getLocation(),
+ e.getMessage() + ";\n this warning may eventually become an error"));
+ }
+
+ // Generate a fake ".o" file containing the command line needed to generate
+ // the real object file.
+ LOG.info("Generating " + outputFile);
+
+ // A cc_fake_binary rule generates fake .o files and a fake target file,
+ // which merely contain instructions on building the real target. We need to
+ // be careful to use a new set of output file names in the instructions, as
+ // to not overwrite the fake output files when someone tries to follow the
+ // instructions. As the real compilation is executed by the test from its
+ // runfiles directory (where writing is forbidden), we patch the command
+ // line to write to $TEST_TMPDIR instead.
+ final String outputPrefix = "$TEST_TMPDIR/";
+ String argv = Joiner.on(' ').join(
+ Iterables.transform(getArgv(outputFile.getExecPath()), new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ String result = ShellEscaper.escapeString(input);
+ if (input.equals(outputFile.getExecPathString())
+ || input.equals(getDotdFile().getSafeExecPath().getPathString())) {
+ result = outputPrefix + result;
+ }
+ return result;
+ }
+ }));
+
+ // Write the command needed to build the real .o file to the fake .o file.
+ // Generate a command to ensure that the output directory exists; otherwise
+ // the compilation would fail.
+ try {
+ // Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for
+ // both.
+ Preconditions.checkState(outputFile.getExecPath().getParentDirectory().equals(
+ getDotdFile().getSafeExecPath().getParentDirectory()));
+ FileSystemUtils.writeContent(outputFile.getPath(), ISO_8859_1,
+ outputFile.getPath().getBaseName() + ": "
+ + "mkdir -p " + outputPrefix + "$(dirname " + outputFile.getExecPath() + ")"
+ + " && " + argv + "\n");
+ } catch (IOException e) {
+ throw new ActionExecutionException("failed to create fake compile command for rule '" +
+ getOwner().getLabel() + ": " + e.getMessage(),
+ this, false);
+ }
+ }
+
+ @Override
+ protected PathFragment getInternalOutputFile() {
+ return tempOutputFile;
+ }
+
+ @Override
+ public String getMnemonic() { return "FakeCppCompile"; }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "fake";
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumptionLocal() {
+ return new ResourceSet(/*memoryMb=*/1, /*cpuUsage=*/0.1, /*ioUsage=*/0.0);
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return executor.getContext(CppCompileActionContext.class).estimateResourceConsumption(this);
+ }
+
+ @Override
+ protected boolean needsIncludeScanning(Executor executor) {
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java
new file mode 100644
index 0000000..f50a1ae
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.vfs.Path;
+
+/**
+ * Stub action to be used as the generating action for FDO files that are extracted from the
+ * FDO zip.
+ *
+ * <p>This is needed because the extraction is currently not a bona fide action, therefore, Blaze
+ * would complain that these files have no generating action if we did not set it to an instance of
+ * this class.
+ */
+public class FdoStubAction extends AbstractAction {
+ public FdoStubAction(ActionOwner owner, Artifact output) {
+ // TODO(bazel-team): Make extracting the zip file a honest-to-God action so that we can do away
+ // with this ugliness.
+ super(owner, ImmutableList.<Artifact>of(), ImmutableList.of(output));
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext) {
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "FdoStubAction";
+ }
+
+ @Override
+ protected String computeKey() {
+ return "fdoStubAction";
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ public void prepare(Path execRoot) {
+ // The superclass would delete the output files here. We can't let that happen, since this
+ // action does not in fact create those files; it is only a placeholder and the actual files
+ // are created *before* the execution phase in FdoSupport.extractFdoZip()
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java
new file mode 100644
index 0000000..911d888
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java
@@ -0,0 +1,679 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.PackageRootResolver;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.skyframe.FileValue;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.ZipFileSystem;
+import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode;
+import com.google.devtools.build.skyframe.SkyFunction;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.zip.ZipException;
+
+/**
+ * Support class for FDO (feedback directed optimization) and LIPO (lightweight inter-procedural
+ * optimization).
+ *
+ * <p>There is a 1:1 relationship between {@link CppConfiguration} objects and {@code FdoSupport}
+ * objects. The FDO support of a build configuration can be retrieved using {@link
+ * CppConfiguration#getFdoSupport()}.
+ *
+ * <p>With respect to thread-safety, the {@link #prepareToBuild} method is not thread-safe, and must
+ * not be called concurrently with other methods on this class.
+ *
+ * <p>Here follows a quick run-down of how FDO/LIPO builds work (for non-FDO/LIPO builds, none
+ * of this applies):
+ *
+ * <p>{@link CppConfiguration#prepareHook} is called before the analysis phase, which calls
+ * {@link #prepareToBuild}, which extracts the FDO .zip (in case we work with an explicitly
+ * generated FDO profile file) or analyzes the .afdo.imports file next to the .afdo file (if
+ * AutoFDO is in effect).
+ *
+ * <p>.afdo.imports files contain one import a line. A line is two paths separated by a colon,
+ * with functions in the second path being referenced by functions in the first path. These are
+ * then put into the imports map. If we do AutoFDO, we don't handle individual .gcda files, so
+ * gcdaFiles will be empty.
+ *
+ * <p>Regular .fdo zip files contain .gcda files (which are added to gcdaFiles) and
+ * .gcda.imports files. There is one .gcda.imports file for every source file and it contains one
+ * path in every line, which can either be a path to a source file that contains a function
+ * referenced by the original source file or the .gcda file for such a referenced file. They
+ * both are added to the imports map.
+ *
+ * <p>If we do LIPO, we create an extra configuration that is called the "LIPO context collector",
+ * whose job it is to collect information that every configured target compiled with LIPO needs.
+ * The top-level target of this configuration is the LIPO context (always a cc_binary) and is an
+ * implicit dependency of every cc_* rule through their :lipo_context_collector attribute. The
+ * collected information is encapsulated in {@link LipoContextProvider}.
+ *
+ * <p>For each C++ compile action in the target configuration, {@link #configureCompilation} is
+ * called, which adds command line options and input files required for the build. There are
+ * three cases:
+ *
+ * <ul>
+ * <li>If we do AutoFDO, the .afdo file and the source files containing the functions imported
+ * by the original source file (as determined from the inputs map) are added.
+ * <li>If we do FDO, the .gcda file corresponding to the source file is added.
+ * <li>If we do LIPO, in addition to the .gcda file corresponding to the source file
+ * (like for FDO) the source files that contain the functions referenced by the source file and
+ * their .gcda files are added, too.
+ * </ul>
+ *
+ * <p>If we do LIPO, the actual C++ compilation context for LIPO compilation actions is pieced
+ * together from the CppCompileContext in LipoContextProvider and that of the rule being compiled.
+ * (see {@link CppCompilationContext#mergeForLipo}) This is so that the include files for the
+ * extra LIPO sources are found and is, strictly speaking, incorrect, since it also changes the
+ * declared include directories of the main source file, which in theory can result in the
+ * compilation passing even though it should fail with undeclared inclusion errors.
+ *
+ * <p>During the actual execution of the C++ compile action, the extra sources also need to be
+ * include scanned, which is the reason why they are {@link IncludeScannable} objects and not
+ * simple artifacts. We currently create these {@link IncludeScannable} objects by creating actual
+ * C++ compile actions in the LIPO context collector configuration which are then never executed.
+ * In fact, these C++ compile actions are never even registered with Skyframe. For this we
+ * propagate a bit from {@code BuildConfiguration.isActionsEnabled} to
+ * {@code CachingAnalysisEnvironment.allowRegisteringActions}, which causes actions to be silently
+ * discarded after configured targets are created.
+ */
+public class FdoSupport implements Serializable {
+
+ /**
+ * Path within profile data .zip files that is considered the root of the
+ * profile information directory tree.
+ */
+ private static final PathFragment ZIP_ROOT = new PathFragment("/");
+
+ /**
+ * Returns true if the give fdoFile represents an AutoFdo profile.
+ */
+ public static final boolean isAutoFdo(String fdoFile) {
+ return CppFileTypes.GCC_AUTO_PROFILE.matches(fdoFile);
+ }
+
+ /**
+ * Coverage information output directory passed to {@code --fdo_instrument},
+ * or {@code null} if FDO instrumentation is disabled.
+ */
+ private final PathFragment fdoInstrument;
+
+ /**
+ * Path of the profile file passed to {@code --fdo_optimize}, or
+ * {@code null} if FDO optimization is disabled. The profile file
+ * can be a coverage ZIP or an AutoFDO feedback file.
+ */
+ private final Path fdoProfile;
+
+ /**
+ * Temporary directory to which the coverage ZIP file is extracted to
+ * (relative to the exec root), or {@code null} if FDO optimization is
+ * disabled. This is used to create artifacts for the extracted files.
+ *
+ * <p>Note that this root is intentionally not registered with the artifact
+ * factory.
+ */
+ private final Root fdoRoot;
+
+ /**
+ * The relative path of the FDO root to the exec root.
+ */
+ private final PathFragment fdoRootExecPath;
+
+ /**
+ * Path of FDO files under the FDO root.
+ */
+ private final PathFragment fdoPath;
+
+ /**
+ * LIPO mode passed to {@code --lipo}. This is only used if
+ * {@code fdoProfile != null}.
+ */
+ private final LipoMode lipoMode;
+
+ /**
+ * Flag indicating whether to use AutoFDO (as opposed to
+ * instrumentation-based FDO).
+ */
+ private final boolean useAutoFdo;
+
+ /**
+ * The {@code .gcda} files that have been extracted from the ZIP file,
+ * relative to the root of the ZIP file.
+ *
+ * <p>Set only in {@link #prepareToBuild}.
+ */
+ private ImmutableSet<PathFragment> gcdaFiles = ImmutableSet.of();
+
+ /**
+ * Multimap from .gcda file base names to auxiliary input files.
+ *
+ * <p>The keys of the multimap are the exec root relative paths of .gcda files
+ * with the extension removed. The values are the lines from the accompanying
+ * .gcda.imports file.
+ *
+ * <p>The contents of the multimap are copied verbatim from the .gcda.imports
+ * files and not yet checked for validity.
+ *
+ * <p>Set only in {@link #prepareToBuild}.
+ */
+ private ImmutableMultimap<PathFragment, Artifact> imports;
+
+ /**
+ * Creates an FDO support object.
+ *
+ * @param fdoInstrument value of the --fdo_instrument option
+ * @param fdoProfile path to the profile file passed to --fdo_optimize option
+ * @param lipoMode value of the --lipo_mode option
+ */
+ public FdoSupport(PathFragment fdoInstrument, Path fdoProfile, LipoMode lipoMode, Path execRoot) {
+ this.fdoInstrument = fdoInstrument;
+ this.fdoProfile = fdoProfile;
+ this.fdoRoot = (fdoProfile == null)
+ ? null
+ : Root.asDerivedRoot(execRoot, execRoot.getRelative("blaze-fdo"));
+ this.fdoRootExecPath = fdoProfile == null
+ ? null
+ : fdoRoot.getExecPath().getRelative(new PathFragment("_fdo").getChild(
+ FileSystemUtils.removeExtension(fdoProfile.getBaseName())));
+ this.fdoPath = fdoProfile == null
+ ? null
+ : new PathFragment("_fdo").getChild(
+ FileSystemUtils.removeExtension(fdoProfile.getBaseName()));
+ this.lipoMode = lipoMode;
+ this.useAutoFdo = fdoProfile != null && isAutoFdo(fdoProfile.getBaseName());
+ }
+
+ public Root getFdoRoot() {
+ return fdoRoot;
+ }
+
+ public void declareSkyframeDependencies(SkyFunction.Environment env, Path execRoot) {
+ if (fdoProfile != null) {
+ if (isLipoEnabled()) {
+ // Incrementality is not supported for LIPO builds, see FdoSupport#scannables.
+ // Ensure that the Skyframe value containing the configuration will not be reused to avoid
+ // incrementality issues.
+ PrecomputedValue.dependOnBuildId(env);
+ return;
+ }
+
+ // IMPORTANT: Keep the following in sync with #prepareToBuild.
+ Path path;
+ if (useAutoFdo) {
+ path = fdoProfile.getParentDirectory().getRelative(
+ fdoProfile.getBaseName() + ".imports");
+ } else {
+ path = fdoProfile;
+ }
+ env.getValue(FileValue.key(RootedPath.toRootedPathMaybeUnderRoot(path,
+ ImmutableList.of(execRoot))));
+ }
+ }
+
+ /**
+ * Prepares the FDO support for building.
+ *
+ * <p>When an {@code --fdo_optimize} compile is requested, unpacks the given
+ * FDO gcda zip file into a clean working directory under execRoot.
+ *
+ * @throws FdoException if the FDO ZIP contains a file of unknown type
+ */
+ @ThreadHostile // must be called before starting the build
+ public void prepareToBuild(Path execRoot, PathFragment genfilesPath,
+ ArtifactFactory artifactDeserializer, PackageRootResolver resolver)
+ throws IOException, FdoException {
+ // The execRoot != null case is only there for testing. We cannot provide a real ZIP file in
+ // tests because ZipFileSystem does not work with a ZIP on an in-memory file system.
+ // IMPORTANT: Keep in sync with #declareSkyframeDependencies to avoid incrementality issues.
+ if (fdoProfile != null && execRoot != null) {
+ Path fdoDirPath = execRoot.getRelative(fdoRootExecPath);
+
+ FileSystemUtils.deleteTreesBelow(fdoDirPath);
+ FileSystemUtils.createDirectoryAndParents(fdoDirPath);
+
+ if (useAutoFdo) {
+ Path fdoImports = fdoProfile.getParentDirectory().getRelative(
+ fdoProfile.getBaseName() + ".imports");
+ if (isLipoEnabled()) {
+ imports = readAutoFdoImports(artifactDeserializer, fdoImports, genfilesPath, resolver);
+ }
+ FileSystemUtils.ensureSymbolicLink(
+ execRoot.getRelative(getAutoProfilePath()), fdoProfile);
+ } else {
+ Path zipFilePath = new ZipFileSystem(fdoProfile).getRootDirectory();
+ if (!zipFilePath.getRelative("blaze-out").isDirectory()) {
+ throw new ZipException("FDO zip files must be zipped directly above 'blaze-out' " +
+ "for the compiler to find the profile");
+ }
+ ImmutableSet.Builder<PathFragment> gcdaFilesBuilder = ImmutableSet.builder();
+ ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder =
+ ImmutableMultimap.builder();
+ extractFdoZip(artifactDeserializer, zipFilePath, fdoDirPath,
+ gcdaFilesBuilder, importsBuilder, resolver);
+ gcdaFiles = gcdaFilesBuilder.build();
+ imports = importsBuilder.build();
+ }
+ }
+ }
+
+ /**
+ * Recursively extracts a directory from the GCDA ZIP file into a target
+ * directory.
+ *
+ * <p>Imports files are not written to disk. Their content is directly added
+ * to an internal data structure.
+ *
+ * <p>The files are written at $EXECROOT/blaze-fdo/_fdo/(base name of profile zip), and the
+ * {@code _fdo} directory there is symlinked to from the exec root, so that the file are also
+ * available at $EXECROOT/_fdo/..., which is their exec path. We need to jump through these
+ * hoops because the FDO root 1. needs to be a source root, thus the exec path of its root is
+ * ".", 2. it must not be equal to the exec root so that the artifact factory does not get
+ * confused, 3. the files under it must be reachable by their exec path from the exec root.
+ *
+ * @throws IOException if any of the I/O operations failed
+ * @throws FdoException if the FDO ZIP contains a file of unknown type
+ */
+ private void extractFdoZip(ArtifactFactory artifactFactory, Path sourceDir,
+ Path targetDir, ImmutableSet.Builder<PathFragment> gcdaFilesBuilder,
+ ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder,
+ PackageRootResolver resolver) throws IOException, FdoException {
+ for (Path sourceFile : sourceDir.getDirectoryEntries()) {
+ Path targetFile = targetDir.getRelative(sourceFile.getBaseName());
+ if (sourceFile.isDirectory()) {
+ targetFile.createDirectory();
+ extractFdoZip(artifactFactory, sourceFile, targetFile, gcdaFilesBuilder, importsBuilder,
+ resolver);
+ } else {
+ if (CppFileTypes.COVERAGE_DATA.matches(sourceFile)) {
+ FileSystemUtils.copyFile(sourceFile, targetFile);
+ gcdaFilesBuilder.add(
+ sourceFile.relativeTo(sourceFile.getFileSystem().getRootDirectory()));
+ } else if (CppFileTypes.COVERAGE_DATA_IMPORTS.matches(sourceFile)) {
+ readCoverageImports(artifactFactory, sourceFile, importsBuilder, resolver);
+ } else {
+ throw new FdoException("FDO ZIP file contained a file of unknown type: "
+ + sourceFile);
+ }
+ }
+ }
+ }
+
+ /**
+ * Reads a .gcda.imports file and stores the imports information.
+ *
+ * @throws FdoException if an auxiliary LIPO input was not found
+ */
+ private void readCoverageImports(ArtifactFactory artifactFactory, Path importsFile,
+ ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder,
+ PackageRootResolver resolver) throws IOException, FdoException {
+ PathFragment key = importsFile.asFragment().relativeTo(ZIP_ROOT);
+ String baseName = key.getBaseName();
+ String ext = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA_IMPORTS.getExtensions());
+ key = key.replaceName(baseName.substring(0, baseName.length() - ext.length()));
+
+ for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) {
+ if (!line.isEmpty()) {
+ // We can't yet fully check the validity of a line. this is done later
+ // when we actually parse the contained paths.
+ PathFragment execPath = new PathFragment(line);
+ if (execPath.isAbsolute()) {
+ throw new FdoException("Absolute paths not allowed in gcda imports file " + importsFile
+ + ": " + execPath);
+ }
+ Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(line), resolver);
+ if (artifact == null) {
+ throw new FdoException("Auxiliary LIPO input not found: " + line);
+ }
+
+ importsBuilder.put(key, artifact);
+ }
+ }
+ }
+
+ /**
+ * Reads a .afdo.imports file and stores the imports information.
+ */
+ private ImmutableMultimap<PathFragment, Artifact> readAutoFdoImports(
+ ArtifactFactory artifactFactory, Path importsFile, PathFragment genFilePath,
+ PackageRootResolver resolver)
+ throws IOException, FdoException {
+ ImmutableMultimap.Builder<PathFragment, Artifact> importBuilder = ImmutableMultimap.builder();
+ for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) {
+ if (!line.isEmpty()) {
+ PathFragment key = new PathFragment(line.substring(0, line.indexOf(':')));
+ if (key.startsWith(genFilePath)) {
+ key = key.relativeTo(genFilePath);
+ }
+ if (key.isAbsolute()) {
+ throw new FdoException("Absolute paths not allowed in afdo imports file " + importsFile
+ + ": " + key);
+ }
+ key = FileSystemUtils.replaceSegments(key, "PROTECTED", "_protected", true);
+ for (String auxFile : line.substring(line.indexOf(':') + 1).split(" ")) {
+ if (auxFile.length() == 0) {
+ continue;
+ }
+ Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(auxFile),
+ resolver);
+ if (artifact == null) {
+ throw new FdoException("Auxiliary LIPO input not found: " + auxFile);
+ }
+ importBuilder.put(key, artifact);
+ }
+ }
+ }
+ return importBuilder.build();
+ }
+
+ /**
+ * Returns the imports from the .afdo.imports file of a source file.
+ *
+ * @param sourceName the source file
+ */
+ private Collection<Artifact> getAutoFdoImports(PathFragment sourceName) {
+ Preconditions.checkState(isLipoEnabled());
+ ImmutableCollection<Artifact> afdoImports = imports.get(sourceName);
+ Preconditions.checkState(afdoImports != null,
+ "AutoFDO import data missing for %s", sourceName);
+ return afdoImports;
+ }
+
+ /**
+ * Returns the imports from the .gcda.imports file of an object file.
+ *
+ * @param objDirectory the object directory of the object file's target
+ * @param objectName the object file
+ */
+ private Iterable<Artifact> getImports(PathFragment objDirectory, PathFragment objectName) {
+ Preconditions.checkState(isLipoEnabled());
+ Preconditions.checkState(imports != null,
+ "Tried to look up imports of uninitialized FDOSupport");
+ PathFragment key = objDirectory.getRelative(FileSystemUtils.removeExtension(objectName));
+ ImmutableCollection<Artifact> importsForObject = imports.get(key);
+ Preconditions.checkState(importsForObject != null, "Import data missing for %s", key);
+ return importsForObject;
+ }
+
+ /**
+ * Configures a compile action builder by adding command line options and
+ * auxiliary inputs according to the FDO configuration. This method does
+ * nothing If FDO is disabled.
+ */
+ @ThreadSafe
+ public void configureCompilation(CppCompileActionBuilder builder, RuleContext ruleContext,
+ AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, final Pattern nocopts,
+ boolean usePic, LipoContextProvider lipoInputProvider) {
+ // It is a bug if this method is called with useLipo if lipo is disabled. However, it is legal
+ // if is is called with !useLipo, even though lipo is enabled.
+ Preconditions.checkArgument(lipoInputProvider == null || isLipoEnabled());
+
+ // FDO is disabled -> do nothing.
+ if ((fdoInstrument == null) && (fdoRoot == null)) {
+ return;
+ }
+
+ List<String> fdoCopts = new ArrayList<>();
+ // Instrumentation phase
+ if (fdoInstrument != null) {
+ fdoCopts.add("-fprofile-generate=" + fdoInstrument.getPathString());
+ if (lipoMode != LipoMode.OFF) {
+ fdoCopts.add("-fripa");
+ }
+ }
+
+ // Optimization phase
+ if (fdoRoot != null) {
+ // Declare dependency on contents of zip file.
+ if (env.getSkyframeEnv().valuesMissing()) {
+ return;
+ }
+ Iterable<Artifact> auxiliaryInputs = getAuxiliaryInputs(
+ ruleContext, env, lipoLabel, sourceName, usePic, lipoInputProvider);
+ builder.addMandatoryInputs(auxiliaryInputs);
+ if (!Iterables.isEmpty(auxiliaryInputs)) {
+ if (useAutoFdo) {
+ fdoCopts.add("-fauto-profile=" + getAutoProfilePath().getPathString());
+ } else {
+ fdoCopts.add("-fprofile-use=" + fdoRootExecPath);
+ }
+ fdoCopts.add("-fprofile-correction");
+ if (lipoInputProvider != null) {
+ fdoCopts.add("-fripa");
+ }
+ }
+ }
+ Iterable<String> filteredCopts = fdoCopts;
+ if (nocopts != null) {
+ // Filter fdoCopts with nocopts if they exist.
+ filteredCopts = Iterables.filter(fdoCopts, new Predicate<String>() {
+ @Override
+ public boolean apply(String copt) {
+ return !nocopts.matcher(copt).matches();
+ }
+ });
+ }
+ builder.addCopts(0, filteredCopts);
+ }
+
+ /**
+ * Returns the auxiliary files that need to be added to the {@link CppCompileAction}.
+ */
+ private Iterable<Artifact> getAuxiliaryInputs(
+ RuleContext ruleContext, AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName,
+ boolean usePic, LipoContextProvider lipoContextProvider) {
+ // If --fdo_optimize was not specified, we don't have any additional inputs.
+ if (fdoProfile == null) {
+ return ImmutableSet.of();
+ } else if (useAutoFdo) {
+ ImmutableSet.Builder<Artifact> auxiliaryInputs = ImmutableSet.builder();
+
+ Artifact artifact = env.getDerivedArtifact(
+ fdoPath.getRelative(getAutoProfileRootRelativePath()), fdoRoot);
+ env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact));
+ auxiliaryInputs.add(artifact);
+ if (lipoContextProvider != null) {
+ auxiliaryInputs.addAll(getAutoFdoImports(sourceName));
+ }
+ return auxiliaryInputs.build();
+ } else {
+ ImmutableSet.Builder<Artifact> auxiliaryInputs = ImmutableSet.builder();
+
+ PathFragment objectName =
+ FileSystemUtils.replaceExtension(sourceName, usePic ? ".pic.o" : ".o");
+
+ auxiliaryInputs.addAll(
+ getGcdaArtifactsForObjectFileName(ruleContext, env, objectName, lipoLabel));
+
+ if (lipoContextProvider != null) {
+ for (Artifact importedFile : getImports(
+ getNonLipoObjDir(ruleContext, lipoLabel), objectName)) {
+ if (CppFileTypes.COVERAGE_DATA.matches(importedFile.getFilename())) {
+ Artifact gcdaArtifact = getGcdaArtifactsForGcdaPath(
+ ruleContext, env, importedFile.getExecPath());
+ if (gcdaArtifact == null) {
+ ruleContext.ruleError(String.format(
+ ".gcda file %s is not in the FDO zip (referenced by source file %s)",
+ importedFile.getExecPath(), sourceName));
+ } else {
+ auxiliaryInputs.add(gcdaArtifact);
+ }
+ } else {
+ auxiliaryInputs.add(importedFile);
+ }
+ }
+ }
+
+ return auxiliaryInputs.build();
+ }
+ }
+
+ /**
+ * Returns the .gcda file artifacts for a .gcda path from the .gcda.imports file or null if the
+ * referenced .gcda file is not in the FDO zip.
+ */
+ private Artifact getGcdaArtifactsForGcdaPath(RuleContext ruleContext,
+ AnalysisEnvironment env, PathFragment gcdaPath) {
+ if (!gcdaFiles.contains(gcdaPath)) {
+ return null;
+ }
+
+ Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaPath), fdoRoot);
+ env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact));
+ return artifact;
+ }
+
+ private PathFragment getNonLipoObjDir(RuleContext ruleContext, Label label) {
+ return ruleContext.getConfiguration().getBinFragment()
+ .getRelative(CppHelper.getObjDirectory(label));
+ }
+
+ /**
+ * Returns a list of .gcda file artifacts for an object file path.
+ *
+ * <p>The resulting set is either empty (because no .gcda file exists for the
+ * given object file) or contains one or two artifacts (the file itself and a
+ * symlink to it).
+ */
+ private ImmutableList<Artifact> getGcdaArtifactsForObjectFileName(RuleContext ruleContext,
+ AnalysisEnvironment env, PathFragment objectFileName, Label lipoLabel) {
+ // We put the .gcda files relative to the location of the .o file in the instrumentation run.
+ String gcdaExt = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA.getExtensions());
+ PathFragment baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt);
+ PathFragment gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName);
+
+ if (!gcdaFiles.contains(gcdaFile)) {
+ // If the object is a .pic.o file and .pic.gcda is not found, we should try finding .gcda too
+ String picoExt = Iterables.getOnlyElement(CppFileTypes.PIC_OBJECT_FILE.getExtensions());
+ baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt, picoExt);
+ if (baseName == null) {
+ // Object file is not .pic.o
+ return ImmutableList.of();
+ }
+ gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName);
+ if (!gcdaFiles.contains(gcdaFile)) {
+ // .gcda file not found
+ return ImmutableList.of();
+ }
+ }
+
+ final Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaFile), fdoRoot);
+ env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact));
+
+ return ImmutableList.of(artifact);
+ }
+
+
+ private PathFragment getAutoProfilePath() {
+ return fdoRootExecPath.getRelative(getAutoProfileRootRelativePath());
+ }
+
+ private PathFragment getAutoProfileRootRelativePath() {
+ return new PathFragment(fdoProfile.getBaseName());
+ }
+
+ /**
+ * Returns whether LIPO is enabled.
+ */
+ @ThreadSafe
+ public boolean isLipoEnabled() {
+ return fdoProfile != null && lipoMode != LipoMode.OFF;
+ }
+
+ /**
+ * Returns whether AutoFDO is enabled.
+ */
+ @ThreadSafe
+ public boolean isAutoFdoEnabled() {
+ return useAutoFdo;
+ }
+
+ /**
+ * Returns an immutable list of command line arguments to add to the linker
+ * command line. If FDO is disabled, and empty list is returned.
+ */
+ @ThreadSafe
+ public ImmutableList<String> getLinkOptions() {
+ return fdoInstrument != null
+ ? ImmutableList.of("-fprofile-generate=" + fdoInstrument.getPathString())
+ : ImmutableList.<String>of();
+ }
+
+ /**
+ * Returns the path of the FDO output tree (relative to the execution root)
+ * containing the .gcda profile files, or null if FDO is not enabled.
+ */
+ @VisibleForTesting
+ public PathFragment getFdoOptimizeDir() {
+ return fdoRootExecPath;
+ }
+
+ /**
+ * Returns the path of the FDO zip containing the .gcda profile files, or null
+ * if FDO is not enabled.
+ */
+ @VisibleForTesting
+ public Path getFdoOptimizeProfile() {
+ return fdoProfile;
+ }
+
+ /**
+ * Returns the path fragment of the instrumentation output dir for gcc when
+ * FDO is enabled, or null if FDO is not enabled.
+ */
+ @ThreadSafe
+ public PathFragment getFdoInstrument() {
+ return fdoInstrument;
+ }
+
+ @VisibleForTesting
+ public void setGcdaFilesForTesting(ImmutableSet<PathFragment> gcdaFiles) {
+ this.gcdaFiles = gcdaFiles;
+ }
+
+ /**
+ * An exception indicating an issue with FDO coverage files.
+ */
+ public static final class FdoException extends Exception {
+ FdoException(String message) {
+ super(message);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java
new file mode 100644
index 0000000..17e2e5c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.List;
+
+/**
+ * A provider for cc_public_library rules to be able to convey the information about the
+ * header target's module map references to the public library target.
+ */
+@Immutable
+public final class HeaderTargetModuleMapProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<CppModuleMap> cppModuleMaps;
+
+ public HeaderTargetModuleMapProvider(Iterable<CppModuleMap> cppModuleMaps) {
+ this.cppModuleMaps = ImmutableList.copyOf(cppModuleMaps);
+ }
+
+ /**
+ * Returns the module maps referenced by cc_public_library's headers target.
+ */
+ public List<CppModuleMap> getCppModuleMaps() {
+ return cppModuleMaps;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java
new file mode 100644
index 0000000..4f2a585
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A provider for cc_library rules to be able to convey the information about which
+ * cc_public_library rules they implement to dependent targets.
+ */
+@Immutable
+public final class ImplementedCcPublicLibrariesProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<Label> implementedCcPublicLibraries;
+
+ public ImplementedCcPublicLibrariesProvider(ImmutableList<Label> implementedCcPublicLibraries) {
+ this.implementedCcPublicLibraries = implementedCcPublicLibraries;
+ }
+
+ /**
+ * Returns the labels for the "$headers" target that are implemented by the target which
+ * implements this interface.
+ */
+ public ImmutableList<Label> getImplementedCcPublicLibraries() {
+ return implementedCcPublicLibraries;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java
new file mode 100644
index 0000000..0b60b45
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java
@@ -0,0 +1,711 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import com.google.common.io.CharStreams;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion.Kind;
+import com.google.devtools.build.lib.rules.cpp.RemoteIncludeExtractor.RemoteParseData;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import javax.annotation.Nullable;
+
+/**
+ * Scans a source file and extracts the literal inclusions it specifies. Does not store results --
+ * repeated requests to the same file will result in repeated scans. Clients should implement a
+ * caching layer in order to avoid unnecessary disk access when requesting an already scanned file.
+ */
+public class IncludeParser implements SkyValue {
+ private static final Logger LOG = Logger.getLogger(IncludeParser.class.getName());
+ private static final boolean LOG_FINE = LOG.isLoggable(Level.FINE);
+ private static final boolean LOG_FINER = LOG.isLoggable(Level.FINER);
+
+ /**
+ * Immutable object representation of the four columns making up a single Rule
+ * in a Hints set. See {@link Hints} for more details.
+ */
+ private static class Rule {
+ private enum Type { PATH, FILE, INCLUDE_QUOTE, INCLUDE_ANGLE; }
+ final Type type;
+ final Pattern pattern;
+ final String findRoot;
+ final Pattern findFilter;
+
+ private Rule(String type, String pattern, String findRoot, Pattern findFilter) {
+ this.type = Type.valueOf(type.trim().toUpperCase());
+ this.pattern = Pattern.compile("^" + pattern + "$");
+ this.findRoot = findRoot;
+ this.findFilter = findFilter;
+ }
+
+ /**
+ * @throws PatternSyntaxException, IllegalArgumentException if bad values
+ * are provided
+ */
+ public Rule(String type, String pattern, String findRoot, String findFilter) {
+ this(type, pattern, findRoot.replace("\\", "$"), Pattern.compile(findFilter));
+ Preconditions.checkArgument((this.type == Type.PATH) || (this.type == Type.FILE));
+ }
+
+ public Rule(String type, String pattern, String findRoot) {
+ this(type, pattern, findRoot, (Pattern) null);
+ Preconditions.checkArgument((this.type == Type.INCLUDE_QUOTE)
+ || (this.type == Type.INCLUDE_ANGLE));
+ }
+
+ @Override public String toString() {
+ return "" + type + " " + pattern + " " + findRoot + " " + findFilter;
+ }
+ }
+
+ /**
+ * This class is a representation of the INCLUDE_HINTS file maintained and
+ * delivered with the remote client. The hints file contains regexp-based rules
+ * to help this simple include scanner cope with computed includes, which
+ * would otherwise require a full preprocessor with symbol support. Instead of
+ * actually processing symbols to evaluate the computed includes, we instead
+ * apply rules to gather inclusions for matching paths.
+ * <p>
+ * The hints file is read, line by line, into a list of rules each of which
+ * encapsulates a line of four columns. Each non-blank, non-comment line has
+ * the format:
+ *
+ * <pre>
+ * "file"|"path" match-pattern find-root find-filter
+ * </pre>
+ *
+ * <p>
+ * The first column specifies whether the line is a rule based on matching
+ * source <em>files</em> (passed directly to gcc as inputs, or transitively
+ * #included by other inputs) or include <em>paths</em> (passed to gcc as
+ * -I, -iquote, or -isystem flags).
+ * <p>
+ * The second column is a regexp for files or paths. Whenever a compiler
+ * argument of the specified type matches that regexp, the rule is taken. (All
+ * matching rules for every path and file on a compiler command line are
+ * followed, and the results are combined.)
+ * <p>
+ * The third column is a point in the local filesystem from which to extract a
+ * recursive listing. (This follows symlinks) Backrefs may be used to refer to
+ * the regexp or its capturing groups. (This is mostly necessary because
+ * --package_path can cause input paths to carry arbitrary prefixes.)
+ * <p>
+ * The fourth column is a regexp applied to each file found by the recursive
+ * listing. All matching files are treated as dependencies.
+ */
+ public static class Hints implements SkyValue {
+
+ private static final Pattern WS_PAT = Pattern.compile("\\s+");
+
+ private final Path workingDir;
+ private final List<Rule> rules = new ArrayList<>();
+ private final ArtifactFactory artifactFactory;
+
+ private final LoadingCache<Artifact, Collection<Artifact>> fileLevelHintsCache =
+ CacheBuilder.newBuilder().build(
+ new CacheLoader<Artifact, Collection<Artifact>>() {
+ @Override
+ public Collection<Artifact> load(Artifact path) {
+ return getHintedInclusions(Rule.Type.FILE, path.getPath(), path.getRoot());
+ }
+ });
+
+ private final LoadingCache<Path, Collection<Artifact>> pathLevelHintsCache =
+ CacheBuilder.newBuilder().build(
+ new CacheLoader<Path, Collection<Artifact>>() {
+ @Override
+ public Collection<Artifact> load(Path path) {
+ return getHintedInclusions(Rule.Type.PATH, path, null);
+ }
+ });
+
+ /**
+ * Constructs a hint set for a given working/exec directory and INCLUDE_HINTS file to read.
+ *
+ * @param workingDir the working/exec directory that processed paths are relative to
+ * @param hintsFile the hints file to read
+ * @throws IOException if the hints file can't be read or parsed
+ */
+ public Hints(Path workingDir, Path hintsFile, ArtifactFactory artifactFactory)
+ throws IOException {
+ this.workingDir = workingDir;
+ this.artifactFactory = artifactFactory;
+ try (InputStream is = hintsFile.getInputStream()) {
+ for (String line : CharStreams.readLines(new InputStreamReader(is, "UTF-8"))) {
+ line = line.trim();
+ if (line.length() == 0 || line.startsWith("#")) {
+ continue;
+ }
+ String[] tokens = WS_PAT.split(line);
+ try {
+ if (tokens.length == 3) {
+ rules.add(new Rule(tokens[0], tokens[1], tokens[2]));
+ } else if (tokens.length == 4) {
+ rules.add(new Rule(tokens[0], tokens[1], tokens[2], tokens[3]));
+ } else {
+ throw new IOException("Malformed hint line: " + line);
+ }
+ } catch (PatternSyntaxException e) {
+ throw new IOException("Malformed hint regex on: " + line + "\n " + e.getMessage());
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Invalid type on: " + line + "\n " + e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the "file" type hinted inclusions for a given path, caching results by path.
+ */
+ public Collection<Artifact> getFileLevelHintedInclusions(Artifact path) {
+ return fileLevelHintsCache.getUnchecked(path);
+ }
+
+ public Collection<Artifact> getPathLevelHintedInclusions(Path path) {
+ return pathLevelHintsCache.getUnchecked(path);
+ }
+
+ /**
+ * Performs the work of matching a given file/path of a specified file/path type against the
+ * hints and returns the expanded paths.
+ */
+ private Collection<Artifact> getHintedInclusions(Rule.Type type, Path path,
+ @Nullable Root sourceRoot) {
+ String pathString = path.getPathString();
+ // Delay creation until we know we need one. Use a TreeSet to make sure that the results are
+ // sorted with a stable order and unique.
+ Set<Path> hints = null;
+ for (final Rule rule : rules) {
+ if (type != rule.type) {
+ continue;
+ }
+ Matcher m = rule.pattern.matcher(pathString);
+ if (!m.matches()) {
+ continue;
+ }
+ if (hints == null) { hints = Sets.newTreeSet(); }
+ Path root = workingDir.getRelative(m.replaceFirst(rule.findRoot));
+ if (LOG_FINE) {
+ LOG.fine("hint for " + rule.type + " " + pathString + " root: " + root);
+ }
+ try {
+ // The assumption is made here that all files specified by this hint are under the same
+ // package path as the original file -- this filesystem tree traversal is completely
+ // ignorant of package paths. This could be violated if there were a hint that resolved to
+ // foo/**/*.h, there was a package foo/bar, and the packages foo and foo/bar were in
+ // different package paths. In that case, this traversal would fail to pick up
+ // foo/bar/**/*.h. No examples of this currently exist in the INCLUDE_HINTS
+ // file.
+ FileSystemUtils.traverseTree(hints, root, new Predicate<Path>() {
+ @Override
+ public boolean apply(Path p) {
+ boolean take = p.isFile() && rule.findFilter.matcher(p.getPathString()).matches();
+ if (LOG_FINER && take) {
+ LOG.finer("hinted include: " + p);
+ }
+ return take;
+ }
+ });
+ } catch (IOException e) {
+ LOG.warning("Error in hint expansion: " + e);
+ }
+ }
+ if (hints != null && !hints.isEmpty()) {
+ // Transform paths into source artifacts (all hints must be to source artifacts).
+ List<Artifact> result = new ArrayList<>(hints.size());
+ for (Path hint : hints) {
+ if (hint.startsWith(workingDir)) {
+ // Paths that are under the execRoot can be resolved as source artifacts as usual. All
+ // include directories are specified relative to the execRoot, and so fall here.
+ result.add(Preconditions.checkNotNull(
+ artifactFactory.resolveSourceArtifact(hint.relativeTo(workingDir)), hint));
+ } else {
+ // The file passed in might not have been under the execRoot, for instance
+ // <workspace>/foo/foo.cc.
+ Preconditions.checkNotNull(sourceRoot, "%s %s", path, hint);
+ Path sourcePath = sourceRoot.getPath();
+ Preconditions.checkState(hint.startsWith(sourcePath),
+ "%s %s %s", hint, path, sourceRoot);
+ result.add(Preconditions.checkNotNull(
+ artifactFactory.getSourceArtifact(hint.relativeTo(sourcePath), sourceRoot)));
+ }
+ }
+ return result;
+ } else {
+ return ImmutableList.of();
+ }
+ }
+
+ private Collection<Inclusion> getHintedInclusions(Artifact path) {
+ String pathString = path.getPath().getPathString();
+ // Delay creation until we know we need one. Use a LinkedHashSet to make sure that the results
+ // are sorted with a stable order and unique.
+ Set<Inclusion> hints = null;
+ for (final Rule rule : rules) {
+ if ((rule.type != Rule.Type.INCLUDE_ANGLE) && (rule.type != Rule.Type.INCLUDE_QUOTE)) {
+ continue;
+ }
+ Matcher m = rule.pattern.matcher(pathString);
+ if (!m.matches()) {
+ continue;
+ }
+ if (hints == null) { hints = Sets.newLinkedHashSet(); }
+ Inclusion inclusion = new Inclusion(rule.findRoot, rule.type == Rule.Type.INCLUDE_QUOTE
+ ? Kind.QUOTE : Kind.ANGLE);
+ hints.add(inclusion);
+ if (LOG_FINE) {
+ LOG.fine("hint for " + rule.type + " " + pathString + " root: " + inclusion);
+ }
+ }
+ if (hints != null && !hints.isEmpty()) {
+ return ImmutableList.copyOf(hints);
+ } else {
+ return ImmutableList.of();
+ }
+ }
+ }
+
+ public Hints getHints() {
+ return hints;
+ }
+
+ /**
+ * An immutable inclusion tuple. This models an {@code #include} or {@code
+ * #include_next} line in a file without the context how this file got
+ * included.
+ */
+ public static class Inclusion {
+ /** The format of the #include in the source file -- quoted, angle bracket, etc. */
+ public enum Kind {
+ /** Quote includes: {@code #include "name"}. */
+ QUOTE,
+
+ /** Angle bracket includes: {@code #include <name>}. */
+ ANGLE,
+
+ /** Quote next includes: {@code #include_next "name"}. */
+ NEXT_QUOTE,
+
+ /** Angle next includes: {@code #include_next <name>}. */
+ NEXT_ANGLE,
+
+ /** Computed or other unhandlable includes: {@code #include HEADER}. */
+ OTHER;
+
+ /**
+ * Returns true if this is an {@code #include_next} inclusion,
+ */
+ public boolean isNext() {
+ return this == NEXT_ANGLE || this == NEXT_QUOTE;
+ }
+ }
+
+ /** The kind of inclusion. */
+ public final Kind kind;
+ /** The relative path of the inclusion. */
+ public final PathFragment pathFragment;
+
+ public Inclusion(String includeTarget, Kind kind) {
+ this.kind = kind;
+ this.pathFragment = new PathFragment(includeTarget);
+ }
+
+ public Inclusion(PathFragment pathFragment, Kind kind) {
+ this.kind = kind;
+ this.pathFragment = Preconditions.checkNotNull(pathFragment);
+ }
+
+ public String getPathString() {
+ return pathFragment.getPathString();
+ }
+
+ @Override
+ public String toString() {
+ return kind.toString() + ":" + pathFragment.getPathString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof Inclusion)) {
+ return false;
+ }
+ Inclusion that = (Inclusion) o;
+ return kind == that.kind && pathFragment.equals(that.pathFragment);
+ }
+
+ @Override
+ public int hashCode() {
+ return pathFragment.hashCode() * 37 + kind.hashCode();
+ }
+ }
+
+ /**
+ * The externally-scoped immutable hints helper that is shared by all scanners.
+ */
+ private final Hints hints;
+
+ /**
+ * A scanner that extracts includes from an individual files remotely, used when scanning files
+ * generated remotely.
+ */
+ private final Supplier<? extends RemoteIncludeExtractor> remoteExtractor;
+
+ /**
+ * Constructs a new FileParser.
+ * @param remoteExtractor a processor that extracts includes from an individual file remotely.
+ * @param hints regexps for converting computed includes into simple strings
+ */
+ public IncludeParser(@Nullable RemoteIncludeExtractor remoteExtractor, Hints hints) {
+ this.hints = hints;
+ this.remoteExtractor = Suppliers.ofInstance(remoteExtractor);
+ }
+
+ /**
+ * Constructs a new FileParser.
+ * @param remoteExtractorSupplier a supplier of a processor that extracts includes from an
+ * individual file remotely.
+ * @param hints regexps for converting computed includes into simple strings
+ */
+ public IncludeParser(Supplier<? extends RemoteIncludeExtractor> remoteExtractorSupplier,
+ Hints hints) {
+ this.hints = hints;
+ this.remoteExtractor = remoteExtractorSupplier;
+ }
+
+ /**
+ * Skips whitespace, \+NL pairs, and block-style / * * / comments. Assumes
+ * line comments are handled outside. Does not handle digraphs, trigraphs or
+ * decahexagraphs.
+ *
+ * @param chars characters to scan
+ * @param pos the starting position
+ * @return the resulting position after skipping whitespace and comments.
+ */
+ protected static int skipWhitespace(char[] chars, int pos, int end) {
+ while (pos < end) {
+ if (Character.isWhitespace(chars[pos])) {
+ pos++;
+ } else if (chars[pos] == '\\' && pos + 1 < end && chars[pos + 1] == '\n') {
+ pos++;
+ } else if (chars[pos] == '/' && pos + 1 < end && chars[pos + 1] == '*') {
+ pos += 2;
+ while (pos < end - 1) {
+ if (chars[pos++] == '*') {
+ if (chars[pos] == '/') {
+ pos++;
+ break; // proper comment end
+ }
+ }
+ }
+ } else { // not whitespace
+ return pos;
+ }
+ }
+ return pos; // pos == len, meaning we fell off the end.
+ }
+
+ /**
+ * Checks for and skips a given token.
+ *
+ * @param chars characters to scan
+ * @param pos the starting position
+ * @param expected the expected token
+ * @return the resulting position if found, otherwise -1
+ */
+ protected static int expect(char[] chars, int pos, int end, String expected) {
+ int si = 0;
+ int expectedLen = expected.length();
+ while (pos < end) {
+ if (si == expectedLen) {
+ return pos;
+ }
+ if (chars[pos++] != expected.charAt(si++)) {
+ return -1;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Finds the index of a given character token from a starting pos.
+ *
+ * @param chars characters to scan
+ * @param pos the starting position
+ * @param echar the character to find
+ * @return the resulting position of echar if found, otherwise -1
+ */
+ private static int indexOf(char[] chars, int pos, int end, char echar) {
+ while (pos < end) {
+ if (chars[pos] == echar) {
+ return pos;
+ }
+ pos++;
+ }
+ return -1;
+ }
+
+ private static final Pattern BS_NL_PAT = Pattern.compile("\\\\" + "\n");
+
+ // Keep this in sync with the auxiliary binary's scanning output format.
+ private static final ImmutableMap<Character, Kind> KIND_MAP = ImmutableMap.of(
+ '"', Kind.QUOTE,
+ '<', Kind.ANGLE,
+ 'q', Kind.NEXT_QUOTE,
+ 'a', Kind.NEXT_ANGLE);
+
+ /**
+ * Processes the output generated by an auxiliary include-scanning binary. Closes the stream upon
+ * completion.
+ *
+ * <p>If a source file has the following include statements:
+ * <pre>
+ * #include <string>
+ * #include "directory/header.h"
+ * </pre>
+ *
+ * <p>Then the output file has the following contents:
+ * <pre>
+ * "directory/header.h
+ * <string
+ * </pre>
+ * <p>Each line of the output is translated into an Inclusion object.
+ */
+ public static List<Inclusion> processIncludes(Object streamName, InputStream is)
+ throws IOException {
+ List<Inclusion> inclusions = new ArrayList<>();
+ InputStreamReader reader = new InputStreamReader(is, ISO_8859_1);
+ try {
+ for (String line : CharStreams.readLines(reader)) {
+ char qchar = line.charAt(0);
+ String name = line.substring(1);
+ Inclusion.Kind kind = KIND_MAP.get(qchar);
+ if (kind == null) {
+ throw new IOException("Illegal inclusion kind '" + qchar + "'");
+ }
+ inclusions.add(new Inclusion(name, kind));
+ }
+ } catch (IOException e) {
+ throw new IOException("Error reading include file " + streamName + ": " + e.getMessage());
+ } finally {
+ reader.close();
+ }
+ return inclusions;
+ }
+
+ @VisibleForTesting
+ Inclusion extractInclusion(String line) {
+ return extractInclusion(line.toCharArray(), 0, line.length());
+ }
+
+ /**
+ * Extracts a new, unresolved an Inclusion from a line of source.
+ *
+ * @param chars the char array containing the line chars to parse
+ * @param lineBegin the position of the first character in the line
+ * @param lineEnd the position of the character after the last
+ * @return the inclusion object if possible, null if none
+ */
+ private Inclusion extractInclusion(char[] chars, int lineBegin, int lineEnd) {
+ // expect WS#WS(include|include_next)WS("name"|<name>|junk)
+ int pos = expectIncludeKeyword(chars, lineBegin, lineEnd);
+ if (pos == -1 || pos == lineEnd) {
+ return null;
+ }
+ boolean isNext = false;
+ int npos = expect(chars, pos, lineEnd, "_next");
+ if (npos >= 0) {
+ isNext = true;
+ pos = npos;
+ }
+ if ((pos = skipWhitespace(chars, pos, lineEnd)) == lineEnd) {
+ return null;
+ }
+ if (chars[pos] == '"' || chars[pos] == '<') {
+ char qchar = chars[pos++];
+ int spos = pos;
+ pos = indexOf(chars, pos + 1, lineEnd, qchar == '<' ? '>' : '"');
+ if (pos < 0) {
+ return null;
+ }
+ if (chars[spos] == '/') {
+ return null; // disallow absolute paths
+ }
+ String name = new String(chars, spos, pos - spos);
+ if (name.contains("\n")) { // strip any \+NL pairs within name
+ name = BS_NL_PAT.matcher(name).replaceAll("");
+ }
+ if (isNext) {
+ return new Inclusion(name, qchar == '"' ? Kind.NEXT_QUOTE : Kind.NEXT_ANGLE);
+ } else {
+ return new Inclusion(name, qchar == '"' ? Kind.QUOTE : Kind.ANGLE);
+ }
+ } else {
+ return createOtherInclusion(new String(chars, pos, lineEnd - pos));
+ }
+ }
+
+ /**
+ * Extracts all inclusions from characters of a file.
+ *
+ * @param chars the file contents to parse & extract inclusions from
+ * @return a new set of inclusions, normalized to the cache
+ */
+ @VisibleForTesting
+ List<Inclusion> extractInclusions(char[] chars) {
+ List<Inclusion> inclusions = new ArrayList<>();
+ int lineBegin = 0; // the first char of each line
+ int end = chars.length; // the file end
+ while (lineBegin < end) {
+ int lineEnd = lineBegin; // the char after the last non-\n in each line
+ // skip to the next \n or after end of buffer, ignoring continuations
+ while (lineEnd < end) {
+ if (chars[lineEnd] == '\n') {
+ break;
+ } else if (chars[lineEnd] == '\\') {
+ lineEnd++;
+ if (chars[lineEnd] == '\n') {
+ lineEnd++;
+ }
+ } else {
+ lineEnd++;
+ }
+ }
+
+ // TODO(bazel-team) handle multiline block comments /* */ for the cases:
+ // /* blah blah blah
+ // lalala */ #include "foo.h"
+ // and:
+ // /* blah
+ // #include "foo.h"
+ // */
+
+ // extract the inclusion, and save only the kind we care about.
+ Inclusion inclusion = extractInclusion(chars, lineBegin, lineEnd);
+ if (inclusion != null) {
+ if (isValidInclusionKind(inclusion.kind)) {
+ inclusions.add(inclusion);
+ } else {
+ //System.err.println("Funky include " + inclusion + " in " + file);
+ }
+ }
+ lineBegin = lineEnd + 1; // next line starts after the previous line
+ }
+ return inclusions;
+ }
+
+ /**
+ * Extracts all inclusions from a given source file.
+ *
+ * @param file the file to parse & extract inclusions from
+ * @param greppedFile if non-null, this file has the already-grepped include lines of file.
+ * @param actionExecutionContext Services in the scope of the action, like the stream to which
+ * scanning messages are printed
+ * @return a new set of inclusions, normalized to the cache
+ */
+ public Collection<Inclusion> extractInclusions(Artifact file, @Nullable Path greppedFile,
+ ActionExecutionContext actionExecutionContext)
+ throws IOException, InterruptedException {
+ Collection<Inclusion> inclusions;
+ if (greppedFile != null) {
+ inclusions = processIncludes(greppedFile, greppedFile.getInputStream());
+ } else {
+ RemoteParseData remoteParseData = remoteExtractor.get() == null
+ ? null
+ : remoteExtractor.get().shouldParseRemotely(file.getPath());
+ if (remoteParseData != null && remoteParseData.shouldParseRemotely()) {
+ inclusions =
+ remoteExtractor.get().extractInclusions(file, actionExecutionContext,
+ remoteParseData);
+ } else {
+ inclusions = extractInclusions(FileSystemUtils.readContentAsLatin1(file.getPath()));
+ }
+ }
+ if (hints != null) {
+ inclusions.addAll(hints.getHintedInclusions(file));
+ }
+ return ImmutableList.copyOf(inclusions);
+ }
+
+ /**
+ * Parses include keyword in the provided char array and returns position
+ * immediately after include keyword or -1 if keyword was not found. Can be
+ * overridden by subclasses.
+ */
+ protected int expectIncludeKeyword(char[] chars, int position, int end) {
+ int pos = expect(chars, skipWhitespace(chars, position, end), end, "#");
+ if (pos > 0) {
+ int npos = skipWhitespace(chars, pos, end);
+ if ((pos = expect(chars, npos, end, "include")) > 0) {
+ return pos;
+ } else if ((pos = expect(chars, npos, end, "import")) > 0) {
+ if (expect(chars, pos, end, "_") == -1) { // Needed to avoid #import_next.
+ return pos;
+ }
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Returns true if we interested in the given inclusion kind. Can be
+ * overridden by the subclass.
+ */
+ protected boolean isValidInclusionKind(Kind kind) {
+ return kind != Kind.OTHER;
+ }
+
+ /**
+ * Returns inclusion object for non-standard inclusion cases or null if
+ * inclusion should be ignored.
+ */
+ protected Inclusion createOtherInclusion(String inclusionContent) {
+ return new Inclusion(inclusionContent, Kind.OTHER);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java
new file mode 100644
index 0000000..f6be877
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+
+/**
+ * Accumulator for problems encountered while reading or validating inclusion
+ * results.
+ */
+class IncludeProblems {
+
+ private StringBuilder message; // null when no problems
+
+ void add(String included) {
+ if (message == null) { message = new StringBuilder(); }
+ message.append("\n '" + included + "'");
+ }
+
+ boolean hasProblems() { return message != null; }
+
+ String getMessage(Action action, Artifact sourceFile) {
+ if (message != null) {
+ return "undeclared inclusion(s) in rule '" + action.getOwner().getLabel() + "':\n"
+ + "this rule is missing dependency declarations for the following files "
+ + "included by '" + sourceFile.prettyPrint() + "':"
+ + message;
+ }
+ return null;
+ }
+
+ void assertProblemFree(Action action, Artifact sourceFile) throws ActionExecutionException {
+ if (hasProblems()) {
+ throw new ActionExecutionException(getMessage(action, sourceFile), action, false);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java
new file mode 100644
index 0000000..9c70090
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * To be implemented by actions (such as C++ compilation steps) whose inputs
+ * can be scanned to discover other implicit inputs (such as C++ header files).
+ *
+ * <p>This is useful for remote execution strategies to be able to compute the
+ * complete set of files that must be distributed in order to execute such an action.
+ */
+public interface IncludeScannable {
+
+ /**
+ * Returns the built-in list of system include paths for the toolchain compiler. All paths in this
+ * list should be relative to the exec directory. They may be absolute if they are also installed
+ * on the remote build nodes or for local compilation.
+ */
+ List<PathFragment> getBuiltInIncludeDirectories();
+
+ /**
+ * Returns an immutable list of "-iquote" include paths that should be used by
+ * the IncludeScanner for this action. GCC searches these paths first, but
+ * only for {@code #include "foo"}, not for {@code #include <foo>}.
+ */
+ List<PathFragment> getQuoteIncludeDirs();
+
+ /**
+ * Returns an immutable list of "-I" include paths that should be used by the
+ * IncludeScanner for this action. GCC searches these paths ahead of the
+ * system include paths, but after "-iquote" include paths.
+ */
+ List<PathFragment> getIncludeDirs();
+
+ /**
+ * Returns an immutable list of "-isystem" include paths that should be used
+ * by the IncludeScanner for this action. GCC searches these paths ahead of
+ * the built-in system include paths, but after all other paths. "-isystem"
+ * paths are treated the same as normal system directories.
+ */
+ List<PathFragment> getSystemIncludeDirs();
+
+ /**
+ * Returns an immutable list of "-include" inclusions specified explicitly on
+ * the command line of this action. GCC will imagine that these files have
+ * been quote-included at the beginning of each source file.
+ */
+ List<String> getCmdlineIncludes();
+
+ /**
+ * Returns an immutable list of sources that the IncludeScanner should scan
+ * for this action.
+ */
+ Collection<Artifact> getIncludeScannerSources();
+
+ /**
+ * Returns additional scannables that need also be scanned when scanning this
+ * scannable. May be empty but not null. This is not evaluated recursively.
+ */
+ Iterable<IncludeScannable> getAuxiliaryScannables();
+
+ /**
+ * Returns a map of generated files:files grepped for headers which may be reached during include
+ * scanning. Generated files which are reached, but not in the key set, must be ignored.
+ *
+ * <p>If grepping of output files is not enabled via --extract_generated_inclusions, keys
+ * should just map to null.
+ */
+ Map<Artifact, Path> getLegalGeneratedScannerFileMap();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java
new file mode 100644
index 0000000..9c00efd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java
@@ -0,0 +1,177 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Scans source files to determine the bounding set of transitively referenced include files.
+ *
+ * <p>Note that include scanning is performance-critical code.
+ */
+public interface IncludeScanner {
+ /**
+ * Processes a source file and a list of includes extracted from command line
+ * flags. Adds all found files to the provided set {@code includes}. This
+ * method takes into account the path- and file-level hints that are part of
+ * this include scanner.
+ */
+ public void process(Artifact source, Map<Artifact, Path> legalOutputPaths,
+ List<String> cmdlineIncludes, Set<Artifact> includes,
+ ActionExecutionContext actionExecutionContext)
+ throws IOException, ExecException, InterruptedException;
+
+ /** Supplies IncludeScanners upon request. */
+ interface IncludeScannerSupplier {
+ /** Returns the possibly shared scanner to be used for a given pair of include paths. */
+ IncludeScanner scannerFor(List<Path> quoteIncludePaths, List<Path> includePaths);
+ }
+
+ /**
+ * Helper class that exists just to provide a static method that prepares the arguments with which
+ * to call an IncludeScanner.
+ */
+ class IncludeScanningPreparer {
+ private IncludeScanningPreparer() {}
+
+ /**
+ * Returns the files transitively included by the source files of the given IncludeScannable.
+ *
+ * @param action IncludeScannable whose sources' transitive includes will be returned.
+ * @param includeScannerSupplier supplies IncludeScanners to actually do the transitive
+ * scanning (and caching results) for a given source file.
+ * @param actionExecutionContext the context for {@code action}.
+ * @param profilerTaskName what the {@link Profiler} should record this call for.
+ */
+ public static Collection<Artifact> scanForIncludedInputs(IncludeScannable action,
+ IncludeScannerSupplier includeScannerSupplier,
+ ActionExecutionContext actionExecutionContext,
+ String profilerTaskName)
+ throws ExecException, InterruptedException, ActionExecutionException {
+
+ Set<Artifact> includes = Sets.newConcurrentHashSet();
+
+ Executor executor = actionExecutionContext.getExecutor();
+ Path execRoot = executor.getExecRoot();
+
+ final List<Path> absoluteBuiltInIncludeDirs = new ArrayList<>();
+
+ Profiler profiler = Profiler.instance();
+ try {
+ profiler.startTask(ProfilerTask.SCANNER, profilerTaskName);
+
+ // We need to scan the action itself, but also the auxiliary scannables
+ // (for LIPO). There is no need to call getAuxiliaryScannables
+ // recursively.
+ for (IncludeScannable scannable :
+ Iterables.concat(ImmutableList.of(action), action.getAuxiliaryScannables())) {
+
+ Map<Artifact, Path> legalOutputPaths = scannable.getLegalGeneratedScannerFileMap();
+ List<PathFragment> includeDirs = new ArrayList<>(scannable.getIncludeDirs());
+ List<PathFragment> quoteIncludeDirs = scannable.getQuoteIncludeDirs();
+ List<String> cmdlineIncludes = scannable.getCmdlineIncludes();
+
+ for (PathFragment pathFragment : scannable.getSystemIncludeDirs()) {
+ includeDirs.add(pathFragment);
+ }
+
+ // Add the system include paths to the list of include paths.
+ for (PathFragment pathFragment : action.getBuiltInIncludeDirectories()) {
+ if (pathFragment.isAbsolute()) {
+ absoluteBuiltInIncludeDirs.add(execRoot.getRelative(pathFragment));
+ }
+ includeDirs.add(pathFragment);
+ }
+
+ IncludeScanner scanner = includeScannerSupplier.scannerFor(
+ relativeTo(execRoot, quoteIncludeDirs),
+ relativeTo(execRoot, includeDirs));
+
+ for (Artifact source : scannable.getIncludeScannerSources()) {
+ // Add all include scanning entry points to the inputs; this is necessary
+ // when we have more than one source to scan from, for example when building
+ // C++ modules.
+ // In that case we have one of two cases:
+ // 1. We compile a header module - there, the .cppmap file is the main source file
+ // (which we do not include-scan, as that would require an extra parser), and
+ // thus already in the input; all headers in the .cppmap file are our entry points
+ // for include scanning, but are not yet in the inputs - they get added here.
+ // 2. We compile an object file that uses a header module; currently using a header
+ // module requires all headers it can reference to be available for the compilation.
+ // The header module can reference headers that are not in the transitive include
+ // closure of the current translation unit. Therefore, {@code CppCompileAction}
+ // adds all headers specified transitively for compiled header modules as include
+ // scanning entry points, and we need to add the entry points to the inputs here.
+ includes.add(source);
+ scanner.process(source, legalOutputPaths, cmdlineIncludes, includes,
+ actionExecutionContext);
+ }
+ }
+ } catch (IOException e) {
+ throw new EnvironmentalExecException(e.getMessage());
+ } finally {
+ profiler.completeTask(ProfilerTask.SCANNER);
+ }
+
+ // Collect inputs and output
+ List<Artifact> inputs = new ArrayList<>();
+ IncludeProblems includeProblems = new IncludeProblems();
+ for (Artifact included : includes) {
+ if (FileSystemUtils.startsWithAny(included.getPath(), absoluteBuiltInIncludeDirs)) {
+ // Skip include files found in absolute include directories. This currently only applies
+ // to grte.
+ continue;
+ }
+ if (included.getRoot().getPath().getParentDirectory() == null) {
+ throw new UserExecException(
+ "illegal absolute path to include file: " + included.getPath());
+ }
+ inputs.add(included);
+ }
+ return inputs;
+ }
+
+ private static List<Path> relativeTo(
+ Path path, Collection<PathFragment> fragments) {
+ List<Path> result = Lists.newArrayListWithCapacity(fragments.size());
+ for (PathFragment fragment : fragments) {
+ result.add(path.getRelative(fragment));
+ }
+ return result;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java
new file mode 100644
index 0000000..69cd26b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionMetadata;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactResolver;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+import java.io.IOException;
+
+/**
+ * Context for actions that do include scanning.
+ */
+public interface IncludeScanningContext extends ActionContext {
+ /**
+ * Extracts the set of include files from a source file.
+ *
+ * @param actionExecutionContext the execution context
+ * @param resourceOwner the resource owner
+ * @param primaryInput the source file to be include scanned
+ * @param primaryOutput the output file where the results should be put
+ */
+ void extractIncludes(ActionExecutionContext actionExecutionContext,
+ ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput)
+ throws IOException, InterruptedException;
+
+ /**
+ * Returns the artifact resolver.
+ */
+ ArtifactResolver getArtifactResolver();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java
new file mode 100644
index 0000000..26175eb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java
@@ -0,0 +1,274 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.Iterator;
+
+/**
+ * Utility types and methods for generating command lines for the linker, given
+ * a CppLinkAction or LinkConfiguration.
+ *
+ * <p>The linker commands, e.g. "ar", may not be functional, i.e.
+ * they may mutate the output file rather than overwriting it.
+ * To avoid this, we need to delete the output file before invoking the
+ * command. But that is not done by this class; deleting the output
+ * file is the responsibility of the classes derived from LinkStrategy.
+ */
+public abstract class Link {
+
+ private Link() {} // uninstantiable
+
+ /** The set of valid linker input files. */
+ public static final FileTypeSet VALID_LINKER_INPUTS = FileTypeSet.of(
+ CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE,
+ CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY,
+ CppFileTypes.OBJECT_FILE, CppFileTypes.PIC_OBJECT_FILE,
+ CppFileTypes.SHARED_LIBRARY, CppFileTypes.VERSIONED_SHARED_LIBRARY,
+ CppFileTypes.INTERFACE_SHARED_LIBRARY);
+
+ /**
+ * These file are supposed to be added using {@code addLibrary()} calls to {@link CppLinkAction}
+ * but will never be expanded to their constituent {@code .o} files. {@link CppLinkAction} checks
+ * that these files are never added as non-libraries.
+ */
+ public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of(
+ CppFileTypes.SHARED_LIBRARY,
+ CppFileTypes.VERSIONED_SHARED_LIBRARY,
+ CppFileTypes.INTERFACE_SHARED_LIBRARY);
+
+ /**
+ * These need special handling when --thin_archive is true. {@link CppLinkAction} checks that
+ * these files are never added as non-libraries.
+ */
+ public static final FileTypeSet ARCHIVE_LIBRARY_FILETYPES = FileTypeSet.of(
+ CppFileTypes.ARCHIVE,
+ CppFileTypes.PIC_ARCHIVE,
+ CppFileTypes.ALWAYS_LINK_LIBRARY,
+ CppFileTypes.ALWAYS_LINK_PIC_LIBRARY);
+
+ public static final FileTypeSet ARCHIVE_FILETYPES = FileTypeSet.of(
+ CppFileTypes.ARCHIVE,
+ CppFileTypes.PIC_ARCHIVE);
+
+ public static final FileTypeSet LINK_LIBRARY_FILETYPES = FileTypeSet.of(
+ CppFileTypes.ALWAYS_LINK_LIBRARY,
+ CppFileTypes.ALWAYS_LINK_PIC_LIBRARY);
+
+
+ /** The set of object files */
+ public static final FileTypeSet OBJECT_FILETYPES = FileTypeSet.of(
+ CppFileTypes.OBJECT_FILE,
+ CppFileTypes.PIC_OBJECT_FILE);
+
+ /**
+ * Prefix that is prepended to command line entries that refer to the output
+ * of cc_fake_binary compile actions. This is a bad hack to signal to the code
+ * in {@code CppLinkAction#executeFake(Executor, FileOutErr)} that it needs
+ * special handling.
+ */
+ public static final String FAKE_OBJECT_PREFIX = "fake:";
+
+ /**
+ * Types of ELF files that can be created by the linker (.a, .so, .lo,
+ * executable).
+ */
+ public enum LinkTargetType {
+ /** A normal static archive. */
+ STATIC_LIBRARY(".a", true),
+
+ /** A static archive with .pic.o object files (compiled with -fPIC). */
+ PIC_STATIC_LIBRARY(".pic.a", true),
+
+ /** An interface dynamic library. */
+ INTERFACE_DYNAMIC_LIBRARY(".ifso", false),
+
+ /** A dynamic library. */
+ DYNAMIC_LIBRARY(".so", false),
+
+ /** A static archive without removal of unused object files. */
+ ALWAYS_LINK_STATIC_LIBRARY(".lo", true),
+
+ /** A PIC static archive without removal of unused object files. */
+ ALWAYS_LINK_PIC_STATIC_LIBRARY(".pic.lo", true),
+
+ /** An executable binary. */
+ EXECUTABLE("", false);
+
+ private final String extension;
+ private final boolean staticLibraryLink;
+
+ private LinkTargetType(String extension, boolean staticLibraryLink) {
+ this.extension = extension;
+ this.staticLibraryLink = staticLibraryLink;
+ }
+
+ public String getExtension() {
+ return extension;
+ }
+
+ public boolean isStaticLibraryLink() {
+ return staticLibraryLink;
+ }
+ }
+
+ /**
+ * The degree of "staticness" of symbol resolution during linking.
+ */
+ public enum LinkStaticness {
+ FULLY_STATIC, // Static binding of all symbols.
+ MOSTLY_STATIC, // Use dynamic binding only for symbols from glibc.
+ DYNAMIC, // Use dynamic binding wherever possible.
+ }
+
+ /**
+ * Types of archive.
+ */
+ public enum ArchiveType {
+ FAT, // Regular archive that includes its members.
+ THIN, // Thin archive that just points to its members.
+ START_END_LIB // A --start-lib ... --end-lib group in the command line.
+ }
+
+ static boolean useStartEndLib(LinkerInput linkerInput, ArchiveType archiveType) {
+ // TODO(bazel-team): Figure out if PicArchives are actually used. For it to be used, both
+ // linkingStatically and linkShared must me true, we must be in opt mode and cpu has to be k8.
+ return archiveType == ArchiveType.START_END_LIB
+ && ARCHIVE_FILETYPES.matches(linkerInput.getArtifact().getFilename())
+ && linkerInput.containsObjectFiles();
+ }
+
+ /**
+ * Replace always used archives with its members. This is used to build the linker cmd line.
+ */
+ public static Iterable<LinkerInput> mergeInputsCmdLine(NestedSet<LibraryToLink> inputs,
+ boolean globalNeedWholeArchive, ArchiveType archiveType) {
+ return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, false);
+ }
+
+ /**
+ * Add in any object files which are implicitly named as inputs by the linker.
+ */
+ public static Iterable<LinkerInput> mergeInputsDependencies(NestedSet<LibraryToLink> inputs,
+ boolean globalNeedWholeArchive, ArchiveType archiveType) {
+ return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, true);
+ }
+
+ /**
+ * On the fly implementation to filter the members.
+ */
+ private static final class FilterMembersForLinkIterable implements Iterable<LinkerInput> {
+ private final boolean globalNeedWholeArchive;
+ private final ArchiveType archiveType;
+ private final boolean deps;
+
+ private final Iterable<LibraryToLink> inputs;
+
+ private FilterMembersForLinkIterable(Iterable<LibraryToLink> inputs,
+ boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) {
+ this.globalNeedWholeArchive = globalNeedWholeArchive;
+ this.archiveType = archiveType;
+ this.deps = deps;
+ this.inputs = CollectionUtils.makeImmutable(inputs);
+ }
+
+ @Override
+ public Iterator<LinkerInput> iterator() {
+ return new FilterMembersForLinkIterator(inputs.iterator(), globalNeedWholeArchive,
+ archiveType, deps);
+ }
+ }
+
+ /**
+ * On the fly implementation to filter the members.
+ */
+ private static final class FilterMembersForLinkIterator extends AbstractIterator<LinkerInput> {
+ private final boolean globalNeedWholeArchive;
+ private final ArchiveType archiveType;
+ private final boolean deps;
+
+ private final Iterator<LibraryToLink> inputs;
+ private Iterator<LinkerInput> delayList = ImmutableList.<LinkerInput>of().iterator();
+
+ private FilterMembersForLinkIterator(Iterator<LibraryToLink> inputs,
+ boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) {
+ this.globalNeedWholeArchive = globalNeedWholeArchive;
+ this.archiveType = archiveType;
+ this.deps = deps;
+ this.inputs = inputs;
+ }
+
+ @Override
+ protected LinkerInput computeNext() {
+ if (delayList.hasNext()) {
+ return delayList.next();
+ }
+
+ while (inputs.hasNext()) {
+ LibraryToLink inputLibrary = inputs.next();
+ Artifact input = inputLibrary.getArtifact();
+ String name = input.getFilename();
+
+ // True if the linker might use the members of this file, i.e., if the file is a thin or
+ // start_end_lib archive (aka static library). Also check if the library contains object
+ // files - otherwise getObjectFiles returns null, which would lead to an NPE in
+ // simpleLinkerInputs.
+ boolean needMembersForLink = archiveType != ArchiveType.FAT
+ && ARCHIVE_LIBRARY_FILETYPES.matches(name) && inputLibrary.containsObjectFiles();
+
+ // True if we will pass the members instead of the original archive.
+ boolean passMembersToLinkCmd = needMembersForLink
+ && (globalNeedWholeArchive || LINK_LIBRARY_FILETYPES.matches(name));
+
+ // If deps is false (when computing the inputs to be passed on the command line), then it's
+ // an if-then-else, i.e., the passMembersToLinkCmd flag decides whether to pass the object
+ // files or the archive itself. This flag in turn is based on whether the archives are fat
+ // or not (thin archives or start_end_lib) - we never expand fat archives, but we do expand
+ // non-fat archives if we need whole-archives for the entire link, or for the specific
+ // library (i.e., if alwayslink=1).
+ //
+ // If deps is true (when computing the inputs to be passed to the action as inputs), then it
+ // becomes more complicated. We always need to pass the members for thin and start_end_lib
+ // archives (needMembersForLink). And we _also_ need to pass the archive file itself unless
+ // it's a start_end_lib archive (unless it's an alwayslink library).
+
+ // A note about ordering: the order in which the object files and the library are returned
+ // does not currently matter - this code results in the library returned first, and the
+ // object files returned after, but only if both are returned, which can only happen if
+ // deps is true, in which case this code only computes the list of inputs for the link
+ // action (so the order isn't critical).
+ if (passMembersToLinkCmd || (deps && needMembersForLink)) {
+ delayList = LinkerInputs.simpleLinkerInputs(inputLibrary.getObjectFiles()).iterator();
+ }
+
+ if (!(passMembersToLinkCmd || (deps && useStartEndLib(inputLibrary, archiveType)))) {
+ return inputLibrary;
+ }
+
+ if (delayList.hasNext()) {
+ return delayList.next();
+ }
+ }
+ return endOfData();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java
new file mode 100644
index 0000000..1dccafa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java
@@ -0,0 +1,1121 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness;
+import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * Represents the command line of a linker invocation. It supports executables and dynamic
+ * libraries as well as static libraries.
+ */
+@Immutable
+public final class LinkCommandLine extends CommandLine {
+ private final BuildConfiguration configuration;
+ private final CppConfiguration cppConfiguration;
+ private final ActionOwner owner;
+ private final Artifact output;
+ @Nullable private final Artifact interfaceOutput;
+ @Nullable private final Artifact symbolCountsOutput;
+ private final ImmutableList<Artifact> buildInfoHeaderArtifacts;
+ private final Iterable<? extends LinkerInput> linkerInputs;
+ private final Iterable<? extends LinkerInput> runtimeInputs;
+ private final LinkTargetType linkTargetType;
+ private final LinkStaticness linkStaticness;
+ private final ImmutableList<String> linkopts;
+ private final ImmutableSet<String> features;
+ private final ImmutableMap<Artifact, Artifact> linkstamps;
+ private final ImmutableList<String> linkstampCompileOptions;
+ @Nullable private final PathFragment runtimeSolibDir;
+ private final boolean nativeDeps;
+ private final boolean useTestOnlyFlags;
+ private final boolean needWholeArchive;
+ private final boolean supportsParamFiles;
+ @Nullable private final Artifact interfaceSoBuilder;
+
+ private LinkCommandLine(
+ BuildConfiguration configuration,
+ ActionOwner owner,
+ Artifact output,
+ @Nullable Artifact interfaceOutput,
+ @Nullable Artifact symbolCountsOutput,
+ ImmutableList<Artifact> buildInfoHeaderArtifacts,
+ Iterable<? extends LinkerInput> linkerInputs,
+ Iterable<? extends LinkerInput> runtimeInputs,
+ LinkTargetType linkTargetType,
+ LinkStaticness linkStaticness,
+ ImmutableList<String> linkopts,
+ ImmutableSet<String> features,
+ ImmutableMap<Artifact, Artifact> linkstamps,
+ ImmutableList<String> linkstampCompileOptions,
+ @Nullable PathFragment runtimeSolibDir,
+ boolean nativeDeps,
+ boolean useTestOnlyFlags,
+ boolean needWholeArchive,
+ boolean supportsParamFiles,
+ Artifact interfaceSoBuilder) {
+ Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY,
+ "you can't link an interface dynamic library directly");
+ if (linkTargetType != LinkTargetType.DYNAMIC_LIBRARY) {
+ Preconditions.checkArgument(interfaceOutput == null,
+ "interface output may only be non-null for dynamic library links");
+ }
+ if (linkTargetType.isStaticLibraryLink()) {
+ Preconditions.checkArgument(linkstamps.isEmpty(),
+ "linkstamps may only be present on dynamic library or executable links");
+ Preconditions.checkArgument(linkStaticness == LinkStaticness.FULLY_STATIC,
+ "static library link must be static");
+ Preconditions.checkArgument(buildInfoHeaderArtifacts.isEmpty(),
+ "build info headers may only be present on dynamic library or executable links");
+ Preconditions.checkArgument(symbolCountsOutput == null,
+ "the symbol counts output must be null for static links");
+ Preconditions.checkArgument(runtimeSolibDir == null,
+ "the runtime solib directory must be null for static links");
+ Preconditions.checkArgument(!nativeDeps,
+ "the native deps flag must be false for static links");
+ Preconditions.checkArgument(!needWholeArchive,
+ "the need whole archive flag must be false for static links");
+ }
+
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.cppConfiguration = configuration.getFragment(CppConfiguration.class);
+ this.owner = Preconditions.checkNotNull(owner);
+ this.output = Preconditions.checkNotNull(output);
+ this.interfaceOutput = interfaceOutput;
+ this.symbolCountsOutput = symbolCountsOutput;
+ this.buildInfoHeaderArtifacts = Preconditions.checkNotNull(buildInfoHeaderArtifacts);
+ this.linkerInputs = Preconditions.checkNotNull(linkerInputs);
+ this.runtimeInputs = Preconditions.checkNotNull(runtimeInputs);
+ this.linkTargetType = Preconditions.checkNotNull(linkTargetType);
+ this.linkStaticness = Preconditions.checkNotNull(linkStaticness);
+ // For now, silently ignore linkopts if this is a static library link.
+ this.linkopts = linkTargetType.isStaticLibraryLink()
+ ? ImmutableList.<String>of()
+ : Preconditions.checkNotNull(linkopts);
+ this.features = Preconditions.checkNotNull(features);
+ this.linkstamps = Preconditions.checkNotNull(linkstamps);
+ this.linkstampCompileOptions = linkstampCompileOptions;
+ this.runtimeSolibDir = runtimeSolibDir;
+ this.nativeDeps = nativeDeps;
+ this.useTestOnlyFlags = useTestOnlyFlags;
+ this.needWholeArchive = needWholeArchive;
+ this.supportsParamFiles = supportsParamFiles;
+ // For now, silently ignore interfaceSoBuilder if we don't build an interface dynamic library.
+ this.interfaceSoBuilder =
+ ((linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) && (interfaceOutput != null))
+ ? Preconditions.checkNotNull(interfaceSoBuilder,
+ "cannot build interface dynamic library without builder")
+ : null;
+ }
+
+ /**
+ * Returns an interface shared object output artifact produced during linking. This only returns
+ * non-null if {@link #getLinkTargetType} is {@code DYNAMIC_LIBRARY} and an interface shared
+ * object was requested.
+ */
+ @Nullable public Artifact getInterfaceOutput() {
+ return interfaceOutput;
+ }
+
+ /**
+ * Returns an artifact containing the number of symbols used per object file passed to the linker.
+ * This is currently a gold only feature, and is only produced for executables. If another target
+ * is being linked, or if symbol counts output is disabled, this will be null.
+ */
+ @Nullable public Artifact getSymbolCountsOutput() {
+ return symbolCountsOutput;
+ }
+
+ /**
+ * Returns the (ordered, immutable) list of header files that contain build info.
+ */
+ public ImmutableList<Artifact> getBuildInfoHeaderArtifacts() {
+ return buildInfoHeaderArtifacts;
+ }
+
+ /**
+ * Returns the (ordered, immutable) list of paths to the linker's input files.
+ */
+ public Iterable<? extends LinkerInput> getLinkerInputs() {
+ return linkerInputs;
+ }
+
+ /**
+ * Returns the runtime inputs to the linker.
+ */
+ public Iterable<? extends LinkerInput> getRuntimeInputs() {
+ return runtimeInputs;
+ }
+
+ /**
+ * Returns the current type of link target set.
+ */
+ public LinkTargetType getLinkTargetType() {
+ return linkTargetType;
+ }
+
+ /**
+ * Returns the "staticness" of the link.
+ */
+ public LinkStaticness getLinkStaticness() {
+ return linkStaticness;
+ }
+
+ /**
+ * Returns the additional linker options for this link.
+ */
+ public ImmutableList<String> getLinkopts() {
+ return linkopts;
+ }
+
+ /**
+ * Returns a (possibly empty) mapping of (C++ source file, .o output file) pairs for source files
+ * that need to be compiled at link time.
+ *
+ * <p>This is used to embed various values from the build system into binaries to identify their
+ * provenance.
+ */
+ public ImmutableMap<Artifact, Artifact> getLinkstamps() {
+ return linkstamps;
+ }
+
+ /**
+ * Returns the location of the C++ runtime solib symlinks. If null, the C++ dynamic runtime
+ * libraries either do not exist (because they do not come from the depot) or they are in the
+ * regular solib directory.
+ */
+ @Nullable public PathFragment getRuntimeSolibDir() {
+ return runtimeSolibDir;
+ }
+
+ /**
+ * Returns true for libraries linked as native dependencies for other languages.
+ */
+ public boolean isNativeDeps() {
+ return nativeDeps;
+ }
+
+ /**
+ * Returns true if this link should use test-specific flags (e.g. $EXEC_ORIGIN as the root for
+ * finding shared libraries or lazy binding); false by default. See bug "Please use
+ * $EXEC_ORIGIN instead of $ORIGIN when linking cc_tests" for further context.
+ */
+ public boolean useTestOnlyFlags() {
+ return useTestOnlyFlags;
+ }
+
+ /**
+ * Splits the link command-line into a part to be written to a parameter file, and the remaining
+ * actual command line to be executed (which references the parameter file). Call {@link
+ * #canBeSplit} first to check if the command-line can be split.
+ *
+ * @throws IllegalStateException if the command-line cannot be split
+ */
+ @VisibleForTesting
+ final Pair<List<String>, List<String>> splitCommandline(PathFragment paramExecPath) {
+ Preconditions.checkState(canBeSplit());
+ List<String> args = getRawLinkArgv();
+ if (linkTargetType.isStaticLibraryLink()) {
+ // Ar link commands can also generate huge command lines.
+ List<String> paramFileArgs = args.subList(1, args.size());
+ List<String> commandlineArgs = new ArrayList<>();
+ commandlineArgs.add(args.get(0));
+
+ commandlineArgs.add("@" + paramExecPath.getPathString());
+ return Pair.of(commandlineArgs, paramFileArgs);
+ } else {
+ // Gcc link commands tend to generate humongous commandlines for some targets, which may
+ // not fit on some remote execution machines. To work around this we will employ the help of
+ // a parameter file and pass any linker options through it.
+ List<String> paramFileArgs = new ArrayList<>();
+ List<String> commandlineArgs = new ArrayList<>();
+ extractArgumentsForParamFile(args, commandlineArgs, paramFileArgs);
+
+ commandlineArgs.add("-Wl,@" + paramExecPath.getPathString());
+ return Pair.of(commandlineArgs, paramFileArgs);
+ }
+ }
+
+ boolean canBeSplit() {
+ if (!supportsParamFiles) {
+ return false;
+ }
+ switch (linkTargetType) {
+ // We currently can't split dynamic library links if they have interface outputs. That was
+ // probably an unintended side effect of the change that introduced interface outputs.
+ case DYNAMIC_LIBRARY:
+ return interfaceOutput == null;
+ case EXECUTABLE:
+ case STATIC_LIBRARY:
+ case PIC_STATIC_LIBRARY:
+ case ALWAYS_LINK_STATIC_LIBRARY:
+ case ALWAYS_LINK_PIC_STATIC_LIBRARY:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ private static void extractArgumentsForParamFile(List<String> args, List<String> commandlineArgs,
+ List<String> paramFileArgs) {
+ // Note, that it is not important that all linker arguments are extracted so that
+ // they can be moved into a parameter file, but the vast majority should.
+ commandlineArgs.add(args.get(0)); // gcc command, must not be moved!
+ int argsSize = args.size();
+ for (int i = 1; i < argsSize; i++) {
+ String arg = args.get(i);
+ if (arg.equals("-Wl,-no-whole-archive")) {
+ paramFileArgs.add("-no-whole-archive");
+ } else if (arg.equals("-Wl,-whole-archive")) {
+ paramFileArgs.add("-whole-archive");
+ } else if (arg.equals("-Wl,--start-group")) {
+ paramFileArgs.add("--start-group");
+ } else if (arg.equals("-Wl,--end-group")) {
+ paramFileArgs.add("--end-group");
+ } else if (arg.equals("-Wl,--start-lib")) {
+ paramFileArgs.add("--start-lib");
+ } else if (arg.equals("-Wl,--end-lib")) {
+ paramFileArgs.add("--end-lib");
+ } else if (arg.equals("--incremental-unchanged")) {
+ paramFileArgs.add(arg);
+ } else if (arg.equals("--incremental-changed")) {
+ paramFileArgs.add(arg);
+ } else if (arg.charAt(0) == '-') {
+ if (arg.startsWith("-l")) {
+ paramFileArgs.add(arg);
+ } else {
+ // Anything else starting with a '-' can stay on the commandline.
+ commandlineArgs.add(arg);
+ if (arg.equals("-o")) {
+ // Special case for '-o': add the following argument as well - it is the output file!
+ commandlineArgs.add(args.get(++i));
+ }
+ }
+ } else if (arg.endsWith(".a") || arg.endsWith(".lo") || arg.endsWith(".so")
+ || arg.endsWith(".ifso") || arg.endsWith(".o")
+ || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(arg)) {
+ // All objects of any kind go into the linker parameters.
+ paramFileArgs.add(arg);
+ } else {
+ // Everything that's left stays conservatively on the commandline.
+ commandlineArgs.add(arg);
+ }
+ }
+ }
+
+ /**
+ * Returns a raw link command for the given link invocation, including both command and
+ * arguments (argv). After any further usage-specific processing, this can be passed to
+ * {@link #finalizeWithLinkstampCommands} to give the final command line.
+ *
+ * @return raw link command line.
+ */
+ public List<String> getRawLinkArgv() {
+ List<String> argv = new ArrayList<>();
+ switch (linkTargetType) {
+ case EXECUTABLE:
+ addCppArgv(argv);
+ break;
+
+ case DYNAMIC_LIBRARY:
+ if (interfaceOutput != null) {
+ argv.add(configuration.getShExecutable().getPathString());
+ argv.add("-c");
+ argv.add("build_iface_so=\"$0\"; impl=\"$1\"; iface=\"$2\"; cmd=\"$3\"; shift 3; "
+ + "\"$cmd\" \"$@\" && \"$build_iface_so\" \"$impl\" \"$iface\"");
+ argv.add(interfaceSoBuilder.getExecPathString());
+ argv.add(output.getExecPathString());
+ argv.add(interfaceOutput.getExecPathString());
+ }
+ addCppArgv(argv);
+ // -pie is not compatible with -shared and should be
+ // removed when the latter is part of the link command. Should we need to further
+ // distinguish between shared libraries and executables, we could add additional
+ // command line / CROSSTOOL flags that distinguish them. But as long as this is
+ // the only relevant use case we're just special-casing it here.
+ Iterables.removeIf(argv, Predicates.equalTo("-pie"));
+ break;
+
+ case STATIC_LIBRARY:
+ case PIC_STATIC_LIBRARY:
+ case ALWAYS_LINK_STATIC_LIBRARY:
+ case ALWAYS_LINK_PIC_STATIC_LIBRARY:
+ // The static library link command follows this template:
+ // ar <cmd> <output_archive> <input_files...>
+ argv.add(cppConfiguration.getArExecutable().getPathString());
+ argv.addAll(
+ cppConfiguration.getArFlags(cppConfiguration.archiveType() == Link.ArchiveType.THIN));
+ argv.add(output.getExecPathString());
+ addInputFileLinkOptions(argv, /*needWholeArchive=*/false,
+ /*includeLinkopts=*/false);
+ break;
+
+ default:
+ throw new IllegalArgumentException();
+ }
+
+ // Fission mode: debug info is in .dwo files instead of .o files. Inform the linker of this.
+ if (!linkTargetType.isStaticLibraryLink() && cppConfiguration.useFission()) {
+ argv.add("-Wl,--gdb-index");
+ }
+
+ return argv;
+ }
+
+ @Override
+ public List<String> arguments() {
+ return finalizeWithLinkstampCommands(getRawLinkArgv());
+ }
+
+ /**
+ * Takes a raw link command line and gives the final link command that will
+ * also first compile any linkstamps necessary. Elements of rawLinkArgv are
+ * shell-escaped.
+ *
+ * @param rawLinkArgv raw link command line
+ *
+ * @return final link command line suitable for execution
+ */
+ public List<String> finalizeWithLinkstampCommands(List<String> rawLinkArgv) {
+ return addLinkstampingToCommand(getLinkstampCompileCommands(""), rawLinkArgv, true);
+ }
+
+ /**
+ * Takes a raw link command line and gives the final link command that will also first compile any
+ * linkstamps necessary. Elements of rawLinkArgv are not shell-escaped.
+ *
+ * @param rawLinkArgv raw link command line
+ * @param outputPrefix prefix to add before the linkstamp outputs' exec paths
+ *
+ * @return final link command line suitable for execution
+ */
+ public List<String> finalizeAlreadyEscapedWithLinkstampCommands(
+ List<String> rawLinkArgv, String outputPrefix) {
+ return addLinkstampingToCommand(getLinkstampCompileCommands(outputPrefix), rawLinkArgv, false);
+ }
+
+ /**
+ * Adds linkstamp compilation to the (otherwise) fully specified link
+ * command if {@link #getLinkstamps} is non-empty.
+ *
+ * <p>Linkstamps were historically compiled implicitly as part of the link
+ * command, but implicit compilation doesn't guarantee consistent outputs.
+ * For example, the command "gcc input.o input.o foo/linkstamp.cc -o myapp"
+ * causes gcc to implicitly run "gcc foo/linkstamp.cc -o /tmp/ccEtJHDB.o",
+ * for some internally decided output path /tmp/ccEtJHDB.o, then add that path
+ * to the linker's command line options. The name of this path can change
+ * even between equivalently specified gcc invocations.
+ *
+ * <p>So now we explicitly compile these files in their own command
+ * invocations before running the link command, thus giving us direct
+ * control over the naming of their outputs. This method adds those extra
+ * steps as necessary.
+ * @param linkstampCommands individual linkstamp compilation commands
+ * @param linkCommand the complete list of link command arguments (after
+ * .params file compacting) for an invocation
+ * @param escapeArgs if true, linkCommand arguments are shell escaped. if
+ * false, arguments are returned as-is
+ *
+ * @return The original argument list if no linkstamps compilation commands
+ * are given, otherwise an expanded list that adds the linkstamp
+ * compilation commands and funnels their outputs into the link step.
+ * Note that these outputs only need to persist for the duration of
+ * the link step.
+ */
+ private static List<String> addLinkstampingToCommand(
+ List<String> linkstampCommands,
+ List<String> linkCommand,
+ boolean escapeArgs) {
+ if (linkstampCommands.isEmpty()) {
+ return linkCommand;
+ } else {
+ List<String> batchCommand = Lists.newArrayListWithCapacity(3);
+ batchCommand.add("/bin/bash");
+ batchCommand.add("-c");
+ batchCommand.add(
+ Joiner.on(" && ").join(linkstampCommands) + " && "
+ + (escapeArgs
+ ? ShellEscaper.escapeJoinAll(linkCommand)
+ : Joiner.on(" ").join(linkCommand)));
+ return ImmutableList.copyOf(batchCommand);
+ }
+ }
+
+ /**
+ * Computes, for each C++ source file in
+ * {@link #getLinkstamps}, the command necessary to compile
+ * that file such that the output is correctly fed into the link command.
+ *
+ * <p>As these options (as well as all others) are taken into account when
+ * computing the action key, they do not directly contain volatile build
+ * information to avoid unnecessary relinking. Instead this information is
+ * passed as an additional header generated by
+ * {@link com.google.devtools.build.lib.rules.cpp.WriteBuildInfoHeaderAction}.
+ *
+ * @param outputPrefix prefix to add before the linkstamp outputs' exec paths
+ * @return a list of shell-escaped compiler commmands, one for each entry
+ * in {@link #getLinkstamps}
+ */
+ public List<String> getLinkstampCompileCommands(String outputPrefix) {
+ if (linkstamps.isEmpty()) {
+ return ImmutableList.of();
+ }
+
+ String compilerCommand = cppConfiguration.getCppExecutable().getPathString();
+ List<String> commands = Lists.newArrayListWithCapacity(linkstamps.size());
+
+ for (Map.Entry<Artifact, Artifact> linkstamp : linkstamps.entrySet()) {
+ List<String> optionList = new ArrayList<>();
+
+ // Defines related to the build info are read from generated headers.
+ for (Artifact header : buildInfoHeaderArtifacts) {
+ optionList.add("-include");
+ optionList.add(header.getExecPathString());
+ }
+
+ String labelReplacement = Matcher.quoteReplacement(
+ isSharedNativeLibrary() ? output.getExecPathString() : Label.print(owner.getLabel()));
+ String outputPathReplacement = Matcher.quoteReplacement(
+ output.getExecPathString());
+ for (String option : linkstampCompileOptions) {
+ optionList.add(option
+ .replaceAll(Pattern.quote("${LABEL}"), labelReplacement)
+ .replaceAll(Pattern.quote("${OUTPUT_PATH}"), outputPathReplacement));
+ }
+
+ optionList.add("-DGPLATFORM=\"" + cppConfiguration + "\"");
+
+ // Needed to find headers included from linkstamps.
+ optionList.add("-I.");
+
+ // Add sysroot.
+ PathFragment sysroot = cppConfiguration.getSysroot();
+ if (sysroot != null) {
+ optionList.add("--sysroot=" + sysroot.getPathString());
+ }
+
+ // Add toolchain compiler options.
+ optionList.addAll(cppConfiguration.getCompilerOptions(features));
+ optionList.addAll(cppConfiguration.getCOptions());
+ optionList.addAll(cppConfiguration.getUnfilteredCompilerOptions(features));
+
+ // For dynamic libraries, produce position independent code.
+ if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY
+ && cppConfiguration.toolchainNeedsPic()) {
+ optionList.add("-fPIC");
+ }
+
+ // Stamp FDO builds with FDO subtype string
+ String fdoBuildStamp = CppHelper.getFdoBuildStamp(cppConfiguration);
+ if (fdoBuildStamp != null) {
+ optionList.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\"");
+ }
+
+ // Add the compilation target.
+ optionList.add("-c");
+ optionList.add(linkstamp.getKey().getExecPathString());
+
+ // Assemble the final command, exempting outputPrefix from shell escaping.
+ commands.add(compilerCommand + " "
+ + ShellEscaper.escapeJoinAll(optionList)
+ + " -o "
+ + outputPrefix
+ + ShellEscaper.escapeString(linkstamp.getValue().getExecPathString()));
+ }
+
+ return commands;
+ }
+
+ /**
+ * Determine the arguments to pass to the C++ compiler when linking.
+ * Add them to the {@code argv} parameter.
+ */
+ private void addCppArgv(List<String> argv) {
+ argv.add(cppConfiguration.getCppExecutable().getPathString());
+
+ // When using gold to link an executable, output the number of used and unused symbols.
+ if (symbolCountsOutput != null) {
+ argv.add("-Wl,--print-symbol-counts=" + symbolCountsOutput.getExecPathString());
+ }
+
+ if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) {
+ argv.add("-shared");
+ }
+
+ // Add the outputs of any associated linkstamp compilations.
+ for (Artifact linkstampOutput : linkstamps.values()) {
+ argv.add(linkstampOutput.getExecPathString());
+ }
+
+ boolean fullyStatic = (linkStaticness == LinkStaticness.FULLY_STATIC);
+ boolean mostlyStatic = (linkStaticness == LinkStaticness.MOSTLY_STATIC);
+ boolean sharedLinkopts =
+ linkTargetType == LinkTargetType.DYNAMIC_LIBRARY
+ || linkopts.contains("-shared")
+ || cppConfiguration.getLinkOptions().contains("-shared");
+
+ if (output != null) {
+ argv.add("-o");
+ String execpath = output.getExecPathString();
+ if (mostlyStatic
+ && linkTargetType == LinkTargetType.EXECUTABLE
+ && cppConfiguration.skipStaticOutputs()) {
+ // Linked binary goes to /dev/null; bogus dependency info in its place.
+ Collections.addAll(argv, "/dev/null", "-MMD", "-MF", execpath); // thanks Ambrose
+ } else {
+ argv.add(execpath);
+ }
+ }
+
+ addInputFileLinkOptions(argv, needWholeArchive, /*includeLinkopts=*/true);
+
+ // Extra toolchain link options based on the output's link staticness.
+ if (fullyStatic) {
+ argv.addAll(cppConfiguration.getFullyStaticLinkOptions(features, sharedLinkopts));
+ } else if (mostlyStatic) {
+ argv.addAll(cppConfiguration.getMostlyStaticLinkOptions(features, sharedLinkopts));
+ } else {
+ argv.addAll(cppConfiguration.getDynamicLinkOptions(features, sharedLinkopts));
+ }
+
+ // Extra test-specific link options.
+ if (useTestOnlyFlags) {
+ argv.addAll(cppConfiguration.getTestOnlyLinkOptions());
+ }
+
+ if (configuration.isCodeCoverageEnabled()) {
+ argv.add("-lgcov");
+ }
+
+ if (linkTargetType == LinkTargetType.EXECUTABLE && cppConfiguration.forcePic()) {
+ argv.add("-pie");
+ }
+
+ argv.addAll(cppConfiguration.getLinkOptions());
+ argv.addAll(cppConfiguration.getFdoSupport().getLinkOptions());
+ }
+
+ private static boolean isDynamicLibrary(LinkerInput linkInput) {
+ Artifact libraryArtifact = linkInput.getArtifact();
+ String name = libraryArtifact.getFilename();
+ return Link.SHARED_LIBRARY_FILETYPES.matches(name) && name.startsWith("lib");
+ }
+
+ private boolean isSharedNativeLibrary() {
+ return nativeDeps && cppConfiguration.shareNativeDeps();
+ }
+
+ /**
+ * When linking a shared library fully or mostly static then we need to link in
+ * *all* dependent files, not just what the shared library needs for its own
+ * code. This is done by wrapping all objects/libraries with
+ * -Wl,-whole-archive and -Wl,-no-whole-archive. For this case the
+ * globalNeedWholeArchive parameter must be set to true. Otherwise only
+ * library objects (.lo) need to be wrapped with -Wl,-whole-archive and
+ * -Wl,-no-whole-archive.
+ */
+ private void addInputFileLinkOptions(List<String> argv, boolean globalNeedWholeArchive,
+ boolean includeLinkopts) {
+ // The Apple ld doesn't support -whole-archive/-no-whole-archive. It
+ // does have -all_load/-noall_load, but -all_load is a global setting
+ // that affects all subsequent files, and -noall_load is simply ignored.
+ // TODO(bazel-team): Not sure what the implications of this are, other than
+ // bloated binaries.
+ boolean macosx = cppConfiguration.getTargetLibc().equals("macosx");
+ if (globalNeedWholeArchive) {
+ argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive");
+ }
+
+ // Used to collect -L and -Wl,-rpath options, ensuring that each used only once.
+ Set<String> libOpts = new LinkedHashSet<>();
+
+ // List of command line parameters to link input files (either directly or using -l).
+ List<String> linkerInputs = new ArrayList<>();
+
+ // List of command line parameters that need to be placed *outside* of
+ // --whole-archive ... --no-whole-archive.
+ List<String> noWholeArchiveInputs = new ArrayList<>();
+
+ PathFragment solibDir = configuration.getBinDirectory().getExecPath()
+ .getRelative(cppConfiguration.getSolibDirectory());
+ String runtimeSolibName = runtimeSolibDir != null ? runtimeSolibDir.getBaseName() : null;
+ boolean runtimeRpath = runtimeSolibDir != null
+ && (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY
+ || (linkTargetType == LinkTargetType.EXECUTABLE
+ && linkStaticness == LinkStaticness.DYNAMIC));
+
+ String rpathRoot = null;
+ List<String> runtimeRpathEntries = new ArrayList<>();
+
+ if (output != null) {
+ String origin =
+ useTestOnlyFlags && cppConfiguration.supportsExecOrigin() ? "$EXEC_ORIGIN/" : "$ORIGIN/";
+ if (runtimeRpath) {
+ runtimeRpathEntries.add("-Wl,-rpath," + origin + runtimeSolibName + "/");
+ }
+
+ // Calculate the correct relative value for the "-rpath" link option (which sets
+ // the search path for finding shared libraries).
+ if (isSharedNativeLibrary()) {
+ // For shared native libraries, special symlinking is applied to ensure C++
+ // runtimes are available under $ORIGIN/_solib_[arch]. So we set the RPATH to find
+ // them.
+ //
+ // Note that we have to do this because $ORIGIN points to different paths for
+ // different targets. In other words, blaze-bin/d1/d2/d3/a_shareddeps.so and
+ // blaze-bin/d4/b_shareddeps.so have different path depths. The first could
+ // reference a standard blaze-bin/_solib_[arch] via $ORIGIN/../../../_solib[arch],
+ // and the second could use $ORIGIN/../_solib_[arch]. But since this is a shared
+ // artifact, both are symlinks to the same place, so
+ // there's no *one* RPATH setting that fits all targets involved in the sharing.
+ rpathRoot = "-Wl,-rpath," + origin + ":"
+ + origin + cppConfiguration.getSolibDirectory() + "/";
+ if (runtimeRpath) {
+ runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/");
+ }
+ } else {
+ // For all other links, calculate the relative path from the output file to _solib_[arch]
+ // (the directory where all shared libraries are stored, which resides under the blaze-bin
+ // directory. In other words, given blaze-bin/my/package/binary, rpathRoot would be
+ // "../../_solib_[arch]".
+ if (runtimeRpath) {
+ runtimeRpathEntries.add("-Wl,-rpath," + origin
+ + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1)
+ + runtimeSolibName + "/");
+ }
+
+ rpathRoot = "-Wl,-rpath,"
+ + origin + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1)
+ + cppConfiguration.getSolibDirectory() + "/";
+
+ if (nativeDeps) {
+ // We also retain the $ORIGIN/ path to solibs that are in _solib_<arch>, as opposed to
+ // the package directory)
+ if (runtimeRpath) {
+ runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/");
+ }
+ rpathRoot += ":" + origin;
+ }
+ }
+ }
+
+ boolean includeSolibDir = false;
+
+ for (LinkerInput input : getLinkerInputs()) {
+ if (isDynamicLibrary(input)) {
+ PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory();
+ Preconditions.checkState(
+ libDir.startsWith(solibDir),
+ "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir);
+ if (libDir.equals(solibDir)) {
+ includeSolibDir = true;
+ }
+ addDynamicInputLinkOptions(input, linkerInputs, libOpts, solibDir, rpathRoot);
+ } else {
+ addStaticInputLinkOptions(input, linkerInputs);
+ }
+ }
+
+ boolean includeRuntimeSolibDir = false;
+
+ for (LinkerInput input : runtimeInputs) {
+ List<String> optionsList = globalNeedWholeArchive
+ ? noWholeArchiveInputs
+ : linkerInputs;
+
+ if (isDynamicLibrary(input)) {
+ PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory();
+ Preconditions.checkState(runtimeSolibDir != null && libDir.equals(runtimeSolibDir),
+ "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir);
+ includeRuntimeSolibDir = true;
+ addDynamicInputLinkOptions(input, optionsList, libOpts, solibDir, rpathRoot);
+ } else {
+ addStaticInputLinkOptions(input, optionsList);
+ }
+ }
+
+ // rpath ordering matters for performance; first add the one where most libraries are found.
+ if (includeSolibDir && rpathRoot != null) {
+ argv.add(rpathRoot);
+ }
+ if (includeRuntimeSolibDir) {
+ argv.addAll(runtimeRpathEntries);
+ }
+ argv.addAll(libOpts);
+
+ // Need to wrap static libraries with whole-archive option
+ for (String option : linkerInputs) {
+ if (!globalNeedWholeArchive && Link.LINK_LIBRARY_FILETYPES.matches(option)) {
+ argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive");
+ argv.add(option);
+ argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive");
+ } else {
+ argv.add(option);
+ }
+ }
+
+ if (globalNeedWholeArchive) {
+ argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive");
+ argv.addAll(noWholeArchiveInputs);
+ }
+
+ if (includeLinkopts) {
+ /*
+ * For backwards compatibility, linkopts come _after_ inputFiles.
+ * This is needed to allow linkopts to contain libraries and
+ * positional library-related options such as
+ * -Wl,--begin-group -lfoo -lbar -Wl,--end-group
+ * or
+ * -Wl,--as-needed -lfoo -Wl,--no-as-needed
+ *
+ * As for the relative order of the three different flavours of linkopts
+ * (global defaults, per-target linkopts, and command-line linkopts),
+ * we have no idea what the right order should be, or if anyone cares.
+ */
+ argv.addAll(linkopts);
+ }
+ }
+
+ /**
+ * Adds command-line options for a dynamic library input file into
+ * options and libOpts.
+ */
+ private void addDynamicInputLinkOptions(LinkerInput input, List<String> options,
+ Set<String> libOpts, PathFragment solibDir, String rpathRoot) {
+ Preconditions.checkState(isDynamicLibrary(input));
+ Preconditions.checkState(
+ !Link.useStartEndLib(input, cppConfiguration.archiveType()));
+
+ Artifact inputArtifact = input.getArtifact();
+ PathFragment libDir = inputArtifact.getExecPath().getParentDirectory();
+ if (rpathRoot != null
+ && !libDir.equals(solibDir)
+ && (runtimeSolibDir == null || !runtimeSolibDir.equals(libDir))) {
+ String dotdots = "";
+ PathFragment commonParent = solibDir;
+ while (!libDir.startsWith(commonParent)) {
+ dotdots += "../";
+ commonParent = commonParent.getParentDirectory();
+ }
+
+ libOpts.add(rpathRoot + dotdots + libDir.relativeTo(commonParent).getPathString());
+ }
+
+ libOpts.add("-L" + inputArtifact.getExecPath().getParentDirectory().getPathString());
+
+ String name = inputArtifact.getFilename();
+ if (CppFileTypes.SHARED_LIBRARY.matches(name)) {
+ String libName = name.replaceAll("(^lib|\\.so$)", "");
+ options.add("-l" + libName);
+ } else {
+ // Interface shared objects have a non-standard extension
+ // that the linker won't be able to find. So use the
+ // filename directly rather than a -l option. Since the
+ // library has an SONAME attribute, this will work fine.
+ options.add(inputArtifact.getExecPathString());
+ }
+ }
+
+ /**
+ * Adds command-line options for a static library or non-library input
+ * into options.
+ */
+ private void addStaticInputLinkOptions(LinkerInput input, List<String> options) {
+ Preconditions.checkState(!isDynamicLibrary(input));
+
+ // start-lib/end-lib library: adds its input object files.
+ if (Link.useStartEndLib(input, cppConfiguration.archiveType())) {
+ Iterable<Artifact> archiveMembers = input.getObjectFiles();
+ if (!Iterables.isEmpty(archiveMembers)) {
+ options.add("-Wl,--start-lib");
+ for (Artifact member : archiveMembers) {
+ options.add(member.getExecPathString());
+ }
+ options.add("-Wl,--end-lib");
+ }
+ // For anything else, add the input directly.
+ } else {
+ Artifact inputArtifact = input.getArtifact();
+ if (input.isFake()) {
+ options.add(Link.FAKE_OBJECT_PREFIX + inputArtifact.getExecPathString());
+ } else {
+ options.add(inputArtifact.getExecPathString());
+ }
+ }
+ }
+
+ /**
+ * A builder for a {@link LinkCommandLine}.
+ */
+ public static final class Builder {
+ // TODO(bazel-team): Pass this in instead of having it here. Maybe move to cc_toolchain.
+ private static final ImmutableList<String> DEFAULT_LINKSTAMP_OPTIONS = ImmutableList.of(
+ // G3_VERSION_INFO and G3_TARGET_NAME are C string literals that normally
+ // contain the label of the target being linked. However, they are set
+ // differently when using shared native deps. In that case, a single .so file
+ // is shared by multiple targets, and its contents cannot depend on which
+ // target(s) were specified on the command line. So in that case we have
+ // to use the (obscure) name of the .so file instead, or more precisely
+ // the path of the .so file relative to the workspace root.
+ "-DG3_VERSION_INFO=\"${LABEL}\"",
+ "-DG3_TARGET_NAME=\"${LABEL}\"",
+
+ // G3_BUILD_TARGET is a C string literal containing the output of this
+ // link. (An undocumented and untested invariant is that G3_BUILD_TARGET is the location of
+ // the executable, either absolutely, or relative to the directory part of BUILD_INFO.)
+ "-DG3_BUILD_TARGET=\"${OUTPUT_PATH}\"");
+
+ private final BuildConfiguration configuration;
+ private final ActionOwner owner;
+
+ @Nullable private Artifact output;
+ @Nullable private Artifact interfaceOutput;
+ @Nullable private Artifact symbolCountsOutput;
+ private ImmutableList<Artifact> buildInfoHeaderArtifacts = ImmutableList.of();
+ private Iterable<? extends LinkerInput> linkerInputs = ImmutableList.of();
+ private Iterable<? extends LinkerInput> runtimeInputs = ImmutableList.of();
+ @Nullable private LinkTargetType linkTargetType;
+ private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC;
+ private ImmutableList<String> linkopts = ImmutableList.of();
+ private ImmutableSet<String> features = ImmutableSet.of();
+ private ImmutableMap<Artifact, Artifact> linkstamps = ImmutableMap.of();
+ private List<String> linkstampCompileOptions = new ArrayList<>();
+ @Nullable private PathFragment runtimeSolibDir;
+ private boolean nativeDeps;
+ private boolean useTestOnlyFlags;
+ private boolean needWholeArchive;
+ private boolean supportsParamFiles;
+ @Nullable private Artifact interfaceSoBuilder;
+
+ public Builder(BuildConfiguration configuration, ActionOwner owner) {
+ this.configuration = configuration;
+ this.owner = owner;
+ }
+
+ public Builder(RuleContext ruleContext) {
+ this(ruleContext.getConfiguration(), ruleContext.getActionOwner());
+ }
+
+ public LinkCommandLine build() {
+ ImmutableList<String> actualLinkstampCompileOptions;
+ if (linkstampCompileOptions.isEmpty()) {
+ actualLinkstampCompileOptions = DEFAULT_LINKSTAMP_OPTIONS;
+ } else {
+ actualLinkstampCompileOptions = ImmutableList.copyOf(
+ Iterables.concat(DEFAULT_LINKSTAMP_OPTIONS, linkstampCompileOptions));
+ }
+ return new LinkCommandLine(configuration, owner, output, interfaceOutput,
+ symbolCountsOutput, buildInfoHeaderArtifacts, linkerInputs, runtimeInputs, linkTargetType,
+ linkStaticness, linkopts, features, linkstamps, actualLinkstampCompileOptions,
+ runtimeSolibDir, nativeDeps, useTestOnlyFlags, needWholeArchive, supportsParamFiles,
+ interfaceSoBuilder);
+ }
+
+ /**
+ * Sets the type of the link. It is an error to try to set this to {@link
+ * LinkTargetType#INTERFACE_DYNAMIC_LIBRARY}. Note that all the static target types (see {@link
+ * LinkTargetType#isStaticLibraryLink}) are equivalent, and there is no check that the output
+ * artifact matches the target type extension.
+ */
+ public Builder setLinkTargetType(LinkTargetType linkTargetType) {
+ Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY);
+ this.linkTargetType = linkTargetType;
+ return this;
+ }
+
+ /**
+ * Sets the primary output artifact. This must be called before calling {@link #build}.
+ */
+ public Builder setOutput(Artifact output) {
+ this.output = output;
+ return this;
+ }
+
+ /**
+ * Sets a list of linker inputs. These get turned into linker options depending on the
+ * staticness and the target type. This call makes an immutable copy of the inputs, if the
+ * provided Iterable isn't already immutable (see {@link CollectionUtils#makeImmutable}).
+ */
+ public Builder setLinkerInputs(Iterable<LinkerInput> linkerInputs) {
+ this.linkerInputs = CollectionUtils.makeImmutable(linkerInputs);
+ return this;
+ }
+
+ public Builder setRuntimeInputs(ImmutableList<LinkerInput> runtimeInputs) {
+ this.runtimeInputs = runtimeInputs;
+ return this;
+ }
+
+ /**
+ * Sets the additional interface output artifact, which is only used for dynamic libraries. The
+ * {@link #build} method throws an exception if the target type is not {@link
+ * LinkTargetType#DYNAMIC_LIBRARY}.
+ */
+ public Builder setInterfaceOutput(Artifact interfaceOutput) {
+ this.interfaceOutput = interfaceOutput;
+ return this;
+ }
+
+ /**
+ * Sets an additional output artifact that contains symbol counts. The {@link #build} method
+ * throws an exception if this is non-null for a static link (see
+ * {@link LinkTargetType#isStaticLibraryLink}).
+ */
+ public Builder setSymbolCountsOutput(Artifact symbolCountsOutput) {
+ this.symbolCountsOutput = symbolCountsOutput;
+ return this;
+ }
+
+ /**
+ * Sets the linker options. These are passed to the linker in addition to the other linker
+ * options like linker inputs, symbol count options, etc. The {@link #build} method
+ * throws an exception if the linker options are non-empty for a static link (see {@link
+ * LinkTargetType#isStaticLibraryLink}).
+ */
+ public Builder setLinkopts(ImmutableList<String> linkopts) {
+ this.linkopts = linkopts;
+ return this;
+ }
+
+ /**
+ * Sets how static the link is supposed to be. For static target types (see {@link
+ * LinkTargetType#isStaticLibraryLink}), the {@link #build} method throws an exception if this
+ * is not {@link LinkStaticness#FULLY_STATIC}. The default setting is {@link
+ * LinkStaticness#FULLY_STATIC}.
+ */
+ public Builder setLinkStaticness(LinkStaticness linkStaticness) {
+ this.linkStaticness = linkStaticness;
+ return this;
+ }
+
+ /**
+ * Sets the binary that should be used to create the interface output for a dynamic library.
+ * This is ignored unless the target type is {@link LinkTargetType#DYNAMIC_LIBRARY} and an
+ * interface output artifact is specified.
+ */
+ public Builder setInterfaceSoBuilder(Artifact interfaceSoBuilder) {
+ this.interfaceSoBuilder = interfaceSoBuilder;
+ return this;
+ }
+
+ /**
+ * Sets the linkstamps. Linkstamps are additional C++ source files that are compiled as part of
+ * the link command. The {@link #build} method throws an exception if the linkstamps are
+ * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}).
+ */
+ public Builder setLinkstamps(ImmutableMap<Artifact, Artifact> linkstamps) {
+ this.linkstamps = linkstamps;
+ return this;
+ }
+
+ /**
+ * Adds the given C++ compiler options to the list of options passed to the linkstamp
+ * compilation.
+ */
+ public Builder addLinkstampCompileOptions(List<String> linkstampCompileOptions) {
+ this.linkstampCompileOptions.addAll(linkstampCompileOptions);
+ return this;
+ }
+
+ /**
+ * The build info header artifacts are generated header files that are used for link stamping.
+ * The {@link #build} method throws an exception if the build info header artifacts are
+ * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}).
+ */
+ public Builder setBuildInfoHeaderArtifacts(ImmutableList<Artifact> buildInfoHeaderArtifacts) {
+ this.buildInfoHeaderArtifacts = buildInfoHeaderArtifacts;
+ return this;
+ }
+
+ /**
+ * Sets the features enabled for the rule.
+ */
+ public Builder setFeatures(ImmutableSet<String> features) {
+ this.features = features;
+ return this;
+ }
+
+ /**
+ * Sets the directory of the dynamic runtime libraries, which is added to the rpath. The {@link
+ * #build} method throws an exception if the runtime dir is non-null for a static link (see
+ * {@link LinkTargetType#isStaticLibraryLink}).
+ */
+ public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) {
+ this.runtimeSolibDir = runtimeSolibDir;
+ return this;
+ }
+
+ /**
+ * Whether the resulting library is intended to be used as a native library from another
+ * programming language. This influences the rpath. The {@link #build} method throws an
+ * exception if this is true for a static link (see {@link LinkTargetType#isStaticLibraryLink}).
+ */
+ public Builder setNativeDeps(boolean nativeDeps) {
+ this.nativeDeps = nativeDeps;
+ return this;
+ }
+
+ /**
+ * Sets whether to use test-specific linker flags, e.g. {@code $EXEC_ORIGIN} instead of
+ * {@code $ORIGIN} in the rpath or lazy binding.
+ */
+ public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) {
+ this.useTestOnlyFlags = useTestOnlyFlags;
+ return this;
+ }
+
+ public Builder setNeedWholeArchive(boolean needWholeArchive) {
+ this.needWholeArchive = needWholeArchive;
+ return this;
+ }
+
+ public Builder setSupportsParamFiles(boolean supportsParamFiles) {
+ this.supportsParamFiles = supportsParamFiles;
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java
new file mode 100644
index 0000000..4f7673e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+/**
+ * A strategy for executing {@link CppLinkAction}s.
+ *
+ * <p>The linker commands, e.g. "ar", are not necessary functional, i.e.
+ * they may mutate the output file rather than overwriting it.
+ * To avoid this, we need to delete the output file before invoking the
+ * command. That must be done by the classes that extend this class.
+ */
+public abstract class LinkStrategy implements CppLinkActionContext {
+ public LinkStrategy() {
+ }
+
+ /** The strategy name, preferably suitable for passing to --link_strategy. */
+ public abstract String linkStrategyName();
+
+ @Override
+ public String strategyLocality(CppLinkAction execOwner) {
+ return linkStrategyName();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java
new file mode 100644
index 0000000..15a8b90
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.Artifact;
+
+/**
+ * Something that appears on the command line of the linker. Since we sometimes expand archive
+ * files to their constituent object files, we need to keep information whether a certain file
+ * contains embedded objects and if so, the list of the object files themselves.
+ */
+public interface LinkerInput {
+ /**
+ * Returns the artifact that is the input of the linker.
+ */
+ Artifact getArtifact();
+
+ /**
+ * Returns the original library to link. If this library is a solib symlink, returns the
+ * artifact the symlink points to, otherwise, the library itself.
+ */
+ Artifact getOriginalLibraryArtifact();
+
+ /**
+ * Whether the input artifact contains object files or is opaque.
+ */
+ boolean containsObjectFiles();
+
+ /**
+ * Returns whether the input artifact is a fake object file or not.
+ */
+ boolean isFake();
+
+ /**
+ * Return the list of object files included in the input artifact, if there are any. It is
+ * legal to call this only when {@link #containsObjectFiles()} returns true.
+ */
+ Iterable<Artifact> getObjectFiles();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java
new file mode 100644
index 0000000..24120ce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java
@@ -0,0 +1,353 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+
+/**
+ * Factory for creating new {@link LinkerInput} objects.
+ */
+public abstract class LinkerInputs {
+ /**
+ * An opaque linker input that is not a library, for example a linker script or an individual
+ * object file.
+ */
+ @ThreadSafety.Immutable
+ public static class SimpleLinkerInput implements LinkerInput {
+ private final Artifact artifact;
+
+ public SimpleLinkerInput(Artifact artifact) {
+ this.artifact = Preconditions.checkNotNull(artifact);
+ }
+
+ @Override
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ @Override
+ public Artifact getOriginalLibraryArtifact() {
+ return artifact;
+ }
+
+ @Override
+ public boolean containsObjectFiles() {
+ return false;
+ }
+
+ @Override
+ public boolean isFake() {
+ return false;
+ }
+
+ @Override
+ public Iterable<Artifact> getObjectFiles() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+
+ if (!(that instanceof SimpleLinkerInput)) {
+ return false;
+ }
+
+ SimpleLinkerInput other = (SimpleLinkerInput) that;
+ return artifact.equals(other.artifact) && isFake() == other.isFake();
+ }
+
+ @Override
+ public int hashCode() {
+ return artifact.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "SimpleLinkerInput(" + artifact.toString() + ")";
+ }
+ }
+
+ /**
+ * A linker input that is a fake object file generated by cc_fake_binary. The contained
+ * artifact must be an object file.
+ */
+ @ThreadSafety.Immutable
+ private static class FakeLinkerInput extends SimpleLinkerInput {
+ private FakeLinkerInput(Artifact artifact) {
+ super(artifact);
+ Preconditions.checkState(Link.OBJECT_FILETYPES.matches(artifact.getFilename()));
+ }
+
+ @Override
+ public boolean isFake() {
+ return true;
+ }
+ }
+
+ /**
+ * A library the user can link to. This is different from a simple linker input in that it also
+ * has a library identifier.
+ */
+ public interface LibraryToLink extends LinkerInput {
+ /**
+ * Returns whether the library is a solib symlink.
+ */
+ boolean isSolibSymlink();
+ }
+
+ /**
+ * This class represents a solib library symlink. Its library identifier is inherited from
+ * the library that it links to.
+ */
+ @ThreadSafety.Immutable
+ public static class SolibLibraryToLink implements LibraryToLink {
+ private final Artifact solibSymlinkArtifact;
+ private final Artifact libraryArtifact;
+
+ private SolibLibraryToLink(Artifact solibSymlinkArtifact, Artifact libraryArtifact) {
+ this.solibSymlinkArtifact = Preconditions.checkNotNull(solibSymlinkArtifact);
+ this.libraryArtifact = libraryArtifact;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("SolibLibraryToLink(%s -> %s",
+ solibSymlinkArtifact.toString(), libraryArtifact.toString());
+ }
+
+ @Override
+ public Artifact getArtifact() {
+ return solibSymlinkArtifact;
+ }
+
+ @Override
+ public boolean containsObjectFiles() {
+ return false;
+ }
+
+ @Override
+ public boolean isFake() {
+ return false;
+ }
+
+ @Override
+ public Iterable<Artifact> getObjectFiles() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Artifact getOriginalLibraryArtifact() {
+ return libraryArtifact;
+ }
+
+ @Override
+ public boolean isSolibSymlink() {
+ return true;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+
+ if (!(that instanceof SolibLibraryToLink)) {
+ return false;
+ }
+
+ SolibLibraryToLink thatSolib = (SolibLibraryToLink) that;
+ return
+ solibSymlinkArtifact.equals(thatSolib.solibSymlinkArtifact) &&
+ libraryArtifact.equals(thatSolib.libraryArtifact);
+ }
+
+ @Override
+ public int hashCode() {
+ return solibSymlinkArtifact.hashCode();
+ }
+ }
+
+ /**
+ * This class represents a library that may contain object files.
+ */
+ @ThreadSafety.Immutable
+ private static class CompoundLibraryToLink implements LibraryToLink {
+ private final Artifact libraryArtifact;
+ private final Iterable<Artifact> objectFiles;
+
+ private CompoundLibraryToLink(Artifact libraryArtifact, Iterable<Artifact> objectFiles) {
+ this.libraryArtifact = Preconditions.checkNotNull(libraryArtifact);
+ this.objectFiles = objectFiles == null ? null : CollectionUtils.makeImmutable(objectFiles);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("CompoundLibraryToLink(%s)", libraryArtifact.toString());
+ }
+
+ @Override
+ public Artifact getArtifact() {
+ return libraryArtifact;
+ }
+
+ @Override
+ public Artifact getOriginalLibraryArtifact() {
+ return libraryArtifact;
+ }
+
+ @Override
+ public boolean containsObjectFiles() {
+ return objectFiles != null;
+ }
+
+ @Override
+ public boolean isFake() {
+ return false;
+ }
+
+ @Override
+ public Iterable<Artifact> getObjectFiles() {
+ Preconditions.checkNotNull(objectFiles);
+ return objectFiles;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+
+ if (!(that instanceof CompoundLibraryToLink)) {
+ return false;
+ }
+
+ return libraryArtifact.equals(((CompoundLibraryToLink) that).libraryArtifact);
+ }
+
+ @Override
+ public int hashCode() {
+ return libraryArtifact.hashCode();
+ }
+
+ @Override
+ public boolean isSolibSymlink() {
+ return false;
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////
+ // Public factory constructors:
+ //////////////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Creates linker input objects for non-library files.
+ */
+ public static Iterable<LinkerInput> simpleLinkerInputs(Iterable<Artifact> input) {
+ return Iterables.transform(input, new Function<Artifact, LinkerInput>() {
+ @Override
+ public LinkerInput apply(Artifact artifact) {
+ return simpleLinkerInput(artifact);
+ }
+ });
+ }
+
+ /**
+ * Creates a linker input for which we do not know what objects files it consists of.
+ */
+ public static LinkerInput simpleLinkerInput(Artifact artifact) {
+ // This precondition check was in place and *most* of the tests passed with them; the only
+ // exception is when you mention a generated .a file in the srcs of a cc_* rule.
+ // Preconditions.checkArgument(!ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType()));
+ return new SimpleLinkerInput(artifact);
+ }
+
+ /**
+ * Creates a fake linker input. The artifact must be an object file.
+ */
+ public static LinkerInput fakeLinkerInput(Artifact artifact) {
+ return new FakeLinkerInput(artifact);
+ }
+
+ /**
+ * Creates input libraries for which we do not know what objects files it consists of.
+ */
+ public static Iterable<LibraryToLink> opaqueLibrariesToLink(Iterable<Artifact> input) {
+ return Iterables.transform(input, new Function<Artifact, LibraryToLink>() {
+ @Override
+ public LibraryToLink apply(Artifact artifact) {
+ return opaqueLibraryToLink(artifact);
+ }
+ });
+ }
+
+ /**
+ * Creates a solib library symlink from the given artifact.
+ */
+ public static LibraryToLink solibLibraryToLink(Artifact solibSymlink, Artifact original) {
+ return new SolibLibraryToLink(solibSymlink, original);
+ }
+
+ /**
+ * Creates an input library for which we do not know what objects files it consists of.
+ */
+ public static LibraryToLink opaqueLibraryToLink(Artifact artifact) {
+ // This precondition check was in place and *most* of the tests passed with them; the only
+ // exception is when you mention a generated .a file in the srcs of a cc_* rule.
+ // It was very useful for proving that this actually works, though.
+ // Preconditions.checkArgument(
+ // !(artifact.getGeneratingAction() instanceof CppLinkAction) ||
+ // !Link.ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType()));
+ return new CompoundLibraryToLink(artifact, null);
+ }
+
+ /**
+ * Creates a library to link with the specified object files.
+ */
+ public static LibraryToLink newInputLibrary(Artifact library, Iterable<Artifact> objectFiles) {
+ return new CompoundLibraryToLink(library, objectFiles);
+ }
+
+ private static final Function<LibraryToLink, Artifact> LIBRARY_TO_NON_SOLIB =
+ new Function<LibraryToLink, Artifact>() {
+ @Override
+ public Artifact apply(LibraryToLink input) {
+ return input.getOriginalLibraryArtifact();
+ }
+ };
+
+ public static Iterable<Artifact> toNonSolibArtifacts(Iterable<LibraryToLink> libraries) {
+ return Iterables.transform(libraries, LIBRARY_TO_NON_SOLIB);
+ }
+
+ /**
+ * Returns the linker input artifacts from a collection of {@link LinkerInput} objects.
+ */
+ public static Iterable<Artifact> toLibraryArtifacts(Iterable<? extends LinkerInput> artifacts) {
+ return Iterables.transform(artifacts, new Function<LinkerInput, Artifact>() {
+ @Override
+ public Artifact apply(LinkerInput input) {
+ return input.getArtifact();
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java
new file mode 100644
index 0000000..8018108
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+/**
+ * This class represents the different linking modes.
+ */
+public enum LinkingMode {
+
+ /**
+ * Everything is linked statically; e.g. {@code gcc -static x.o libfoo.a
+ * libbar.a -lm}. Specified by {@code -static} in linkopts.
+ */
+ FULLY_STATIC,
+
+ /**
+ * Link binaries statically except for system libraries
+ * e.g. {@code gcc x.o libfoo.a libbar.a -lm}. Specified by {@code linkstatic=1}.
+ *
+ * <p>This mode applies to executables.
+ */
+ MOSTLY_STATIC,
+
+ /**
+ * Same as MOSTLY_STATIC, but for shared libraries.
+ */
+ MOSTLY_STATIC_LIBRARIES,
+
+ /**
+ * All libraries are linked dynamically (if a dynamic version is available),
+ * e.g. {@code gcc x.o libfoo.so libbar.so -lm}. Specified by {@code
+ * linkstatic=0}.
+ */
+ DYNAMIC;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java
new file mode 100644
index 0000000..a9ffea8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.Map;
+
+/**
+ * Provides LIPO context information to the LIPO-enabled target configuration.
+ *
+ * <p>This is a rollup of the data collected in the LIPO context collector configuration.
+ * Each target in the LIPO context collector configuration has a {@link TransitiveLipoInfoProvider}
+ * which is used to transitively collect the data, then the {@code cc_binary} that is referred to
+ * in {@code --lipo_context} puts the collected data into {@link LipoContextProvider}, of which
+ * there is only one in any given build.
+ */
+@Immutable
+public final class LipoContextProvider implements TransitiveInfoProvider {
+
+ private final CppCompilationContext cppCompilationContext;
+
+ private final ImmutableMap<Artifact, IncludeScannable> includeScannables;
+ public LipoContextProvider(CppCompilationContext cppCompilationContext,
+ Map<Artifact, IncludeScannable> scannables) {
+ this.cppCompilationContext = cppCompilationContext;
+ this.includeScannables = ImmutableMap.copyOf(scannables);
+ }
+
+ /**
+ * Returns merged compilation context for the whole LIPO subtree.
+ */
+ public CppCompilationContext getLipoContext() {
+ return cppCompilationContext;
+ }
+
+ /**
+ * Returns the map from source artifact to the include scannable object representing
+ * the corresponding FDO source input file.
+ */
+ public ImmutableMap<Artifact, IncludeScannable> getIncludeScannables() {
+ return includeScannables;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java
new file mode 100644
index 0000000..80ee23d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.common.options.OptionsClassProvider;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Run gcc locally by delegating to spawn.
+ */
+@ExecutionStrategy(name = { "local" },
+ contextType = CppCompileActionContext.class)
+public class LocalGccStrategy implements CppCompileActionContext {
+ private static final Reply CANNED_REPLY = new Reply() {
+ @Override
+ public byte[] getContents() {
+ throw new IllegalStateException("Remotely computed data requested for local action");
+ }
+ };
+
+ public LocalGccStrategy(OptionsClassProvider options) {
+ }
+
+ @Override
+ public String strategyLocality() {
+ return "local";
+ }
+
+ public static void updateEnv(CppCompileAction action, Map<String, String> env) {
+ // We cannot locally execute an action that does not expect to output a .d file, since we would
+ // have no way to tell what files that it included were used during compilation.
+ env.put("INTERCEPT_LOCALLY_EXECUTABLE", action.getDotdFile().artifact() == null ? "0" : "1");
+ }
+
+ @Override
+ public boolean needsIncludeScanning() {
+ return false;
+ }
+
+ @Override
+ public Collection<? extends ActionInput> findAdditionalInputs(CppCompileAction action,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public CppCompileActionContext.Reply execWithReply(
+ CppCompileAction action, ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException {
+ Map<String, String> env = new HashMap<>();
+ env.putAll(action.getEnvironment());
+ updateEnv(action, env);
+ actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic())
+ .exec(new BaseSpawn.Local(action.getArgv(), env, action),
+ actionExecutionContext);
+ return null;
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(CppCompileAction action) {
+ return action.estimateResourceConsumptionLocal();
+ }
+
+ @Override
+ public Collection<Artifact> getScannedIncludeFiles(
+ CppCompileAction action, ActionExecutionContext actionExecutionContext) {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Reply getReplyFromException(ExecException e, CppCompileAction action) {
+ return CANNED_REPLY;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java
new file mode 100644
index 0000000..3e7c863
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+
+import java.util.List;
+
+/**
+ * A link strategy that runs the linking step on the local host.
+ *
+ * <p>The set of input files necessary to successfully complete the link is the middleman-expanded
+ * set of the action's dependency inputs (which includes crosstool and libc dependencies, as
+ * defined by {@link com.google.devtools.build.lib.rules.cpp.CppHelper#getCrosstoolInputsForLink
+ * CppHelper.getCrosstoolInputsForLink}).
+ */
+@ExecutionStrategy(contextType = CppLinkActionContext.class, name = { "local" })
+public final class LocalLinkStrategy extends LinkStrategy {
+
+ public LocalLinkStrategy() {
+ }
+
+ @Override
+ public void exec(CppLinkAction action, ActionExecutionContext actionExecutionContext)
+ throws ExecException, ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ List<String> argv =
+ action.prepareCommandLine(executor.getExecRoot(), null);
+ executor.getSpawnActionContext(action.getMnemonic()).exec(
+ new BaseSpawn.Local(argv, ImmutableMap.<String, String>of(), action),
+ actionExecutionContext);
+ }
+
+ @Override
+ public String linkStrategyName() {
+ return "local";
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(CppLinkAction action) {
+ return action.estimateResourceConsumptionLocal();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java
new file mode 100644
index 0000000..87a0712
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/** Parses a single file for its (direct) includes, possibly using a remote service. */
+public interface RemoteIncludeExtractor extends ActionContext {
+ /** Result of checking if this object should be used to parse a given file. */
+ interface RemoteParseData {
+ boolean shouldParseRemotely();
+ }
+
+ /**
+ * Returns whether to use this object to parse the given file for includes. The returned data
+ * should be passed to {@link #extractInclusions} to direct its behavior.
+ */
+ RemoteParseData shouldParseRemotely(Path file);
+
+ /**
+ * Extracts all inclusions from a given source file, possibly using a remote service.
+ *
+ * @param file the file from which to parse and extract inclusions.
+ * @param actionExecutionContext services in the scope of the action. Like the Err/Out stream
+ * outputs.
+ * @param remoteParseData the returned value of {@link #shouldParseRemotely}.
+ * @return a collection of inclusions, normalized to the cache
+ */
+ public Collection<Inclusion> extractInclusions(Artifact file,
+ ActionExecutionContext actionExecutionContext, RemoteParseData remoteParseData)
+ throws IOException, InterruptedException;
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java
new file mode 100644
index 0000000..120ba86
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java
@@ -0,0 +1,234 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+
+/**
+ * Creates mangled symlinks in the solib directory for all shared libraries.
+ * Libraries that have a potential to contain SONAME field rely on the mangled
+ * symlink to the parent directory instead.
+ *
+ * Such symlinks are used by the linker to ensure that all rpath entries can be
+ * specified relative to the $ORIGIN.
+ */
+public final class SolibSymlinkAction extends AbstractAction {
+
+ private final Artifact library;
+ private final Path target;
+ private final Artifact symlink;
+
+ private SolibSymlinkAction(ActionOwner owner, Artifact library, Artifact symlink) {
+ super(owner, ImmutableList.of(library), ImmutableList.of(symlink));
+
+ Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename()));
+ this.library = Preconditions.checkNotNull(library);
+ this.symlink = Preconditions.checkNotNull(symlink);
+ this.target = library.getPath();
+ }
+
+ @Override
+ protected void deleteOutputs(Path execRoot) throws IOException {
+ // Do not delete outputs if action does not intend to do anything.
+ if (target != null) {
+ super.deleteOutputs(execRoot);
+ }
+ }
+
+ @Override
+ public void execute(
+ ActionExecutionContext actionExecutionContext) throws ActionExecutionException {
+ Path mangledPath = symlink.getPath();
+ try {
+ FileSystemUtils.createDirectoryAndParents(mangledPath.getParentDirectory());
+ mangledPath.createSymbolicLink(target);
+ } catch (IOException e) {
+ throw new ActionExecutionException("failed to create _solib symbolic link '"
+ + symlink.prettyPrint() + "' to target '" + target + "'", e, this, false);
+ }
+ }
+
+ @Override
+ public Artifact getPrimaryInput() {
+ return library;
+ }
+
+ @Override
+ public Artifact getPrimaryOutput() {
+ return symlink;
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.0);
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addPath(symlink.getPath());
+ if (target != null) {
+ f.addPath(target);
+ }
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public String getMnemonic() { return "SolibSymlink"; }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "local";
+ }
+
+ @Override
+ protected String getRawProgressMessage() { return null; }
+
+ /**
+ * Replaces shared library artifact with mangled symlink and creates related
+ * symlink action. For artifacts that should retain filename (e.g. libraries
+ * with SONAME tag), link is created to the parent directory instead.
+ *
+ * This action is performed to minimize number of -rpath entries used during
+ * linking process (by essentially "collecting" as many shared libraries as
+ * possible in the single directory), since we will be paying quadratic price
+ * for each additional entry on the -rpath.
+ *
+ * @param ruleContext rule context, that requested symlink.
+ * @param library Shared library artifact that needs to be mangled.
+ * @param preserveName whether to preserve the name of the library
+ * @param prefixConsumer whether to prefix the output artifact name with the label of the
+ * consumer
+ * @return mangled symlink artifact.
+ */
+ public static LibraryToLink getDynamicLibrarySymlink(final RuleContext ruleContext,
+ final Artifact library,
+ boolean preserveName,
+ boolean prefixConsumer,
+ BuildConfiguration configuration) {
+ PathFragment mangledName = getMangledName(
+ ruleContext, library.getRootRelativePath(), preserveName, prefixConsumer,
+ configuration.getFragment(CppConfiguration.class));
+ return getDynamicLibrarySymlinkInternal(ruleContext, library, mangledName, configuration);
+ }
+
+ /**
+ * Version of {@link #getDynamicLibrarySymlink} for the special case of C++ runtime libraries.
+ * These are handled differently than other libraries: neither their names nor directories are
+ * mangled, i.e. libstdc++.so.6 is symlinked from _solib_[arch]/libstdc++.so.6
+ */
+ public static LibraryToLink getCppRuntimeSymlink(RuleContext ruleContext, Artifact library,
+ String solibDirOverride, BuildConfiguration configuration) {
+ PathFragment solibDir = new PathFragment(solibDirOverride != null
+ ? solibDirOverride
+ : configuration.getFragment(CppConfiguration.class).getSolibDirectory());
+ PathFragment symlinkName = solibDir.getRelative(library.getRootRelativePath().getBaseName());
+ return getDynamicLibrarySymlinkInternal(ruleContext, library, symlinkName, configuration);
+ }
+
+ /**
+ * Internal implementation that takes a pre-determined symlink name; supports both the
+ * generic {@link #getDynamicLibrarySymlink} and the specialized {@link #getCppRuntimeSymlink}.
+ */
+ private static LibraryToLink getDynamicLibrarySymlinkInternal(RuleContext ruleContext,
+ Artifact library, PathFragment symlinkName, BuildConfiguration configuration) {
+ Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename()));
+ Preconditions.checkArgument(!library.getRootRelativePath().getSegment(0).startsWith("_solib_"));
+
+ // Ignore libraries that are already represented by the symlinks.
+ Root root = configuration.getBinDirectory();
+ Artifact symlink = ruleContext.getAnalysisEnvironment().getDerivedArtifact(symlinkName, root);
+ ruleContext.registerAction(
+ new SolibSymlinkAction(ruleContext.getActionOwner(), library, symlink));
+ return LinkerInputs.solibLibraryToLink(symlink, library);
+ }
+
+ /**
+ * Returns the name of the symlink that will be created for a library, given
+ * its name.
+ *
+ * @param ruleContext rule context that requests symlink
+ * @param libraryPath the root-relative path of the library
+ * @param preserveName true if filename should be preserved
+ * @param prefixConsumer true if the result should be prefixed with the label of the consumer
+ * @returns root relative path name
+ */
+ public static PathFragment getMangledName(RuleContext ruleContext,
+ PathFragment libraryPath,
+ boolean preserveName,
+ boolean prefixConsumer,
+ CppConfiguration cppConfiguration) {
+ String escapedRulePath = Actions.escapedPath(
+ "_" + ruleContext.getLabel());
+ String soname = getDynamicLibrarySoname(libraryPath, preserveName);
+ PathFragment solibDir = new PathFragment(cppConfiguration.getSolibDirectory());
+ if (preserveName) {
+ String escapedLibraryPath =
+ Actions.escapedPath("_" + libraryPath.getParentDirectory().getPathString());
+ PathFragment mangledDir = solibDir.getRelative(prefixConsumer
+ ? escapedRulePath + "__" + escapedLibraryPath
+ : escapedLibraryPath);
+ return mangledDir.getRelative(soname);
+ } else {
+ return solibDir.getRelative(prefixConsumer
+ ? escapedRulePath + "__" + soname
+ : soname);
+ }
+ }
+
+ /**
+ * Compute the SONAME to use for a dynamic library. This name is basically the
+ * name of the shared library in its final symlinked location.
+ *
+ * @param libraryPath name of the shared library that needs to be mangled
+ * @param preserveName true if filename should be preserved, false - mangled
+ * @return soname to embed in the dynamic library
+ */
+ public static String getDynamicLibrarySoname(PathFragment libraryPath,
+ boolean preserveName) {
+ String mangledName;
+ if (preserveName) {
+ mangledName = libraryPath.getBaseName();
+ } else {
+ mangledName = "lib" + Actions.escapedPath(libraryPath.getPathString());
+ }
+ return mangledName;
+ }
+
+ @Override
+ public boolean shouldReportPathPrefixConflict(Action action) {
+ return false; // Always ignore path prefix conflict for the SolibSymlinkAction.
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java
new file mode 100644
index 0000000..4094124
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A target that can contribute profiling information to LIPO C++ compilations.
+ *
+ * <p>This is used in the LIPO context collector tree to collect data from the transitive
+ * closure of the :lipo_context_collector target. It is eventually passed to the configured
+ * targets in the target configuration through {@link LipoContextProvider}.
+ */
+@Immutable
+public final class TransitiveLipoInfoProvider implements TransitiveInfoProvider {
+ public static final TransitiveLipoInfoProvider EMPTY =
+ new TransitiveLipoInfoProvider(
+ NestedSetBuilder.<IncludeScannable>emptySet(Order.STABLE_ORDER));
+
+ private final NestedSet<IncludeScannable> includeScannables;
+
+ public TransitiveLipoInfoProvider(NestedSet<IncludeScannable> includeScannables) {
+ this.includeScannables = includeScannables;
+ }
+
+ /**
+ * Returns the include scannables in the transitive closure.
+ *
+ * <p>This is used for constructing the path fragment -> include scannable map in the
+ * LIPO-enabled target configuration.
+ */
+ public NestedSet<IncludeScannable> getTransitiveIncludeScannables() {
+ return includeScannables;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java
new file mode 100644
index 0000000..58b3330
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java
@@ -0,0 +1,194 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.cpp;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.BuildInfoHelper;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * An action that creates a C++ header containing the build information in the
+ * form of #define directives.
+ */
+public final class WriteBuildInfoHeaderAction extends AbstractFileWriteAction {
+ private static final String GUID = "b0798174-1352-4a54-854a-9785aaea491b";
+
+ private final ImmutableList<Artifact> valueArtifacts;
+
+ private final boolean writeVolatileInfo;
+ private final boolean writeStableInfo;
+
+ /**
+ * Creates an action that writes a C++ header with the build information.
+ *
+ * <p>It reads the set of build info keys from an action context that is usually contributed
+ * to Bazel by the workspace status module, and the value associated with said keys from the
+ * workspace status files (stable and volatile) written by the workspace status action.
+ *
+ * <p>Without input artifacts this action uses redacted build information.
+ * @param inputs Artifacts that contain build information, or an empty
+ * collection to use redacted build information
+ * @param output the C++ header Artifact created by this action
+ * @param writeVolatileInfo whether to write the volatile part of the build
+ * information to the generated header
+ * @param writeStableInfo whether to write the non-volatile part of the
+ * build information to the generated header
+ */
+ public WriteBuildInfoHeaderAction(Collection<Artifact> inputs,
+ Artifact output, boolean writeVolatileInfo, boolean writeStableInfo) {
+ super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER,
+ inputs, output, /*makeExecutable=*/false);
+ valueArtifacts = ImmutableList.copyOf(inputs);
+ if (!inputs.isEmpty()) {
+ // With non-empty inputs we should not generate both volatile and non-volatile data
+ // in the same header file.
+ Preconditions.checkState(writeVolatileInfo ^ writeStableInfo);
+ }
+ Preconditions.checkState(
+ output.isConstantMetadata() == (writeVolatileInfo && !inputs.isEmpty()));
+
+ this.writeVolatileInfo = writeVolatileInfo;
+ this.writeStableInfo = writeStableInfo;
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor)
+ throws IOException {
+ WorkspaceStatusAction.Context context =
+ executor.getContext(WorkspaceStatusAction.Context.class);
+
+ final Map<String, WorkspaceStatusAction.Key> keys = new LinkedHashMap<>();
+ if (writeVolatileInfo) {
+ keys.putAll(context.getVolatileKeys());
+ }
+
+ if (writeStableInfo) {
+ keys.putAll(context.getStableKeys());
+ }
+
+ final Map<String, String> values = new LinkedHashMap<>();
+ for (Artifact valueFile : valueArtifacts) {
+ values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath()));
+ }
+
+ final boolean redacted = valueArtifacts.isEmpty();
+
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ Writer writer = new OutputStreamWriter(out, UTF_8);
+
+ for (Map.Entry<String, WorkspaceStatusAction.Key> key : keys.entrySet()) {
+ if (!key.getValue().isInLanguage("C++")) {
+ continue;
+ }
+
+ String value = redacted ? key.getValue().getRedactedValue()
+ : values.containsKey(key.getKey()) ? values.get(key.getKey())
+ : key.getValue().getDefaultValue();
+
+ switch (key.getValue().getType()) {
+ case VERBATIM:
+ case INTEGER:
+ break;
+
+ case STRING:
+ value = quote(value);
+ break;
+
+ default:
+ throw new IllegalStateException();
+ }
+ define(writer, key.getKey(), value);
+
+ }
+ writer.flush();
+ }
+ };
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addBoolean(writeStableInfo);
+ f.addBoolean(writeVolatileInfo);
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public boolean executeUnconditionally() {
+ // Note: isVolatile must return true if executeUnconditionally can ever return true
+ // for this instance.
+ return isUnconditional();
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return isUnconditional();
+ }
+
+ private boolean isUnconditional() {
+ // Because of special handling in the MetadataHandler, changed volatile build
+ // information does not trigger relinking of all libraries that have
+ // linkstamps. But we do want to regenerate the header in case libraries are
+ // relinked because of other reasons.
+ // Without inputs the contents of the header do not change, so there is no
+ // point in executing the action again in that case.
+ return writeVolatileInfo && !Iterables.isEmpty(getInputs());
+ }
+
+ /**
+ * Quote a string with double quotes.
+ */
+ private String quote(String string) {
+ // TODO(bazel-team): This is doesn't really work if the string contains quotes. Or a newline.
+ // Or a backslash. Or anything unusual, really.
+ return "\"" + string + "\"";
+ }
+
+ /**
+ * Write a preprocessor define directive to a Writer.
+ */
+ private void define(Writer writer, String name, String value) throws IOException {
+ writer.write("#define ");
+ writer.write(name);
+ writer.write(' ');
+ writer.write(value);
+ writer.write('\n');
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java
new file mode 100644
index 0000000..f3b302f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.extra;
+
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Implementation for the 'action_listener' rule.
+ */
+public final class ActionListener implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ // This rule doesn't produce any output when listed as a build target.
+ // Only when used via the --experimental_action_listener flag,
+ // this rule instructs the build system to add additional outputs.
+
+ List<ExtraActionSpec> extraActions;
+
+ Multimap<String, ExtraActionSpec> extraActionMap;
+
+ Set<String> mnemonics = Sets.newHashSet(
+ ruleContext.attributes().get("mnemonics", Type.STRING_LIST));
+ extraActions = retrieveAndValidateExtraActions(ruleContext);
+ ImmutableSortedKeyListMultimap.Builder<String, ExtraActionSpec>
+ extraActionMapBuilder = ImmutableSortedKeyListMultimap.builder();
+ for (String mnemonic : mnemonics) {
+ extraActionMapBuilder.putAll(mnemonic, extraActions);
+ }
+ extraActionMap = extraActionMapBuilder.build();
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY))
+ .add(ExtraActionMapProvider.class, new ExtraActionMapProvider(extraActionMap))
+ .build();
+ }
+
+ /**
+ * Loads the targets listed in the 'extra_actions' attribute of this rule.
+ * Validates these targets to be extra_actions indeed. And checks if the
+ * blaze version number is in the range of the blaze_version restrictions on the rule.
+ */
+ private List<ExtraActionSpec> retrieveAndValidateExtraActions(RuleContext ruleContext) {
+ List<ExtraActionSpec> extraActions = new ArrayList<>();
+ for (TransitiveInfoCollection prerequisite :
+ ruleContext.getPrerequisites("extra_actions", Mode.TARGET)) {
+ ExtraActionSpec spec = prerequisite.getProvider(ExtraActionSpec.class);
+ if (spec == null) {
+ ruleContext.attributeError("extra_actions", String.format("target %s is not an "
+ + "extra_action rule", prerequisite.getLabel().toString()));
+ } else {
+ extraActions.add(spec);
+ }
+ }
+ if (extraActions.size() == 0) {
+ ruleContext.attributeWarning("extra_actions",
+ "No extra_action is specified for this version of blaze.");
+ }
+ return extraActions;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java
new file mode 100644
index 0000000..2b53a1f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java
@@ -0,0 +1,246 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.extra;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactResolver;
+import com.google.devtools.build.lib.actions.DelegateSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Action used by extra_action rules to create an action that shadows an existing action. Runs a
+ * command-line using {@link SpawnActionContext} for executions.
+ */
+public final class ExtraAction extends SpawnAction {
+ private final Action shadowedAction;
+ private final boolean createDummyOutput;
+ private final Artifact extraActionInfoFile;
+ private final ImmutableMap<PathFragment, Artifact> runfilesManifests;
+ private final ImmutableSet<Artifact> extraActionInputs;
+ private boolean inputsKnown;
+
+ public ExtraAction(ActionOwner owner,
+ ImmutableSet<Artifact> extraActionInputs,
+ Map<PathFragment, Artifact> runfilesManifests,
+ Artifact extraActionInfoFile,
+ Collection<Artifact> outputs,
+ Action shadowedAction,
+ boolean createDummyOutput,
+ CommandLine argv,
+ Map<String, String> environment,
+ String progressMessage,
+ String mnemonic) {
+ super(owner,
+ createInputs(shadowedAction.getInputs(), extraActionInputs),
+ outputs,
+ AbstractAction.DEFAULT_RESOURCE_SET,
+ argv, environment, progressMessage, mnemonic);
+ this.extraActionInfoFile = extraActionInfoFile;
+ this.shadowedAction = shadowedAction;
+ this.runfilesManifests = ImmutableMap.copyOf(runfilesManifests);
+ this.createDummyOutput = createDummyOutput;
+
+ this.extraActionInputs = extraActionInputs;
+ inputsKnown = shadowedAction.inputsKnown();
+ if (createDummyOutput) {
+ // extra action file & dummy file
+ Preconditions.checkArgument(outputs.size() == 2);
+ }
+ }
+
+ @Override
+ public boolean discoversInputs() {
+ return shadowedAction.discoversInputs();
+ }
+
+ @Override
+ public void discoverInputs(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Preconditions.checkState(discoversInputs(), this);
+ if (getContext(actionExecutionContext.getExecutor()).isRemotable(getMnemonic(),
+ isRemotable())) {
+ // If we're running remotely, we need to update our inputs to take account of any additional
+ // inputs the shadowed action may need to do its work.
+ if (shadowedAction.discoversInputs() && shadowedAction instanceof AbstractAction) {
+ updateInputs(
+ ((AbstractAction) shadowedAction).getInputFilesForExtraAction(actionExecutionContext));
+ }
+ }
+ }
+
+ @Override
+ public boolean inputsKnown() {
+ return inputsKnown;
+ }
+
+ private static NestedSet<Artifact> createInputs(
+ Iterable<Artifact> shadowedActionInputs, ImmutableSet<Artifact> extraActionInputs) {
+ NestedSetBuilder<Artifact> result = new NestedSetBuilder<>(Order.STABLE_ORDER);
+ if (shadowedActionInputs instanceof NestedSet) {
+ result.addTransitive((NestedSet<Artifact>) shadowedActionInputs);
+ } else {
+ result.addAll(shadowedActionInputs);
+ }
+ return result.addAll(extraActionInputs).build();
+ }
+
+ private void updateInputs(Iterable<Artifact> shadowedActionInputs) {
+ synchronized (this) {
+ setInputs(createInputs(shadowedActionInputs, extraActionInputs));
+ inputsKnown = true;
+ }
+ }
+
+ @Override
+ public void updateInputsFromCache(ArtifactResolver artifactResolver,
+ Collection<PathFragment> inputPaths) {
+ // We update the inputs directly from the shadowed action.
+ Set<PathFragment> extraActionPathFragments =
+ ImmutableSet.copyOf(Artifact.asPathFragments(extraActionInputs));
+ shadowedAction.updateInputsFromCache(artifactResolver,
+ Collections2.filter(inputPaths, Predicates.in(extraActionPathFragments)));
+ Preconditions.checkState(shadowedAction.inputsKnown(), "%s %s", this, shadowedAction);
+ updateInputs(shadowedAction.getInputs());
+ }
+
+ /**
+ * @InheritDoc
+ *
+ * This method calls in to {@link AbstractAction#getInputFilesForExtraAction} and
+ * {@link Action#getExtraActionInfo} of the action being shadowed from the thread executing this
+ * ExtraAction. It assumes these methods are safe to call from a different thread than the thread
+ * responsible for the execution of the action being shadowed.
+ */
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ // PHASE 1: generate .xa file containing protocol buffer describing
+ // the action being shadowed
+
+ // We call the getExtraActionInfo command only at execution time
+ // so actions can store information only known at execution time into the
+ // protocol buffer.
+ ExtraActionInfo info = shadowedAction.getExtraActionInfo().build();
+ try (OutputStream out = extraActionInfoFile.getPath().getOutputStream()) {
+ info.writeTo(out);
+ } catch (IOException e) {
+ throw new ActionExecutionException(e.getMessage(), e, this, false);
+ }
+ Executor executor = actionExecutionContext.getExecutor();
+
+ // PHASE 2: execution of extra_action.
+
+ if (getContext(executor).isRemotable(getMnemonic(), isRemotable())) {
+ try {
+ getContext(executor).exec(getExtraActionSpawn(), actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException(this);
+ }
+ } else {
+ super.execute(actionExecutionContext);
+ }
+
+ // PHASE 3: create dummy output.
+ // If the user didn't specify output, we need to create dummy output
+ // to make blaze schedule this action.
+ if (createDummyOutput) {
+ for (Artifact output : getOutputs()) {
+ try {
+ FileSystemUtils.touchFile(output.getPath());
+ } catch (IOException e) {
+ throw new ActionExecutionException(e.getMessage(), e, this, false);
+ }
+ }
+ }
+ synchronized (this) {
+ inputsKnown = true;
+ }
+ }
+
+ /**
+ * The spawn command for ExtraAction needs to be slightly modified from
+ * regular SpawnActions:
+ * -the extraActionInfo file needs to be added to the list of inputs.
+ * -the extraActionInfo file that is an output file of this task is created
+ * before the SpawnAction so should not be listed as one of its outputs.
+ */
+ // TODO(bazel-team): Add more tests that execute this code path!
+ private Spawn getExtraActionSpawn() {
+ final Spawn base = super.getSpawn();
+ return new DelegateSpawn(base) {
+ @Override public Iterable<? extends ActionInput> getInputFiles() {
+ return Iterables.concat(base.getInputFiles(), ImmutableSet.of(extraActionInfoFile));
+ }
+
+ @Override public List<? extends ActionInput> getOutputFiles() {
+ return Lists.newArrayList(
+ Iterables.filter(getOutputs(), new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact item) {
+ return item != extraActionInfoFile;
+ }
+ }));
+ }
+
+ @Override public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() {
+ ImmutableMap.Builder<PathFragment, Artifact> builder = ImmutableMap.builder();
+ builder.putAll(super.getRunfilesManifests());
+ builder.putAll(runfilesManifests);
+ return builder.build();
+ }
+
+ @Override public String getMnemonic() { return ExtraAction.this.getMnemonic(); }
+ };
+ }
+
+ /**
+ * Returns the action this extra action is 'shadowing'.
+ */
+ public Action getShadowedAction() {
+ return shadowedAction;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java
new file mode 100644
index 0000000..8040ee0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.extra;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.CommandHelper;
+import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.MakeVariableExpander;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.List;
+
+/**
+ * Factory for 'extra_action'.
+ */
+public final class ExtraActionFactory implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext context) {
+ // This rule doesn't produce any output when listed as a build target.
+ // Only when used via the --experimental_action_listener flag,
+ // this rule instructs the build system to add additional outputs.
+ List<Artifact> resolvedData = Lists.newArrayList();
+
+ Iterable<FilesToRunProvider> tools =
+ context.getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class);
+ CommandHelper commandHelper = new CommandHelper(
+ context, tools, ImmutableMap.<Label, Iterable<Artifact>>of());
+
+ resolvedData.addAll(context.getPrerequisiteArtifacts("data", Mode.DATA).list());
+ List<String>outputTemplates =
+ context.attributes().get("out_templates", Type.STRING_LIST);
+
+ String command = commandHelper.resolveCommandAndExpandLabels(false, true);
+ // This is a bit of a hack. We want to run the MakeVariableExpander first, so we expand $ on
+ // variables that are expanded below with $$, which gets reverted to $ by the
+ // MakeVariableExpander. This allows us to expand package-specific make variables in the
+ // package where the extra action is defined, and then later replace the owner-specific make
+ // variables when the extra action is instantiated.
+ command = command.replace("$(EXTRA_ACTION_FILE)", "$$(EXTRA_ACTION_FILE)");
+ command = command.replace("$(ACTION_ID)", "$$(ACTION_ID)");
+ command = command.replace("$(OWNER_LABEL_DIGEST)", "$$(OWNER_LABEL_DIGEST)");
+ command = command.replace("$(output ", "$$(output ");
+ try {
+ command = MakeVariableExpander.expand(
+ command, new ConfigurationMakeVariableContext(
+ context.getTarget().getPackage(), context.getConfiguration()));
+ } catch (MakeVariableExpander.ExpansionException e) {
+ context.ruleError(String.format("Unable to expand make variables: %s",
+ e.getMessage()));
+ }
+
+ boolean requiresActionOutput =
+ context.attributes().get("requires_action_output", Type.BOOLEAN);
+
+ ExtraActionSpec spec = new ExtraActionSpec(
+ commandHelper.getResolvedTools(),
+ commandHelper.getRemoteRunfileManifestMap(),
+ resolvedData,
+ outputTemplates,
+ command,
+ context.getLabel(),
+ requiresActionOutput);
+
+ return new RuleConfiguredTargetBuilder(context)
+ .addProvider(ExtraActionSpec.class, spec)
+ .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java
new file mode 100644
index 0000000..ffeebe0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.extra;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Provides an action type -> set of extra actions to run map.
+ */
+@Immutable
+public final class ExtraActionMapProvider implements TransitiveInfoProvider {
+ private final ImmutableMultimap<String, ExtraActionSpec> extraActionMap;
+
+ public ExtraActionMapProvider(Multimap<String, ExtraActionSpec> extraActionMap) {
+ this.extraActionMap = ImmutableMultimap.copyOf(extraActionMap);
+ }
+
+ /**
+ * Returns the extra action map.
+ */
+ public ImmutableMultimap<String, ExtraActionSpec> getExtraActionMap() {
+ return extraActionMap;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java
new file mode 100644
index 0000000..40a063e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java
@@ -0,0 +1,220 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.extra;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.CommandHelper;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The specification for a particular extra action type.
+ */
+@Immutable
+public final class ExtraActionSpec implements TransitiveInfoProvider {
+ private final ImmutableList<Artifact> resolvedTools;
+ private final ImmutableMap<PathFragment, Artifact> manifests;
+ private final ImmutableList<Artifact> resolvedData;
+ private final ImmutableList<String> outputTemplates;
+ private final String command;
+ private final boolean requiresActionOutput;
+ private final Label label;
+
+ ExtraActionSpec(
+ Iterable<Artifact> resolvedTools,
+ Map<PathFragment, Artifact> manifests,
+ Iterable<Artifact> resolvedData,
+ Iterable<String> outputTemplates,
+ String command,
+ Label label,
+ boolean requiresActionOutput) {
+ this.resolvedTools = ImmutableList.copyOf(resolvedTools);
+ this.manifests = ImmutableMap.copyOf(manifests);
+ this.resolvedData = ImmutableList.copyOf(resolvedData);
+ this.outputTemplates = ImmutableList.copyOf(outputTemplates);
+ this.command = command;
+ this.label = label;
+ this.requiresActionOutput = requiresActionOutput;
+ }
+
+ public Label getLabel() {
+ return label;
+ }
+
+ /**
+ * Adds an extra_action to the action graph based on the action to shadow.
+ */
+ public Collection<Artifact> addExtraAction(RuleContext owningRule,
+ Action actionToShadow) {
+ Collection<Artifact> extraActionOutputs = new LinkedHashSet<>();
+ ImmutableSet.Builder<Artifact> extraActionInputs = ImmutableSet.builder();
+
+ ActionOwner owner = actionToShadow.getOwner();
+ Label ownerLabel = owner.getLabel();
+ if (requiresActionOutput) {
+ extraActionInputs.addAll(actionToShadow.getOutputs());
+ }
+ extraActionInputs.addAll(resolvedTools);
+ extraActionInputs.addAll(resolvedData);
+
+ boolean createDummyOutput = false;
+
+ for (String outputTemplate : outputTemplates) {
+ // We create output for the extra_action based on the 'out_template' attribute.
+ // See {link #getExtraActionOutputArtifact} for supported variables.
+ extraActionOutputs.add(getExtraActionOutputArtifact(owningRule, actionToShadow,
+ owner, outputTemplate));
+ }
+ // extra_action has no output, we need to create some dummy output to keep the build up-to-date.
+ if (extraActionOutputs.size() == 0) {
+ createDummyOutput = true;
+ extraActionOutputs.add(getExtraActionOutputArtifact(owningRule, actionToShadow,
+ owner, "$(ACTION_ID).dummy"));
+ }
+
+ // We generate a file containing a protocol buffer describing the action that is being shadowed.
+ // It is up to each action being shadowed to decide what contents to store here.
+ Artifact extraActionInfoFile = getExtraActionOutputArtifact(owningRule, actionToShadow,
+ owner, "$(ACTION_ID).xa");
+ extraActionOutputs.add(extraActionInfoFile);
+
+ // Expand extra_action specific variables from the provided command-line.
+ // See {@link #createExpandedCommand} for list of supported variables.
+ String command = createExpandedCommand(owningRule, actionToShadow, owner, extraActionInfoFile);
+
+ Map<String, String> env = owningRule.getConfiguration().getDefaultShellEnvironment();
+
+ List<String> argv = CommandHelper.buildCommandLine(owningRule,
+ command, extraActionInputs, ".extra_action_script.sh");
+
+ String commandMessage = String.format("Executing extra_action %s on %s", label, ownerLabel);
+ owningRule.registerAction(new ExtraAction(
+ actionToShadow.getOwner(),
+ extraActionInputs.build(),
+ manifests,
+ extraActionInfoFile,
+ extraActionOutputs,
+ actionToShadow,
+ createDummyOutput,
+ CommandLine.of(argv, false),
+ env,
+ commandMessage,
+ label.getName()));
+
+ return extraActionOutputs;
+ }
+
+ /**
+ * Expand extra_action specific variables:
+ * $(EXTRA_ACTION_FILE): expands to a path of the file containing a protocol buffer
+ * describing the action being shadowed.
+ * $(output <out_template>): expands the output template to the execPath of the file.
+ * e.g. $(output $(ACTION_ID).out) ->
+ * <build_path>/extra_actions/bar/baz/devtools/build/test_A41234.out
+ */
+ private String createExpandedCommand(RuleContext owningRule,
+ Action action, ActionOwner owner, Artifact extraActionInfoFile) {
+ String realCommand = command.replace(
+ "$(EXTRA_ACTION_FILE)", extraActionInfoFile.getExecPathString());
+
+ for (String outputTemplate : outputTemplates) {
+ String outFile = getExtraActionOutputArtifact(owningRule, action, owner, outputTemplate)
+ .getExecPathString();
+ realCommand = realCommand.replace("$(output " + outputTemplate + ")", outFile);
+ }
+ return realCommand;
+ }
+
+ /**
+ * Creates an output artifact for the extra_action based on the output_template.
+ * The path will be in the following form:
+ * <output dir>/<target-configuration-specific-path>/extra_actions/<extra_action_label>/ +
+ * <configured_target_label>/<expanded_template>
+ *
+ * The template can use the following variables:
+ * $(ACTION_ID): a unique id for the extra_action.
+ *
+ * Sample:
+ * extra_action: foo/bar:extra
+ * template: $(ACTION_ID).analysis
+ * target: foo/bar:main
+ * expands to: output/configuration/extra_actions/\
+ * foo/bar/extra/foo/bar/4683026f7ac1dd1a873ccc8c3d764132.analysis
+ */
+ private Artifact getExtraActionOutputArtifact(RuleContext owningRule, Action action,
+ ActionOwner owner, String template) {
+ String actionId = getActionId(owner, action);
+
+ template = template.replace("$(ACTION_ID)", actionId);
+ template = template.replace("$(OWNER_LABEL_DIGEST)", getOwnerDigest(owner));
+
+ PathFragment rootRelativePath = getRootRelativePath(template, owner);
+ return owningRule.getAnalysisEnvironment().getDerivedArtifact(rootRelativePath,
+ owningRule.getConfiguration().getOutputDirectory());
+ }
+
+ private PathFragment getRootRelativePath(String template, ActionOwner owner) {
+ PathFragment extraActionPackageFragment = label.getPackageFragment();
+ PathFragment extraActionPrefix = extraActionPackageFragment.getRelative(label.getName());
+
+ PathFragment ownerFragment = owner.getLabel().getPackageFragment();
+ return new PathFragment("extra_actions").getRelative(extraActionPrefix)
+ .getRelative(ownerFragment).getRelative(template);
+ }
+
+ /**
+ * Calculates a digest representing the owner label. We use the digest instead of the
+ * original value as the original value might lead to a filename that is too long.
+ * By using a digest, tools can deterministically find all extra_action outputs for a given
+ * target, without having to open every file in the package.
+ */
+ private static String getOwnerDigest(ActionOwner owner) {
+ Fingerprint f = new Fingerprint();
+ f.addString(owner.getLabel().toString());
+ return f.hexDigestAndReset();
+ }
+
+ /**
+ * Creates a unique id for the action shadowed by this extra_action.
+ *
+ * We need to have a unique id for the extra_action to use. We build this
+ * from the owner's label and the shadowed action id (which is only
+ * guaranteed to be unique per target). Together with the subfolder
+ * matching the original target's package name, we believe this is enough
+ * of a uniqueness guarantee.
+ */
+ @VisibleForTesting
+ public static String getActionId(ActionOwner owner, Action action) {
+ Fingerprint f = new Fingerprint();
+ f.addString(owner.getLabel().toString());
+ f.addString(action.getKey());
+ return f.hexDigestAndReset();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java b/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java
new file mode 100644
index 0000000..cb297d8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.filegroup;
+
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.CompilationHelper;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.MiddlemanProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Iterator;
+
+/**
+ * ConfiguredTarget for "filegroup".
+ */
+public class Filegroup implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ NestedSet<Artifact> filesToBuild = NestedSetBuilder.wrap(Order.STABLE_ORDER,
+ ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list());
+ NestedSet<Artifact> middleman = CompilationHelper.getAggregatingMiddleman(
+ ruleContext, Actions.escapeLabel(ruleContext.getLabel()), filesToBuild);
+
+ InstrumentedFilesCollector instrumentedFilesCollector =
+ new InstrumentedFilesCollector(ruleContext,
+ // what do *we* know about whether this is a source file or not
+ new InstrumentationSpec(FileTypeSet.ANY_FILE, "srcs", "deps", "data"),
+ InstrumentedFilesCollector.NO_METADATA_COLLECTOR, filesToBuild);
+
+ RunfilesProvider runfilesProvider = RunfilesProvider.withData(
+ new Runfiles.Builder()
+ .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES)
+ .build(),
+ // If you're visiting a filegroup as data, then we also visit its data as data.
+ new Runfiles.Builder().addTransitiveArtifacts(filesToBuild)
+ .addDataDeps(ruleContext).build());
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .add(RunfilesProvider.class, runfilesProvider)
+ .setFilesToBuild(filesToBuild)
+ .setRunfilesSupport(null, getExecutable(filesToBuild))
+ .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl(
+ instrumentedFilesCollector))
+ .add(MiddlemanProvider.class, new MiddlemanProvider(middleman))
+ .add(FilegroupPathProvider.class,
+ new FilegroupPathProvider(getFilegroupPath(ruleContext)))
+ .build();
+ }
+
+ /*
+ * Returns the single executable output of this filegroup. Returns
+ * {@code null} if there are multiple outputs or the single output is not
+ * considered an executable.
+ */
+ private Artifact getExecutable(NestedSet<Artifact> filesToBuild) {
+ Iterator<Artifact> it = filesToBuild.iterator();
+ if (it.hasNext()) {
+ Artifact out = it.next();
+ if (!it.hasNext()) {
+ return out;
+ }
+ }
+ return null;
+ }
+
+ private PathFragment getFilegroupPath(RuleContext ruleContext) {
+ String attr = ruleContext.attributes().get("path", Type.STRING);
+ if (attr.isEmpty()) {
+ return PathFragment.EMPTY_FRAGMENT;
+ } else {
+ return ruleContext.getLabel().getPackageFragment().getRelative(attr);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java b/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java
new file mode 100644
index 0000000..370be07
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.filegroup;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A transitive info provider for dependent targets to query {@code path} attributes.
+ */
+@Immutable
+public final class FilegroupPathProvider implements TransitiveInfoProvider {
+ private final PathFragment pathFragment;
+
+ public FilegroupPathProvider(PathFragment pathFragment) {
+ this.pathFragment = pathFragment;
+ }
+
+ /**
+ * Returns the value of the {@code path} attribute or the empty fragment if it is not present.
+ */
+ public PathFragment getFilegroupPath() {
+ return pathFragment;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java
new file mode 100644
index 0000000..056b61e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.fileset;
+
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * Action context for fileset collection actions.
+ */
+public interface FilesetActionContext extends ActionContext {
+
+ /**
+ * Returns a thread pool for fileset symlink tree creation.
+ */
+ ThreadPoolExecutor getFilesetPool();
+
+ /**
+ * Returns the name of the workspace the build is run in.
+ */
+ String getWorkspaceName();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java
new file mode 100644
index 0000000..9c03129
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java
@@ -0,0 +1,101 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.fileset;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BlazeExecutor;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.events.Reporter;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Context for Fileset manifest actions. It currently only provides a ThreadPoolExecutor.
+ *
+ * <p>Fileset is a legacy, google-internal mechanism to make parts of the source tree appear as a
+ * tree in the output directory.
+ */
+@ExecutionStrategy(contextType = FilesetActionContext.class)
+public final class FilesetActionContextImpl implements FilesetActionContext {
+ // TODO(bazel-team): it would be nice if this weren't shipped in Bazel at all.
+
+ /**
+ * Factory class.
+ */
+ public static class Provider implements ActionContextProvider {
+ private FilesetActionContextImpl impl;
+ private final Reporter reporter;
+ private final ThreadPoolExecutor filesetPool;
+
+ public Provider(Reporter reporter, String workspaceName) {
+ this.reporter = reporter;
+ this.filesetPool = newFilesetPool(100);
+ this.impl = new FilesetActionContextImpl(filesetPool, workspaceName);
+ }
+
+ private static ThreadPoolExecutor newFilesetPool(int threads) {
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 3L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>());
+ // Do not consume threads when not in use.
+ pool.allowCoreThreadTimeOut(true);
+ pool.setThreadFactory(new ThreadFactoryBuilder().setNameFormat("Fileset worker %d").build());
+ return pool;
+ }
+
+ @Override
+ public Iterable<ActionContext> getActionContexts() {
+ return ImmutableList.<ActionContext>of(impl);
+ }
+
+ @Override
+ public void executorCreated(Iterable<ActionContext> usedStrategies) {}
+
+ @Override
+ public void executionPhaseStarting(
+ ActionInputFileCache actionInputFileCache,
+ ActionGraph actionGraph,
+ Iterable<Artifact> topLevelArtifacts) {}
+
+ @Override
+ public void executionPhaseEnding() {
+ BlazeExecutor.shutdownHelperPool(reporter, filesetPool, "Fileset");
+ }
+ }
+
+ private final ThreadPoolExecutor filesetPool;
+ private final String workspaceName;
+
+ private FilesetActionContextImpl(ThreadPoolExecutor filesetPool, String workspaceName) {
+ this.filesetPool = filesetPool;
+ this.workspaceName = workspaceName;
+ }
+
+ @Override
+ public ThreadPoolExecutor getFilesetPool() {
+ return filesetPool;
+ }
+
+ @Override
+ public String getWorkspaceName() {
+ return workspaceName;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java
new file mode 100644
index 0000000..d523edc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java
@@ -0,0 +1,218 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.fileset;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.syntax.FilesetEntry;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * FilesetLinks manages the set of links added to a Fileset. If two links conflict, the first wins.
+ *
+ * <p>FilesetLinks is FileSystem-aware. For example, if you first create a link
+ * (a/b/c, foo), a subsequent call to link (a/b, bar) is a no-op.
+ * This is because the first link requires us to create a directory "a/b",
+ * so "a/b" cannot also link to "bar".
+ *
+ * <p>TODO(bazel-team): Consider warning if we have such a conflict; we don't do that currently.
+ */
+public interface FilesetLinks {
+
+ /**
+ * Get late directory information for a source.
+ *
+ * @param src The source to search for.
+ * @return The late directory info, or null if none was found.
+ */
+ public LateDirectoryInfo getLateDirectoryInfo(PathFragment src);
+
+ public boolean putLateDirectoryInfo(PathFragment src, LateDirectoryInfo lateDir);
+
+ /**
+ * Add specified file as a symlink.
+ *
+ * The behavior when the target file is a symlink depends on the
+ * symlinkBehavior parameter (see comments for FilesetEntry.SymlinkBehavior).
+ *
+ * @param src The root-relative symlink path.
+ * @param target The symlink target.
+ */
+ public void addFile(PathFragment src, Path target, String metadata,
+ FilesetEntry.SymlinkBehavior symlinkBehavior)
+ throws IOException;
+
+ /**
+ * Add all late directories as symlinks. This function should be called only
+ * after all recursions have completed, but before getData or getSymlinks are
+ * called.
+ */
+ public void addLateDirectories() throws IOException;
+
+ /**
+ * Adds the given symlink to the tree.
+ *
+ * @param fromFrag The root-relative symlink path.
+ * @param toFrag The symlink target.
+ * @return true iff the symlink was added.
+ */
+ public boolean addLink(PathFragment fromFrag, PathFragment toFrag, String dataVal);
+
+ /**
+ * @return The unmodifiable map of symlinks.
+ */
+ public Map<PathFragment, PathFragment> getSymlinks();
+
+ /**
+ * @return The unmodifiable map of metadata.
+ */
+ public Map<PathFragment, String> getData();
+
+ /**
+ * A data structure for containing all the information about a directory that
+ * is late-added. This means the directory is skipped unless we need to
+ * recurse into it later. If the directory is never recursed into, we will
+ * create a symlink directly to it.
+ */
+ public static final class LateDirectoryInfo {
+ // The constructors are private. Use the factory functions below to create
+ // instances of this class.
+
+ /** Construct a stub LateDirectoryInfo object. */
+ private LateDirectoryInfo() {
+ this.added = new AtomicBoolean(true);
+
+ // Shut up the compiler.
+ this.target = null;
+ this.src = null;
+ this.pkgMode = SubpackageMode.IGNORE;
+ this.metadata = null;
+ this.symlinkBehavior = null;
+ }
+
+ /** Construct a normal LateDirectoryInfo object. */
+ private LateDirectoryInfo(Path target, PathFragment src, SubpackageMode pkgMode,
+ String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) {
+ this.target = target;
+ this.src = src;
+ this.pkgMode = pkgMode;
+ this.metadata = metadata;
+ this.symlinkBehavior = symlinkBehavior;
+ this.added = new AtomicBoolean(false);
+ }
+
+ /** @return The target path for the symlink. The target is the referent. */
+ public Path getTarget() {
+ return target;
+ }
+
+ /**
+ * @return The source path for the symlink. The source is the place the
+ * symlink will be written. */
+ public PathFragment getSrc() {
+ return src;
+ }
+
+ /**
+ * @return Whether we should show a warning if we cross a package boundary
+ * when recursing into this directory.
+ */
+ public SubpackageMode getPkgMode() {
+ return pkgMode;
+ }
+
+ /**
+ * @return The metadata we will write into the manifest if we symlink to
+ * this directory.
+ */
+ public String getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return How to perform the symlinking if the source happens to be a
+ * symlink itself.
+ */
+ public FilesetEntry.SymlinkBehavior getTargetSymlinkBehavior() {
+ return Preconditions.checkNotNull(symlinkBehavior,
+ "should not call this method on stub instances");
+ }
+
+ /**
+ * Atomically checks if the late directory has been added to the manifest
+ * and marks it as added. If this function returns true, it is the
+ * responsibility of the caller to recurse into the late directory.
+ * Otherwise, some other caller has already, or is in the process of
+ * recursing into it.
+ * @return Whether the caller should recurse into the late directory.
+ */
+ public boolean shouldAdd() {
+ return !added.getAndSet(true);
+ }
+
+ /**
+ * Create a stub LateDirectoryInfo that is already marked as added.
+ * @return The new LateDirectoryInfo object.
+ */
+ public static LateDirectoryInfo createStub() {
+ return new LateDirectoryInfo();
+ }
+
+ /**
+ * Create a LateDirectoryInfo object with the specified attributes.
+ * @param target The directory to which the symlinks will refer.
+ * @param src The location at which to create the symlink.
+ * @param pkgMode How to handle recursion into another package.
+ * @param metadata The metadata for the directory to write into the
+ * manifest if we symlink it directly.
+ * @return The new LateDirectoryInfo object.
+ */
+ public static LateDirectoryInfo create(Path target, PathFragment src, SubpackageMode pkgMode,
+ String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) {
+ return new LateDirectoryInfo(target, src, pkgMode, metadata, symlinkBehavior);
+ }
+
+ /**
+ * The target directory to which the symlink will point.
+ * Note this is a real path on the filesystem and can't be compared to src
+ * or any source (key) in the links map.
+ */
+ private final Path target;
+
+ /** The referent of the symlink. */
+ private final PathFragment src;
+
+ /** Whether to show cross package boundary warnings / errors. */
+ private final SubpackageMode pkgMode;
+
+ /** The metadata to write into the manifest file. */
+ private final String metadata;
+
+ /** How to perform the symlinking if the source happens to be a symlink itself. */
+ private final FilesetEntry.SymlinkBehavior symlinkBehavior;
+
+ /** Whether the directory has already been recursed into. */
+ private final AtomicBoolean added;
+ }
+
+ /** How to handle filesets that cross subpackages. */
+ public static enum SubpackageMode {
+ ERROR, WARNING, IGNORE;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java
new file mode 100644
index 0000000..6b70aab
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.fileset;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Information needed by a Fileset to do the right thing when it depends on another Fileset.
+ */
+public interface FilesetProvider extends TransitiveInfoProvider {
+ Artifact getFilesetInputManifest();
+ PathFragment getFilesetLinkDir();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java
new file mode 100644
index 0000000..db13bdb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.fileset;
+
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.io.IOException;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * An interface which contains a method to compute a symlink mapping.
+ */
+public interface SymlinkTraversal {
+
+ /**
+ * Adds symlinks to the given FilesetLinks.
+ *
+ * @throws IOException if a filesystem operation fails.
+ * @throws InterruptedException if the traversal is interrupted.
+ */
+ void addSymlinks(EventHandler eventHandler, FilesetLinks links, ThreadPoolExecutor filesetPool)
+ throws IOException, InterruptedException;
+
+ /**
+ * Add the traversal's fingerprint to the given Fingerprint.
+ * @param fp the Fingerprint to combine.
+ */
+ void fingerprint(Fingerprint fp);
+
+ /**
+ * @return true iff this traversal must be executed unconditionally.
+ */
+ boolean executeUnconditionally();
+
+ /**
+ * Returns true if it's ever possible that {@link #executeUnconditionally}
+ * could evaluate to true during the lifetime of this instance, false
+ * otherwise.
+ */
+ boolean isVolatile();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java
new file mode 100644
index 0000000..69dc41b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java
@@ -0,0 +1,237 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+
+/**
+ * A helper class for compiling Java targets. This helper does not rely on the
+ * presence of rule-specific attributes.
+ */
+public class BaseJavaCompilationHelper {
+ /**
+ * Also see DeployArchiveBuilder.SINGLEJAR_MAX_MEMORY. We don't expect that anyone has more
+ * than ~500,000 files in a source jar, so 256 MB of memory should be plenty.
+ */
+ private static final String SINGLEJAR_MAX_MEMORY = "-Xmx256m";
+
+ private final RuleContext ruleContext;
+
+ public BaseJavaCompilationHelper(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ /**
+ * Returns the artifacts required to invoke {@code javahome} relative binary
+ * in the action.
+ */
+ public static NestedSet<Artifact> getHostJavabaseInputs(RuleContext ruleContext) {
+ // This must have a different name than above, because the middleman creation uses the rule's
+ // configuration, although it should use the host configuration.
+ return AnalysisUtils.getMiddlemanFor(ruleContext, ":host_jdk");
+ }
+
+ private static final ImmutableList<String> SOURCE_JAR_COMMAND_LINE_ARGS = ImmutableList.of(
+ "--compression",
+ "--normalize",
+ "--exclude_build_data",
+ "--warn_duplicate_resources");
+
+ private CommandLine sourceJarCommandLine(JavaSemantics semantics, Artifact outputJar,
+ Iterable<Artifact> resources, Iterable<Artifact> resourceJars) {
+ CustomCommandLine.Builder args = CustomCommandLine.builder();
+ args.addExecPath("--output", outputJar);
+ args.add(SOURCE_JAR_COMMAND_LINE_ARGS);
+ args.addExecPaths("--sources", resourceJars);
+ args.add("--resources");
+ for (Artifact resource : resources) {
+ args.addPaths("%s:%s", resource.getExecPath(),
+ semantics.getJavaResourcePath(resource.getRootRelativePath()));
+ }
+ return args.build();
+ }
+
+ /**
+ * Creates an Action that packages files into a Jar file.
+ *
+ * @param semantics delegate semantics for java.
+ * @param resources the resources to put into the Jar.
+ * @param resourceJars the resource jars to merge into the jar
+ * @param outputJar the Jar to create
+ */
+ public void createSourceJarAction(JavaSemantics semantics, Collection<Artifact> resources,
+ Collection<Artifact> resourceJars, Artifact outputJar) {
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .addOutput(outputJar)
+ .addInputs(resources)
+ .addInputs(resourceJars)
+ .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext))
+ .setJarExecutable(
+ ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(),
+ ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST),
+ ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY))
+ .setCommandLine(sourceJarCommandLine(semantics, outputJar, resources, resourceJars))
+ .useParameterFile(ParameterFileType.SHELL_QUOTED)
+ .setProgressMessage("Building source jar " + outputJar.prettyPrint())
+ .setMnemonic("JavaSourceJar")
+ .build(ruleContext));
+ }
+
+ /**
+ * Returns the langtools jar Artifact.
+ */
+ protected final Artifact getLangtoolsJar() {
+ return ruleContext.getHostPrerequisiteArtifact("$java_langtools");
+ }
+
+ /**
+ * Returns the JavaBuilder jar Artifact.
+ */
+ protected final Artifact getJavaBuilderJar() {
+ return ruleContext.getPrerequisiteArtifact("$javabuilder", Mode.HOST);
+ }
+
+ /**
+ * Returns the javac bootclasspath artifacts.
+ */
+ protected final Iterable<Artifact> getBootClasspath() {
+ return ruleContext.getPrerequisiteArtifacts("$javac_bootclasspath", Mode.HOST).list();
+ }
+
+ private Artifact getIjarArtifact(Artifact jar, boolean addPrefix) {
+ if (addPrefix) {
+ PathFragment ruleBase = ruleContext.getLabel().getPackageFragment().getRelative(
+ ruleContext.getLabel().getName()).getRelative("_ijars");
+ PathFragment artifactDirFragment = jar.getRootRelativePath().getParentDirectory();
+ String ijarBasename = FileSystemUtils.removeExtension(jar.getFilename()) + "-ijar.jar";
+ return getAnalysisEnvironment().getDerivedArtifact(
+ ruleBase.getRelative(artifactDirFragment).getRelative(ijarBasename),
+ getConfiguration().getGenfilesDirectory());
+ } else {
+ return derivedArtifact(jar, "", "-ijar.jar");
+ }
+ }
+
+ /**
+ * Creates the Action that creates ijars from Jar files.
+ *
+ * @param inputJar the Jar to create the ijar for
+ * @param addPrefix whether to prefix the path of the generated ijar with the package and
+ * name of the current rule
+ * @return the Artifact to create with the Action
+ */
+ protected Artifact createIjarAction(final Artifact inputJar, boolean addPrefix) {
+ Artifact interfaceJar = getIjarArtifact(inputJar, addPrefix);
+ final FilesToRunProvider ijarTarget =
+ ruleContext.getExecutablePrerequisite("$ijar", Mode.HOST);
+ if (!ruleContext.hasErrors()) {
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .addInput(inputJar)
+ .addOutput(interfaceJar)
+ .setExecutable(ijarTarget)
+ .addArgument(inputJar.getExecPathString())
+ .addArgument(interfaceJar.getExecPathString())
+ .setProgressMessage("Extracting interface " + ruleContext.getLabel())
+ .setMnemonic("JavaIjar")
+ .build(ruleContext));
+ }
+ return interfaceJar;
+ }
+
+ protected final JavaCompileAction.Builder createJavaCompileActionBuilder(
+ JavaSemantics semantics) {
+ JavaCompileAction.Builder builder = new JavaCompileAction.Builder(ruleContext, semantics);
+ builder.setJavaExecutable(
+ ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable());
+ builder.setJavaBaseInputs(BaseJavaCompilationHelper.getHostJavabaseInputs(ruleContext));
+ return builder;
+ }
+
+ public RuleContext getRuleContext() {
+ return ruleContext;
+ }
+
+ public AnalysisEnvironment getAnalysisEnvironment() {
+ return ruleContext.getAnalysisEnvironment();
+ }
+
+ protected BuildConfiguration getConfiguration() {
+ return ruleContext.getConfiguration();
+ }
+
+ protected JavaConfiguration getJavaConfiguration() {
+ return ruleContext.getFragment(JavaConfiguration.class);
+ }
+
+ protected PathFragment outputDir(Artifact outputJar) {
+ return workDir(outputJar, "_files");
+ }
+
+ /**
+ * Produces a derived directory where source files generated by annotation processors should be
+ * stored.
+ */
+ protected PathFragment sourceGenDir(Artifact outputJar) {
+ return workDir(outputJar, "_sourcegenfiles");
+ }
+
+ protected PathFragment tempDir(Artifact outputJar) {
+ return workDir(outputJar, "_temp");
+ }
+
+ /**
+ * For an output jar and a suffix, produces a derived directory under
+ * {@code bin} directory with a given suffix.
+ */
+ private PathFragment workDir(Artifact outputJar, String suffix) {
+ PathFragment path = outputJar.getRootRelativePath();
+ String basename = FileSystemUtils.removeExtension(path.getBaseName()) + suffix;
+ path = path.replaceName(basename);
+ return getConfiguration().getBinDirectory().getExecPath().getRelative(path);
+ }
+
+ /**
+ * Creates a derived artifact from the given artifact by adding the given
+ * prefix and removing the extension and replacing it by the given suffix.
+ * The new artifact will have the same root as the given one.
+ */
+ protected Artifact derivedArtifact(Artifact artifact, String prefix, String suffix) {
+ return derivedArtifact(artifact, prefix, suffix, artifact.getRoot());
+ }
+
+ protected Artifact derivedArtifact(Artifact artifact, String prefix, String suffix, Root root) {
+ PathFragment path = artifact.getRootRelativePath();
+ String basename = FileSystemUtils.removeExtension(path.getBaseName()) + suffix;
+ path = path.replaceName(prefix + basename);
+ return getAnalysisEnvironment().getDerivedArtifact(path, root);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java b/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java
new file mode 100644
index 0000000..053b9e7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * A class to describe how build information should be translated into the generated properties
+ * file.
+ */
+public interface BuildInfoPropertiesTranslator {
+
+ /** Translate build information into a property file. */
+ public void translate(Map<String, String> buildInfo, Properties properties);
+
+ /**
+ * Returns a unique key for this translator to be used by the
+ * {@link com.google.devtools.build.lib.actions.Action#getKey()} method
+ */
+ public String computeKey();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java b/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java
new file mode 100644
index 0000000..6510a49
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+
+/**
+ * Represents common aspects of all JVM targeting configured targets.
+ */
+public final class ClasspathConfiguredFragment {
+
+ private final NestedSet<Artifact> runtimeClasspath;
+ private final NestedSet<Artifact> compileTimeClasspath;
+ private final ImmutableList<Artifact> bootClasspath;
+
+ /**
+ * Initializes the runtime and compile time classpaths for this target. This method
+ * should be called during {@code initializationHook()} once a {@link JavaTargetAttributes}
+ * object for this target is fully initialized.
+ *
+ * @param attributes the processed attributes of this Java target
+ * @param isNeverLink whether to leave runtimeClasspath empty
+ */
+ public ClasspathConfiguredFragment(JavaCompilationArtifacts javaArtifacts,
+ JavaTargetAttributes attributes, boolean isNeverLink) {
+ if (!isNeverLink) {
+ runtimeClasspath = getRuntimeClasspathList(attributes, javaArtifacts);
+ } else {
+ runtimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER);
+ }
+ compileTimeClasspath = attributes.getCompileTimeClassPath();
+ bootClasspath = attributes.getBootClassPath();
+ }
+
+ public ClasspathConfiguredFragment() {
+ runtimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER);
+ compileTimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER);
+ bootClasspath = ImmutableList.of();
+ }
+
+ /**
+ * Returns the runtime class path. It consists of the concatenation of the
+ * instrumentation class path, output jars and the runtime time class path of
+ * the transitive dependencies of this rule.
+ *
+ * @param attributes the processed attributes of this Java target
+ *
+ * @return a {@List} of artifacts that comprise the runtime class path.
+ */
+ private NestedSet<Artifact> getRuntimeClasspathList(
+ JavaTargetAttributes attributes, JavaCompilationArtifacts javaArtifacts) {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.naiveLinkOrder();
+ builder.addAll(javaArtifacts.getRuntimeJars());
+ builder.addTransitive(attributes.getRuntimeClassPath());
+ return builder.build();
+ }
+
+ /**
+ * Returns the classpath to be passed to the JVM when running a target containing this fragment.
+ */
+ public NestedSet<Artifact> getRuntimeClasspath() {
+ return runtimeClasspath;
+ }
+
+ /**
+ * Returns the classpath to be passed to the Java compiler when compiling a target containing this
+ * fragment.
+ */
+ public NestedSet<Artifact> getCompileTimeClasspath() {
+ return compileTimeClasspath;
+ }
+
+ /**
+ * Returns the classpath to be passed as a boot classpath to the Java compiler when compiling
+ * a target containing this fragment.
+ */
+ public ImmutableList<Artifact> getBootClasspath() {
+ return bootClasspath;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java
new file mode 100644
index 0000000..b9fe186
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java
@@ -0,0 +1,256 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.collect.IterablesChain;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Utility for configuring an action to generate a deploy archive.
+ */
+public class DeployArchiveBuilder {
+ /**
+ * Memory consumption of SingleJar is about 250 bytes per entry in the output file. Unfortunately,
+ * the JVM tends to kill the process with an OOM long before we're at the limit. In the most
+ * recent example, 400 MB of memory was enough for about 500,000 entries.
+ */
+ private static final String SINGLEJAR_MAX_MEMORY = "-Xmx1600m";
+
+ private final RuleContext ruleContext;
+
+ private final IterablesChain.Builder<Artifact> runtimeJarsBuilder = IterablesChain.builder();
+
+ private final JavaSemantics semantics;
+
+ private JavaTargetAttributes attributes;
+ private boolean includeBuildData;
+ private Compression compression = Compression.UNCOMPRESSED;
+ @Nullable private Artifact runfilesMiddleman;
+ private Artifact outputJar;
+ @Nullable private String javaStartClass;
+ private ImmutableList<String> deployManifestLines = ImmutableList.of();
+ @Nullable private Artifact launcher;
+
+ /**
+ * Type of compression to apply to output archive.
+ */
+ public enum Compression {
+
+ /** Output should be compressed */
+ COMPRESSED,
+
+ /** Output should not be compressed */
+ UNCOMPRESSED;
+ }
+
+ /**
+ * Creates a builder using the configuration of the rule as the action configuration.
+ */
+ public DeployArchiveBuilder(JavaSemantics semantics, RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ this.semantics = semantics;
+ }
+
+ /**
+ * Sets the processed attributes of the rule generating the deploy archive.
+ */
+ public DeployArchiveBuilder setAttributes(JavaTargetAttributes attributes) {
+ this.attributes = attributes;
+ return this;
+ }
+
+ /**
+ * Sets whether to include build-data.properties in the deploy archive.
+ */
+ public DeployArchiveBuilder setIncludeBuildData(boolean includeBuildData) {
+ this.includeBuildData = includeBuildData;
+ return this;
+ }
+
+ /**
+ * Sets whether to enable compression of the output deploy archive.
+ */
+ public DeployArchiveBuilder setCompression(Compression compress) {
+ this.compression = Preconditions.checkNotNull(compress);
+ return this;
+ }
+
+ /**
+ * Sets additional dependencies to be added to the action that creates the
+ * deploy jar so that we force the runtime dependencies to be built.
+ */
+ public DeployArchiveBuilder setRunfilesMiddleman(@Nullable Artifact runfilesMiddleman) {
+ this.runfilesMiddleman = runfilesMiddleman;
+ return this;
+ }
+
+ /**
+ * Sets the artifact to create with the action.
+ */
+ public DeployArchiveBuilder setOutputJar(Artifact outputJar) {
+ this.outputJar = Preconditions.checkNotNull(outputJar);
+ return this;
+ }
+
+ /**
+ * Sets the class to launch the Java application.
+ */
+ public DeployArchiveBuilder setJavaStartClass(@Nullable String javaStartClass) {
+ this.javaStartClass = javaStartClass;
+ return this;
+ }
+
+ /**
+ * Adds additional jars that should be on the classpath at runtime.
+ */
+ public DeployArchiveBuilder addRuntimeJars(Iterable<Artifact> jars) {
+ this.runtimeJarsBuilder.add(jars);
+ return this;
+ }
+
+ /**
+ * Sets the list of extra lines to add to the archive's MANIFEST.MF file.
+ */
+ public DeployArchiveBuilder setDeployManifestLines(Iterable<String> deployManifestLines) {
+ this.deployManifestLines = ImmutableList.copyOf(deployManifestLines);
+ return this;
+ }
+
+ /**
+ * Sets the optional launcher to be used as the executable for this deploy
+ * JAR
+ */
+ public DeployArchiveBuilder setLauncher(@Nullable Artifact launcher) {
+ this.launcher = launcher;
+ return this;
+ }
+
+ public static CustomCommandLine.Builder defaultSingleJarCommandLine(Artifact outputJar,
+ String javaMainClass,
+ ImmutableList<String> deployManifestLines, Iterable<Artifact> buildInfoFiles,
+ ImmutableList<Artifact> classpathResources,
+ Iterable<Artifact> runtimeClasspath, boolean includeBuildData,
+ Compression compress, Artifact launcher) {
+
+ CustomCommandLine.Builder args = CustomCommandLine.builder();
+ args.addExecPath("--output", outputJar);
+ if (compress == Compression.COMPRESSED) {
+ args.add("--compression");
+ }
+ args.add("--normalize");
+ if (javaMainClass != null) {
+ args.add("--main_class");
+ args.add(javaMainClass);
+ }
+
+ if (!deployManifestLines.isEmpty()) {
+ args.add("--deploy_manifest_lines");
+ args.add(deployManifestLines);
+ }
+
+ if (buildInfoFiles != null) {
+ for (Artifact artifact : buildInfoFiles) {
+ args.addExecPath("--build_info_file", artifact);
+ }
+ }
+ if (!includeBuildData) {
+ args.add("--exclude_build_data");
+ }
+ if (launcher != null) {
+ args.add("--java_launcher");
+ args.add(launcher.getExecPathString());
+ }
+
+ args.addExecPaths("--classpath_resources", classpathResources);
+ args.addExecPaths("--sources", runtimeClasspath);
+ return args;
+ }
+
+ /**
+ * Builds the action as configured.
+ */
+ public void build() {
+ ImmutableList<Artifact> classpathResources = attributes.getClassPathResources();
+ Set<String> classPathResourceNames = new HashSet<>();
+ for (Artifact artifact : classpathResources) {
+ String name = artifact.getExecPath().getBaseName();
+ if (!classPathResourceNames.add(name)) {
+ ruleContext.attributeError("classpath_resources",
+ "entries must have different file names (duplicate: " + name + ")");
+ return;
+ }
+ }
+
+ IterablesChain<Artifact> runtimeJars = runtimeJarsBuilder.build();
+
+ IterablesChain.Builder<Artifact> inputs = IterablesChain.builder();
+ inputs.add(attributes.getArchiveInputs(true));
+
+ inputs.add(ImmutableList.copyOf(runtimeJars));
+ if (runfilesMiddleman != null) {
+ inputs.addElement(runfilesMiddleman);
+ }
+
+ final ImmutableList<Artifact> buildInfoArtifacts =
+ ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, JavaBuildInfoFactory.KEY);
+ inputs.add(buildInfoArtifacts);
+
+ Iterable<Artifact> runtimeClasspath = Iterables.concat(
+ runtimeJars,
+ attributes.getRuntimeClassPathForArchive());
+
+ if (launcher != null) {
+ inputs.addElement(launcher);
+ }
+
+ CommandLine commandLine = semantics.buildSingleJarCommandLine(ruleContext.getConfiguration(),
+ outputJar, javaStartClass, deployManifestLines, buildInfoArtifacts, classpathResources,
+ runtimeClasspath, includeBuildData, compression, launcher);
+
+ List<String> jvmArgs = ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY);
+ ResourceSet resourceSet =
+ new ResourceSet(/*memoryMb = */200.0, /*cpuUsage = */.2, /*ioUsage=*/.2);
+
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .addInputs(inputs.build())
+ .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext))
+ .addOutput(outputJar)
+ .setResources(resourceSet)
+ .setJarExecutable(
+ ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(),
+ ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST),
+ jvmArgs)
+ .setCommandLine(commandLine)
+ .useParameterFile(ParameterFileType.SHELL_QUOTED)
+ .setProgressMessage("Building deploy jar " + outputJar.prettyPrint())
+ .setMnemonic("JavaDeployJar")
+ .build(ruleContext));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java
new file mode 100644
index 0000000..5b2e106
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A provider that returns the direct dependencies of a target. Used for strict dependency
+ * checking.
+ */
+@Immutable
+public final class DirectDependencyProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<Dependency> strictDependencies;
+
+ public DirectDependencyProvider(Iterable<Dependency> strictDependencies) {
+ this.strictDependencies = ImmutableList.copyOf(strictDependencies);
+ }
+
+ /**
+ * @returns the direct (strict) dependencies of this provider. All symbols that are directly
+ * reachable from the sources of the provider should be available in one these artifacts.
+ */
+ public Iterable<Dependency> getStrictDependencies() {
+ return strictDependencies;
+ }
+
+ /**
+ * A pair of label and its generated list of artifacts.
+ */
+ public static class Dependency {
+ private final Label label;
+
+ // TODO(bazel-team): change this to Artifacts
+ private final Iterable<String> fileExecPaths;
+
+ public Dependency(Label label, Iterable<String> fileExecPaths) {
+ this.label = label;
+ this.fileExecPaths = fileExecPaths;
+ }
+
+ public Label getLabel() {
+ return label;
+ }
+
+ public Iterable<String> getDependencyOutputs() {
+ return fileExecPaths;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java b/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java
new file mode 100644
index 0000000..df6a325
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.util.Map;
+import java.util.Properties;
+
+/** The generic implementation of {@link BuildInfoPropertiesTranslator} */
+public class GenericBuildInfoPropertiesTranslator implements
+ BuildInfoPropertiesTranslator {
+
+ private static final String GUID = "e71fe4a8-11af-4ec0-9b38-1d3e7f542f51";
+
+ // syntax is %ID% for a property that depends on the ID key, %ID|default% to
+ // always add the property with the "default" key, %% is to add a percent sign
+ private final Map<String, String> translationKeys;
+
+ /**
+ * Create a generic translator, for each key,value pair in {@code translationKeys}, the key
+ * represents the property key that will be written and the value, its value. Inside value every
+ * %ID% is replaced by the corresponding build information with the same ID key. The property
+ * won't be added if it's depends on an unresolved build information. Adding a property can
+ * be forced even if a build information is missing by specifying a default value using the
+ * %ID|default% syntax. Finally to add a percent sign, just use the %% syntax.
+ */
+ public GenericBuildInfoPropertiesTranslator(Map<String, String> translationKeys) {
+ this.translationKeys = translationKeys;
+ }
+
+ @Override
+ public String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addStringMap(translationKeys);
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public void translate(Map<String, String> buildInfo, Properties properties) {
+ for (Map.Entry<String, String> entry : translationKeys.entrySet()) {
+ String translatedValue = translateValue(entry.getValue(), buildInfo);
+ if (translatedValue != null) {
+ properties.put(entry.getKey(), translatedValue);
+ }
+ }
+ }
+
+ private String translateValue(String valueDescription, Map<String, String> buildInfo) {
+ String[] split = valueDescription.split("%");
+ StringBuffer result = new StringBuffer();
+ boolean isInsideKey = false;
+ for (String key : split) {
+ if (isInsideKey) {
+ if (key.isEmpty()) {
+ result.append("%"); // empty key means %%
+ } else {
+ String defaultValue = null;
+ int i = key.lastIndexOf('|');
+ if (i >= 0) {
+ defaultValue = key.substring(i + 1);
+ key = key.substring(0, i);
+ }
+ if (buildInfo.containsKey(key)) {
+ result.append(buildInfo.get(key));
+ } else if (defaultValue != null) {
+ result.append(defaultValue);
+ } else { // we haven't found the requested key so we ignore the whole value
+ return null;
+ }
+ }
+ } else {
+ result.append(key);
+ }
+ isInsideKey = !isInsideKey;
+ }
+ return result.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java
new file mode 100644
index 0000000..e957f49
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java
@@ -0,0 +1,359 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import static com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression.COMPRESSED;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.cpp.CppHelper;
+import com.google.devtools.build.lib.rules.cpp.LinkerInput;
+import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * An implementation of java_binary.
+ */
+public class JavaBinary implements RuleConfiguredTargetFactory {
+ private static final PathFragment CPP_RUNTIMES = new PathFragment("_cpp_runtimes");
+
+ private final JavaSemantics semantics;
+
+ protected JavaBinary(JavaSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ final JavaCommon common = new JavaCommon(ruleContext, semantics);
+ DeployArchiveBuilder deployArchiveBuilder = new DeployArchiveBuilder(semantics, ruleContext);
+ Runfiles.Builder runfilesBuilder = new Runfiles.Builder();
+ List<String> jvmFlags = new ArrayList<>();
+
+ common.initializeJavacOpts();
+ JavaTargetAttributes.Builder attributesBuilder = common.initCommon();
+ attributesBuilder.addClassPathResources(
+ ruleContext.getPrerequisiteArtifacts("classpath_resources", Mode.TARGET).list());
+
+ List<String> userJvmFlags = common.getJvmFlags();
+
+ ruleContext.checkSrcsSamePackage(true);
+ boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN);
+ List<TransitiveInfoCollection> deps =
+ Lists.newArrayList(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY));
+ semantics.checkRule(ruleContext, common);
+ String mainClass = semantics.getMainClass(ruleContext, common);
+ String originalMainClass = mainClass;
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ // Collect the transitive dependencies.
+ JavaCompilationHelper helper = new JavaCompilationHelper(
+ ruleContext, semantics, common.getJavacOpts(), attributesBuilder);
+ helper.addLibrariesToAttributes(deps);
+ helper.addProvidersToAttributes(common.compilationArgsFromSources(), /* isNeverLink */ false);
+ attributesBuilder.addNativeLibraries(
+ collectNativeLibraries(common.targetsTreatedAsDeps(ClasspathType.BOTH)));
+
+ // deploy_env is valid for java_binary, but not for java_test.
+ if (ruleContext.getRule().isAttrDefined("deploy_env", Type.LABEL_LIST)) {
+ for (JavaRuntimeClasspathProvider envTarget : ruleContext.getPrerequisites(
+ "deploy_env", Mode.TARGET, JavaRuntimeClasspathProvider.class)) {
+ attributesBuilder.addExcludedArtifacts(envTarget.getRuntimeClasspath());
+ }
+ }
+
+ Artifact srcJar =
+ ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_SOURCE_JAR);
+
+ Artifact classJar =
+ ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_CLASS_JAR);
+
+ ImmutableList<Artifact> srcJars = ImmutableList.of(srcJar);
+
+ Artifact launcher = semantics.getLauncher(ruleContext, common, deployArchiveBuilder,
+ runfilesBuilder, jvmFlags, attributesBuilder);
+ JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder();
+ Artifact instrumentationMetadata =
+ helper.createInstrumentationMetadata(classJar, javaArtifactsBuilder);
+
+ NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder();
+ Artifact executable = null;
+ if (createExecutable) {
+ executable = ruleContext.createOutputArtifact(); // the artifact for the rule itself
+ filesBuilder.add(classJar).add(executable);
+
+ if (ruleContext.getConfiguration().isCodeCoverageEnabled()) {
+ mainClass = semantics.addCoverageSupport(helper, attributesBuilder,
+ executable, instrumentationMetadata, javaArtifactsBuilder, mainClass);
+ }
+ } else {
+ filesBuilder.add(classJar);
+ }
+
+ JavaTargetAttributes attributes = helper.getAttributes();
+ List<Artifact> nativeLibraries = attributes.getNativeLibraries();
+ if (!nativeLibraries.isEmpty()) {
+ jvmFlags.add("-Djava.library.path=" + JavaCommon.javaLibraryPath(nativeLibraries));
+ }
+
+ JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class);
+ if (attributes.hasMessages()) {
+ helper.addTranslations(semantics.translate(ruleContext, javaConfig,
+ attributes.getMessages()));
+ }
+
+ if (attributes.hasSourceFiles() || attributes.hasSourceJars()
+ || attributes.hasResources() || attributes.hasClassPathResources()) {
+ // We only want to add a jar to the classpath of a dependent rule if it has content.
+ javaArtifactsBuilder.addRuntimeJar(classJar);
+ }
+
+ // Any JAR files should be added to the collection of runtime jars.
+ javaArtifactsBuilder.addRuntimeJars(attributes.getJarFiles());
+
+ Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder);
+
+ common.setJavaCompilationArtifacts(javaArtifactsBuilder.build());
+
+ // The gensrcJar is only created if the target uses annotation processing. Otherwise,
+ // it is null, and the source jar action will not depend on the compile action.
+ Artifact gensrcJar = helper.createGensrcJar(classJar);
+
+ helper.createCompileAction(classJar, gensrcJar, outputDepsProto, instrumentationMetadata);
+ helper.createSourceJarAction(srcJar, gensrcJar);
+
+ common.setClassPathFragment(new ClasspathConfiguredFragment(
+ common.getJavaCompilationArtifacts(), attributes, false));
+
+ // Collect the action inputs for the runfiles collector here because we need to access the
+ // analysis environment, and that may no longer be safe when the runfiles collector runs.
+ Iterable<Artifact> dynamicRuntimeActionInputs =
+ CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs();
+
+
+ Iterables.addAll(jvmFlags, semantics.getJvmFlags(ruleContext, common, launcher, userJvmFlags));
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ if (createExecutable) {
+ // Create a shell stub for a Java application
+ semantics.createStubAction(ruleContext, common, jvmFlags, executable, mainClass,
+ common.getJavaBinSubstitution(launcher));
+ }
+
+ NestedSet<Artifact> transitiveSourceJars = collectTransitiveSourceJars(common, srcJar);
+
+ // TODO(bazel-team): if (getOptions().sourceJars) then make this a dummy prerequisite for the
+ // DeployArchiveAction ? Needs a few changes there as we can't pass inputs
+ helper.createSourceJarAction(semantics, ImmutableList.<Artifact>of(),
+ transitiveSourceJars.toCollection(),
+ ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_SOURCE_JAR));
+
+ RuleConfiguredTargetBuilder builder =
+ new RuleConfiguredTargetBuilder(ruleContext);
+
+ semantics.addProviders(ruleContext, common, jvmFlags, classJar, srcJar, gensrcJar,
+ ImmutableMap.<Artifact, Artifact>of(), helper, filesBuilder, builder);
+
+ NestedSet<Artifact> filesToBuild = filesBuilder.build();
+
+ collectDefaultRunfiles(runfilesBuilder, ruleContext, common, filesToBuild, launcher,
+ dynamicRuntimeActionInputs);
+ Runfiles defaultRunfiles = runfilesBuilder.build();
+
+ RunfilesSupport runfilesSupport = createExecutable
+ ? runfilesSupport = RunfilesSupport.withExecutable(
+ ruleContext, defaultRunfiles, executable,
+ semantics.getExtraArguments(ruleContext, common))
+ : null;
+
+ RunfilesProvider runfilesProvider = RunfilesProvider.withData(
+ defaultRunfiles,
+ new Runfiles.Builder().merge(runfilesSupport).build());
+
+ ImmutableList<String> deployManifestLines =
+ getDeployManifestLines(ruleContext, originalMainClass);
+
+ Artifact deployJar =
+ ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_JAR);
+
+ deployArchiveBuilder
+ .setOutputJar(deployJar)
+ .setJavaStartClass(mainClass)
+ .setDeployManifestLines(deployManifestLines)
+ .setAttributes(attributes)
+ .addRuntimeJars(common.getJavaCompilationArtifacts().getRuntimeJars())
+ .setIncludeBuildData(true)
+ .setRunfilesMiddleman(
+ runfilesSupport == null ? null : runfilesSupport.getRunfilesMiddleman())
+ .setCompression(COMPRESSED)
+ .setLauncher(launcher);
+
+ deployArchiveBuilder.build();
+
+ common.addTransitiveInfoProviders(builder, filesToBuild, classJar);
+
+ return builder
+ .setFilesToBuild(filesToBuild)
+ .add(RunfilesProvider.class, runfilesProvider)
+ .setRunfilesSupport(runfilesSupport, executable)
+ .add(JavaRuntimeClasspathProvider.class,
+ new JavaRuntimeClasspathProvider(common.getRuntimeClasspath()))
+ .add(JavaSourceJarsProvider.class,
+ new JavaSourceJarsProvider(transitiveSourceJars, srcJars))
+ .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider(
+ JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars))
+ .build();
+ }
+
+ // Create the deploy jar and make it dependent on the runfiles middleman if an executable is
+ // created. Do not add the deploy jar to files to build, so we will only build it when it gets
+ // requested.
+ private ImmutableList<String> getDeployManifestLines(RuleContext ruleContext,
+ String originalMainClass) {
+ ImmutableList.Builder<String> builder = ImmutableList.<String>builder()
+ .addAll(ruleContext.attributes().get("deploy_manifest_lines", Type.STRING_LIST));
+ if (ruleContext.getConfiguration().isCodeCoverageEnabled()) {
+ builder.add("Coverage-Main-Class: " + originalMainClass);
+ }
+ return builder.build();
+ }
+
+ private void collectDefaultRunfiles(Runfiles.Builder builder, RuleContext ruleContext,
+ JavaCommon common, NestedSet<Artifact> filesToBuild, Artifact launcher,
+ Iterable<Artifact> dynamicRuntimeActionInputs) {
+ // Convert to iterable: filesToBuild has a different order.
+ builder.addArtifacts((Iterable<Artifact>) filesToBuild);
+ builder.addArtifacts(common.getJavaCompilationArtifacts().getRuntimeJars());
+ if (launcher != null) {
+ final TransitiveInfoCollection defaultLauncher =
+ JavaHelper.launcherForTarget(semantics, ruleContext);
+ final Artifact defaultLauncherArtifact =
+ JavaHelper.launcherArtifactForTarget(semantics, ruleContext);
+ if (!defaultLauncherArtifact.equals(launcher)) {
+ builder.addArtifact(launcher);
+
+ // N.B. The "default launcher" referred to here is the launcher target specified through
+ // an attribute or flag. We wish to retain the runfiles of the default launcher, *except*
+ // for the original cc_binary artifact, because we've swapped it out with our custom
+ // launcher. Hence, instead of calling builder.addTarget(), or adding an odd method
+ // to Runfiles.Builder, we "unravel" the call and manually add things to the builder.
+ // Because the NestedSet representing each target's launcher runfiles is re-built here,
+ // we may see increased memory consumption for representing the target's runfiles.
+ Runfiles runfiles =
+ defaultLauncher.getProvider(RunfilesProvider.class)
+ .getDefaultRunfiles();
+ NestedSetBuilder<Artifact> unconditionalArtifacts = NestedSetBuilder.compileOrder();
+ for (Artifact a : runfiles.getUnconditionalArtifacts()) {
+ if (!a.equals(defaultLauncherArtifact)) {
+ unconditionalArtifacts.add(a);
+ }
+ }
+ builder.addTransitiveArtifacts(unconditionalArtifacts.build());
+ builder.addSymlinks(runfiles.getSymlinks());
+ builder.addRootSymlinks(runfiles.getRootSymlinks());
+ builder.addPruningManifests(runfiles.getPruningManifests());
+ } else {
+ builder.addTarget(defaultLauncher, RunfilesProvider.DEFAULT_RUNFILES);
+ }
+ }
+
+ semantics.addRunfilesForBinary(ruleContext, launcher, builder);
+ builder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES);
+ builder.add(ruleContext, JavaRunfilesProvider.TO_RUNFILES);
+
+ List<? extends TransitiveInfoCollection> runtimeDeps =
+ ruleContext.getPrerequisites("runtime_deps", Mode.TARGET);
+ builder.addTargets(runtimeDeps, JavaRunfilesProvider.TO_RUNFILES);
+ builder.addTargets(runtimeDeps, RunfilesProvider.DEFAULT_RUNFILES);
+ semantics.addDependenciesForRunfiles(ruleContext, builder);
+
+ if (ruleContext.getConfiguration().isCodeCoverageEnabled()) {
+ Artifact instrumentedJar = common.getJavaCompilationArtifacts().getInstrumentedJar();
+ if (instrumentedJar != null) {
+ builder.addArtifact(instrumentedJar);
+ }
+ }
+
+ builder.addArtifacts((Iterable<Artifact>) common.getRuntimeClasspath());
+
+ // Add the JDK files if it comes from the source repository (see java_stub_template.txt).
+ TransitiveInfoCollection javabaseTarget = ruleContext.getPrerequisite(":jvm", Mode.HOST);
+ if (javabaseTarget != null) {
+ builder.addArtifacts(
+ (Iterable<Artifact>) javabaseTarget.getProvider(FileProvider.class).getFilesToBuild());
+
+ // Add symlinks to the C++ runtime libraries under a path that can be built
+ // into the Java binary without having to embed the crosstool, gcc, and grte
+ // version information contained within the libraries' package paths.
+ for (Artifact lib : dynamicRuntimeActionInputs) {
+ PathFragment path = CPP_RUNTIMES.getRelative(lib.getExecPath().getBaseName());
+ builder.addSymlink(path, lib);
+ }
+ }
+ }
+
+ private NestedSet<Artifact> collectTransitiveSourceJars(JavaCommon common, Artifact srcJar) {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+
+ builder.add(srcJar);
+ for (JavaSourceJarsProvider dep : common.getDependencies(JavaSourceJarsProvider.class)) {
+ builder.addTransitive(dep.getTransitiveSourceJars());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Collects the native libraries in the transitive closure of the deps.
+ *
+ * @param deps the dependencies to be included as roots of the transitive closure.
+ * @return the native libraries found in the transitive closure of the deps.
+ */
+ public static Collection<Artifact> collectNativeLibraries(
+ Iterable<? extends TransitiveInfoCollection> deps) {
+ NestedSet<LinkerInput> linkerInputs = new NativeLibraryNestedSetBuilder()
+ .addJavaTargets(deps)
+ .build();
+ ImmutableList.Builder<Artifact> result = ImmutableList.builder();
+ for (LinkerInput linkerInput : linkerInputs) {
+ result.add(linkerInput.getArtifact());
+ }
+
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java
new file mode 100644
index 0000000..442b85b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.rules.java.WriteBuildInfoPropertiesAction.TimestampFormatter;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Java build info creation - generates properties file that contain the corresponding build-info
+ * data.
+ */
+public abstract class JavaBuildInfoFactory implements BuildInfoFactory {
+ public static final BuildInfoKey KEY = new BuildInfoKey("Java");
+
+ static final PathFragment BUILD_INFO_NONVOLATILE_PROPERTIES_NAME =
+ new PathFragment("build-info-nonvolatile.properties");
+ static final PathFragment BUILD_INFO_VOLATILE_PROPERTIES_NAME =
+ new PathFragment("build-info-volatile.properties");
+ static final PathFragment BUILD_INFO_REDACTED_PROPERTIES_NAME =
+ new PathFragment("build-info-redacted.properties");
+
+ private static final DateTimeFormatter DEFAULT_TIME_FORMAT =
+ DateTimeFormat.forPattern("EEE MMM d HH:mm:ss yyyy");
+
+ // A default formatter that returns a date in UTC format.
+ private static final TimestampFormatter DEFAULT_FORMATTER = new TimestampFormatter() {
+ @Override
+ public String format(long timestamp) {
+ return new DateTime(timestamp, DateTimeZone.UTC).toString(DEFAULT_TIME_FORMAT) + " ("
+ + timestamp / 1000 + ')';
+ }
+ };
+
+ @Override
+ public final BuildInfoCollection create(BuildInfoContext context, BuildConfiguration config,
+ Artifact stableStatus, Artifact volatileStatus) {
+ WriteBuildInfoPropertiesAction redactedInfo = getHeader(context,
+ config,
+ BUILD_INFO_REDACTED_PROPERTIES_NAME,
+ Artifact.NO_ARTIFACTS,
+ createRedactedTranslator(),
+ true,
+ true);
+ WriteBuildInfoPropertiesAction nonvolatileInfo = getHeader(context,
+ config,
+ BUILD_INFO_NONVOLATILE_PROPERTIES_NAME,
+ ImmutableList.of(stableStatus),
+ createNonVolatileTranslator(),
+ false,
+ true);
+ WriteBuildInfoPropertiesAction volatileInfo = getHeader(context,
+ config,
+ BUILD_INFO_VOLATILE_PROPERTIES_NAME,
+ ImmutableList.of(volatileStatus),
+ createVolatileTranslator(),
+ true,
+ false);
+ List<Action> actions = new ArrayList<Action>(3);
+ actions.add(redactedInfo);
+ actions.add(nonvolatileInfo);
+ actions.add(volatileInfo);
+ return new BuildInfoCollection(actions,
+ ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()),
+ ImmutableList.of(redactedInfo.getPrimaryOutput()));
+ }
+
+ /**
+ * Creates a {@link BuildInfoPropertiesTranslator} to use for volatile keys.
+ */
+ protected abstract BuildInfoPropertiesTranslator createVolatileTranslator();
+
+ /**
+ * Creates a {@link BuildInfoPropertiesTranslator} to use for non-volatile keys.
+ */
+ protected abstract BuildInfoPropertiesTranslator createNonVolatileTranslator();
+
+ /**
+ * Creates a {@link BuildInfoPropertiesTranslator} to use for redacted version of the build
+ * informations.
+ */
+ protected abstract BuildInfoPropertiesTranslator createRedactedTranslator();
+
+ /**
+ * Specifies the {@link TimestampFormatter} to use to output dates in the properties file.
+ */
+ protected TimestampFormatter getTimestampFormatter() {
+ return DEFAULT_FORMATTER;
+ }
+
+ private WriteBuildInfoPropertiesAction getHeader(BuildInfoContext context,
+ BuildConfiguration config,
+ PathFragment propertyFileName,
+ ImmutableList<Artifact> inputs,
+ BuildInfoPropertiesTranslator translator,
+ boolean includeVolatile,
+ boolean includeNonVolatile) {
+ Root outputPath = config.getIncludeDirectory();
+ final Artifact output = context.getBuildInfoArtifact(propertyFileName, outputPath,
+ includeVolatile && !inputs.isEmpty() ? BuildInfoType.NO_REBUILD
+ : BuildInfoType.FORCE_REBUILD_IF_CHANGED);
+ return new WriteBuildInfoPropertiesAction(inputs,
+ output,
+ translator,
+ includeVolatile,
+ includeNonVolatile,
+ getTimestampFormatter());
+ }
+
+ @Override
+ public final BuildInfoKey getKey() {
+ return KEY;
+ }
+
+ @Override
+ public boolean isEnabled(BuildConfiguration config) {
+ return config.hasFragment(JavaConfiguration.class);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java
new file mode 100644
index 0000000..5218f3f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl;
+
+/**
+ * A target that provides C++ libraries to be linked into Java targets.
+ */
+@Immutable
+public final class JavaCcLinkParamsProvider implements TransitiveInfoProvider {
+ private final CcLinkParamsStoreImpl store;
+
+ public JavaCcLinkParamsProvider(CcLinkParamsStore store) {
+ this.store = new CcLinkParamsStoreImpl(store);
+ }
+
+ public CcLinkParamsStore getLinkParams() {
+ return store;
+ }
+
+ public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS =
+ new Function<TransitiveInfoCollection, CcLinkParamsStore>() {
+ @Override
+ public CcLinkParamsStore apply(TransitiveInfoCollection input) {
+ JavaCcLinkParamsProvider provider = input.getProvider(
+ JavaCcLinkParamsProvider.class);
+ return provider == null ? null : provider.getLinkParams();
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java
new file mode 100644
index 0000000..c55a74e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java
@@ -0,0 +1,651 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToCompileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.Util;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CppCompilationContext;
+import com.google.devtools.build.lib.rules.cpp.LinkerInput;
+import com.google.devtools.build.lib.rules.java.DirectDependencyProvider.Dependency;
+import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType;
+import com.google.devtools.build.lib.rules.test.BaselineCoverageAction;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class to create configured targets for Java rules.
+ */
+public class JavaCommon {
+ private static final Function<TransitiveInfoCollection, Label> GET_COLLECTION_LABEL =
+ new Function<TransitiveInfoCollection, Label>() {
+ @Override
+ public Label apply(TransitiveInfoCollection collection) {
+ return collection.getLabel();
+ }
+ };
+
+ /**
+ * Collects all metadata files generated by Java compilation actions.
+ */
+ private static final LocalMetadataCollector JAVA_METADATA_COLLECTOR =
+ new LocalMetadataCollector() {
+ @Override
+ public void collectMetadataArtifacts(Iterable<Artifact> objectFiles,
+ AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) {
+ for (Artifact artifact : objectFiles) {
+ Action action = analysisEnvironment.getLocalGeneratingAction(artifact);
+ if (action instanceof JavaCompileAction) {
+ addOutputs(metadataFilesBuilder, action, JavaSemantics.COVERAGE_METADATA);
+ }
+ }
+ }
+ };
+
+ private ClasspathConfiguredFragment classpathFragment = new ClasspathConfiguredFragment();
+ private JavaCompilationArtifacts javaArtifacts = JavaCompilationArtifacts.EMPTY;
+ private ImmutableList<String> javacOpts;
+
+ // Targets treated as deps in compilation time, runtime time and both
+ private final ImmutableMap<ClasspathType, ImmutableList<TransitiveInfoCollection>>
+ targetsTreatedAsDeps;
+
+ private ImmutableList<Artifact> sources = ImmutableList.of();
+ private ImmutableList<JavaPluginInfoProvider> activePlugins = ImmutableList.of();
+
+ private final RuleContext ruleContext;
+ private final JavaSemantics semantics;
+
+ public JavaCommon(RuleContext ruleContext, JavaSemantics semantics) {
+ this(ruleContext, semantics,
+ collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.COMPILE_ONLY),
+ collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.RUNTIME_ONLY),
+ collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.BOTH));
+ }
+
+ public JavaCommon(RuleContext ruleContext,
+ JavaSemantics semantics,
+ ImmutableList<TransitiveInfoCollection> compileDeps,
+ ImmutableList<TransitiveInfoCollection> runtimeDeps,
+ ImmutableList<TransitiveInfoCollection> bothDeps) {
+ this.ruleContext = ruleContext;
+ this.semantics = semantics;
+ this.targetsTreatedAsDeps = ImmutableMap.of(
+ ClasspathType.COMPILE_ONLY, compileDeps,
+ ClasspathType.RUNTIME_ONLY, runtimeDeps,
+ ClasspathType.BOTH, bothDeps);
+ }
+
+ public void setClassPathFragment(ClasspathConfiguredFragment classpathFragment) {
+ this.classpathFragment = classpathFragment;
+ }
+
+ public void setJavaCompilationArtifacts(JavaCompilationArtifacts javaArtifacts) {
+ this.javaArtifacts = javaArtifacts;
+ }
+
+ public JavaCompilationArtifacts getJavaCompilationArtifacts() {
+ return javaArtifacts;
+ }
+
+ public ImmutableList<Artifact> getProcessorClasspathJars() {
+ Set<Artifact> processorClasspath = new LinkedHashSet<>();
+ for (JavaPluginInfoProvider plugin : activePlugins) {
+ for (Artifact classpathJar : plugin.getProcessorClasspath()) {
+ processorClasspath.add(classpathJar);
+ }
+ }
+ return ImmutableList.copyOf(processorClasspath);
+ }
+
+ public ImmutableList<String> getProcessorClassNames() {
+ Set<String> processorNames = new LinkedHashSet<>();
+ for (JavaPluginInfoProvider plugin : activePlugins) {
+ processorNames.addAll(plugin.getProcessorClasses());
+ }
+ return ImmutableList.copyOf(processorNames);
+ }
+
+ /**
+ * Creates the java.library.path from a list of the native libraries.
+ * Concatenates the parent directories of the shared libraries into a Java
+ * search path. Each relative path entry is prepended with "${JAVA_RUNFILES}/"
+ * so it can be resolved at runtime.
+ *
+ * @param sharedLibraries a collection of native libraries to create the java
+ * library path from
+ * @return a String containing the ":" separated java library path
+ */
+ public static String javaLibraryPath(Collection<Artifact> sharedLibraries) {
+ StringBuilder buffer = new StringBuilder();
+ Set<PathFragment> entries = new HashSet<>();
+ for (Artifact sharedLibrary : sharedLibraries) {
+ PathFragment entry = sharedLibrary.getRootRelativePath().getParentDirectory();
+ if (entries.add(entry)) {
+ if (buffer.length() > 0) {
+ buffer.append(':');
+ }
+ buffer.append("${JAVA_RUNFILES}/" + Constants.RUNFILES_PREFIX + "/");
+ buffer.append(entry.getPathString());
+ }
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Collects Java compilation arguments for this target.
+ *
+ * @param recursive Whether to scan dependencies recursively.
+ * @param isNeverLink Whether the target has the 'neverlink' attr.
+ */
+ JavaCompilationArgs collectJavaCompilationArgs(boolean recursive, boolean isNeverLink,
+ Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources) {
+ ClasspathType type = isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH;
+ JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder()
+ .merge(getJavaCompilationArtifacts(), isNeverLink)
+ .addTransitiveTargets(getExports(ruleContext), recursive, type);
+ if (recursive) {
+ builder
+ .addTransitiveTargets(targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY), recursive, type)
+ .addTransitiveTargets(getRuntimeDeps(ruleContext), recursive, ClasspathType.RUNTIME_ONLY)
+ .addSourcesTransitiveCompilationArgs(compilationArgsFromSources, recursive, type);
+ }
+ return builder.build();
+ }
+
+ /**
+ * Collects Java dependency artifacts for this target.
+ *
+ * @param outDeps output (compile-time) dependency artifact of this target
+ */
+ NestedSet<Artifact> collectCompileTimeDependencyArtifacts(Artifact outDeps) {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+ if (outDeps != null) {
+ builder.add(outDeps);
+ }
+
+ for (JavaCompilationArgsProvider provider : AnalysisUtils.getProviders(
+ getExports(ruleContext), JavaCompilationArgsProvider.class)) {
+ builder.addTransitive(provider.getCompileTimeJavaDependencyArtifacts());
+ }
+ return builder.build();
+ }
+
+ public static List<TransitiveInfoCollection> getExports(RuleContext ruleContext) {
+ // We need to check here because there are classes inheriting from this class that implement
+ // rules that don't have this attribute.
+ if (ruleContext.getRule().getRuleClassObject().hasAttr("exports", Type.LABEL_LIST)) {
+ return ImmutableList.copyOf(ruleContext.getPrerequisites("exports", Mode.TARGET));
+ } else {
+ return ImmutableList.of();
+ }
+ }
+
+ /**
+ * Sanity checks the given runtime dependencies, and emits errors if there is a problem.
+ * Also called by {@link #initCommon()} for the current target's runtime dependencies.
+ */
+ public void checkRuntimeDeps(List<TransitiveInfoCollection> runtimeDepInfo) {
+ for (TransitiveInfoCollection c : runtimeDepInfo) {
+ JavaNeverlinkInfoProvider neverLinkedness =
+ c.getProvider(JavaNeverlinkInfoProvider.class);
+ if (neverLinkedness == null) {
+ continue;
+ }
+ boolean reportError = !ruleContext.getConfiguration().getAllowRuntimeDepsOnNeverLink();
+ if (neverLinkedness.isNeverlink()) {
+ String msg = String.format("neverlink dep %s not allowed in runtime deps", c.getLabel());
+ if (reportError) {
+ ruleContext.attributeError("runtime_deps", msg);
+ } else {
+ ruleContext.attributeWarning("runtime_deps", msg);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns transitive Java native libraries.
+ *
+ * @see JavaNativeLibraryProvider
+ */
+ protected NestedSet<LinkerInput> collectTransitiveJavaNativeLibraries() {
+ NativeLibraryNestedSetBuilder builder = new NativeLibraryNestedSetBuilder();
+ builder.addJavaTargets(targetsTreatedAsDeps(ClasspathType.BOTH));
+
+ if (ruleContext.getRule().isAttrDefined("data", Type.LABEL_LIST)) {
+ builder.addJavaTargets(ruleContext.getPrerequisites("data", Mode.DATA));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Collects transitive source jars for the current rule.
+ *
+ * @param targetSrcJar The source jar artifact corresponding to the output of the current rule.
+ * @return A nested set containing all of the source jar artifacts on which the current rule
+ * transitively depends.
+ */
+ public NestedSet<Artifact> collectTransitiveSourceJars(Artifact targetSrcJar) {
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
+
+ builder.add(targetSrcJar);
+ for (JavaSourceJarsProvider dep : getDependencies(JavaSourceJarsProvider.class)) {
+ builder.addTransitive(dep.getTransitiveSourceJars());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Collects transitive C++ dependencies.
+ */
+ protected CppCompilationContext collectTransitiveCppDeps() {
+ CppCompilationContext.Builder builder = new CppCompilationContext.Builder(ruleContext);
+ for (TransitiveInfoCollection dep : targetsTreatedAsDeps(ClasspathType.BOTH)) {
+ CppCompilationContext context =
+ dep.getProvider(CppCompilationContext.class);
+ if (context != null) {
+ builder.mergeDependentContext(context);
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Collects labels of targets and artifacts reached transitively via the "exports" attribute.
+ */
+ protected NestedSet<Label> collectTransitiveExports() {
+ NestedSetBuilder<Label> builder = NestedSetBuilder.stableOrder();
+ List<TransitiveInfoCollection> currentRuleExports = getExports(ruleContext);
+
+ builder.addAll(Iterables.transform(currentRuleExports, GET_COLLECTION_LABEL));
+
+ for (TransitiveInfoCollection dep : currentRuleExports) {
+ JavaExportsProvider exportsProvider = dep.getProvider(JavaExportsProvider.class);
+
+ if (exportsProvider != null) {
+ builder.addTransitive(exportsProvider.getTransitiveExports());
+ }
+ }
+
+ return builder.build();
+ }
+
+ public final void initializeJavacOpts() {
+ initializeJavacOpts(semantics.getExtraJavacOpts(ruleContext));
+ }
+
+ public final void initializeJavacOpts(Iterable<String> extraJavacOpts) {
+ javacOpts = ImmutableList.copyOf(Iterables.concat(
+ JavaToolchainProvider.getDefaultJavacOptions(ruleContext),
+ ruleContext.getTokenizedStringListAttr("javacopts"), extraJavacOpts));
+ }
+
+ /**
+ * Returns the string that the stub should use to determine the JVM
+ * @param launcher if non-null, the cc_binary used to launch the Java Virtual Machine
+ */
+ public String getJavaBinSubstitution(@Nullable Artifact launcher) {
+ PathFragment javaExecutable;
+
+ if (launcher != null) {
+ javaExecutable = launcher.getRootRelativePath();
+ } else {
+ javaExecutable = ruleContext.getFragment(Jvm.class).getJavaExecutable();
+ }
+
+ String pathPrefix =
+ javaExecutable.isAbsolute() ? "" : "${JAVA_RUNFILES}/" + Constants.RUNFILES_PREFIX + "/";
+ return "JAVABIN=${JAVABIN:-" + pathPrefix + javaExecutable.getPathString() + "}";
+ }
+
+ /**
+ * Heuristically determines the name of the primary Java class for this
+ * executable, based on the rule name and the "srcs" list.
+ *
+ * <p>(This is expected to be the class containing the "main" method for a
+ * java_binary, or a JUnit Test class for a java_test.)
+ *
+ * @param sourceFiles the source files for this rule
+ * @return a fully qualified Java class name, or null if none could be
+ * determined.
+ */
+ public String determinePrimaryClass(Collection<Artifact> sourceFiles) {
+ if (!sourceFiles.isEmpty()) {
+ String mainSource = ruleContext.getTarget().getName() + ".java";
+ for (Artifact sourceFile : sourceFiles) {
+ PathFragment path = sourceFile.getRootRelativePath();
+ if (path.getBaseName().equals(mainSource)) {
+ return JavaUtil.getJavaFullClassname(FileSystemUtils.removeExtension(path));
+ }
+ }
+ }
+ // Last resort: Use the name and package name of the target.
+ // TODO(bazel-team): this should be fixed to use a source file from the dependencies to
+ // determine the package of the Java class.
+ return JavaUtil.getJavaFullClassname(Util.getWorkspaceRelativePath(ruleContext.getTarget()));
+ }
+
+ /**
+ * Gets the value of the "jvm_flags" attribute combining it with the default
+ * options and expanding any make variables.
+ */
+ public List<String> getJvmFlags() {
+ List<String> jvmFlags = new ArrayList<>();
+ jvmFlags.addAll(ruleContext.getFragment(JavaConfiguration.class).getDefaultJvmFlags());
+ jvmFlags.addAll(ruleContext.expandedMakeVariablesList("jvm_flags"));
+ return jvmFlags;
+ }
+
+ private static List<TransitiveInfoCollection> getRuntimeDeps(RuleContext ruleContext) {
+ // We need to check here because there are classes inheriting from this class that implement
+ // rules that don't have this attribute.
+ if (ruleContext.getRule().getRuleClassObject().hasAttr("runtime_deps", Type.LABEL_LIST)) {
+ return ImmutableList.copyOf(ruleContext.getPrerequisites("runtime_deps", Mode.TARGET));
+ } else {
+ return ImmutableList.of();
+ }
+ }
+
+ public JavaTargetAttributes.Builder initCommon() {
+ return initCommon(Collections.<Artifact>emptySet());
+ }
+
+ /**
+ * Initialize the common actions and build various collections of artifacts
+ * for the initializationHook() methods of the subclasses.
+ *
+ * <p>Note that not all subclasses call this method.
+ *
+ * @return the processed attributes
+ */
+ public JavaTargetAttributes.Builder initCommon(Collection<Artifact> extraSrcs) {
+ Preconditions.checkState(javacOpts != null);
+ sources = ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list();
+ activePlugins = collectPlugins();
+
+ JavaTargetAttributes.Builder javaTargetAttributes = new JavaTargetAttributes.Builder(semantics);
+ processSrcs(javaTargetAttributes, javacOpts);
+ javaTargetAttributes.addSourceArtifacts(extraSrcs);
+ processRuntimeDeps(javaTargetAttributes);
+
+ semantics.commonDependencyProcessing(ruleContext, javaTargetAttributes,
+ targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY));
+
+ // Check that we have do not have both sources and jars.
+ if ((javaTargetAttributes.hasSourceFiles() || javaTargetAttributes.hasSourceJars())
+ && javaTargetAttributes.hasJarFiles()) {
+ ruleContext.attributeWarning("srcs", "cannot use both Java sources - source "
+ + "jars or source files - and precompiled jars");
+ }
+
+ if (disallowDepsWithoutSrcs(ruleContext.getRule().getRuleClass())
+ && ruleContext.attributes().get("srcs", Type.LABEL_LIST).isEmpty()
+ && ruleContext.getRule().isAttributeValueExplicitlySpecified("deps")) {
+ ruleContext.attributeError("deps", "deps not allowed without srcs; move to runtime_deps?");
+ }
+
+ javaTargetAttributes.addResources(semantics.collectResources(ruleContext));
+ addPlugins(javaTargetAttributes);
+
+ javaTargetAttributes.setRuleKind(ruleContext.getRule().getRuleClass());
+ javaTargetAttributes.setTargetLabel(ruleContext.getLabel());
+
+ return javaTargetAttributes;
+ }
+
+ private boolean disallowDepsWithoutSrcs(String ruleClass) {
+ return ruleClass.equals("java_library")
+ || ruleClass.equals("java_binary")
+ || ruleClass.equals("java_test");
+ }
+
+ public ImmutableList<? extends TransitiveInfoCollection> targetsTreatedAsDeps(
+ ClasspathType type) {
+ return targetsTreatedAsDeps.get(type);
+ }
+
+ private static ImmutableList<TransitiveInfoCollection> collectTargetsTreatedAsDeps(
+ RuleContext ruleContext, JavaSemantics semantics, ClasspathType type) {
+ ImmutableList.Builder<TransitiveInfoCollection> builder = new Builder<>();
+
+ if (!type.equals(ClasspathType.COMPILE_ONLY)) {
+ builder.addAll(getRuntimeDeps(ruleContext));
+ builder.addAll(getExports(ruleContext));
+ }
+ builder.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET));
+
+ semantics.collectTargetsTreatedAsDeps(ruleContext, builder);
+
+ // Implicitly add dependency on java launcher cc_binary when --java_launcher= is enabled,
+ // or when launcher attribute is specified in a build rule.
+ TransitiveInfoCollection launcher = JavaHelper.launcherForTarget(semantics, ruleContext);
+ if (launcher != null) {
+ builder.add(launcher);
+ }
+
+ return builder.build();
+ }
+
+ public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder,
+ NestedSet<Artifact> filesToBuild, @Nullable Artifact classJar) {
+ InstrumentedFilesCollector instrumentedFilesCollector =
+ new InstrumentedFilesCollector(ruleContext, semantics.getCoverageInstrumentationSpec(),
+ JAVA_METADATA_COLLECTOR, filesToBuild);
+
+ builder
+ .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl(
+ instrumentedFilesCollector))
+ .add(FilesToCompileProvider.class,
+ new FilesToCompileProvider(getFilesToCompile(classJar)))
+ .add(JavaExportsProvider.class, new JavaExportsProvider(collectTransitiveExports()));
+
+ if (!TargetUtils.isTestRule(ruleContext.getTarget())) {
+ ImmutableList<Artifact> baselineCoverageArtifacts =
+ BaselineCoverageAction.getBaselineCoverageArtifacts(ruleContext,
+ instrumentedFilesCollector.getInstrumentedFiles());
+ builder.setBaselineCoverageArtifacts(baselineCoverageArtifacts);
+ }
+ }
+
+ /**
+ * Processes the sources of this target, adding them as messages, proper
+ * sources or to the list of targets treated as deps as required.
+ */
+ private void processSrcs(JavaTargetAttributes.Builder attributes,
+ ImmutableList<String> javacOpts) {
+ for (MessageBundleProvider srcItem : ruleContext.getPrerequisites(
+ "srcs", Mode.TARGET, MessageBundleProvider.class)) {
+ attributes.addMessages(srcItem.getMessages());
+ }
+
+ attributes.addSourceArtifacts(sources);
+
+ addCompileTimeClassPathEntriesMaybeThroughIjar(attributes, javacOpts);
+ }
+
+ /**
+ * Processes the transitive runtime_deps of this target.
+ */
+ private void processRuntimeDeps(JavaTargetAttributes.Builder attributes) {
+ List<TransitiveInfoCollection> runtimeDepInfo = getRuntimeDeps(ruleContext);
+ checkRuntimeDeps(runtimeDepInfo);
+ JavaCompilationArgs args = JavaCompilationArgs.builder()
+ .addTransitiveTargets(runtimeDepInfo, true, ClasspathType.RUNTIME_ONLY)
+ .build();
+ attributes.addRuntimeClassPathEntries(args.getRuntimeJars());
+ attributes.addInstrumentationMetadataEntries(args.getInstrumentationMetadata());
+ }
+
+ public Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources() {
+ return ruleContext.getPrerequisites("srcs", Mode.TARGET,
+ SourcesJavaCompilationArgsProvider.class);
+ }
+
+ /**
+ * Adds jars in the given group of entries to the compile time classpath after
+ * using ijar to create jar interfaces for the generated jars.
+ */
+ private void addCompileTimeClassPathEntriesMaybeThroughIjar(
+ JavaTargetAttributes.Builder attributes, ImmutableList<String> javacOpts) {
+ JavaCompilationHelper helper = new JavaCompilationHelper(
+ ruleContext, semantics, javacOpts, attributes);
+ for (FileProvider provider : ruleContext
+ .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) {
+ Iterable<Artifact> jarFiles = helper.filterGeneratedJarsThroughIjar(
+ FileType.filter(provider.getFilesToBuild(), JavaSemantics.JAR));
+ List<Artifact> jarsWithOwners = Lists.newArrayList(jarFiles);
+ attributes.addDirectCompileTimeClassPathEntries(jarsWithOwners);
+ attributes.addCompileTimeJarFiles(jarsWithOwners);
+ }
+ }
+
+ /**
+ * Adds information about the annotation processors that should be run for this java target to
+ * the target attributes.
+ */
+ private void addPlugins(JavaTargetAttributes.Builder attributes) {
+ for (JavaPluginInfoProvider plugin : activePlugins) {
+ for (String name : plugin.getProcessorClasses()) {
+ attributes.addProcessorName(name);
+ }
+ // Now get the plugin-libraries runtime classpath.
+ attributes.addProcessorPath(plugin.getProcessorClasspath());
+ }
+ }
+
+ private ImmutableList<JavaPluginInfoProvider> collectPlugins() {
+ List<JavaPluginInfoProvider> result = new ArrayList<>();
+ Iterables.addAll(result, getPluginInfoProvidersForAttribute(":java_plugins", Mode.HOST));
+ Iterables.addAll(result, getPluginInfoProvidersForAttribute("plugins", Mode.HOST));
+ Iterables.addAll(result, getPluginInfoProvidersForAttribute("deps", Mode.TARGET));
+ return ImmutableList.copyOf(result);
+ }
+
+ Iterable<JavaPluginInfoProvider> getPluginInfoProvidersForAttribute(String attribute,
+ Mode mode) {
+ if (ruleContext.getRule().getRuleClassObject().hasAttr(attribute, Type.LABEL_LIST)) {
+ return ruleContext.getPrerequisites(attribute, mode, JavaPluginInfoProvider.class);
+ }
+ return ImmutableList.of();
+ }
+
+ /**
+ * Gets all the deps.
+ */
+ public final Iterable<? extends TransitiveInfoCollection> getDependencies() {
+ return targetsTreatedAsDeps(ClasspathType.BOTH);
+ }
+
+ /**
+ * Gets all the deps that implement a particular provider.
+ */
+ public final <P extends TransitiveInfoProvider> Iterable<P> getDependencies(
+ Class<P> provider) {
+ return AnalysisUtils.getProviders(getDependencies(), provider);
+ }
+
+ /**
+ * Returns true if and only if this target has the neverlink attribute set to
+ * 1, or false if the neverlink attribute does not exist (for example, on
+ * *_binary targets)
+ *
+ * @return the value of the neverlink attribute.
+ */
+ public final boolean isNeverLink() {
+ return ruleContext.getRule().isAttrDefined("neverlink", Type.BOOLEAN) &&
+ ruleContext.attributes().get("neverlink", Type.BOOLEAN);
+ }
+
+ private ImmutableList<Artifact> getFilesToCompile(Artifact classJar) {
+ if (classJar == null) {
+ // Some subclasses don't produce jars
+ return ImmutableList.of();
+ }
+ return ImmutableList.of(classJar);
+ }
+
+ public ImmutableList<Dependency> computeStrictDepsFromJavaAttributes(
+ JavaTargetAttributes javaTargetAttributes) {
+ Multimap<Label, String> depMap = HashMultimap.<Label, String>create();
+ for (Artifact jar : javaTargetAttributes.getDirectJars()) {
+ depMap.put(Preconditions.checkNotNull(jar.getOwner()),
+ jar.getExecPathString());
+ }
+ ImmutableList.Builder<Dependency> depOuts = ImmutableList.builder();
+ for (Label label : depMap.keySet()) {
+ depOuts.add(new Dependency(label, depMap.get(label)));
+ }
+ return depOuts.build();
+ }
+
+ public ImmutableList<Artifact> getSrcsArtifacts() {
+ return sources;
+ }
+
+ public ImmutableList<String> getJavacOpts() {
+ return javacOpts;
+ }
+
+ public ImmutableList<Artifact> getBootClasspath() {
+ return classpathFragment.getBootClasspath();
+ }
+
+ public NestedSet<Artifact> getRuntimeClasspath() {
+ return classpathFragment.getRuntimeClasspath();
+ }
+
+ public NestedSet<Artifact> getCompileTimeClasspath() {
+ return classpathFragment.getCompileTimeClasspath();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java
new file mode 100644
index 0000000..145d646
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java
@@ -0,0 +1,301 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.util.FileType;
+
+import java.util.Collection;
+
+/**
+ * A container of Java compilation artifacts.
+ */
+public final class JavaCompilationArgs {
+ // TODO(bazel-team): It would be desirable to use LinkOrderNestedSet here so that
+ // parents-before-deps is preserved for graphs that are not trees. However, the legacy
+ // JavaLibraryCollector implemented naive link ordering and many targets in the
+ // depot depend on the consistency of left-to-right ordering that is not provided by
+ // LinkOrderNestedSet. They simply list their local dependencies before
+ // other targets that may use conflicting dependencies, and the local deps
+ // appear earlier on the classpath, as desired. Behavior of LinkOrderNestedSet
+ // can be very unintuitive in case of conflicting orders, because the order is
+ // decided by the rightmost branch in such cases. For example, if A depends on {junit4,
+ // B}, B depends on {C, D}, C depends on {junit3}, and D depends on {junit4},
+ // the classpath of A will have junit3 before junit4.
+ private final NestedSet<Artifact> runtimeJars;
+ private final NestedSet<Artifact> compileTimeJars;
+ private final NestedSet<Artifact> instrumentationMetadata;
+
+ public static final JavaCompilationArgs EMPTY_ARGS = new JavaCompilationArgs(
+ NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER),
+ NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER),
+ NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER));
+
+ private JavaCompilationArgs(NestedSet<Artifact> runtimeJars,
+ NestedSet<Artifact> compileTimeJars,
+ NestedSet<Artifact> instrumentationMetadata) {
+ this.runtimeJars = runtimeJars;
+ this.compileTimeJars = compileTimeJars;
+ this.instrumentationMetadata = instrumentationMetadata;
+ }
+
+ /**
+ * Returns transitive runtime jars.
+ */
+ public NestedSet<Artifact> getRuntimeJars() {
+ return runtimeJars;
+ }
+
+ /**
+ * Returns transitive compile-time jars.
+ */
+ public NestedSet<Artifact> getCompileTimeJars() {
+ return compileTimeJars;
+ }
+
+ /**
+ * Returns transitive instrumentation metadata jars.
+ */
+ public NestedSet<Artifact> getInstrumentationMetadata() {
+ return instrumentationMetadata;
+ }
+
+ /**
+ * Returns a new builder instance.
+ */
+ public static final Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Builder for {@link JavaCompilationArgs}.
+ *
+ *
+ */
+ public static final class Builder {
+ private final NestedSetBuilder<Artifact> runtimeJarsBuilder =
+ NestedSetBuilder.naiveLinkOrder();
+ private final NestedSetBuilder<Artifact> compileTimeJarsBuilder =
+ NestedSetBuilder.naiveLinkOrder();
+ private final NestedSetBuilder<Artifact> instrumentationMetadataBuilder =
+ NestedSetBuilder.naiveLinkOrder();
+
+ /**
+ * Use {@code TransitiveJavaCompilationArgs#builder()} to instantiate the builder.
+ */
+ private Builder() {
+ }
+
+ /**
+ * Legacy method for dealing with objects which construct
+ * {@link JavaCompilationArtifacts} objects.
+ */
+ // TODO(bazel-team): Remove when we get rid of JavaCompilationArtifacts.
+ public Builder merge(JavaCompilationArtifacts other, boolean isNeverLink) {
+ if (!isNeverLink) {
+ addRuntimeJars(other.getRuntimeJars());
+ }
+ addCompileTimeJars(other.getCompileTimeJars());
+ addInstrumentationMetadata(other.getInstrumentationMetadata());
+ return this;
+ }
+
+ /**
+ * Legacy method for dealing with objects which construct
+ * {@link JavaCompilationArtifacts} objects.
+ */
+ public Builder merge(JavaCompilationArtifacts other) {
+ return merge(other, false);
+ }
+
+ public Builder addRuntimeJar(Artifact runtimeJar) {
+ this.runtimeJarsBuilder.add(runtimeJar);
+ return this;
+ }
+
+ public Builder addRuntimeJars(Iterable<Artifact> runtimeJars) {
+ this.runtimeJarsBuilder.addAll(runtimeJars);
+ return this;
+ }
+
+ public Builder addCompileTimeJar(Artifact compileTimeJar) {
+ this.compileTimeJarsBuilder.add(compileTimeJar);
+ return this;
+ }
+
+ public Builder addCompileTimeJars(Iterable<Artifact> compileTimeJars) {
+ this.compileTimeJarsBuilder.addAll(compileTimeJars);
+ return this;
+ }
+
+ public Builder addInstrumentationMetadata(Artifact instrumentationMetadata) {
+ this.instrumentationMetadataBuilder.add(instrumentationMetadata);
+ return this;
+ }
+
+ public Builder addInstrumentationMetadata(Collection<Artifact> instrumentationMetadata) {
+ this.instrumentationMetadataBuilder.addAll(instrumentationMetadata);
+ return this;
+ }
+
+ public Builder addTransitiveCompilationArgs(
+ JavaCompilationArgsProvider dep, boolean recursive, ClasspathType type) {
+ JavaCompilationArgs args = recursive
+ ? dep.getRecursiveJavaCompilationArgs()
+ : dep.getJavaCompilationArgs();
+ addTransitiveArgs(args, type);
+ return this;
+ }
+
+ public Builder addTransitiveCompilationArgs(
+ SourcesJavaCompilationArgsProvider dep, boolean recursive, ClasspathType type) {
+ JavaCompilationArgs args;
+ if (recursive) {
+ args = dep.getRecursiveJavaCompilationArgs();
+ } else {
+ args = dep.getJavaCompilationArgs();
+ }
+ addTransitiveArgs(args, type);
+ return this;
+ }
+
+ public Builder addSourcesTransitiveCompilationArgs(
+ Iterable<? extends SourcesJavaCompilationArgsProvider> deps,
+ boolean recursive,
+ ClasspathType type) {
+ for (SourcesJavaCompilationArgsProvider dep : deps) {
+ addTransitiveCompilationArgs(dep, recursive, type);
+ }
+
+ return this;
+ }
+
+ /**
+ * Merges the artifacts of another target.
+ */
+ public Builder addTransitiveTarget(TransitiveInfoCollection dep, boolean recursive,
+ ClasspathType type) {
+ JavaCompilationArgsProvider provider = dep.getProvider(JavaCompilationArgsProvider.class);
+ if (provider != null) {
+ addTransitiveCompilationArgs(provider, recursive, type);
+ return this;
+ } else {
+ NestedSet<Artifact> filesToBuild =
+ dep.getProvider(FileProvider.class).getFilesToBuild();
+ for (Artifact jar : FileType.filter(filesToBuild, JavaSemantics.JAR)) {
+ addCompileTimeJar(jar);
+ addRuntimeJar(jar);
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Merges the artifacts of a collection of targets.
+ */
+ public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps,
+ boolean recursive, ClasspathType type) {
+ for (TransitiveInfoCollection dep : deps) {
+ addTransitiveTarget(dep, recursive, type);
+ }
+ return this;
+ }
+
+ /**
+ * Merges the artifacts of a collection of targets.
+ */
+ public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps,
+ boolean recursive) {
+ return addTransitiveTargets(deps, recursive, ClasspathType.BOTH);
+ }
+
+ /**
+ * Merges the artifacts of a collection of targets.
+ */
+ public Builder addTransitiveDependencies(Iterable<JavaCompilationArgsProvider> deps,
+ boolean recursive) {
+ for (JavaCompilationArgsProvider dep : deps) {
+ addTransitiveDependency(dep, recursive, ClasspathType.BOTH);
+ }
+ return this;
+ }
+
+ /**
+ * Merges the artifacts of another target.
+ */
+ private Builder addTransitiveDependency(JavaCompilationArgsProvider dep, boolean recursive,
+ ClasspathType type) {
+ JavaCompilationArgs args = recursive
+ ? dep.getRecursiveJavaCompilationArgs()
+ : dep.getJavaCompilationArgs();
+ addTransitiveArgs(args, type);
+ return this;
+ }
+
+ /**
+ * Merges the artifacts of a collection of targets.
+ */
+ public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps) {
+ return addTransitiveTargets(deps, /*recursive=*/true, ClasspathType.BOTH);
+ }
+
+ /**
+ * Includes the contents of another instance of JavaCompilationArgs.
+ *
+ * @param args the JavaCompilationArgs instance
+ * @param type the classpath(s) to consider
+ */
+ public Builder addTransitiveArgs(JavaCompilationArgs args, ClasspathType type) {
+ if (!ClasspathType.RUNTIME_ONLY.equals(type)) {
+ compileTimeJarsBuilder.addTransitive(args.getCompileTimeJars());
+ }
+ if (!ClasspathType.COMPILE_ONLY.equals(type)) {
+ runtimeJarsBuilder.addTransitive(args.getRuntimeJars());
+ }
+ instrumentationMetadataBuilder.addTransitive(
+ args.getInstrumentationMetadata());
+ return this;
+ }
+
+ /**
+ * Builds a {@link JavaCompilationArgs} object.
+ */
+ public JavaCompilationArgs build() {
+ return new JavaCompilationArgs(
+ runtimeJarsBuilder.build(),
+ compileTimeJarsBuilder.build(),
+ instrumentationMetadataBuilder.build());
+ }
+ }
+
+ /**
+ * Enum to specify transitive compilation args traversal
+ */
+ public static enum ClasspathType {
+ /* treat the same for compile time and runtime */
+ BOTH,
+
+ /* Only include on compile classpath */
+ COMPILE_ONLY,
+
+ /* Only include on runtime classpath */
+ RUNTIME_ONLY;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java
new file mode 100644
index 0000000..1958ada
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java
@@ -0,0 +1,94 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * An interface for objects that provide information on how to include them in
+ * Java builds.
+ */
+@Immutable
+public final class JavaCompilationArgsProvider implements TransitiveInfoProvider {
+ private final JavaCompilationArgs javaCompilationArgs;
+ private final JavaCompilationArgs recursiveJavaCompilationArgs;
+ private final NestedSet<Artifact> compileTimeJavaDepArtifacts;
+ private final NestedSet<Artifact> runTimeJavaDepArtifacts;
+
+ public JavaCompilationArgsProvider(JavaCompilationArgs javaCompilationArgs,
+ JavaCompilationArgs recursiveJavaCompilationArgs,
+ NestedSet<Artifact> compileTimeJavaDepArtifacts,
+ NestedSet<Artifact> runTimeJavaDepArtifacts) {
+ this.javaCompilationArgs = javaCompilationArgs;
+ this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs;
+ this.compileTimeJavaDepArtifacts = compileTimeJavaDepArtifacts;
+ this.runTimeJavaDepArtifacts = runTimeJavaDepArtifacts;
+ }
+
+ public JavaCompilationArgsProvider(JavaCompilationArgs javaCompilationArgs,
+ JavaCompilationArgs recursiveJavaCompilationArgs) {
+ this.javaCompilationArgs = javaCompilationArgs;
+ this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs;
+ this.compileTimeJavaDepArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER);
+ this.runTimeJavaDepArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER);
+ }
+
+ /**
+ * Returns non-recursively collected Java compilation information for
+ * building this target (called when strict_java_deps = 1).
+ *
+ * <p>Note that some of the parameters are still collected from the complete
+ * transitive closure. The non-recursive collection applies mainly to
+ * compile-time jars.
+ */
+ public JavaCompilationArgs getJavaCompilationArgs() {
+ return javaCompilationArgs;
+ }
+
+ /**
+ * Returns recursively collected Java compilation information for building
+ * this target (called when strict_java_deps = 0).
+ */
+ public JavaCompilationArgs getRecursiveJavaCompilationArgs() {
+ return recursiveJavaCompilationArgs;
+ }
+
+ /**
+ * Returns non-recursively collected Java dependency artifacts for
+ * computing a restricted classpath when building this target (called when
+ * strict_java_deps = 1).
+ *
+ * <p>Note that dependency artifacts are needed only when non-recursive
+ * compilation args do not provide a safe super-set of dependencies.
+ * Non-strict targets such as proto_library, always collecting their
+ * transitive closure of deps, do not need to provide dependency artifacts.
+ */
+ public NestedSet<Artifact> getCompileTimeJavaDependencyArtifacts() {
+ return compileTimeJavaDepArtifacts;
+ }
+
+ /**
+ * Returns Java dependency artifacts for computing a restricted run-time
+ * classpath (called when strict_java_deps = 1).
+ */
+ public NestedSet<Artifact> getRunTimeJavaDependencyArtifacts() {
+ return runTimeJavaDepArtifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java
new file mode 100644
index 0000000..98ccbac
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java
@@ -0,0 +1,148 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A collection of artifacts for java compilations. It concisely describes the
+ * outputs of a java-related rule, with runtime jars, compile-time jars,
+ * unfiltered compile-time jars (these are run through ijar if they are
+ * dependent upon by another target), source ijars, and instrumentation
+ * manifests. Not all rules generate all kinds of artifacts. Each java-related
+ * rule should add both a runtime jar and either a compile-time jar or an
+ * unfiltered compile-time jar.
+ *
+ * <p>An instance of this class only collects the data for the current target,
+ * not for the transitive closure of targets, so these still need to be
+ * collected using some other mechanism, such as the {@link
+ * JavaCompilationArgsProvider}.
+ */
+@Immutable
+public final class JavaCompilationArtifacts {
+
+ public static final JavaCompilationArtifacts EMPTY = new Builder().build();
+
+ private final ImmutableList<Artifact> runtimeJars;
+ private final ImmutableList<Artifact> compileTimeJars;
+ private final ImmutableList<Artifact> instrumentationMetadata;
+ private final Artifact compileTimeDependencyArtifact;
+ private final Artifact runTimeDependencyArtifact;
+ private final Artifact instrumentedJar;
+
+ private JavaCompilationArtifacts(ImmutableList<Artifact> runtimeJars,
+ ImmutableList<Artifact> compileTimeJars,
+ ImmutableList<Artifact> instrumentationMetadata,
+ Artifact compileTimeDependencyArtifact, Artifact runTimeDependencyArtifact,
+ Artifact instrumentedJar) {
+ this.runtimeJars = runtimeJars;
+ this.compileTimeJars = compileTimeJars;
+ this.instrumentationMetadata = instrumentationMetadata;
+ this.compileTimeDependencyArtifact = compileTimeDependencyArtifact;
+ this.runTimeDependencyArtifact = runTimeDependencyArtifact;
+ this.instrumentedJar = instrumentedJar;
+ }
+
+ public ImmutableList<Artifact> getRuntimeJars() {
+ return runtimeJars;
+ }
+
+ public ImmutableList<Artifact> getCompileTimeJars() {
+ return compileTimeJars;
+ }
+
+ public ImmutableList<Artifact> getInstrumentationMetadata() {
+ return instrumentationMetadata;
+ }
+
+ public Artifact getCompileTimeDependencyArtifact() {
+ return compileTimeDependencyArtifact;
+ }
+
+ public Artifact getRunTimeDependencyArtifact() {
+ return runTimeDependencyArtifact;
+ }
+
+ public Artifact getInstrumentedJar() {
+ return instrumentedJar;
+ }
+
+ /**
+ * A builder for {@link JavaCompilationArtifacts}.
+ */
+ public static final class Builder {
+ private final Set<Artifact> runtimeJars = new LinkedHashSet<>();
+ private final Set<Artifact> compileTimeJars = new LinkedHashSet<>();
+ private final Set<Artifact> instrumentationMetadata = new LinkedHashSet<>();
+ private Artifact compileTimeDependencies;
+ private Artifact runTimeDependencies;
+ private Artifact instrumentedJar;
+
+ public JavaCompilationArtifacts build() {
+ return new JavaCompilationArtifacts(ImmutableList.copyOf(runtimeJars),
+ ImmutableList.copyOf(compileTimeJars),
+ ImmutableList.copyOf(instrumentationMetadata),
+ compileTimeDependencies, runTimeDependencies, instrumentedJar);
+ }
+
+ public Builder addRuntimeJar(Artifact jar) {
+ this.runtimeJars.add(jar);
+ return this;
+ }
+
+ public Builder addRuntimeJars(Iterable<Artifact> jars) {
+ Iterables.addAll(this.runtimeJars, jars);
+ return this;
+ }
+
+ public Builder addCompileTimeJar(Artifact jar) {
+ this.compileTimeJars.add(jar);
+ return this;
+ }
+
+ public Builder addCompileTimeJars(Iterable<Artifact> jars) {
+ Iterables.addAll(this.compileTimeJars, jars);
+ return this;
+ }
+
+ public Builder addInstrumentationMetadata(Artifact instrumentationMetadata) {
+ this.instrumentationMetadata.add(instrumentationMetadata);
+ return this;
+ }
+
+ public Builder setCompileTimeDependencies(@Nullable Artifact compileTimeDependencies) {
+ this.compileTimeDependencies = compileTimeDependencies;
+ return this;
+ }
+
+ public Builder setRunTimeDependencies(@Nullable Artifact runTimeDependencies) {
+ this.runTimeDependencies = runTimeDependencies;
+ return this;
+ }
+
+ public Builder setInstrumentedJar(@Nullable Artifact instrumentedJar) {
+ this.instrumentedJar = instrumentedJar;
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java
new file mode 100644
index 0000000..181dd12
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java
@@ -0,0 +1,436 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import static com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode.OFF;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class for compiling Java targets. It contains method to create the
+ * various intermediate Artifacts for using ijars and source ijars.
+ * <p>
+ * Also supports the creation of resource and source only Jars.
+ */
+public class JavaCompilationHelper extends BaseJavaCompilationHelper {
+ private Artifact outputDepsProtoArtifact;
+ private JavaTargetAttributes.Builder attributes;
+ private JavaTargetAttributes builtAttributes;
+ private final ImmutableList<String> customJavacOpts;
+ private final List<Artifact> translations = new ArrayList<>();
+ private boolean translationsFrozen = false;
+ private final JavaSemantics semantics;
+
+ public JavaCompilationHelper(RuleContext ruleContext, JavaSemantics semantics,
+ ImmutableList<String> javacOpts, JavaTargetAttributes.Builder attributes) {
+ super(ruleContext);
+ this.attributes = attributes;
+ this.customJavacOpts = javacOpts;
+ this.semantics = semantics;
+ }
+
+ public JavaCompilationHelper(RuleContext ruleContext, JavaSemantics semantics,
+ JavaTargetAttributes.Builder attributes) {
+ this(ruleContext, semantics, getDefaultJavacOptsFromRule(ruleContext), attributes);
+ }
+
+ public JavaTargetAttributes getAttributes() {
+ if (builtAttributes == null) {
+ builtAttributes = attributes.build();
+ }
+ return builtAttributes;
+ }
+
+ /**
+ * Creates the Action that compiles Java source files.
+ *
+ * @param outputJar the class jar Artifact to create with the Action
+ * @param gensrcOutputJar the generated sources jar Artifact to create with the Action
+ * (null if no sources will be generated).
+ * @param outputDepsProto the compiler-generated jdeps file to create with the Action
+ * (null if not requested)
+ * @param outputMetadata metadata file (null if no instrumentation is needed).
+ */
+ public void createCompileAction(Artifact outputJar, @Nullable Artifact gensrcOutputJar,
+ @Nullable Artifact outputDepsProto, @Nullable Artifact outputMetadata) {
+ JavaTargetAttributes attributes = getAttributes();
+ List<String> javacOpts = getJavacOpts();
+ JavaCompileAction.Builder builder = createJavaCompileActionBuilder(semantics);
+ builder.setClasspathEntries(attributes.getCompileTimeClassPath());
+ builder.addResources(attributes.getResources());
+ builder.addClasspathResources(attributes.getClassPathResources());
+ // Only add default bootclasspath entries if not explicitly set in attributes.
+ if (!attributes.getBootClassPath().isEmpty()) {
+ builder.setBootclasspathEntries(attributes.getBootClassPath());
+ } else {
+ builder.setBootclasspathEntries(getBootClasspath());
+ }
+ builder.setLangtoolsJar(getLangtoolsJar());
+ builder.setJavaBuilderJar(getJavaBuilderJar());
+ builder.addTranslations(getTranslations());
+ builder.setOutputJar(outputJar);
+ builder.setGensrcOutputJar(gensrcOutputJar);
+ builder.setOutputDepsProto(outputDepsProto);
+ builder.setMetadata(outputMetadata);
+ builder.addSourceFiles(attributes.getSourceFiles());
+ builder.addSourceJars(attributes.getSourceJars());
+ builder.setJavacOpts(javacOpts);
+ builder.setCompressJar(true);
+ builder.setClassDirectory(outputDir(outputJar));
+ builder.setSourceGenDirectory(sourceGenDir(outputJar));
+ builder.setTempDirectory(tempDir(outputJar));
+ builder.addProcessorPaths(attributes.getProcessorPath());
+ builder.addProcessorNames(attributes.getProcessorNames());
+ builder.setStrictJavaDeps(attributes.getStrictJavaDeps());
+ builder.addDirectJars(attributes.getDirectJars());
+ builder.addCompileTimeDependencyArtifacts(attributes.getCompileTimeDependencyArtifacts());
+ builder.setRuleKind(attributes.getRuleKind());
+ builder.setTargetLabel(attributes.getTargetLabel());
+ getAnalysisEnvironment().registerAction(builder.build());
+ }
+
+ /**
+ * Creates the Action that compiles Java source files and optionally instruments them for
+ * coverage.
+ *
+ * @param outputJar the class jar Artifact to create with the Action
+ * @param gensrcJar the generated sources jar Artifact to create with the Action
+ * @param outputDepsProto the compiler-generated jdeps file to create with the Action
+ * @param javaArtifactsBuilder the build to store the instrumentation metadata in
+ */
+ public void createCompileActionWithInstrumentation(Artifact outputJar, Artifact gensrcJar,
+ Artifact outputDepsProto, JavaCompilationArtifacts.Builder javaArtifactsBuilder) {
+ createCompileAction(outputJar, gensrcJar, outputDepsProto,
+ createInstrumentationMetadata(outputJar, javaArtifactsBuilder));
+ }
+
+ /**
+ * Creates the instrumentation metadata artifact if needed.
+ *
+ * @return the instrumentation metadata artifact or null if instrumentation is
+ * disabled
+ */
+ public Artifact createInstrumentationMetadata(Artifact outputJar,
+ JavaCompilationArtifacts.Builder javaArtifactsBuilder) {
+ // If we need to instrument the jar, add additional output (the coverage metadata file) to the
+ // JavaCompileAction.
+ Artifact instrumentationMetadata = null;
+ if (shouldInstrumentJar()) {
+ instrumentationMetadata = semantics.createInstrumentationMetadataArtifact(
+ getAnalysisEnvironment(), outputJar);
+
+ if (instrumentationMetadata != null) {
+ javaArtifactsBuilder.addInstrumentationMetadata(instrumentationMetadata);
+ }
+ }
+ return instrumentationMetadata;
+ }
+
+ private boolean shouldInstrumentJar() {
+ // TODO(bazel-team): What about source jars?
+ return getConfiguration().isCodeCoverageEnabled() && attributes.hasSourceFiles() &&
+ getConfiguration().getInstrumentationFilter().isIncluded(
+ getRuleContext().getLabel().toString());
+ }
+
+ /**
+ * Returns the artifact for a jar file containing source files that were generated by an
+ * annotation processor or null if no annotation processors are used.
+ */
+ public Artifact createGensrcJar(@Nullable Artifact outputJar) {
+ if (!usesAnnotationProcessing()) {
+ return null;
+ }
+ return getAnalysisEnvironment().getDerivedArtifact(
+ FileSystemUtils.appendWithoutExtension(outputJar.getRootRelativePath(), "-gensrc"),
+ outputJar.getRoot());
+ }
+
+ /**
+ * Returns whether this target uses annotation processing.
+ */
+ private boolean usesAnnotationProcessing() {
+ JavaTargetAttributes attributes = getAttributes();
+ return getJavacOpts().contains("-processor") || !attributes.getProcessorNames().isEmpty();
+ }
+
+ public Artifact getOutputDepsProtoArtifact() {
+ return outputDepsProtoArtifact;
+ }
+ /**
+ * Creates the jdeps file artifact if needed. Returns null if the target can't emit dependency
+ * information (i.e there is no compilation step, the target acts as an alias).
+ *
+ * @param outputJar output jar artifact used to derive the name
+ * @return the jdeps file artifact or null if the target can't generate such a file
+ */
+ public Artifact createOutputDepsProtoArtifact(Artifact outputJar,
+ JavaCompilationArtifacts.Builder builder) {
+ if (!generatesOutputDeps()) {
+ return null;
+ }
+
+ outputDepsProtoArtifact = getAnalysisEnvironment().getDerivedArtifact(
+ FileSystemUtils.replaceExtension(outputJar.getRootRelativePath(), ".jdeps"),
+ outputJar.getRoot());
+
+ builder.setRunTimeDependencies(outputDepsProtoArtifact);
+ return outputDepsProtoArtifact;
+ }
+
+ /**
+ * Returns whether this target emits dependency information. Compilation must occur, so certain
+ * targets acting as aliases have to be filtered out.
+ */
+ private boolean generatesOutputDeps() {
+ return getJavaConfiguration().getGenerateJavaDeps() &&
+ (attributes.hasSourceFiles() || attributes.hasSourceJars());
+ }
+
+ /**
+ * Creates an Action that packages all of the resources into a Jar. This
+ * includes the declared resources, the classpath resources and the translated
+ * messages.
+ *
+ * <p>The resource jar artifact is derived from the given original jar, by
+ * prepending the given prefix and appending the given suffix. The new jar
+ * uses the same root as the original jar.
+ */
+ // TODO(bazel-team): Extract this method to make it easier to create simple
+ // zip/jar archives without having to first create a JavaCompilationhelper and
+ // JavaTargetAttributes.
+ public Artifact createResourceJarAction(Artifact resourceJar) {
+ JavaTargetAttributes attributes = getAttributes();
+ JavaCompileAction.Builder builder = createJavaCompileActionBuilder(semantics);
+ builder.setOutputJar(resourceJar);
+ builder.addResources(attributes.getResources());
+ builder.addClasspathResources(attributes.getClassPathResources());
+ builder.setLangtoolsJar(getLangtoolsJar());
+ builder.addTranslations(getTranslations());
+ builder.setCompressJar(true);
+ builder.setClassDirectory(outputDir(resourceJar));
+ builder.setTempDirectory(tempDir(resourceJar));
+ builder.setJavaBuilderJar(getJavaBuilderJar());
+ getAnalysisEnvironment().registerAction(builder.build());
+ return resourceJar;
+ }
+
+ /**
+ * Creates an Action that packages the Java source files into a Jar. If {@code gensrcJar} is
+ * non-null, includes the contents of the {@code gensrcJar} with the output source jar.
+ *
+ * @param outputJar the Artifact to create with the Action
+ * @param gensrcJar the generated sources jar Artifact that should be included with the
+ * sources in the output Artifact. May be null.
+ */
+ public void createSourceJarAction(Artifact outputJar, @Nullable Artifact gensrcJar) {
+ JavaTargetAttributes attributes = getAttributes();
+ Collection<Artifact> resourceJars = new ArrayList<>(attributes.getSourceJars());
+ if (gensrcJar != null) {
+ resourceJars.add(gensrcJar);
+ }
+ createSourceJarAction(semantics, attributes.getSourceFiles(), resourceJars, outputJar);
+ }
+
+ /**
+ * Creates the actions that produce the interface jar. Adds the jar artifacts to the given
+ * JavaCompilationArtifacts builder.
+ */
+ public void createCompileTimeJarAction(Artifact runtimeJar,
+ @Nullable Artifact runtimeDeps, JavaCompilationArtifacts.Builder builder) {
+ Artifact jar = getJavaConfiguration().getUseIjars()
+ ? createIjarAction(runtimeJar, false)
+ : runtimeJar;
+ Artifact deps = runtimeDeps;
+
+ builder.addCompileTimeJar(jar);
+ builder.setCompileTimeDependencies(deps);
+ }
+
+ /**
+ * Creates actions that create ijars from generated jars that are an input to
+ * the Java target.
+ *
+ * @return the generated ijars or original jars that are not generated by a
+ * genrule
+ */
+ public Iterable<Artifact> filterGeneratedJarsThroughIjar(Iterable<Artifact> jars) {
+ if (!getJavaConfiguration().getUseIjars()) {
+ return jars;
+ }
+ // We need to copy this list in order to avoid generating a new action each time the iterator
+ // is enumerated
+ return ImmutableList.copyOf(Iterables.transform(jars, new Function<Artifact, Artifact>() {
+ @Override
+ public Artifact apply(Artifact jar) {
+ return !jar.isSourceArtifact() ? createIjarAction(jar, true) : jar;
+ }
+ }));
+ }
+
+ private void addArgsAndJarsToAttributes(JavaCompilationArgs args, Iterable<Artifact> directJars) {
+ // Can only be non-null when isStrict() returns true.
+ if (directJars != null) {
+ attributes.addDirectCompileTimeClassPathEntries(directJars);
+ attributes.addDirectJars(directJars);
+ }
+
+ attributes.merge(args);
+ }
+
+ private void addLibrariesToAttributesInternal(Iterable<? extends TransitiveInfoCollection> deps) {
+ JavaCompilationArgs args = JavaCompilationArgs.builder()
+ .addTransitiveTargets(deps).build();
+
+ NestedSet<Artifact> directJars = isStrict()
+ ? getNonRecursiveCompileTimeJarsFromCollection(deps)
+ : null;
+ addArgsAndJarsToAttributes(args, directJars);
+ }
+
+ private void addProvidersToAttributesInternal(
+ Iterable<? extends SourcesJavaCompilationArgsProvider> deps, boolean isNeverLink) {
+ JavaCompilationArgs args = JavaCompilationArgs.builder()
+ .addSourcesTransitiveCompilationArgs(deps, true,
+ isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH)
+ .build();
+
+ NestedSet<Artifact> directJars = isStrict()
+ ? getNonRecursiveCompileTimeJarsFromProvider(deps, isNeverLink)
+ : null;
+ addArgsAndJarsToAttributes(args, directJars);
+ }
+
+ private boolean isStrict() {
+ return getStrictJavaDeps() != OFF;
+ }
+
+ private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromCollection(
+ Iterable<? extends TransitiveInfoCollection> deps) {
+ JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder();
+ builder.addTransitiveTargets(deps, /*recursive=*/false);
+ return builder.build().getCompileTimeJars();
+ }
+
+ private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromProvider(
+ Iterable<? extends SourcesJavaCompilationArgsProvider> deps, boolean isNeverLink) {
+ return JavaCompilationArgs.builder()
+ .addSourcesTransitiveCompilationArgs(deps, false,
+ isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH)
+ .build().getCompileTimeJars();
+ }
+
+ private void addDependencyArtifactsToAttributes(
+ Iterable<? extends TransitiveInfoCollection> deps) {
+ NestedSetBuilder<Artifact> compileTimeBuilder = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Artifact> runTimeBuilder = NestedSetBuilder.stableOrder();
+ for (JavaCompilationArgsProvider provider : AnalysisUtils.getProviders(
+ deps, JavaCompilationArgsProvider.class)) {
+ compileTimeBuilder.addTransitive(provider.getCompileTimeJavaDependencyArtifacts());
+ runTimeBuilder.addTransitive(provider.getRunTimeJavaDependencyArtifacts());
+ }
+ attributes.addCompileTimeDependencyArtifacts(compileTimeBuilder.build());
+ attributes.addRuntimeDependencyArtifacts(runTimeBuilder.build());
+ }
+
+ /**
+ * Adds the compile time and runtime Java libraries in the transitive closure
+ * of the deps to the attributes.
+ *
+ * @param deps the dependencies to be included as roots of the transitive
+ * closure
+ */
+ public void addLibrariesToAttributes(Iterable<? extends TransitiveInfoCollection> deps) {
+ // Enforcing strict Java dependencies: when the --strict_java_deps flag is
+ // WARN or ERROR, or is DEFAULT and strict_java_deps attribute is unset,
+ // we use a stricter javac compiler to perform direct deps checks.
+ attributes.setStrictJavaDeps(getStrictJavaDeps());
+ addLibrariesToAttributesInternal(deps);
+
+ JavaClasspathMode classpathMode = getJavaConfiguration().getReduceJavaClasspath();
+ if (isStrict() && classpathMode != JavaClasspathMode.OFF) {
+ addDependencyArtifactsToAttributes(deps);
+ }
+ }
+
+ public void addProvidersToAttributes(Iterable<? extends SourcesJavaCompilationArgsProvider> deps,
+ boolean isNeverLink) {
+ // see addLibrariesToAttributes() for explanation
+ attributes.setStrictJavaDeps(getStrictJavaDeps());
+ addProvidersToAttributesInternal(deps, isNeverLink);
+ }
+
+ /**
+ * Determines whether to enable strict_java_deps.
+ *
+ * @return filtered command line flag value, defaulting to ERROR
+ */
+ public StrictDepsMode getStrictJavaDeps() {
+ return getJavaConfiguration().getFilteredStrictJavaDeps();
+ }
+
+ /**
+ * Gets the value of the "javacopts" attribute combining them with the
+ * default options. If the current rule has no javacopts attribute, this
+ * method only returns the default options.
+ */
+ @VisibleForTesting
+ ImmutableList<String> getJavacOpts() {
+ return customJavacOpts;
+ }
+
+ /**
+ * Obtains the standard list of javac opts needed to build {@code rule}.
+ *
+ * This method must only be called during initialization.
+ *
+ * @param ruleContext a rule context
+ * @return a list of options to provide to javac
+ */
+ private static ImmutableList<String> getDefaultJavacOptsFromRule(RuleContext ruleContext) {
+ return ImmutableList.copyOf(Iterables.concat(
+ JavaToolchainProvider.getDefaultJavacOptions(ruleContext),
+ ruleContext.getTokenizedStringListAttr("javacopts")));
+ }
+
+ public void addTranslations(Collection<Artifact> translations) {
+ Preconditions.checkArgument(!translationsFrozen);
+ this.translations.addAll(translations);
+ }
+
+ private ImmutableList<Artifact> getTranslations() {
+ translationsFrozen = true;
+ return ImmutableList.copyOf(translations);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
new file mode 100644
index 0000000..655270b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
@@ -0,0 +1,1021 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ParameterFile;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
+import com.google.devtools.build.lib.actions.extra.JavaCompileInfo;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomArgv;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomMultiArgv;
+import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.ImmutableIterable;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+
+/**
+ * Action that represents a Java compilation.
+ */
+@ThreadCompatible
+public class JavaCompileAction extends AbstractAction {
+
+ private static final String GUID = "786e174d-ed97-4e79-9f61-ae74430714cf";
+
+ private static final ResourceSet LOCAL_RESOURCES =
+ new ResourceSet(750 /*MB*/, 0.5 /*CPU*/, 0.0 /*IO*/);
+
+ private final CommandLine javaCompileCommandLine;
+ private final CommandLine commandLine;
+
+ /**
+ * The directory in which generated classfiles are placed.
+ * May be erased/created by the JavaBuilder.
+ */
+ private final PathFragment classDirectory;
+
+ private final Artifact outputJar;
+
+ /**
+ * The list of classpath entries to specify to javac.
+ */
+ private final NestedSet<Artifact> classpath;
+
+ /**
+ * The list of classpath entries to search for annotation processors.
+ */
+ private final ImmutableList<Artifact> processorPath;
+
+ /**
+ * The list of annotation processor classes to run.
+ */
+ private final ImmutableList<String> processorNames;
+
+ /**
+ * The translation messages.
+ */
+ private final ImmutableList<Artifact> messages;
+
+ /**
+ * The set of resources to put into the jar.
+ */
+ private final ImmutableList<Artifact> resources;
+
+ /**
+ * The set of classpath resources to put into the jar.
+ */
+ private final ImmutableList<Artifact> classpathResources;
+
+ /**
+ * The set of files which contain lists of additional Java source files to
+ * compile.
+ */
+ private final ImmutableList<Artifact> sourceJars;
+
+ /**
+ * The set of explicit Java source files to compile.
+ */
+ private final ImmutableList<Artifact> sourceFiles;
+
+ /**
+ * The compiler options to pass to javac.
+ */
+ private final ImmutableList<String> javacOpts;
+
+ /**
+ * The subset of classpath jars provided by direct dependencies.
+ */
+ private final ImmutableList<Artifact> directJars;
+
+ /**
+ * The level of strict dependency checks (off, warnings, or errors).
+ */
+ private final BuildConfiguration.StrictDepsMode strictJavaDeps;
+
+ /**
+ * The set of .deps artifacts provided by direct dependencies.
+ */
+ private final ImmutableList<Artifact> compileTimeDependencyArtifacts;
+
+ /**
+ * The java semantics to get the list of action outputs.
+ */
+ private final JavaSemantics semantics;
+
+ /**
+ * Constructs an action to compile a set of Java source files to class files.
+ *
+ * @param owner the action owner, typically a java_* RuleConfiguredTarget.
+ * @param baseInputs the set of the input artifacts of the compile action
+ * without the parameter file action;
+ * @param outputs the outputs of the action
+ * @param javaCompileCommandLine the command line for the java library
+ * builder - it's actually written to the parameter file, but other
+ * parts (for example, ide_build_info) need access to the data
+ * @param commandLine the actual invocation command line
+ */
+ private JavaCompileAction(ActionOwner owner,
+ Iterable<Artifact> baseInputs,
+ Collection<Artifact> outputs,
+ CommandLine javaCompileCommandLine,
+ CommandLine commandLine,
+ PathFragment classDirectory,
+ Artifact outputJar,
+ NestedSet<Artifact> classpath,
+ List<Artifact> processorPath,
+ Artifact langtoolsJar,
+ Artifact javaBuilderJar,
+ List<String> processorNames,
+ Collection<Artifact> messages,
+ Collection<Artifact> resources,
+ Collection<Artifact> classpathResources,
+ Collection<Artifact> sourceJars,
+ Collection<Artifact> sourceFiles,
+ List<String> javacOpts,
+ Collection<Artifact> directJars,
+ BuildConfiguration.StrictDepsMode strictJavaDeps,
+ Collection<Artifact> compileTimeDependencyArtifacts,
+ JavaSemantics semantics) {
+ super(owner, Iterables.concat(ImmutableList.of(
+ classpath, processorPath, messages, resources,
+ classpathResources, sourceJars, sourceFiles, compileTimeDependencyArtifacts,
+ ImmutableList.of(langtoolsJar, javaBuilderJar), baseInputs)),
+ outputs);
+ this.javaCompileCommandLine = javaCompileCommandLine;
+ this.commandLine = commandLine;
+
+ this.classDirectory = Preconditions.checkNotNull(classDirectory);
+ this.outputJar = outputJar;
+ this.classpath = classpath;
+ this.processorPath = ImmutableList.copyOf(processorPath);
+ this.processorNames = ImmutableList.copyOf(processorNames);
+ this.messages = ImmutableList.copyOf(messages);
+ this.resources = ImmutableList.copyOf(resources);
+ this.classpathResources = ImmutableList.copyOf(classpathResources);
+ this.sourceJars = ImmutableList.copyOf(sourceJars);
+ this.sourceFiles = ImmutableList.copyOf(sourceFiles);
+ this.javacOpts = ImmutableList.copyOf(javacOpts);
+ this.directJars = ImmutableList.copyOf(directJars);
+ this.strictJavaDeps = strictJavaDeps;
+ this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts);
+ this.semantics = semantics;
+ }
+
+ /**
+ * Returns the given (passed to constructor) source files.
+ */
+ @VisibleForTesting
+ public Collection<Artifact> getSourceFiles() {
+ return sourceFiles;
+ }
+
+ /**
+ * Returns the list of paths that represent the resources to be added to the
+ * jar.
+ */
+ @VisibleForTesting
+ public Collection<Artifact> getResources() {
+ return resources;
+ }
+
+ /**
+ * Returns the list of paths that represents the classpath.
+ */
+ @VisibleForTesting
+ public Iterable<Artifact> getClasspath() {
+ return classpath;
+ }
+
+ /**
+ * Returns the list of paths that represents the source jars.
+ */
+ @VisibleForTesting
+ public Collection<Artifact> getSourceJars() {
+ return sourceJars;
+ }
+
+ /**
+ * Returns the list of paths that represents the processor path.
+ */
+ @VisibleForTesting
+ public List<Artifact> getProcessorpath() {
+ return processorPath;
+ }
+
+ @VisibleForTesting
+ public List<String> getJavacOpts() {
+ return javacOpts;
+ }
+
+ @VisibleForTesting
+ public Collection<Artifact> getDirectJars() {
+ return directJars;
+ }
+
+ @VisibleForTesting
+ public Collection<Artifact> getCompileTimeDependencyArtifacts() {
+ return compileTimeDependencyArtifacts;
+ }
+
+ @VisibleForTesting
+ public BuildConfiguration.StrictDepsMode getStrictJavaDepsMode() {
+ return strictJavaDeps;
+ }
+
+ public PathFragment getClassDirectory() {
+ return classDirectory;
+ }
+
+ /**
+ * Returns the list of class names of processors that should
+ * be run.
+ */
+ @VisibleForTesting
+ public List<String> getProcessorNames() {
+ return processorNames;
+ }
+
+ /**
+ * Returns the output jar artifact that gets generated by archiving the
+ * results of the Java compilation and the declared resources.
+ */
+ public Artifact getOutputJar() {
+ return outputJar;
+ }
+
+ @Override
+ public Artifact getPrimaryOutput() {
+ return getOutputJar();
+ }
+
+ /**
+ * Constructs a command line that can be used to invoke the
+ * JavaBuilder.
+ *
+ * <p>Do not use this method, except for testing (and for the in-process
+ * strategy).
+ */
+ @VisibleForTesting
+ public Iterable<String> buildCommandLine() {
+ return javaCompileCommandLine.arguments();
+ }
+
+ /**
+ * Returns the command and arguments for a java compile action.
+ */
+ public List<String> getCommand() {
+ return ImmutableList.copyOf(commandLine.arguments());
+ }
+
+ @Override
+ @ThreadCompatible
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ try {
+ List<ActionInput> outputs = new ArrayList<>();
+ outputs.addAll(getOutputs());
+ // Add a few useful side-effect output files to the list to retrieve.
+ // TODO(bazel-team): Just make these Artifacts.
+ PathFragment classDirectory = getClassDirectory();
+ outputs.addAll(semantics.getExtraJavaCompileOutputs(classDirectory));
+ outputs.add(ActionInputHelper.fromPath(classDirectory.getChild("srclist").getPathString()));
+
+ try {
+ // Make sure the directories exist, else the distributor will bomb.
+ Path classDirectoryPath = executor.getExecRoot().getRelative(getClassDirectory());
+ FileSystemUtils.createDirectoryAndParents(classDirectoryPath);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException(e.getMessage());
+ }
+
+ final ImmutableList<ActionInput> finalOutputs = ImmutableList.copyOf(outputs);
+ Spawn spawn = new BaseSpawn(getCommand(), ImmutableMap.<String, String>of(),
+ ImmutableMap.<String, String>of(), this, LOCAL_RESOURCES) {
+ @Override
+ public Collection<? extends ActionInput> getOutputFiles() {
+ return finalOutputs;
+ }
+ };
+
+ executor.getSpawnActionContext(getMnemonic()).exec(spawn, actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException("Java compilation in rule '" + getOwner().getLabel() + "'",
+ executor.getVerboseFailures(), this);
+ }
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addStrings(commandLine.arguments());
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public String describeKey() {
+ StringBuilder message = new StringBuilder();
+ for (String arg : ShellEscaper.escapeAll(commandLine.arguments())) {
+ message.append(" Command-line argument: ");
+ message.append(arg);
+ message.append('\n');
+ }
+ return message.toString();
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "Javac";
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ int count = sourceFiles.size();
+ if (count == 0) { // nothing to compile, just bundling resources and messages
+ count = resources.size() + classpathResources.size() + messages.size();
+ }
+ return "Building " + outputJar.prettyPrint() + " (" + count + " files)";
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return getContext(executor).strategyLocality(getMnemonic(), true);
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ SpawnActionContext context = getContext(executor);
+ if (context.isRemotable(getMnemonic(), true)) {
+ return ResourceSet.ZERO;
+ }
+ return LOCAL_RESOURCES;
+ }
+
+ protected SpawnActionContext getContext(Executor executor) {
+ return executor.getSpawnActionContext(getMnemonic());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ result.append("JavaBuilder ");
+ Joiner.on(' ').appendTo(result, commandLine.arguments());
+ return result.toString();
+ }
+
+ @Override
+ public ExtraActionInfo.Builder getExtraActionInfo() {
+ JavaCompileInfo.Builder info = JavaCompileInfo.newBuilder();
+ info.addAllSourceFile(Artifact.toExecPaths(getSourceFiles()));
+ info.addAllClasspath(Artifact.toExecPaths(getClasspath()));
+ info.addClasspath(getClassDirectory().getPathString());
+ info.addAllSourcepath(Artifact.toExecPaths(getSourceJars()));
+ info.addAllJavacOpt(getJavacOpts());
+ info.addAllProcessor(getProcessorNames());
+ info.addAllProcessorpath(Artifact.toExecPaths(getProcessorpath()));
+ info.setOutputjar(getOutputJar().getExecPathString());
+
+ return super.getExtraActionInfo()
+ .setExtension(JavaCompileInfo.javaCompileInfo, info.build());
+ }
+
+ /**
+ * Creates an instance.
+ *
+ * @param configuration the build configuration, which provides the default options and the path
+ * to the compiler, etc.
+ * @param classDirectory the directory in which generated classfiles are placed relative to the
+ * exec root
+ * @param sourceGenDirectory the directory where source files generated by annotation processors
+ * should be stored.
+ * @param tempDirectory a directory in which the library builder can store temporary files
+ * relative to the exec root
+ * @param outputJar output jar
+ * @param compressJar if true compress the output jar
+ * @param outputDepsProto the proto file capturing dependency information
+ * @param classpath the complete classpath, the directory in which generated classfiles are placed
+ * @param processorPath the classpath where javac should search for annotation processors
+ * @param processorNames the classes that javac should use as annotation processors
+ * @param messages the message files for translation
+ * @param resources the set of resources to put into the jar
+ * @param classpathResources the set of classpath resources to put into the jar
+ * @param sourceJars the set of jars containing additional source files to compile
+ * @param sourceFiles the set of explicit Java source files to compile
+ * @param javacOpts the compiler options to pass to javac
+ */
+ private static CustomCommandLine.Builder javaCompileCommandLine(
+ final JavaSemantics semantics,
+ final BuildConfiguration configuration,
+ final PathFragment classDirectory,
+ final PathFragment sourceGenDirectory,
+ PathFragment tempDirectory,
+ Artifact outputJar,
+ Artifact gensrcOutputJar,
+ boolean compressJar,
+ Artifact outputDepsProto,
+ final NestedSet<Artifact> classpath,
+ List<Artifact> processorPath,
+ Artifact langtoolsJar,
+ Artifact javaBuilderJar,
+ List<String> processorNames,
+ Collection<Artifact> messages,
+ Collection<Artifact> resources,
+ Collection<Artifact> classpathResources,
+ Collection<Artifact> sourceJars,
+ Collection<Artifact> sourceFiles,
+ List<String> javacOpts,
+ final Collection<Artifact> directJars,
+ BuildConfiguration.StrictDepsMode strictJavaDeps,
+ Collection<Artifact> compileTimeDependencyArtifacts,
+ String ruleKind,
+ Label targetLabel) {
+ Preconditions.checkNotNull(classDirectory);
+ Preconditions.checkNotNull(tempDirectory);
+ Preconditions.checkNotNull(langtoolsJar);
+ Preconditions.checkNotNull(javaBuilderJar);
+
+ CustomCommandLine.Builder result = CustomCommandLine.builder();
+
+ result.add("--classdir").addPath(classDirectory);
+
+ result.add("--tempdir").addPath(tempDirectory);
+
+ if (outputJar != null) {
+ result.addExecPath("--output", outputJar);
+ }
+
+ if (gensrcOutputJar != null) {
+ result.add("--sourcegendir").addPath(sourceGenDirectory);
+ result.addExecPath("--generated_sources_output", gensrcOutputJar);
+ }
+
+ if (compressJar) {
+ result.add("--compress_jar");
+ }
+
+ if (outputDepsProto != null) {
+ result.addExecPath("--output_deps_proto", outputDepsProto);
+ }
+
+ result.add("--classpath").add(new CustomArgv() {
+ @Override
+ public String argv() {
+ List<PathFragment> classpathEntries = new ArrayList<>();
+ for (Artifact classpathArtifact : classpath) {
+ classpathEntries.add(classpathArtifact.getExecPath());
+ }
+ classpathEntries.add(classDirectory);
+ return Joiner.on(configuration.getHostPathSeparator()).join(classpathEntries);
+ }
+ });
+
+ if (!processorPath.isEmpty()) {
+ result.addJoinExecPaths("--processorpath",
+ configuration.getHostPathSeparator(), processorPath);
+ }
+
+ if (!processorNames.isEmpty()) {
+ result.add("--processors", processorNames);
+ }
+
+ if (!messages.isEmpty()) {
+ result.add("--messages");
+ for (Artifact message : messages) {
+ addAsResourcePrefixedExecPath(semantics, message, result);
+ }
+ }
+
+ if (!resources.isEmpty()) {
+ result.add("--resources");
+ for (Artifact resource : resources) {
+ addAsResourcePrefixedExecPath(semantics, resource, result);
+ }
+ }
+
+ if (!classpathResources.isEmpty()) {
+ result.addExecPaths("--classpath_resources", classpathResources);
+ }
+
+ if (!sourceJars.isEmpty()) {
+ result.addExecPaths("--source_jars", sourceJars);
+ }
+
+ result.addExecPaths("--sources", sourceFiles);
+
+ if (!javacOpts.isEmpty()) {
+ result.add("--javacopts", javacOpts);
+ }
+
+ // strict_java_deps controls whether the mapping from jars to targets is
+ // written out and whether we try to minimize the compile-time classpath.
+ if (strictJavaDeps != BuildConfiguration.StrictDepsMode.OFF) {
+ result.add("--strict_java_deps");
+ result.add((semantics.useStrictJavaDeps(configuration) ? strictJavaDeps
+ : BuildConfiguration.StrictDepsMode.OFF).toString());
+ result.add(new CustomMultiArgv() {
+ @Override
+ public Iterable<String> argv() {
+ return addJarsToTargets(classpath, directJars);
+ }
+ });
+
+ if (configuration.getFragment(JavaConfiguration.class).getReduceJavaClasspath()
+ == JavaClasspathMode.JAVABUILDER) {
+ result.add("--reduce_classpath");
+
+ if (!compileTimeDependencyArtifacts.isEmpty()) {
+ result.addExecPaths("--deps_artifacts", compileTimeDependencyArtifacts);
+ }
+ }
+ }
+
+ if (ruleKind != null) {
+ result.add("--rule_kind");
+ result.add(ruleKind);
+ }
+ if (targetLabel != null) {
+ result.add("--target_label");
+ if (targetLabel.getPackageIdentifier().getRepository().isDefault()) {
+ result.add(targetLabel.toString());
+ } else {
+ // @-prefixed strings will be assumed to be filenames and expanded by
+ // {@link JavaLibraryBuildRequest}, so add an extra &at; to escape it.
+ result.add("@" + targetLabel);
+ }
+ }
+
+ return result;
+ }
+
+ private static void addAsResourcePrefixedExecPath(JavaSemantics semantics,
+ Artifact artifact, CustomCommandLine.Builder builder) {
+ PathFragment execPath = artifact.getExecPath();
+ PathFragment resourcePath = semantics.getJavaResourcePath(artifact.getRootRelativePath());
+ if (execPath.equals(resourcePath)) {
+ builder.addPaths(":%s", resourcePath);
+ } else {
+ // execPath must end with resourcePath in all cases
+ PathFragment rootPrefix = trimTail(execPath, resourcePath);
+ builder.addPaths("%s:%s", rootPrefix, resourcePath);
+ }
+ }
+
+ /**
+ * Returns the root-part of a given path by trimming off the end specified by
+ * a given tail. Assumes that the tail is known to match, and simply relies on
+ * the segment lengths.
+ */
+ private static PathFragment trimTail(PathFragment path, PathFragment tail) {
+ return path.subFragment(0, path.segmentCount() - tail.segmentCount());
+ }
+
+ /**
+ * Builds the list of mappings between jars on the classpath and their
+ * originating targets names.
+ */
+ private static ImmutableList<String> addJarsToTargets(
+ NestedSet<Artifact> classpath, Collection<Artifact> directJars) {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (Artifact jar : classpath) {
+ builder.add(directJars.contains(jar)
+ ? "--direct_dependency"
+ : "--indirect_dependency");
+ builder.add(jar.getExecPathString());
+ Label label = getTargetName(jar);
+ builder.add(label.getPackageIdentifier().getRepository().isDefault()
+ ? label.toString()
+ : label.toPathFragment().toString());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Gets the name of the target that produced the given jar artifact.
+ *
+ * When specifying jars directly in the "srcs" attribute of a rule (mostly
+ * for third_party libraries), there is no generating action, so we just
+ * return the jar name in label form.
+ */
+ private static Label getTargetName(Artifact jar) {
+ return Preconditions.checkNotNull(jar.getOwner(), jar);
+ }
+
+ /**
+ * The actual command line executed for a compile action.
+ */
+ private static CommandLine spawnCommandLine(PathFragment javaExecutable, Artifact javaBuilderJar,
+ Artifact langtoolsJar, Artifact paramFile, ImmutableList<String> javaBuilderJvmFlags) {
+ Preconditions.checkNotNull(langtoolsJar);
+ Preconditions.checkNotNull(javaBuilderJar);
+ return CustomCommandLine.builder()
+ .addPath(javaExecutable)
+ // Langtools jar is placed on the boot classpath so that it can override classes
+ // in the JRE. Typically this has no effect since langtools.jar does not have
+ // classes in common with rt.jar. However, it is necessary when using a version
+ // of javac.jar generated via ant from the langtools build.xml that is of a
+ // different version than AND has an overlap in contents with the default
+ // run-time (eg while upgrading the Java version).
+ .addPaths("-Xbootclasspath/p:%s", langtoolsJar.getExecPath())
+ .add(javaBuilderJvmFlags)
+ .addExecPath("-jar", javaBuilderJar)
+ .addPaths("@%s", paramFile.getExecPath())
+ .build();
+ }
+
+ /**
+ * Builder class to construct Java compile actions.
+ */
+ public static class Builder {
+ private final ActionOwner owner;
+ private final AnalysisEnvironment analysisEnvironment;
+ private final BuildConfiguration configuration;
+ private final JavaSemantics semantics;
+
+ private PathFragment javaExecutable;
+ private List<Artifact> javabaseInputs = ImmutableList.of();
+ private Artifact outputJar;
+ private Artifact gensrcOutputJar;
+ private Artifact outputDepsProto;
+ private Artifact paramFile;
+ private Artifact metadata;
+ private final Collection<Artifact> sourceFiles = new ArrayList<>();
+ private final Collection<Artifact> sourceJars = new ArrayList<>();
+ private final Collection<Artifact> resources = new ArrayList<>();
+ private final Collection<Artifact> classpathResources = new ArrayList<>();
+ private final Collection<Artifact> translations = new LinkedHashSet<>();
+ private BuildConfiguration.StrictDepsMode strictJavaDeps =
+ BuildConfiguration.StrictDepsMode.OFF;
+ private final Collection<Artifact> directJars = new ArrayList<>();
+ private final Collection<Artifact> compileTimeDependencyArtifacts = new ArrayList<>();
+ private List<String> javacOpts = new ArrayList<>();
+ private boolean compressJar;
+ private NestedSet<Artifact> classpathEntries =
+ NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER);
+ private ImmutableList<Artifact> bootclasspathEntries = ImmutableList.of();
+ private Artifact javaBuilderJar;
+ private Artifact langtoolsJar;
+ private PathFragment classDirectory;
+ private PathFragment sourceGenDirectory;
+ private PathFragment tempDirectory;
+ private final List<Artifact> processorPath = new ArrayList<>();
+ private final List<String> processorNames = new ArrayList<>();
+ private String ruleKind;
+ private Label targetLabel;
+
+ /**
+ * Creates a Builder from an owner and a build configuration.
+ */
+ public Builder(ActionOwner owner, AnalysisEnvironment analysisEnvironment,
+ BuildConfiguration configuration, JavaSemantics semantics) {
+ this.owner = owner;
+ this.analysisEnvironment = analysisEnvironment;
+ this.configuration = configuration;
+ this.semantics = semantics;
+ }
+
+ /**
+ * Creates a Builder from an owner and a build configuration.
+ */
+ public Builder(RuleContext ruleContext, JavaSemantics semantics) {
+ this(ruleContext.getActionOwner(), ruleContext.getAnalysisEnvironment(),
+ ruleContext.getConfiguration(), semantics);
+ }
+
+ public JavaCompileAction build() {
+ // TODO(bazel-team): all the params should be calculated before getting here, and the various
+ // aggregation code below should go away.
+ List<String> jcopts = new ArrayList<>(javacOpts);
+ JavaConfiguration javaConfiguration = configuration.getFragment(JavaConfiguration.class);
+ if (javaConfiguration.getJavaWarns().size() > 0) {
+ jcopts.add("-Xlint:" + Joiner.on(',').join(javaConfiguration.getJavaWarns()));
+ }
+ if (!bootclasspathEntries.isEmpty()) {
+ jcopts.add("-bootclasspath");
+ jcopts.add(
+ Artifact.joinExecPaths(configuration.getHostPathSeparator(), bootclasspathEntries));
+ }
+ List<String> internedJcopts = new ArrayList<>();
+ for (String jcopt : jcopts) {
+ internedJcopts.add(StringCanonicalizer.intern(jcopt));
+ }
+
+ // Invariant: if strictJavaDeps is OFF, then directJars and
+ // dependencyArtifacts are ignored
+ if (strictJavaDeps == BuildConfiguration.StrictDepsMode.OFF) {
+ directJars.clear();
+ compileTimeDependencyArtifacts.clear();
+ }
+
+ // Invariant: if experimental_java_classpath is not set to 'javabuilder',
+ // dependencyArtifacts are ignored
+ if (javaConfiguration.getReduceJavaClasspath() != JavaClasspathMode.JAVABUILDER) {
+ compileTimeDependencyArtifacts.clear();
+ }
+
+ if (paramFile == null) {
+ paramFile = analysisEnvironment.getDerivedArtifact(
+ ParameterFile.derivePath(outputJar.getRootRelativePath()),
+ configuration.getBinDirectory());
+ }
+
+ // ImmutableIterable is safe to use here because we know that neither of the components of
+ // the Iterable.concat() will change. Without ImmutableIterable, AbstractAction will
+ // waste memory by making a preventive copy of the iterable.
+ Iterable<Artifact> baseInputs = ImmutableIterable.from(Iterables.concat(
+ javabaseInputs,
+ bootclasspathEntries,
+ ImmutableList.of(paramFile)));
+
+ Preconditions.checkState(javaExecutable != null, owner);
+ Preconditions.checkState(javaExecutable.isAbsolute() ^ !javabaseInputs.isEmpty(),
+ javaExecutable);
+
+ Collection<Artifact> outputs;
+ ImmutableList.Builder<Artifact> outputsBuilder = ImmutableList.builder();
+ outputsBuilder.add(outputJar);
+ if (metadata != null) {
+ outputsBuilder.add(metadata);
+ }
+ if (gensrcOutputJar != null) {
+ outputsBuilder.add(gensrcOutputJar);
+ }
+ if (outputDepsProto != null) {
+ outputsBuilder.add(outputDepsProto);
+ }
+ outputs = outputsBuilder.build();
+
+ CustomCommandLine.Builder paramFileContentsBuilder = javaCompileCommandLine(
+ semantics,
+ configuration,
+ classDirectory,
+ sourceGenDirectory,
+ tempDirectory,
+ outputJar,
+ gensrcOutputJar,
+ compressJar,
+ outputDepsProto,
+ classpathEntries,
+ processorPath,
+ langtoolsJar,
+ javaBuilderJar,
+ processorNames,
+ translations,
+ resources,
+ classpathResources,
+ sourceJars,
+ sourceFiles,
+ internedJcopts,
+ directJars,
+ strictJavaDeps,
+ compileTimeDependencyArtifacts,
+ ruleKind,
+ targetLabel);
+ semantics.buildJavaCommandLine(outputs, configuration, paramFileContentsBuilder);
+ CommandLine paramFileContents = paramFileContentsBuilder.build();
+ Action parameterFileWriteAction = new ParameterFileWriteAction(owner, paramFile,
+ paramFileContents, ParameterFile.ParameterFileType.UNQUOTED, ISO_8859_1);
+ analysisEnvironment.registerAction(parameterFileWriteAction);
+
+ CommandLine javaBuilderCommandLine = spawnCommandLine(
+ javaExecutable,
+ javaBuilderJar,
+ langtoolsJar,
+ paramFile,
+ javaConfiguration.getDefaultJavaBuilderJvmFlags());
+
+ return new JavaCompileAction(owner,
+ baseInputs,
+ outputs,
+ paramFileContents,
+ javaBuilderCommandLine,
+ classDirectory,
+ outputJar,
+ classpathEntries,
+ processorPath,
+ langtoolsJar,
+ javaBuilderJar,
+ processorNames,
+ translations,
+ resources,
+ classpathResources,
+ sourceJars,
+ sourceFiles,
+ internedJcopts,
+ directJars,
+ strictJavaDeps,
+ compileTimeDependencyArtifacts,
+
+ semantics);
+ }
+
+ public Builder setParameterFile(Artifact paramFile) {
+ this.paramFile = paramFile;
+ return this;
+ }
+
+ public Builder setJavaExecutable(PathFragment javaExecutable) {
+ this.javaExecutable = javaExecutable;
+ return this;
+ }
+
+ public Builder setJavaBaseInputs(Iterable<Artifact> javabaseInputs) {
+ this.javabaseInputs = ImmutableList.copyOf(javabaseInputs);
+ return this;
+ }
+
+ public Builder setOutputJar(Artifact outputJar) {
+ this.outputJar = outputJar;
+ return this;
+ }
+
+ public Builder setGensrcOutputJar(Artifact gensrcOutputJar) {
+ this.gensrcOutputJar = gensrcOutputJar;
+ return this;
+ }
+
+ public Builder setOutputDepsProto(Artifact outputDepsProto) {
+ this.outputDepsProto = outputDepsProto;
+ return this;
+ }
+
+ public Builder setMetadata(Artifact metadata) {
+ this.metadata = metadata;
+ return this;
+ }
+
+ public Builder addSourceFile(Artifact sourceFile) {
+ sourceFiles.add(sourceFile);
+ return this;
+ }
+
+ public Builder addSourceFiles(Collection<Artifact> sourceFiles) {
+ this.sourceFiles.addAll(sourceFiles);
+ return this;
+ }
+
+ public Builder addSourceJars(Collection<Artifact> sourceJars) {
+ this.sourceJars.addAll(sourceJars);
+ return this;
+ }
+
+ public Builder addResources(Collection<Artifact> resources) {
+ this.resources.addAll(resources);
+ return this;
+ }
+
+ public Builder addClasspathResources(Collection<Artifact> classpathResources) {
+ this.classpathResources.addAll(classpathResources);
+ return this;
+ }
+
+ public Builder addTranslations(Collection<Artifact> translations) {
+ this.translations.addAll(translations);
+ return this;
+ }
+
+ /**
+ * Sets the strictness of Java dependency checking, see {@link
+ * com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode}.
+ */
+ public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictDeps) {
+ strictJavaDeps = strictDeps;
+ return this;
+ }
+
+ /**
+ * Accumulates the given jar artifacts as being provided by direct dependencies.
+ */
+ public Builder addDirectJars(Collection<Artifact> directJars) {
+ Iterables.addAll(this.directJars, directJars);
+ return this;
+ }
+
+ public Builder addCompileTimeDependencyArtifacts(Collection<Artifact> dependencyArtifacts) {
+ Iterables.addAll(this.compileTimeDependencyArtifacts, dependencyArtifacts);
+ return this;
+ }
+
+ public Builder setJavacOpts(Iterable<String> copts) {
+ this.javacOpts = ImmutableList.copyOf(copts);
+ return this;
+ }
+
+ public Builder setCompressJar(boolean compressJar) {
+ this.compressJar = compressJar;
+ return this;
+ }
+
+ public Builder setClasspathEntries(NestedSet<Artifact> classpathEntries) {
+ this.classpathEntries = classpathEntries;
+ return this;
+ }
+
+ public Builder setBootclasspathEntries(Iterable<Artifact> bootclasspathEntries) {
+ this.bootclasspathEntries = ImmutableList.copyOf(bootclasspathEntries);
+ return this;
+ }
+
+ public Builder setClassDirectory(PathFragment classDirectory) {
+ this.classDirectory = classDirectory;
+ return this;
+ }
+
+ /**
+ * Sets the directory where source files generated by annotation processors should be stored.
+ */
+ public Builder setSourceGenDirectory(PathFragment sourceGenDirectory) {
+ this.sourceGenDirectory = sourceGenDirectory;
+ return this;
+ }
+
+ public Builder setTempDirectory(PathFragment tempDirectory) {
+ this.tempDirectory = tempDirectory;
+ return this;
+ }
+
+ public Builder addProcessorPaths(Collection<Artifact> processorPaths) {
+ this.processorPath.addAll(processorPaths);
+ return this;
+ }
+
+ public Builder addProcessorNames(Collection<String> processorNames) {
+ this.processorNames.addAll(processorNames);
+ return this;
+ }
+
+ public Builder setLangtoolsJar(Artifact langtoolsJar) {
+ this.langtoolsJar = langtoolsJar;
+ return this;
+ }
+
+ public Builder setJavaBuilderJar(Artifact javaBuilderJar) {
+ this.javaBuilderJar = javaBuilderJar;
+ return this;
+ }
+
+ public Builder setRuleKind(String ruleKind) {
+ this.ruleKind = ruleKind;
+ return this;
+ }
+
+ public Builder setTargetLabel(Label targetLabel) {
+ this.targetLabel = targetLabel;
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
new file mode 100644
index 0000000..e1c6dc2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
@@ -0,0 +1,260 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.common.options.TriState;
+
+import java.util.List;
+
+/**
+ * A java compiler configuration containing the flags required for compilation.
+ */
+@Immutable
+@SkylarkModule(name = "java_configuration", doc = "A java compiler configuration")
+public final class JavaConfiguration extends Fragment {
+ /**
+ * Values for the --experimental_java_classpath option
+ */
+ public static enum JavaClasspathMode {
+ /** Use full transitive classpaths, the default behavior. */
+ OFF,
+ /** JavaBuilder computes the reduced classpath before invoking javac. */
+ JAVABUILDER,
+ /** Blaze computes the reduced classpath before invoking JavaBuilder. */
+ BLAZE
+ }
+
+ private final ImmutableList<String> commandLineJavacFlags;
+ private final Label javaLauncherLabel;
+ private final Label javaBuilderTop;
+ private final ImmutableList<String> defaultJavaBuilderJvmOpts;
+ private final Label javaLangtoolsJar;
+ private final boolean useIjars;
+ private final boolean generateJavaDeps;
+ private final JavaClasspathMode experimentalJavaClasspath;
+ private final ImmutableList<String> javaWarns;
+ private final ImmutableList<String> defaultJvmFlags;
+ private final ImmutableList<String> checkedConstraints;
+ private final StrictDepsMode strictJavaDeps;
+ private final Label javacBootclasspath;
+ private final ImmutableList<String> javacOpts;
+ private final TriState bundleTranslations;
+ private final ImmutableList<Label> translationTargets;
+ private final String javaCpu;
+
+ private final String cacheKey;
+ private Label javaToolchain;
+
+ JavaConfiguration(boolean generateJavaDeps,
+ List<String> defaultJvmFlags, JavaOptions javaOptions, Label javaToolchain, String javaCpu,
+ ImmutableList<String> defaultJavaBuilderJvmOpts) throws InvalidConfigurationException {
+ this.commandLineJavacFlags =
+ ImmutableList.copyOf(JavaHelper.tokenizeJavaOptions(javaOptions.javacOpts));
+ this.javaLauncherLabel = javaOptions.javaLauncher;
+ this.javaBuilderTop = javaOptions.javaBuilderTop;
+ this.defaultJavaBuilderJvmOpts = defaultJavaBuilderJvmOpts;
+ this.javaLangtoolsJar = javaOptions.javaLangtoolsJar;
+ this.useIjars = javaOptions.useIjars;
+ this.generateJavaDeps = generateJavaDeps;
+ this.experimentalJavaClasspath = javaOptions.experimentalJavaClasspath;
+ this.javaWarns = ImmutableList.copyOf(javaOptions.javaWarns);
+ this.defaultJvmFlags = ImmutableList.copyOf(defaultJvmFlags);
+ this.checkedConstraints = ImmutableList.copyOf(javaOptions.checkedConstraints);
+ this.strictJavaDeps = javaOptions.strictJavaDeps;
+ this.javacBootclasspath = javaOptions.javacBootclasspath;
+ this.javacOpts = ImmutableList.copyOf(javaOptions.javacOpts);
+ this.bundleTranslations = javaOptions.bundleTranslations;
+ this.javaCpu = javaCpu;
+ this.javaToolchain = javaToolchain;
+
+ ImmutableList.Builder<Label> translationsBuilder = ImmutableList.builder();
+ for (String s : javaOptions.translationTargets) {
+ try {
+ Label label = Label.parseAbsolute(s);
+ translationsBuilder.add(label);
+ } catch (SyntaxException e) {
+ throw new InvalidConfigurationException("Invalid translations target '" + s + "', make " +
+ "sure it uses correct absolute path syntax.", e);
+ }
+ }
+ this.translationTargets = translationsBuilder.build();
+
+ this.cacheKey = Joiner.on(" ").join(commandLineJavacFlags);
+ }
+
+ @SkylarkCallable(name = "default_javac_flags", structField = true,
+ doc = "The default flags for the Java compiler.")
+ // TODO(bazel-team): this is the command-line passed options, we should remove from skylark
+ // probably.
+ public List<String> getDefaultJavacFlags() {
+ return commandLineJavacFlags;
+ }
+
+ @Override
+ public String cacheKey() {
+ return cacheKey;
+ }
+
+ @Override
+ public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
+ if ((bundleTranslations == TriState.YES) && translationTargets.isEmpty()) {
+ reporter.handle(Event.error("Translations enabled, but no message translations specified. " +
+ "Use '--message_translations' to select the message translations to use"));
+ }
+ }
+
+ @Override
+ public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) {
+ globalMakeEnvBuilder.put("JAVA_TRANSLATIONS", buildTranslations() ? "1" : "0");
+ globalMakeEnvBuilder.put("JAVA_CPU", javaCpu);
+ }
+
+ /**
+ * Returns the Java cpu.
+ */
+ public String getJavaCpu() {
+ return javaCpu;
+ }
+
+ /**
+ * Returns the default javabuilder jar
+ */
+ public Label getDefaultJavaBuilderJar() {
+ return javaBuilderTop;
+ }
+
+ /**
+ * Returns the default JVM flags to be used when invoking javabuilder.
+ */
+ public ImmutableList<String> getDefaultJavaBuilderJvmFlags() {
+ return defaultJavaBuilderJvmOpts;
+ }
+
+ /**
+ * Returns the default java langtools jar
+ */
+ public Label getDefaultJavaLangtoolsJar() {
+ return javaLangtoolsJar;
+ }
+
+ /**
+ * Returns true iff Java compilation should use ijars.
+ */
+ public boolean getUseIjars() {
+ return useIjars;
+ }
+
+ /**
+ * Returns true iff dependency information is generated after compilation.
+ */
+ public boolean getGenerateJavaDeps() {
+ return generateJavaDeps;
+ }
+
+ public JavaClasspathMode getReduceJavaClasspath() {
+ return experimentalJavaClasspath;
+ }
+
+ /**
+ * Returns the extra warnings enabled for Java compilation.
+ */
+ public List<String> getJavaWarns() {
+ return javaWarns;
+ }
+
+ public List<String> getDefaultJvmFlags() {
+ return defaultJvmFlags;
+ }
+
+ public List<String> getCheckedConstraints() {
+ return checkedConstraints;
+ }
+
+ public StrictDepsMode getStrictJavaDeps() {
+ return strictJavaDeps;
+ }
+
+ public StrictDepsMode getFilteredStrictJavaDeps() {
+ StrictDepsMode strict = getStrictJavaDeps();
+ switch (strict) {
+ case STRICT:
+ case DEFAULT:
+ return StrictDepsMode.ERROR;
+ default: // OFF, WARN, ERROR
+ return strict;
+ }
+ }
+
+ /**
+ * @return proper label only if --java_launcher= is specified, otherwise null.
+ */
+ public Label getJavaLauncherLabel() {
+ return javaLauncherLabel;
+ }
+
+ public Label getJavacBootclasspath() {
+ return javacBootclasspath;
+ }
+
+ public List<String> getJavacOpts() {
+ return javacOpts;
+ }
+
+ @Override
+ public String getName() {
+ return "Java";
+ }
+
+ /**
+ * Returns the raw translation targets.
+ */
+ public List<Label> getTranslationTargets() {
+ return translationTargets;
+ }
+
+ /**
+ * Returns true if the we should build translations.
+ */
+ public boolean buildTranslations() {
+ return (bundleTranslations != TriState.NO) && !translationTargets.isEmpty();
+ }
+
+ /**
+ * Returns whether translations were explicitly disabled.
+ */
+ public boolean isTranslationsDisabled() {
+ return bundleTranslations == TriState.NO;
+ }
+
+ /**
+ * Returns the label of the default java_toolchain rule
+ */
+ public Label getToolchainLabel() {
+ return javaToolchain;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java
new file mode 100644
index 0000000..53fdfdf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.RedirectChaser;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A loader that creates JavaConfiguration instances based on JavaBuilder configurations and
+ * command-line options.
+ */
+public class JavaConfigurationLoader implements ConfigurationFragmentFactory {
+ private final JavaCpuSupplier cpuSupplier;
+
+ public JavaConfigurationLoader(JavaCpuSupplier cpuSupplier) {
+ this.cpuSupplier = cpuSupplier;
+ }
+
+ @Override
+ public JavaConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions)
+ throws InvalidConfigurationException {
+ JavaOptions javaOptions = buildOptions.get(JavaOptions.class);
+
+ Label javaToolchain = RedirectChaser.followRedirects(env, javaOptions.javaToolchain,
+ "java_toolchain");
+ return create(javaOptions, javaToolchain, cpuSupplier.getJavaCpu(buildOptions, env));
+ }
+
+ @Override
+ public Class<? extends Fragment> creates() {
+ return JavaConfiguration.class;
+ }
+
+ public JavaConfiguration create(JavaOptions javaOptions, Label javaToolchain, String javaCpu)
+ throws InvalidConfigurationException {
+
+ boolean generateJavaDeps = javaOptions.javaDeps ||
+ javaOptions.experimentalJavaClasspath != JavaClasspathMode.OFF;
+
+ ImmutableList<String> defaultJavaBuilderJvmOpts = ImmutableList.<String>builder()
+ .addAll(getJavacJvmOptions())
+ .addAll(JavaHelper.tokenizeJavaOptions(javaOptions.javaBuilderJvmOpts))
+ .build();
+
+ return new JavaConfiguration(generateJavaDeps, javaOptions.jvmOpts, javaOptions,
+ javaToolchain, javaCpu, defaultJavaBuilderJvmOpts);
+ }
+
+ /**
+ * This method returns the list of JVM options when invoking the java compiler.
+ *
+ * <p>TODO(bazel-team): Maybe we should put those options in the java_toolchain rule.
+ */
+ protected ImmutableList<String> getJavacJvmOptions() {
+ return ImmutableList.of("-client");
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java
new file mode 100644
index 0000000..5492abf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+
+/**
+ * Determines the CPU to be used for Java compilation from the build options and the
+ * configuration environment.
+ */
+public interface JavaCpuSupplier {
+ /**
+ * Returns the Java CPU based on the buiold options and the configuration environment.
+ */
+ String getJavaCpu(BuildOptions buildOptions, ConfigurationEnvironment env)
+ throws InvalidConfigurationException;
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java
new file mode 100644
index 0000000..52857f1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * The collection of labels of exported targets and artifacts reached via "exports" attribute
+ * transitively.
+ */
+@Immutable
+public final class JavaExportsProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Label> transitiveExports;
+
+ public JavaExportsProvider(NestedSet<Label> transitiveExports) {
+ this.transitiveExports = transitiveExports;
+ }
+
+ /**
+ * Returns the labels of exported targets and artifacts reached transitively through the "exports"
+ * attribute.
+ */
+ public NestedSet<Label> getTransitiveExports() {
+ return transitiveExports;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java
new file mode 100644
index 0000000..b2a7ca0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java
@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.shell.ShellUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Utility methods for use by Java-related parts of Bazel.
+ */
+// TODO(bazel-team): Merge with JavaUtil.
+public abstract class JavaHelper {
+
+ private JavaHelper() {}
+
+ /**
+ * Returns the java launcher implementation for the given target, if any.
+ * A null return value means "use the JDK launcher".
+ */
+ public static TransitiveInfoCollection launcherForTarget(JavaSemantics semantics,
+ RuleContext ruleContext) {
+ String launcher = filterLauncherForTarget(semantics, ruleContext);
+ return (launcher == null) ? null : ruleContext.getPrerequisite(launcher, Mode.TARGET);
+ }
+
+ /**
+ * Returns the java launcher artifact for the given target, if any.
+ * A null return value means "use the JDK launcher".
+ */
+ public static Artifact launcherArtifactForTarget(JavaSemantics semantics,
+ RuleContext ruleContext) {
+ String launcher = filterLauncherForTarget(semantics, ruleContext);
+ return (launcher == null) ? null : ruleContext.getPrerequisiteArtifact(launcher, Mode.TARGET);
+ }
+
+ /**
+ * Control structure abstraction for safely extracting a prereq from the launcher attribute
+ * or --java_launcher flag.
+ */
+ private static String filterLauncherForTarget(JavaSemantics semantics, RuleContext ruleContext) {
+ // BUILD rule "launcher" attribute
+ if (ruleContext.getRule().isAttrDefined("launcher", Type.LABEL)
+ && ruleContext.attributes().get("launcher", Type.LABEL) != null) {
+ if (ruleContext.attributes().get("launcher", Type.LABEL)
+ .equals(JavaSemantics.JDK_LAUNCHER_LABEL)) {
+ return null;
+ }
+ return "launcher";
+ }
+ // Blaze flag --java_launcher
+ JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class);
+ if (ruleContext.getRule().isAttrDefined(":java_launcher", Type.LABEL)
+ && ((javaConfig.getJavaLauncherLabel() != null
+ && !javaConfig.getJavaLauncherLabel().equals(JavaSemantics.JDK_LAUNCHER_LABEL))
+ || semantics.forceUseJavaLauncherTarget(ruleContext))) {
+ return ":java_launcher";
+ }
+ return null;
+ }
+
+ /**
+ * Javac options require special processing - People use them and expect the
+ * options to be tokenized.
+ */
+ public static List<String> tokenizeJavaOptions(Iterable<String> inOpts) {
+ // Ideally, this would be in the options parser. Unfortunately,
+ // the options parser can't handle a converter that expands
+ // from a value X into a List<X> and allow-multiple at the
+ // same time.
+ List<String> result = new ArrayList<>();
+ for (String current : inOpts) {
+ try {
+ ShellUtils.tokenize(result, current);
+ } catch (ShellUtils.TokenizationException ex) {
+ // Tokenization failed; this likely means that the user
+ // did not want tokenization to happen on his argument.
+ // (Any tokenization where we should produce an error
+ // has already been done by the shell that invoked
+ // blaze). Therefore, pass the argument through to
+ // the tool, so that we can see the original error.
+ result.add(current);
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java
new file mode 100644
index 0000000..f978f98
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java
@@ -0,0 +1,201 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParams;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
+import com.google.devtools.build.lib.rules.cpp.CppCompilationContext;
+import com.google.devtools.build.lib.rules.cpp.LinkerInput;
+import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType;
+
+/**
+ * An implementation for the "java_import" rule.
+ */
+public class JavaImport implements RuleConfiguredTargetFactory {
+ private final JavaSemantics semantics;
+
+ protected JavaImport(JavaSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ ImmutableList<Artifact> srcJars = ImmutableList.of();
+ ImmutableList<Artifact> jars = collectJars(ruleContext);
+ Artifact srcJar = ruleContext.getPrerequisiteArtifact("srcjar", Mode.TARGET);
+
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ ImmutableList<TransitiveInfoCollection> targets = ImmutableList.copyOf(
+ ruleContext.getPrerequisites("exports", Mode.TARGET));
+ final JavaCommon common = new JavaCommon(
+ ruleContext, semantics, targets, targets, targets);
+ semantics.checkRule(ruleContext, common);
+
+ // No need for javac options - no compilation happening here.
+ JavaCompilationHelper helper = new JavaCompilationHelper(ruleContext, semantics,
+ ImmutableList.<String>of(), new JavaTargetAttributes.Builder(semantics));
+ ImmutableMap.Builder<Artifact, Artifact> compilationToRuntimeJarMap = ImmutableMap.builder();
+ ImmutableList<Artifact> interfaceJars =
+ processWithIjar(jars, helper, compilationToRuntimeJarMap);
+
+ common.setJavaCompilationArtifacts(collectJavaArtifacts(jars, interfaceJars));
+
+ CppCompilationContext transitiveCppDeps = common.collectTransitiveCppDeps();
+ NestedSet<LinkerInput> transitiveJavaNativeLibraries =
+ common.collectTransitiveJavaNativeLibraries();
+
+ JavaCompilationArgs javaCompilationArgs = common.collectJavaCompilationArgs(
+ false, common.isNeverLink(), compilationArgsFromSources());
+ JavaCompilationArgs recursiveJavaCompilationArgs = common.collectJavaCompilationArgs(
+ true, common.isNeverLink(), compilationArgsFromSources());
+ NestedSet<Artifact> transitiveJavaSourceJars =
+ collectTransitiveJavaSourceJars(ruleContext, srcJar);
+ if (srcJar != null) {
+ srcJars = ImmutableList.of(srcJar);
+ }
+
+ // The "neverlink" attribute is transitive, so if it is enabled, we don't add any
+ // runfiles from this target or its dependencies.
+ Runfiles runfiles = common.isNeverLink() ?
+ Runfiles.EMPTY :
+ new Runfiles.Builder()
+ // add the jars to the runfiles
+ .addArtifacts(common.getJavaCompilationArtifacts().getRuntimeJars())
+ .addTargets(targets, RunfilesProvider.DEFAULT_RUNFILES)
+ .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES)
+ .addTargets(targets, JavaRunfilesProvider.TO_RUNFILES)
+ .add(ruleContext, JavaRunfilesProvider.TO_RUNFILES)
+ .build();
+
+ CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() {
+ @Override
+ protected void collect(CcLinkParams.Builder builder, boolean linkingStatically,
+ boolean linkShared) {
+ Iterable<? extends TransitiveInfoCollection> deps =
+ common.targetsTreatedAsDeps(ClasspathType.BOTH);
+ builder.addTransitiveTargets(deps);
+ builder.addTransitiveLangTargets(deps, JavaCcLinkParamsProvider.TO_LINK_PARAMS);
+ }
+ };
+ RuleConfiguredTargetBuilder ruleBuilder =
+ new RuleConfiguredTargetBuilder(ruleContext);
+ NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder();
+ filesBuilder.addAll(jars);
+
+ semantics.addProviders(
+ ruleContext, common, ImmutableList.<String>of(), null,
+ srcJar, null, compilationToRuntimeJarMap.build(), helper, filesBuilder, ruleBuilder);
+
+ NestedSet<Artifact> filesToBuild = filesBuilder.build();
+
+ common.addTransitiveInfoProviders(ruleBuilder, filesToBuild, null);
+ return ruleBuilder
+ .setFilesToBuild(filesToBuild)
+ .add(JavaNeverlinkInfoProvider.class, new JavaNeverlinkInfoProvider(common.isNeverLink()))
+ .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles))
+ .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore))
+ .add(JavaCompilationArgsProvider.class, new JavaCompilationArgsProvider(
+ javaCompilationArgs, recursiveJavaCompilationArgs))
+ .add(JavaNativeLibraryProvider.class, new JavaNativeLibraryProvider(
+ transitiveJavaNativeLibraries))
+ .add(CppCompilationContext.class, transitiveCppDeps)
+ .add(JavaSourceJarsProvider.class, new JavaSourceJarsProvider(
+ transitiveJavaSourceJars, srcJars))
+ .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider(
+ JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveJavaSourceJars))
+ .build();
+ }
+
+ private NestedSet<Artifact> collectTransitiveJavaSourceJars(RuleContext ruleContext,
+ Artifact srcJar) {
+ NestedSetBuilder<Artifact> transitiveJavaSourceJarBuilder =
+ NestedSetBuilder.stableOrder();
+ if (srcJar != null) {
+ transitiveJavaSourceJarBuilder.add(srcJar);
+ }
+ for (JavaSourceJarsProvider other :
+ ruleContext.getPrerequisites("exports", Mode.TARGET, JavaSourceJarsProvider.class)) {
+ transitiveJavaSourceJarBuilder.addTransitive(other.getTransitiveSourceJars());
+ }
+ return transitiveJavaSourceJarBuilder.build();
+ }
+
+ private JavaCompilationArtifacts collectJavaArtifacts(
+ ImmutableList<Artifact> jars,
+ ImmutableList<Artifact> interfaceJars) {
+ JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder();
+ javaArtifactsBuilder.addRuntimeJars(jars);
+ // interfaceJars Artifacts have proper owner labels
+ javaArtifactsBuilder.addCompileTimeJars(interfaceJars);
+ return javaArtifactsBuilder.build();
+ }
+
+ private ImmutableList<Artifact> collectJars(RuleContext ruleContext) {
+ ImmutableList.Builder<Artifact> jarsBuilder = ImmutableList.builder();
+ for (TransitiveInfoCollection info : ruleContext.getPrerequisites("jars", Mode.TARGET)) {
+ if (info.getProvider(JavaCompilationArgsProvider.class) != null) {
+ ruleContext.attributeError("jars", "should not refer to Java rules");
+ }
+ for (Artifact jar : info.getProvider(FileProvider.class).getFilesToBuild()) {
+ if (!JavaSemantics.JAR.matches(jar.getFilename())) {
+ ruleContext.attributeError("jars", jar.getFilename() + " is not a .jar file");
+ } else {
+ jarsBuilder.add(jar);
+ }
+ }
+ }
+ return jarsBuilder.build();
+ }
+
+ private ImmutableList<Artifact> processWithIjar(ImmutableList<Artifact> jars,
+ JavaCompilationHelper helper,
+ ImmutableMap.Builder<Artifact, Artifact> compilationToRuntimeJarMap) {
+ ImmutableList.Builder<Artifact> interfaceJarsBuilder = ImmutableList.builder();
+ for (Artifact jar : jars) {
+ Artifact ijar = helper.createIjarAction(jar, true);
+ interfaceJarsBuilder.add(ijar);
+ compilationToRuntimeJarMap.put(ijar, jar);
+ }
+ return interfaceJarsBuilder.build();
+ }
+
+ private Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources() {
+ return ImmutableList.of();
+ }
+
+ private ImmutableList<String> getJavaConstraints(RuleContext ruleContext) {
+ return ImmutableList.copyOf(ruleContext.attributes().get("constraints", Type.STRING_LIST));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java
new file mode 100644
index 0000000..b153b58
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * A base rule for building the java_import rule.
+ */
+@BlazeRule(name = "$java_import_base",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.RuleBase.class })
+public class JavaImportBaseRule implements RuleDefinition {
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ .add(attr(":host_jdk", LABEL)
+ .cfg(HOST)
+ .value(JavaSemantics.HOST_JDK))
+ /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(jars) -->
+ The list of JAR files provided to Java targets that depend on this target.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("jars", LABEL_LIST)
+ .mandatory()
+ .nonEmpty()
+ .allowedFileTypes(JavaSemantics.JAR))
+ /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(srcjar) -->
+ A JAR file that contains source code for the compiled JAR files.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("srcjar", LABEL)
+ .allowedFileTypes(JavaSemantics.SOURCE_JAR, JavaSemantics.JAR)
+ .direct_compile_time_input())
+ .removeAttribute("deps") // only exports are allowed; nothing is compiled
+ /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(neverlink) -->
+ Only use this library for compilation and not at runtime.
+ ${SYNOPSIS}
+ Useful if the library will be provided by the runtime environment
+ during execution. Examples of libraries like this are IDE APIs
+ for IDE plug-ins or <code>tools.jar</code> for anything running on
+ a standard JDK.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("neverlink", BOOLEAN).value(false))
+ /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(constraints) -->
+ Extra constraints imposed on this rule as a Java library.
+ ${SYNOPSIS}
+ See <a href="#java_library.constraints">java_library.constraints</a>.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("constraints", STRING_LIST)
+ .orderIndependent()
+ .nonconfigurable("used in Attribute.validityPredicate implementations (loading time)"))
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = java_import, TYPE = LIBRARY, FAMILY = Java) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+ <p>This rule allows the use of precompiled JAR files as libraries for
+ <code><a href="#java_library">java_library</a></code> rules.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java
new file mode 100644
index 0000000..1831ef0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java
@@ -0,0 +1,244 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParams;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
+import com.google.devtools.build.lib.rules.cpp.CppCompilationContext;
+import com.google.devtools.build.lib.rules.cpp.LinkerInput;
+import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Implementation for the java_library rule.
+ */
+public class JavaLibrary implements RuleConfiguredTargetFactory {
+ private final JavaSemantics semantics;
+
+ protected JavaLibrary(JavaSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ JavaCommon common = new JavaCommon(ruleContext, semantics);
+ RuleConfiguredTargetBuilder builder = init(ruleContext, common);
+ return builder != null ? builder.build() : null;
+ }
+
+ public RuleConfiguredTargetBuilder init(RuleContext ruleContext, final JavaCommon common) {
+ common.initializeJavacOpts();
+ JavaTargetAttributes.Builder attributesBuilder = common.initCommon();
+
+ // Collect the transitive dependencies.
+ JavaCompilationHelper helper = new JavaCompilationHelper(
+ ruleContext, semantics, common.getJavacOpts(), attributesBuilder);
+ helper.addLibrariesToAttributes(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY));
+ helper.addProvidersToAttributes(common.compilationArgsFromSources(), common.isNeverLink());
+
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ semantics.checkRule(ruleContext, common);
+
+ JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder();
+
+ if (ruleContext.hasErrors()) {
+ common.setJavaCompilationArtifacts(JavaCompilationArtifacts.EMPTY);
+ return null;
+ }
+
+ JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class);
+ NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder();
+
+ JavaTargetAttributes attributes = helper.getAttributes();
+ if (attributes.hasJarFiles()) {
+ // This rule is repackaging some source jars as a java library.
+ Set<Artifact> jarFiles = attributes.getJarFiles();
+ javaArtifactsBuilder.addRuntimeJars(jarFiles);
+ javaArtifactsBuilder.addCompileTimeJars(attributes.getCompileTimeJarFiles());
+
+ filesBuilder.addAll(jarFiles);
+ }
+ if (attributes.hasMessages()) {
+ helper.addTranslations(semantics.translate(ruleContext, javaConfig,
+ attributes.getMessages()));
+ }
+
+ ruleContext.checkSrcsSamePackage(true);
+
+ Artifact jar = null;
+
+ Artifact srcJar = ruleContext.getImplicitOutputArtifact(
+ JavaSemantics.JAVA_LIBRARY_SOURCE_JAR);
+
+ Artifact classJar = ruleContext.getImplicitOutputArtifact(
+ JavaSemantics.JAVA_LIBRARY_CLASS_JAR);
+
+ if (attributes.hasSourceFiles() || attributes.hasSourceJars() || attributes.hasResources()
+ || attributes.hasMessages()) {
+ // We only want to add a jar to the classpath of a dependent rule if it has content.
+ javaArtifactsBuilder.addRuntimeJar(classJar);
+ jar = classJar;
+ }
+
+ filesBuilder.add(classJar);
+
+ // The gensrcJar is only created if the target uses annotation processing. Otherwise,
+ // it is null, and the source jar action will not depend on the compile action.
+ Artifact gensrcJar = helper.createGensrcJar(classJar);
+
+ Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder);
+
+ helper.createCompileActionWithInstrumentation(classJar, gensrcJar, outputDepsProto,
+ javaArtifactsBuilder);
+ helper.createSourceJarAction(srcJar, gensrcJar);
+
+ if ((attributes.hasSourceFiles() || attributes.hasSourceJars()) && jar != null) {
+ helper.createCompileTimeJarAction(jar, outputDepsProto,
+ javaArtifactsBuilder);
+ }
+
+ common.setJavaCompilationArtifacts(javaArtifactsBuilder.build());
+ common.setClassPathFragment(new ClasspathConfiguredFragment(
+ common.getJavaCompilationArtifacts(), attributes, common.isNeverLink()));
+ CppCompilationContext transitiveCppDeps = common.collectTransitiveCppDeps();
+
+ NestedSet<Artifact> transitiveSourceJars = common.collectTransitiveSourceJars(srcJar);
+
+ // If sources are empty, treat this library as a forwarding node for dependencies.
+ JavaCompilationArgs javaCompilationArgs = common.collectJavaCompilationArgs(
+ false, common.isNeverLink(), common.compilationArgsFromSources());
+ JavaCompilationArgs recursiveJavaCompilationArgs = common.collectJavaCompilationArgs(
+ true, common.isNeverLink(), common.compilationArgsFromSources());
+ NestedSet<Artifact> compileTimeJavaDepArtifacts = common.collectCompileTimeDependencyArtifacts(
+ common.getJavaCompilationArtifacts().getCompileTimeDependencyArtifact());
+ NestedSet<Artifact> runTimeJavaDepArtifacts = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ NestedSet<LinkerInput> transitiveJavaNativeLibraries =
+ common.collectTransitiveJavaNativeLibraries();
+
+ ImmutableList<String> exportedProcessorClasses = ImmutableList.of();
+ NestedSet<Artifact> exportedProcessorClasspath =
+ NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER);
+ ImmutableList.Builder<String> processorClasses = ImmutableList.builder();
+ NestedSetBuilder<Artifact> processorClasspath = NestedSetBuilder.naiveLinkOrder();
+ for (JavaPluginInfoProvider provider : Iterables.concat(
+ common.getPluginInfoProvidersForAttribute("exported_plugins", Mode.HOST),
+ common.getPluginInfoProvidersForAttribute("exports", Mode.TARGET))) {
+ processorClasses.addAll(provider.getProcessorClasses());
+ processorClasspath.addTransitive(provider.getProcessorClasspath());
+ }
+ exportedProcessorClasses = processorClasses.build();
+ exportedProcessorClasspath = processorClasspath.build();
+
+ CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() {
+ @Override
+ protected void collect(CcLinkParams.Builder builder, boolean linkingStatically,
+ boolean linkShared) {
+ Iterable<? extends TransitiveInfoCollection> deps =
+ common.targetsTreatedAsDeps(ClasspathType.BOTH);
+ builder.addTransitiveTargets(deps);
+ builder.addTransitiveLangTargets(deps, JavaCcLinkParamsProvider.TO_LINK_PARAMS);
+ }
+ };
+
+ // The "neverlink" attribute is transitive, so we don't add any
+ // runfiles from this target or its dependencies.
+ Runfiles runfiles = Runfiles.EMPTY;
+ if (!common.isNeverLink()) {
+ Runfiles.Builder runfilesBuilder = new Runfiles.Builder().addArtifacts(
+ common.getJavaCompilationArtifacts().getRuntimeJars());
+
+
+ runfilesBuilder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES);
+ runfilesBuilder.add(ruleContext, JavaRunfilesProvider.TO_RUNFILES);
+
+ List<TransitiveInfoCollection> depsForRunfiles = new ArrayList<>();
+ if (ruleContext.getRule().isAttrDefined("runtime_deps", Type.LABEL_LIST)) {
+ depsForRunfiles.addAll(ruleContext.getPrerequisites("runtime_deps", Mode.TARGET));
+ }
+ if (ruleContext.getRule().isAttrDefined("exports", Type.LABEL_LIST)) {
+ depsForRunfiles.addAll(ruleContext.getPrerequisites("exports", Mode.TARGET));
+ }
+
+ runfilesBuilder.addTargets(depsForRunfiles, RunfilesProvider.DEFAULT_RUNFILES);
+ runfilesBuilder.addTargets(depsForRunfiles, JavaRunfilesProvider.TO_RUNFILES);
+
+ TransitiveInfoCollection launcher = JavaHelper.launcherForTarget(semantics, ruleContext);
+ if (launcher != null) {
+ runfilesBuilder.addTarget(launcher, RunfilesProvider.DATA_RUNFILES);
+ }
+
+ semantics.addRunfilesForLibrary(ruleContext, runfilesBuilder);
+ runfiles = runfilesBuilder.build();
+ }
+
+ RuleConfiguredTargetBuilder builder =
+ new RuleConfiguredTargetBuilder(ruleContext);
+
+ semantics.addProviders(
+ ruleContext, common, ImmutableList.<String>of(), classJar, srcJar, gensrcJar,
+ ImmutableMap.<Artifact, Artifact>of(), helper, filesBuilder, builder);
+
+ NestedSet<Artifact> filesToBuild = filesBuilder.build();
+ common.addTransitiveInfoProviders(builder, filesToBuild, classJar);
+
+ builder
+ .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles))
+ .setFilesToBuild(filesToBuild)
+ .add(JavaNeverlinkInfoProvider.class, new JavaNeverlinkInfoProvider(common.isNeverLink()))
+ .add(CppCompilationContext.class, transitiveCppDeps)
+ .add(JavaCompilationArgsProvider.class, new JavaCompilationArgsProvider(
+ javaCompilationArgs, recursiveJavaCompilationArgs,
+ compileTimeJavaDepArtifacts, runTimeJavaDepArtifacts))
+ .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore))
+ .add(JavaNativeLibraryProvider.class, new JavaNativeLibraryProvider(
+ transitiveJavaNativeLibraries))
+ .add(JavaSourceJarsProvider.class, new JavaSourceJarsProvider(
+ transitiveSourceJars, ImmutableList.of(srcJar)))
+ .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider(
+ JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars))
+ // TODO(bazel-team): this should only happen for java_plugin
+ .add(JavaPluginInfoProvider.class, new JavaPluginInfoProvider(
+ exportedProcessorClasses, exportedProcessorClasspath));
+
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ return builder;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java
new file mode 100644
index 0000000..a28d7fd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java
@@ -0,0 +1,382 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import static com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode.OFF;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder;
+import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
+import com.google.devtools.build.lib.rules.cpp.CcSpecificLinkParamsProvider;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class to create Java compile actions in a way that is consistent with java_library. Rules that
+ * generate source files and emulate java_library on top of that should use this class
+ * instead of the lower-level API in JavaCompilationHelper.
+ *
+ * <p>Rules that want to use this class are required to have an implicit dependency on the
+ * Java compiler.
+ */
+public final class JavaLibraryHelper {
+ /**
+ * Function for extracting the {@link JavaCompilationArgs} - note that it also handles .jar files.
+ */
+ private static final Function<TransitiveInfoCollection, JavaCompilationArgsProvider>
+ TO_COMPILATION_ARGS = new Function<TransitiveInfoCollection, JavaCompilationArgsProvider>() {
+ @Override
+ public JavaCompilationArgsProvider apply(TransitiveInfoCollection target) {
+ return forTarget(target);
+ }
+ };
+
+ /**
+ * Contains the providers as well as the compilation outputs.
+ */
+ public static final class Info {
+ private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers;
+ private final JavaCompilationArtifacts compilationArtifacts;
+
+ private Info(Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers,
+ JavaCompilationArtifacts compilationArtifacts) {
+ this.providers = Collections.unmodifiableMap(providers);
+ this.compilationArtifacts = compilationArtifacts;
+ }
+
+ public Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> getProviders() {
+ return providers;
+ }
+
+ public JavaCompilationArtifacts getCompilationArtifacts() {
+ return compilationArtifacts;
+ }
+ }
+
+ private final RuleContext ruleContext;
+ private final BuildConfiguration configuration;
+
+ private Artifact output;
+ private final List<Artifact> sourceJars = new ArrayList<>();
+ /**
+ * Contains all the dependencies; these are treated as both compile-time and runtime dependencies.
+ * Some of these may not be complete configured targets; for backwards compatibility with some
+ * existing code, we sometimes only have pretend dependencies that only have a single {@link
+ * JavaCompilationArgsProvider}.
+ */
+ private final List<TransitiveInfoCollection> deps = new ArrayList<>();
+ private ImmutableList<String> javacOpts = ImmutableList.of();
+
+ private StrictDepsMode strictDepsMode = StrictDepsMode.OFF;
+ private JavaClasspathMode classpathMode = JavaClasspathMode.OFF;
+ private boolean emitProviders = true;
+
+ public JavaLibraryHelper(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ this.configuration = ruleContext.getConfiguration();
+ this.classpathMode = ruleContext.getFragment(JavaConfiguration.class).getReduceJavaClasspath();
+ }
+
+ /**
+ * Sets the final output jar; if this is not set, then the {@link #build} method throws an {@link
+ * IllegalStateException}. Note that this class may generate not just the output itself, but also
+ * a number of additional intermediate files and outputs.
+ */
+ public JavaLibraryHelper setOutput(Artifact output) {
+ this.output = output;
+ return this;
+ }
+
+ /**
+ * Adds the given source jars. Any .java files in these jars will be compiled.
+ */
+ public JavaLibraryHelper addSourceJars(Iterable<Artifact> sourceJars) {
+ Iterables.addAll(this.sourceJars, sourceJars);
+ return this;
+ }
+
+ /**
+ * Adds the given source jars. Any .java files in these jars will be compiled.
+ */
+ public JavaLibraryHelper addSourceJars(Artifact... sourceJars) {
+ return this.addSourceJars(Arrays.asList(sourceJars));
+ }
+
+ /**
+ * Adds the given compilation args as deps. Avoid this method, and prefer {@link #addDeps}
+ * instead; this method only exists for backward compatibility and may be removed at any time.
+ */
+ public JavaLibraryHelper addProcessedDeps(JavaCompilationArgs... deps) {
+ for (JavaCompilationArgs dep : deps) {
+ this.deps.add(toTransitiveInfoCollection(dep));
+ }
+ return this;
+ }
+
+ private static TransitiveInfoCollection toTransitiveInfoCollection(
+ final JavaCompilationArgs args) {
+ return new TransitiveInfoCollection() {
+ @Override
+ public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) {
+ if (JavaCompilationArgsProvider.class.equals(provider)) {
+ return provider.cast(new JavaCompilationArgsProvider(args, args));
+ }
+ return null;
+ }
+
+ @Override
+ public Label getLabel() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BuildConfiguration getConfiguration() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object get(String providerKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ /**
+ * Adds the given targets as deps. These are used as both compile-time and runtime dependencies.
+ */
+ public JavaLibraryHelper addDeps(Iterable<? extends TransitiveInfoCollection> deps) {
+ for (TransitiveInfoCollection dep : deps) {
+ Preconditions.checkArgument(dep.getConfiguration() == null
+ || dep.getConfiguration().equals(configuration));
+ this.deps.add(dep);
+ }
+ return this;
+ }
+
+ /**
+ * Sets the compiler options.
+ */
+ public JavaLibraryHelper setJavacOpts(Iterable<String> javacOpts) {
+ this.javacOpts = ImmutableList.copyOf(javacOpts);
+ return this;
+ }
+
+ /**
+ * Sets the mode that determines how strictly dependencies are checked.
+ */
+ public JavaLibraryHelper setStrictDepsMode(StrictDepsMode strictDepsMode) {
+ this.strictDepsMode = strictDepsMode;
+ return this;
+ }
+
+ /**
+ * Disables all providers, i.e., the resulting {@link Info} object will not contain any providers.
+ * Avoid this method - having this class compute the providers ensures consistency among all
+ * clients of this code.
+ */
+ public JavaLibraryHelper noProviders() {
+ this.emitProviders = false;
+ return this;
+ }
+
+ /**
+ * Creates the compile actions and providers.
+ */
+ public Info build(JavaSemantics semantics) {
+ Preconditions.checkState(output != null, "must have an output file; use setOutput()");
+ JavaTargetAttributes.Builder attributes = new JavaTargetAttributes.Builder(semantics);
+ attributes.addSourceJars(sourceJars);
+ addDepsToAttributes(attributes);
+ attributes.setStrictJavaDeps(strictDepsMode);
+ attributes.setRuleKind(ruleContext.getRule().getRuleClass());
+ attributes.setTargetLabel(ruleContext.getLabel());
+
+ if (isStrict() && classpathMode != JavaClasspathMode.OFF) {
+ addDependencyArtifactsToAttributes(attributes);
+ }
+
+ JavaCompilationArtifacts.Builder artifactsBuilder = new JavaCompilationArtifacts.Builder();
+ JavaCompilationHelper helper =
+ new JavaCompilationHelper(ruleContext, semantics, javacOpts, attributes);
+ Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(output, artifactsBuilder);
+ helper.createCompileAction(output, null, outputDepsProto, null);
+ helper.createCompileTimeJarAction(output, outputDepsProto, artifactsBuilder);
+ artifactsBuilder.addRuntimeJar(output);
+ JavaCompilationArtifacts compilationArtifacts = artifactsBuilder.build();
+
+ Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers =
+ new LinkedHashMap<>();
+ if (emitProviders) {
+ providers.put(JavaCompilationArgsProvider.class,
+ collectJavaCompilationArgs(compilationArtifacts));
+ providers.put(JavaSourceJarsProvider.class,
+ new JavaSourceJarsProvider(collectTransitiveJavaSourceJars(), sourceJars));
+ providers.put(JavaRunfilesProvider.class, collectJavaRunfiles(compilationArtifacts));
+ providers.put(JavaCcLinkParamsProvider.class,
+ new JavaCcLinkParamsProvider(createJavaCcLinkParamsStore()));
+ }
+ return new Info(providers, compilationArtifacts);
+ }
+
+ private void addDepsToAttributes(JavaTargetAttributes.Builder attributes) {
+ NestedSet<Artifact> directJars = null;
+ if (isStrict()) {
+ directJars = getNonRecursiveCompileTimeJarsFromDeps();
+ if (directJars != null) {
+ attributes.addDirectCompileTimeClassPathEntries(directJars);
+ attributes.addDirectJars(directJars);
+ }
+ }
+
+ JavaCompilationArgs args = JavaCompilationArgs.builder()
+ .addTransitiveDependencies(transformDeps(), true).build();
+ attributes.addCompileTimeClassPathEntries(args.getCompileTimeJars());
+ attributes.addRuntimeClassPathEntries(args.getRuntimeJars());
+ attributes.addInstrumentationMetadataEntries(args.getInstrumentationMetadata());
+ }
+
+ private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromDeps() {
+ JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder();
+ builder.addTransitiveDependencies(transformDeps(), false);
+ return builder.build().getCompileTimeJars();
+ }
+
+ private void addDependencyArtifactsToAttributes(JavaTargetAttributes.Builder attributes) {
+ NestedSetBuilder<Artifact> compileTimeBuilder = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Artifact> runTimeBuilder = NestedSetBuilder.stableOrder();
+ for (JavaCompilationArgsProvider dep : transformDeps()) {
+ compileTimeBuilder.addTransitive(dep.getCompileTimeJavaDependencyArtifacts());
+ runTimeBuilder.addTransitive(dep.getRunTimeJavaDependencyArtifacts());
+ }
+ attributes.addCompileTimeDependencyArtifacts(compileTimeBuilder.build());
+ attributes.addRuntimeDependencyArtifacts(runTimeBuilder.build());
+ }
+
+ private Iterable<JavaCompilationArgsProvider> transformDeps() {
+ return Iterables.transform(deps, TO_COMPILATION_ARGS);
+ }
+
+ private static JavaCompilationArgsProvider forTarget(TransitiveInfoCollection target) {
+ if (target.getProvider(JavaCompilationArgsProvider.class) != null) {
+ // If the target has JavaCompilationArgs, we use those.
+ return target.getProvider(JavaCompilationArgsProvider.class);
+ } else {
+ // Otherwise we look for any jar files. It would be good to remove this, and require
+ // intermediate java_import rules in these cases.
+ NestedSet<Artifact> filesToBuild =
+ target.getProvider(FileProvider.class).getFilesToBuild();
+ final List<Artifact> jars = new ArrayList<>();
+ Iterables.addAll(jars, FileType.filter(filesToBuild, JavaSemantics.JAR));
+ JavaCompilationArgs args = JavaCompilationArgs.builder()
+ .addCompileTimeJars(jars)
+ .addRuntimeJars(jars)
+ .build();
+ return new JavaCompilationArgsProvider(args, args);
+ }
+ }
+
+ private boolean isStrict() {
+ return strictDepsMode != OFF;
+ }
+
+ private JavaCompilationArgsProvider collectJavaCompilationArgs(
+ JavaCompilationArtifacts compilationArtifacts) {
+ JavaCompilationArgs javaCompilationArgs =
+ collectJavaCompilationArgs(compilationArtifacts, false);
+ JavaCompilationArgs recursiveJavaCompilationArgs =
+ collectJavaCompilationArgs(compilationArtifacts, true);
+ return new JavaCompilationArgsProvider(javaCompilationArgs, recursiveJavaCompilationArgs);
+ }
+
+ /**
+ * Get compilation arguments for java compilation action.
+ *
+ * @param recursive a boolean specifying whether to get transitive
+ * dependencies
+ * @return java compilation args
+ */
+ private JavaCompilationArgs collectJavaCompilationArgs(
+ JavaCompilationArtifacts compilationArtifacts, boolean recursive) {
+ return JavaCompilationArgs.builder()
+ .merge(compilationArtifacts)
+ .addTransitiveDependencies(transformDeps(), recursive)
+ .build();
+ }
+
+ private NestedSet<Artifact> collectTransitiveJavaSourceJars() {
+ NestedSetBuilder<Artifact> transitiveJavaSourceJarBuilder =
+ NestedSetBuilder.<Artifact>stableOrder();
+ transitiveJavaSourceJarBuilder.addAll(sourceJars);
+ for (JavaSourceJarsProvider other : ruleContext.getPrerequisites(
+ "deps", Mode.TARGET, JavaSourceJarsProvider.class)) {
+ transitiveJavaSourceJarBuilder.addTransitive(other.getTransitiveSourceJars());
+ }
+ return transitiveJavaSourceJarBuilder.build();
+ }
+
+ private JavaRunfilesProvider collectJavaRunfiles(
+ JavaCompilationArtifacts javaCompilationArtifacts) {
+ Runfiles runfiles = new Runfiles.Builder()
+ // Compiled templates as well, for API.
+ .addArtifacts(javaCompilationArtifacts.getRuntimeJars())
+ .addTargets(deps, JavaRunfilesProvider.TO_RUNFILES)
+ .build();
+ return new JavaRunfilesProvider(runfiles);
+ }
+
+ private CcLinkParamsStore createJavaCcLinkParamsStore() {
+ return new CcLinkParamsStore() {
+ @Override
+ protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {
+ builder.addTransitiveLangTargets(
+ deps,
+ JavaCcLinkParamsProvider.TO_LINK_PARAMS);
+ builder.addTransitiveTargets(deps);
+ // TODO(bazel-team): This may need to be optional for some clients of this class.
+ builder.addTransitiveLangTargets(
+ deps,
+ CcSpecificLinkParamsProvider.TO_LINK_PARAMS);
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java
new file mode 100644
index 0000000..8be42c0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.rules.cpp.LinkerInput;
+
+/**
+ * A target that provides native libraries in the transitive closure of its deps that are needed for
+ * executing Java code.
+ */
+@Immutable
+public final class JavaNativeLibraryProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<LinkerInput> transitiveJavaNativeLibraries;
+
+ public JavaNativeLibraryProvider(
+ NestedSet<LinkerInput> transitiveJavaNativeLibraries) {
+ this.transitiveJavaNativeLibraries = transitiveJavaNativeLibraries;
+ }
+
+ /**
+ * Collects native libraries in the transitive closure of its deps that are needed for executing
+ * Java code.
+ */
+ public NestedSet<LinkerInput> getTransitiveJavaNativeLibraries() {
+ return transitiveJavaNativeLibraries;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java
new file mode 100644
index 0000000..75b36c1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A {@link TransitiveInfoProvider} that provides information about whether a Java archive
+ * is neverlink.
+ */
+@Immutable
+public final class JavaNeverlinkInfoProvider implements TransitiveInfoProvider {
+ private final boolean isNeverLink;
+
+ public JavaNeverlinkInfoProvider(boolean isNeverLink) {
+ this.isNeverLink = isNeverLink;
+ }
+
+ public boolean isNeverlink() {
+ return isNeverLink;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java
new file mode 100644
index 0000000..f7ef0c7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java
@@ -0,0 +1,350 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsConverter;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode;
+import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
+import com.google.devtools.build.lib.analysis.config.FragmentOptions;
+import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.common.options.Converters.StringSetConverter;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.TriState;
+
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Command-line options for building Java targets
+ */
+public class JavaOptions extends FragmentOptions {
+ // Defaults value for options
+ static final String DEFAULT_LANGTOOLS_BOOTCLASSPATH = "//tools/jdk:bootclasspath";
+ static final String DEFAULT_LANGTOOLS = "//tools/jdk:langtools";
+ static final String DEFAULT_JAVABUILDER = "//tools:java/JavaBuilder_deploy.jar";
+ static final String DEFAULT_SINGLEJAR = "//tools:java/SingleJar_deploy.jar";
+ static final String DEFAULT_JAVABASE = "//tools/jdk:jdk";
+ static final String DEFAULT_IJAR = "//tools:java/ijar";
+ static final String DEFAULT_TOOLCHAIN = "//tools/jdk:toolchain";
+
+ /**
+ * Converter for the --javawarn option.
+ */
+ public static class JavacWarnConverter extends StringSetConverter {
+ public JavacWarnConverter() {
+ super("all",
+ "cast",
+ "-cast",
+ "deprecation",
+ "-deprecation",
+ "divzero",
+ "-divzero",
+ "empty",
+ "-empty",
+ "fallthrough",
+ "-fallthrough",
+ "finally",
+ "-finally",
+ "none",
+ "options",
+ "-options",
+ "overrides",
+ "-overrides",
+ "path",
+ "-path",
+ "processing",
+ "-processing",
+ "rawtypes",
+ "-rawtypes",
+ "serial",
+ "-serial",
+ "unchecked",
+ "-unchecked"
+ );
+ }
+ }
+
+ /**
+ * Converter for the --experimental_java_classpath option.
+ */
+ public static class JavaClasspathModeConverter extends EnumConverter<JavaClasspathMode> {
+ public JavaClasspathModeConverter() {
+ super(JavaClasspathMode.class, "Java classpath reduction strategy");
+ }
+ }
+
+ @Option(name = "javabase",
+ defaultValue = DEFAULT_JAVABASE,
+ category = "version",
+ help = "JAVABASE used for the JDK invoked by Blaze. This is the "
+ + "JAVABASE which will be used to execute external Java "
+ + "commands.")
+ public String javaBase;
+
+ @Option(name = "java_toolchain",
+ defaultValue = DEFAULT_TOOLCHAIN,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "The name of the toolchain rule for Java. Default is " + DEFAULT_TOOLCHAIN)
+ public Label javaToolchain;
+
+ @Option(name = "host_javabase",
+ defaultValue = DEFAULT_JAVABASE,
+ category = "version",
+ help = "JAVABASE used for the host JDK. This is the JAVABASE which is used to execute "
+ + " tools during a build.")
+ public String hostJavaBase;
+
+ @Option(name = "javacopt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional options to pass to javac.")
+ public List<String> javacOpts;
+
+ @Option(name = "jvmopt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional options to pass to the Java VM. These options will get added to the "
+ + "VM startup options of each java_binary target.")
+ public List<String> jvmOpts;
+
+ @Option(name = "javawarn",
+ converter = JavacWarnConverter.class,
+ defaultValue = "",
+ category = "flags",
+ allowMultiple = true,
+ help = "Additional javac warnings to enable when compiling Java source files.")
+ public List<String> javaWarns;
+
+ @Option(name = "use_ijars",
+ defaultValue = "true",
+ category = "strategy",
+ help = "If enabled, this option causes Java compilation to use interface jars. "
+ + "This will result in faster incremental compilation, "
+ + "but error messages can be different.")
+ public boolean useIjars;
+
+ @Deprecated
+ @Option(name = "use_src_ijars",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "No-op. Kept here for backwards compatibility.")
+ public boolean useSourceIjars;
+
+ @Deprecated
+ @Option(name = "experimental_incremental_ijars",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "No-op. Kept here for backwards compatibility.")
+ public boolean incrementalIjars;
+
+ @Option(name = "java_deps",
+ defaultValue = "true",
+ category = "strategy",
+ help = "Generate dependency information (for now, compile-time classpath) per Java target.")
+ public boolean javaDeps;
+
+ @Option(name = "experimental_java_deps",
+ defaultValue = "false",
+ category = "experimental",
+ expansion = "--java_deps",
+ deprecationWarning = "Use --java_deps instead")
+ public boolean experimentalJavaDeps;
+
+ @Option(name = "experimental_java_classpath",
+ allowMultiple = false,
+ defaultValue = "javabuilder",
+ converter = JavaClasspathModeConverter.class,
+ category = "semantics",
+ help = "Enables reduced classpaths for Java compilations.")
+ public JavaClasspathMode experimentalJavaClasspath;
+
+ @Option(name = "java_cpu",
+ defaultValue = "null",
+ category = "semantics",
+ help = "The Java target CPU. Default is k8.")
+ public String javaCpu;
+
+ @Option(name = "java_debug",
+ defaultValue = "null",
+ category = "testing",
+ expansion = {"--test_arg=--wrapper_script_flag=--debug", "--test_output=streamed",
+ "--test_strategy=exclusive", "--test_timeout=9999", "--nocache_test_results"},
+ help = "Causes the Java virtual machine of a java test to wait for a connection from a "
+ + "JDWP-compliant debugger (such as jdb) before starting the test. Implies "
+ + "-test_output=streamed."
+ )
+ public Void javaTestDebug;
+
+ @Option(name = "strict_java_deps",
+ allowMultiple = false,
+ defaultValue = "default",
+ converter = StrictDepsConverter.class,
+ category = "semantics",
+ help = "If true, checks that a Java target explicitly declares all directly used "
+ + "targets as dependencies.")
+ public StrictDepsMode strictJavaDeps;
+
+ @Option(name = "javabuilder_top",
+ defaultValue = DEFAULT_JAVABUILDER,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "Label of the filegroup that contains the JavaBuilder jar.")
+ public Label javaBuilderTop;
+
+ @Option(name = "javabuilder_jvmopt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "undocumented",
+ help = "Additional options to pass to the JVM when invoking JavaBuilder.")
+ public List<String> javaBuilderJvmOpts;
+
+ @Option(name = "singlejar_top",
+ defaultValue = DEFAULT_SINGLEJAR,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "Label of the filegroup that contains the SingleJar jar.")
+ public Label singleJarTop;
+
+ @Option(name = "ijar_top",
+ defaultValue = DEFAULT_IJAR,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "Label of the filegroup that contains the ijar binary.")
+ public Label iJarTop;
+
+ @Option(name = "java_langtools",
+ defaultValue = DEFAULT_LANGTOOLS,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "Label of the rule that produces the Java langtools jar.")
+ public Label javaLangtoolsJar;
+
+ @Option(name = "javac_bootclasspath",
+ defaultValue = DEFAULT_LANGTOOLS_BOOTCLASSPATH,
+ category = "version",
+ converter = LabelConverter.class,
+ help = "Label of the rule that produces the bootclasspath jars for javac to use.")
+ public Label javacBootclasspath;
+
+ @Option(name = "java_launcher",
+ defaultValue = "null",
+ converter = LabelConverter.class,
+ category = "semantics",
+ help = "If enabled, a specific Java launcher is used. "
+ + "The \"launcher\" attribute overrides this flag. ")
+ public Label javaLauncher;
+
+ @Option(name = "translations",
+ defaultValue = "auto",
+ category = "semantics",
+ help = "Translate Java messages; bundle all translations into the jar "
+ + "for each affected rule.")
+ public TriState bundleTranslations;
+
+ @Option(name = "message_translations",
+ defaultValue = "",
+ category = "semantics",
+ allowMultiple = true,
+ help = "The message translations used for translating messages in Java targets.")
+ public List<String> translationTargets;
+
+ @Option(name = "check_constraint",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "checking",
+ help = "Check the listed constraint.")
+ public List<String> checkedConstraints;
+
+ @Override
+ public FragmentOptions getHost(boolean fallback) {
+ JavaOptions host = (JavaOptions) getDefault();
+
+ host.javaBase = hostJavaBase;
+ host.jvmOpts = ImmutableList.of("-client", "-XX:ErrorFile=/dev/stderr");
+
+ host.javacOpts = javacOpts;
+ host.javaLangtoolsJar = javaLangtoolsJar;
+ host.javaBuilderTop = javaBuilderTop;
+ host.javaToolchain = javaToolchain;
+ host.singleJarTop = singleJarTop;
+ host.iJarTop = iJarTop;
+
+ // Java builds often contain complicated code generators for which
+ // incremental build performance is important.
+ host.useIjars = useIjars;
+
+ host.javaDeps = javaDeps;
+ host.experimentalJavaClasspath = experimentalJavaClasspath;
+
+ return host;
+ }
+
+ @Override
+ public void addAllLabels(Multimap<String, Label> labelMap) {
+ addOptionalLabel(labelMap, "jdk", javaBase);
+ addOptionalLabel(labelMap, "jdk", hostJavaBase);
+ if (javaLauncher != null) {
+ labelMap.put("java_launcher", javaLauncher);
+ }
+ labelMap.put("javabuilder", javaBuilderTop);
+ labelMap.put("singlejar", singleJarTop);
+ labelMap.put("ijar", iJarTop);
+ labelMap.put("java_toolchain", javaToolchain);
+ labelMap.putAll("translation", getTranslationLabels());
+ }
+
+ @Override
+ public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) {
+ Set<Label> jdkLabels = new LinkedHashSet<>();
+ DefaultsPackage.parseAndAdd(jdkLabels, javaBase);
+ DefaultsPackage.parseAndAdd(jdkLabels, hostJavaBase);
+ Map<String, Set<Label>> result = new HashMap<>();
+ result.put("JDK", jdkLabels);
+ result.put("JAVA_LANGTOOLS", ImmutableSet.of(javaLangtoolsJar));
+ result.put("JAVAC_BOOTCLASSPATH", ImmutableSet.of(javacBootclasspath));
+ result.put("JAVABUILDER", ImmutableSet.of(javaBuilderTop));
+ result.put("SINGLEJAR", ImmutableSet.of(singleJarTop));
+ result.put("IJAR", ImmutableSet.of(iJarTop));
+ result.put("JAVA_TOOLCHAIN", ImmutableSet.of(javaToolchain));
+
+ return result;
+ }
+
+ private Set<Label> getTranslationLabels() {
+ Set<Label> result = new LinkedHashSet<>();
+ for (String s : translationTargets) {
+ try {
+ Label label = Label.parseAbsolute(s);
+ result.add(label);
+ } catch (SyntaxException e) {
+ // We ignore this exception here - it will cause an error message at a later time.
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java
new file mode 100644
index 0000000..526d52c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for the java_plugin rule.
+ */
+public class JavaPlugin implements RuleConfiguredTargetFactory {
+
+ private final JavaSemantics semantics;
+
+ protected JavaPlugin(JavaSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ JavaLibrary javaLibrary = new JavaLibrary(semantics);
+ JavaCommon common = new JavaCommon(ruleContext, semantics);
+ RuleConfiguredTargetBuilder builder = javaLibrary.init(ruleContext, common);
+ if (builder == null) {
+ return null;
+ }
+ builder.add(JavaPluginInfoProvider.class, new JavaPluginInfoProvider(
+ getProcessorClasses(ruleContext), common.getRuntimeClasspath()));
+ return builder.build();
+ }
+
+ /**
+ * Returns the class that should be passed to javac in order
+ * to run the annotation processor this class represents.
+ */
+ private ImmutableList<String> getProcessorClasses(RuleContext ruleContext) {
+ if (ruleContext.getRule().isAttributeValueExplicitlySpecified("processor_class")) {
+ return ImmutableList.of(ruleContext.attributes().get("processor_class", Type.STRING));
+ }
+ return ImmutableList.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java
new file mode 100644
index 0000000..520a228
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Provider for users of Java plugins.
+ */
+@Immutable
+public final class JavaPluginInfoProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<String> processorClasses;
+ private final NestedSet<Artifact> processorClasspath;
+
+ public JavaPluginInfoProvider(ImmutableList<String> processorClasses,
+ NestedSet<Artifact> processorClasspath) {
+ this.processorClasses = processorClasses;
+ this.processorClasspath = processorClasspath;
+ }
+
+ /**
+ * Returns the class that should be passed to javac in order
+ * to run the annotation processor this class represents.
+ */
+ public ImmutableList<String> getProcessorClasses() {
+ return processorClasses;
+ }
+
+ /**
+ * Returns the artifacts to add to the runtime classpath for this plugin.
+ */
+ public NestedSet<Artifact> getProcessorClasspath() {
+ return processorClasspath;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java
new file mode 100644
index 0000000..fd90011
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Provides the fully qualified name of the primary class to invoke for java targets.
+ */
+@Immutable
+public final class JavaPrimaryClassProvider implements TransitiveInfoProvider {
+
+ private final String primaryClass;
+
+ public JavaPrimaryClassProvider(String primaryClass) {
+ this.primaryClass = primaryClass;
+ }
+
+ /**
+ * Returns either the Java class whose main() method is to be invoked (when
+ * use_testrunner=0) or the Java subclass of junit.framework.Test that
+ * is to be tested by the test runner class (when use_testrunner=1).
+ *
+ * @return a fully qualified Java class name, or null if none could be
+ * determined.
+ */
+ public String getPrimaryClass() {
+ return primaryClass;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java
new file mode 100644
index 0000000..b742d62
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Function;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * A {@link TransitiveInfoProvider} that supplies runfiles for Java dependencies.
+ */
+@Immutable
+public final class JavaRunfilesProvider implements TransitiveInfoProvider {
+ private final Runfiles runfiles;
+
+ public JavaRunfilesProvider(Runfiles runfiles) {
+ this.runfiles = runfiles;
+ }
+
+ public Runfiles getRunfiles() {
+ return runfiles;
+ }
+
+ /**
+ * Returns a function that gets the Java runfiles from a {@link TransitiveInfoCollection} or
+ * the empty runfiles instance if it does not contain that provider.
+ */
+ public static final Function<TransitiveInfoCollection, Runfiles> TO_RUNFILES =
+ new Function<TransitiveInfoCollection, Runfiles>() {
+ @Override
+ public Runfiles apply(TransitiveInfoCollection input) {
+ JavaRunfilesProvider provider = input.getProvider(JavaRunfilesProvider.class);
+ return provider == null
+ ? Runfiles.EMPTY
+ : provider.getRunfiles();
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java
new file mode 100644
index 0000000..c8090df
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Provider for the runtime classpath contributions of a Java binary.
+ *
+ * Used to exclude already-available artifacts from related binaries
+ * (e.g. plugins).
+ */
+@Immutable
+public final class JavaRuntimeClasspathProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> runtimeClasspath;
+
+ public JavaRuntimeClasspathProvider(NestedSet<Artifact> runtimeClasspath) {
+ this.runtimeClasspath = runtimeClasspath;
+ }
+
+ /**
+ * Returns the artifacts included on the runtime classpath of this binary.
+ */
+ public NestedSet<Artifact> getRuntimeClasspath() {
+ return runtimeClasspath;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java
new file mode 100644
index 0000000..64b6214
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java
@@ -0,0 +1,351 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.Runfiles.Builder;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel;
+import com.google.devtools.build.lib.packages.Attribute.LateBoundLabelList;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression;
+import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Pluggable Java compilation semantics.
+ */
+public interface JavaSemantics {
+
+ public static final LibraryLanguage LANGUAGE = new LibraryLanguage("Java");
+
+ public static final SafeImplicitOutputsFunction JAVA_LIBRARY_CLASS_JAR =
+ fromTemplates("lib%{name}.jar");
+ public static final SafeImplicitOutputsFunction JAVA_LIBRARY_SOURCE_JAR =
+ fromTemplates("lib%{name}-src.jar");
+ public static final SafeImplicitOutputsFunction JAVA_BINARY_CLASS_JAR =
+ fromTemplates("%{name}.jar");
+ public static final SafeImplicitOutputsFunction JAVA_BINARY_SOURCE_JAR =
+ fromTemplates("%{name}-src.jar");
+ public static final SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_JAR =
+ fromTemplates("%{name}_deploy.jar");
+ public static final SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_SOURCE_JAR =
+ fromTemplates("%{name}_deploy-src.jar");
+
+ public static final FileType JAVA_SOURCE = FileType.of(".java");
+ public static final FileType JAR = FileType.of(".jar");
+ public static final FileType PROPERTIES = FileType.of(".properties");
+ public static final FileType SOURCE_JAR = FileType.of(".srcjar");
+ // TODO(bazel-team): Rename this metadata extension to something meaningful.
+ public static final FileType COVERAGE_METADATA = FileType.of(".em");
+
+ /**
+ * Label to the Java Toolchain rule. It is resolved from a label given in the java options.
+ */
+ static final String JAVA_TOOLCHAIN_LABEL = "//tools/defaults:java_toolchain";
+
+ public static final LateBoundLabel<BuildConfiguration> JAVA_TOOLCHAIN =
+ new LateBoundLabel<BuildConfiguration>(JAVA_TOOLCHAIN_LABEL) {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(JavaConfiguration.class).getToolchainLabel();
+ }
+ };
+
+ /**
+ * Name of the output group used for source jars.
+ */
+ public static final String SOURCE_JARS_OUTPUT_GROUP = "source_jars";
+
+ /**
+ * Label of a pseudo-filegroup that contains all jdk files for all
+ * configurations, as specified on the command-line.
+ */
+ public static final String JDK_LABEL = "//tools/defaults:jdk";
+
+ /**
+ * Label of a pseudo-filegroup that contains the boot-classpath entries.
+ */
+ public static final String JAVAC_BOOTCLASSPATH_LABEL = "//tools/defaults:javac_bootclasspath";
+
+ /**
+ * Label of the JavaBuilder JAR used for compiling Java source code.
+ */
+ public static final String JAVABUILDER_LABEL = "//tools/defaults:javabuilder";
+
+ /**
+ * Label of the SingleJar JAR used for creating deploy jars.
+ */
+ public static final String SINGLEJAR_LABEL = "//tools/defaults:singlejar";
+
+ /**
+ * Label of pseudo-cc_binary that tells Blaze a java target's JAVABIN is never to be replaced by
+ * the contents of --java_launcher; only the JDK's launcher will ever be used.
+ */
+ public static final Label JDK_LAUNCHER_LABEL =
+ Label.parseAbsoluteUnchecked("//third_party/java/jdk:jdk_launcher");
+
+ /**
+ * Implementation for the :jvm attribute.
+ */
+ public static final LateBoundLabel<BuildConfiguration> JVM =
+ new LateBoundLabel<BuildConfiguration>(JDK_LABEL) {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(Jvm.class).getJvmLabel();
+ }
+ };
+
+ /**
+ * Implementation for the :host_jdk attribute.
+ */
+ public static final LateBoundLabel<BuildConfiguration> HOST_JDK =
+ new LateBoundLabel<BuildConfiguration>(JDK_LABEL) {
+ @Override
+ public boolean useHostConfiguration() {
+ return true;
+ }
+
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(Jvm.class).getJvmLabel();
+ }
+ };
+
+ /**
+ * Implementation for the :java_launcher attribute. Note that the Java launcher is disabled by
+ * default, so it returns null for the configuration-independent default value.
+ */
+ public static final LateBoundLabel<BuildConfiguration> JAVA_LAUNCHER =
+ new LateBoundLabel<BuildConfiguration>() {
+ @Override
+ public Label getDefault(Rule rule, BuildConfiguration configuration) {
+ return configuration.getFragment(JavaConfiguration.class).getJavaLauncherLabel();
+ }
+ };
+
+ public static final LateBoundLabelList<BuildConfiguration> JAVA_PLUGINS =
+ new LateBoundLabelList<BuildConfiguration>() {
+ @Override
+ public List<Label> getDefault(Rule rule, BuildConfiguration configuration) {
+ return ImmutableList.copyOf(configuration.getPlugins());
+ }
+ };
+
+ public static final String IJAR_LABEL = "//tools/defaults:ijar";
+
+ /**
+ * Verifies if the rule contains and errors.
+ *
+ * <p>Errors should be signaled through {@link RuleContext}.
+ */
+ void checkRule(RuleContext ruleContext, JavaCommon javaCommon);
+
+ /**
+ * Returns the main class of a Java binary.
+ */
+ String getMainClass(RuleContext ruleContext, JavaCommon javaCommon);
+
+ /**
+ * Returns the resources contributed by a Java rule (usually the contents of the
+ * {@code resources} attribute)
+ */
+ ImmutableList<Artifact> collectResources(RuleContext ruleContext);
+
+ /**
+ * Creates the instrumentation metadata artifact for the specified output .jar .
+ */
+ @Nullable Artifact createInstrumentationMetadataArtifact(
+ AnalysisEnvironment analysisEnvironment, Artifact outputJar);
+
+ /**
+ * May add extra command line options to the Java compile command line.
+ */
+ void buildJavaCommandLine(Collection<Artifact> outputs, BuildConfiguration configuration,
+ CustomCommandLine.Builder result);
+
+
+ /**
+ * Constructs the command line to call SingleJar to join all artifacts from
+ * {@code classpath} (java code) and {@code resources} into {@code output}.
+ */
+ CustomCommandLine buildSingleJarCommandLine(BuildConfiguration configuration,
+ Artifact output, String mainClass, ImmutableList<String> manifestLines,
+ Iterable<Artifact> buildInfoFiles, ImmutableList<Artifact> resources,
+ Iterable<Artifact> classpath, boolean includeBuildData,
+ Compression compression, Artifact launcher);
+
+ /**
+ * Creates the action that writes the Java executable stub script.
+ */
+ void createStubAction(RuleContext ruleContext, final JavaCommon javaCommon,
+ List<String> jvmFlags, Artifact executable, String javaStartClass,
+ String javaExecutable);
+
+ /**
+ * Adds extra runfiles for a {@code java_binary} rule.
+ */
+ void addRunfilesForBinary(RuleContext ruleContext, Artifact launcher,
+ Runfiles.Builder runfilesBuilder);
+
+ /**
+ * Adds extra runfiles for a {@code java_library} rule.
+ */
+ void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder);
+
+ /**
+ * Returns the coverage instrumentation specification to be used in Java rules.
+ */
+ InstrumentationSpec getCoverageInstrumentationSpec();
+
+ /**
+ * Returns the additional options to be passed to javac.
+ */
+ Iterable<String> getExtraJavacOpts(RuleContext ruleContext);
+
+ /**
+ * Add additional targets to be treated as direct dependencies.
+ */
+ void collectTargetsTreatedAsDeps(
+ RuleContext ruleContext, ImmutableList.Builder<TransitiveInfoCollection> builder);
+
+ /**
+ * Enables coverage support for the java target - adds instrumented jar to the classpath and
+ * modifies main class.
+ *
+ * @return new main class
+ */
+ String addCoverageSupport(JavaCompilationHelper helper,
+ JavaTargetAttributes.Builder attributes,
+ Artifact executable, Artifact instrumentationMetadata,
+ JavaCompilationArtifacts.Builder javaArtifactsBuilder, String mainClass);
+
+ /**
+ * Return the JVM flags to be used in a Java binary.
+ */
+ Iterable<String> getJvmFlags(RuleContext ruleContext, JavaCommon javaCommon,
+ Artifact launcher, List<String> userJvmFlags);
+
+ /**
+ * Adds extra providers to a Java target.
+ */
+ void addProviders(RuleContext ruleContext,
+ JavaCommon javaCommon,
+ List<String> jvmFlags,
+ Artifact classJar,
+ Artifact srcJar,
+ Artifact gensrcJar,
+ ImmutableMap<Artifact, Artifact> compilationToRuntimeJarMap,
+ JavaCompilationHelper helper,
+ NestedSetBuilder<Artifact> filesBuilder,
+ RuleConfiguredTargetBuilder ruleBuilder);
+
+ /**
+ * Tell if a build with the given configuration should use strict java dependencies. This method
+ * enforces strict java dependencies off if it returns false.
+ */
+ boolean useStrictJavaDeps(BuildConfiguration configuration);
+
+ /**
+ * Translates XMB messages to translations artifact suitable for Java targets.
+ */
+ Collection<Artifact> translate(RuleContext ruleContext, JavaConfiguration javaConfig,
+ List<Artifact> messages);
+
+ /**
+ * Get the launcher artifact for a java binary, creating the necessary actions for it.
+ *
+ * @param ruleContext The rule context
+ * @param common The common helper class.
+ * @param deployArchiveBuilder the builder to construct the deploy archive action (mutable).
+ * @param runfilesBuilder the builder to construct the list of runfiles (mutable).
+ * @param jvmFlags the list of flags to pass to the JVM when running the Java binary (mutable).
+ * @param attributesBuilder the builder to construct the list of attributes of this target
+ * (mutable).
+ * @return the launcher as an artifact
+ */
+ Artifact getLauncher(final RuleContext ruleContext, final JavaCommon common,
+ DeployArchiveBuilder deployArchiveBuilder, Runfiles.Builder runfilesBuilder,
+ List<String> jvmFlags, JavaTargetAttributes.Builder attributesBuilder);
+
+ /**
+ * Add extra dependencies for runfiles of a Java binary.
+ */
+ void addDependenciesForRunfiles(RuleContext ruleContext, Builder builder);
+
+ /**
+ * Determines if we should enforce the use of the :java_launcher target to determine the java
+ * launcher artifact even if the --java_launcher option was not specified.
+ */
+ boolean forceUseJavaLauncherTarget(RuleContext ruleContext);
+
+ /**
+ * Add a source artifact to a {@link JavaTargetAttributes.Builder}. It is called when a source
+ * artifact is processed but is not matched by default patterns in the
+ * {@link JavaTargetAttributes.Builder#addSourceArtifacts(Iterable)} method. The semantics can
+ * then detect its custom artifact types and add it to the builder.
+ */
+ void addArtifactToJavaTargetAttribute(JavaTargetAttributes.Builder builder, Artifact srcArtifact);
+
+ /**
+ * Works on the list of dependencies of a java target to builder the {@link JavaTargetAttributes}.
+ * This work is performed in {@link JavaCommon} for all java targets.
+ */
+ void commonDependencyProcessing(RuleContext ruleContext, JavaTargetAttributes.Builder attributes,
+ Collection<? extends TransitiveInfoCollection> deps);
+
+ /**
+ * Returns an list of {@link ActionInput} that the {@link JavaCompileAction} generates and
+ * that should be cached.
+ */
+ Collection<ActionInput> getExtraJavaCompileOutputs(PathFragment classDirectory);
+
+ /**
+ * Takes the path of a Java resource and tries to determine the Java
+ * root relative path of the resource.
+ *
+ * @param path the root relative path of the resource.
+ * @return the Java root relative path of the resource of the root
+ * relative path of the resource if no Java root relative path can be
+ * determined.
+ */
+ PathFragment getJavaResourcePath(PathFragment path);
+
+ /**
+ * @return a list of extra arguments to appends to the runfiles support.
+ */
+ List<String> getExtraArguments(RuleContext ruleContext, JavaCommon javaCommon);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java
new file mode 100644
index 0000000..2bb3597
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * The collection of source jars from the transitive closure.
+ */
+@Immutable
+public final class JavaSourceJarsProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> transitiveSourceJars;
+ private final ImmutableList<Artifact> sourceJars;
+
+ public JavaSourceJarsProvider(NestedSet<Artifact> transitiveSourceJars,
+ Iterable<Artifact> sourceJars) {
+ this.transitiveSourceJars = transitiveSourceJars;
+ this.sourceJars = ImmutableList.copyOf(sourceJars);
+ }
+
+ /**
+ * Returns all the source jars in the transitive closure, that can be reached by a chain of
+ * JavaSourceJarsProvider instances.
+ */
+ public NestedSet<Artifact> getTransitiveSourceJars() {
+ return transitiveSourceJars;
+ }
+
+ /**
+ * Return the source jars that are to be built when the target is on the command line.
+ */
+ public ImmutableList<Artifact> getSourceJars() {
+ return sourceJars;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java
new file mode 100644
index 0000000..a7fc497
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java
@@ -0,0 +1,603 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.IterablesChain;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * An object that captures the temporary state we need to pass around while
+ * the initialization hook for a java rule is running.
+ */
+public class JavaTargetAttributes {
+
+ private static void checkJar(Artifact classPathEntry) {
+ if (!JavaSemantics.JAR.matches(classPathEntry.getFilename())) {
+ throw new IllegalArgumentException(
+ "not a jar file: " + classPathEntry.prettyPrint());
+ }
+ }
+
+ /**
+ * A builder class for JavaTargetAttributes.
+ */
+ public static class Builder {
+
+ // The order of source files is important, and there must not be duplicates.
+ // Unfortunately, there is no interface in Java that represents a collection
+ // without duplicates that has a stable and deterministic iteration order,
+ // but is not sorted according to a property of the elements. Thus we are
+ // stuck with Set.
+ private final Set<Artifact> sourceFiles = new LinkedHashSet<>();
+ private final Set<Artifact> jarFiles = new LinkedHashSet<>();
+ private final Set<Artifact> compileTimeJarFiles = new LinkedHashSet<>();
+
+ private final NestedSetBuilder<Artifact> runtimeClassPath =
+ NestedSetBuilder.naiveLinkOrder();
+
+ private final NestedSetBuilder<Artifact> compileTimeClassPath =
+ NestedSetBuilder.naiveLinkOrder();
+
+ private final List<Artifact> bootClassPath = new ArrayList<>();
+ private final List<Artifact> nativeLibraries = new ArrayList<>();
+
+ private final Set<Artifact> processorPath = new LinkedHashSet<>();
+ private final Set<String> processorNames = new LinkedHashSet<>();
+
+ private final List<Artifact> resources = new ArrayList<>();
+ private final List<Artifact> messages = new ArrayList<>();
+ private final List<Artifact> instrumentationMetadata = new ArrayList<>();
+ private final List<Artifact> sourceJars = new ArrayList<>();
+
+ private final List<Artifact> classPathResources = new ArrayList<>();
+
+ private BuildConfiguration.StrictDepsMode strictJavaDeps =
+ BuildConfiguration.StrictDepsMode.OFF;
+ private final List<Artifact> directJars = new ArrayList<>();
+ private final List<Artifact> compileTimeDependencyArtifacts = new ArrayList<>();
+ private final List<Artifact> runtimeDependencyArtifacts = new ArrayList<>();
+ private String ruleKind;
+ private Label targetLabel;
+
+ private final NestedSetBuilder<Artifact> excludedArtifacts =
+ NestedSetBuilder.naiveLinkOrder();
+
+ private boolean built = false;
+
+ private final JavaSemantics semantics;
+
+ public Builder(JavaSemantics semantics) {
+ this.semantics = semantics;
+ }
+
+ public Builder addSourceArtifacts(Iterable<Artifact> sourceArtifacts) {
+ Preconditions.checkArgument(!built);
+ for (Artifact srcArtifact : sourceArtifacts) {
+ String srcFilename = srcArtifact.getExecPathString();
+ if (JavaSemantics.JAR.matches(srcFilename)) {
+ runtimeClassPath.add(srcArtifact);
+ jarFiles.add(srcArtifact);
+ } else if (JavaSemantics.SOURCE_JAR.matches(srcFilename)) {
+ sourceJars.add(srcArtifact);
+ } else if (JavaSemantics.PROPERTIES.matches(srcFilename)) {
+ // output files of the message compiler
+ resources.add(srcArtifact);
+ } else if (JavaSemantics.JAVA_SOURCE.matches(srcFilename)) {
+ sourceFiles.add(srcArtifact);
+ } else {
+ // try specific cases from the semantics.
+ semantics.addArtifactToJavaTargetAttribute(this, srcArtifact);
+ }
+ }
+ return this;
+ }
+
+ public Builder addSourceFiles(Iterable<Artifact> sourceFiles) {
+ Preconditions.checkArgument(!built);
+ for (Artifact artifact : sourceFiles) {
+ if (JavaSemantics.JAVA_SOURCE.matches(artifact.getFilename())) {
+ this.sourceFiles.add(artifact);
+ }
+ }
+ return this;
+ }
+
+ public Builder merge(JavaCompilationArgs context) {
+ Preconditions.checkArgument(!built);
+ addCompileTimeClassPathEntries(context.getCompileTimeJars());
+ addRuntimeClassPathEntries(context.getRuntimeJars());
+ addInstrumentationMetadataEntries(context.getInstrumentationMetadata());
+ return this;
+ }
+
+ public Builder addSourceJars(Collection<Artifact> sourceJars) {
+ Preconditions.checkArgument(!built);
+ this.sourceJars.addAll(sourceJars);
+ return this;
+ }
+
+ public Builder addSourceJar(Artifact sourceJar) {
+ Preconditions.checkArgument(!built);
+ this.sourceJars.add(sourceJar);
+ return this;
+ }
+
+ public Builder addCompileTimeJarFiles(Iterable<Artifact> jars) {
+ Preconditions.checkArgument(!built);
+ Iterables.addAll(compileTimeJarFiles, jars);
+ return this;
+ }
+
+ public Builder addRuntimeClassPathEntry(Artifact classPathEntry) {
+ Preconditions.checkArgument(!built);
+ checkJar(classPathEntry);
+ runtimeClassPath.add(classPathEntry);
+ return this;
+ }
+
+ public Builder addRuntimeClassPathEntries(NestedSet<Artifact> classPathEntries) {
+ Preconditions.checkArgument(!built);
+ runtimeClassPath.addTransitive(classPathEntries);
+ return this;
+ }
+
+ public Builder addCompileTimeClassPathEntries(NestedSet<Artifact> entries) {
+ Preconditions.checkArgument(!built);
+ compileTimeClassPath.addTransitive(entries);
+ return this;
+ }
+
+ public Builder addDirectCompileTimeClassPathEntries(Iterable<Artifact> entries) {
+ Preconditions.checkArgument(!built);
+ // The other version is preferred as it is more memory-efficient.
+ for (Artifact classPathEntry : entries) {
+ compileTimeClassPath.add(classPathEntry);
+ }
+ return this;
+ }
+
+ public Builder setRuleKind(String ruleKind) {
+ Preconditions.checkArgument(!built);
+ this.ruleKind = ruleKind;
+ return this;
+ }
+
+ public Builder setTargetLabel(Label targetLabel) {
+ Preconditions.checkArgument(!built);
+ this.targetLabel = targetLabel;
+ return this;
+ }
+
+ /**
+ * Sets the bootclasspath to be passed to the Java compiler.
+ *
+ * <p>If this method is called, then the bootclasspath specified in this JavaTargetAttributes
+ * instance overrides the default bootclasspath.
+ */
+ public Builder setBootClassPath(List<Artifact> jars) {
+ Preconditions.checkArgument(!built);
+ Preconditions.checkArgument(!jars.isEmpty());
+ Preconditions.checkState(bootClassPath.isEmpty());
+ bootClassPath.addAll(jars);
+ return this;
+ }
+
+ public Builder addExcludedArtifacts(NestedSet<Artifact> toExclude) {
+ Preconditions.checkArgument(!built);
+ excludedArtifacts.addTransitive(toExclude);
+ return this;
+ }
+
+ /**
+ * Controls how strict the javac compiler will be in checking correct use of
+ * direct dependencies.
+ *
+ * @param strictDeps one of WARN, ERROR or OFF
+ */
+ public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictDeps) {
+ Preconditions.checkArgument(!built);
+ strictJavaDeps = strictDeps;
+ return this;
+ }
+
+ /**
+ * In tandem with strictJavaDeps, directJars represents a subset of the
+ * compile-time, classpath jars that were provided by direct dependencies.
+ * When strictJavaDeps is OFF, there is no need to provide directJars, and
+ * no extra information is passed to javac. When strictJavaDeps is set to
+ * WARN or ERROR, the compiler command line will include extra flags to
+ * indicate the warning/error policy and to map the classpath jars to direct
+ * or transitive dependencies, using the information in directJars. The extra
+ * flags are formatted like this (same for --indirect_dependency):
+ * --direct_dependency
+ * foo/bar/lib.jar
+ * //java/com/google/foo:bar
+ *
+ * @param directJars
+ */
+ public Builder addDirectJars(Iterable<Artifact> directJars) {
+ Preconditions.checkArgument(!built);
+ Iterables.addAll(this.directJars, directJars);
+ return this;
+ }
+
+ public Builder addCompileTimeDependencyArtifacts(Iterable<Artifact> dependencyArtifacts) {
+ Preconditions.checkArgument(!built);
+ Iterables.addAll(this.compileTimeDependencyArtifacts, dependencyArtifacts);
+ return this;
+ }
+
+ public Builder addRuntimeDependencyArtifacts(Iterable<Artifact> dependencyArtifacts) {
+ Preconditions.checkArgument(!built);
+ Iterables.addAll(this.runtimeDependencyArtifacts, dependencyArtifacts);
+ return this;
+ }
+
+ public Builder addInstrumentationMetadataEntries(Iterable<Artifact> metadataEntries) {
+ Preconditions.checkArgument(!built);
+ Iterables.addAll(instrumentationMetadata, metadataEntries);
+ return this;
+ }
+
+ public Builder addNativeLibrary(Artifact nativeLibrary) {
+ Preconditions.checkArgument(!built);
+ String name = nativeLibrary.getFilename();
+ if (CppFileTypes.INTERFACE_SHARED_LIBRARY.matches(name)) {
+ return this;
+ }
+ if (!(CppFileTypes.SHARED_LIBRARY.matches(name)
+ || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(name))) {
+ throw new IllegalArgumentException("not a shared library :" + nativeLibrary.prettyPrint());
+ }
+ nativeLibraries.add(nativeLibrary);
+ return this;
+ }
+
+ public Builder addNativeLibraries(Iterable<Artifact> nativeLibraries) {
+ Preconditions.checkArgument(!built);
+ for (Artifact nativeLibrary : nativeLibraries) {
+ addNativeLibrary(nativeLibrary);
+ }
+ return this;
+ }
+
+ public Builder addMessages(Collection<Artifact> messages) {
+ Preconditions.checkArgument(!built);
+ this.messages.addAll(messages);
+ return this;
+ }
+
+ public Builder addMessage(Artifact messagesArtifact) {
+ Preconditions.checkArgument(!built);
+ this.messages.add(messagesArtifact);
+ return this;
+ }
+
+ public Builder addResources(Collection<Artifact> resources) {
+ Preconditions.checkArgument(!built);
+ this.resources.addAll(resources);
+ return this;
+ }
+
+ public Builder addResource(Artifact resource) {
+ Preconditions.checkArgument(!built);
+ resources.add(resource);
+ return this;
+ }
+
+ public Builder addProcessorName(String processor) {
+ Preconditions.checkArgument(!built);
+ processorNames.add(processor);
+ return this;
+ }
+
+ public Builder addProcessorPath(Iterable<Artifact> jars) {
+ Preconditions.checkArgument(!built);
+ Iterables.addAll(processorPath, jars);
+ return this;
+ }
+
+ public Builder addClassPathResources(List<Artifact> classPathResources) {
+ Preconditions.checkArgument(!built);
+ this.classPathResources.addAll(classPathResources);
+ return this;
+ }
+
+ public Builder addClassPathResource(Artifact classPathResource) {
+ Preconditions.checkArgument(!built);
+ this.classPathResources.add(classPathResource);
+ return this;
+ }
+
+ public JavaTargetAttributes build() {
+ built = true;
+ return new JavaTargetAttributes(
+ sourceFiles,
+ jarFiles,
+ compileTimeJarFiles,
+ runtimeClassPath,
+ compileTimeClassPath,
+ bootClassPath,
+ nativeLibraries,
+ processorPath,
+ processorNames,
+ resources,
+ messages,
+ sourceJars,
+ classPathResources,
+ directJars,
+ compileTimeDependencyArtifacts,
+ ruleKind,
+ targetLabel,
+ excludedArtifacts,
+ strictJavaDeps);
+ }
+
+ // TODO(bazel-team): Remove these 5 methods.
+ @Deprecated
+ public Set<Artifact> getSourceFiles() {
+ return sourceFiles;
+ }
+
+ @Deprecated
+ public boolean hasSourceFiles() {
+ return !sourceFiles.isEmpty();
+ }
+
+ @Deprecated
+ public List<Artifact> getInstrumentationMetadata() {
+ return instrumentationMetadata;
+ }
+
+ @Deprecated
+ public boolean hasSourceJars() {
+ return !sourceJars.isEmpty();
+ }
+
+ @Deprecated
+ public boolean hasJarFiles() {
+ return !jarFiles.isEmpty();
+ }
+ }
+
+ //
+ // -------------------------- END OF BUILDER CLASS -------------------------
+ //
+
+ private final ImmutableSet<Artifact> sourceFiles;
+ private final ImmutableSet<Artifact> jarFiles;
+ private final ImmutableSet<Artifact> compileTimeJarFiles;
+
+ private final NestedSet<Artifact> runtimeClassPath;
+ private final NestedSet<Artifact> compileTimeClassPath;
+
+ private final ImmutableList<Artifact> bootClassPath;
+ private final ImmutableList<Artifact> nativeLibraries;
+
+ private final ImmutableSet<Artifact> processorPath;
+ private final ImmutableSet<String> processorNames;
+
+ private final ImmutableList<Artifact> resources;
+ private final ImmutableList<Artifact> messages;
+ private final ImmutableList<Artifact> sourceJars;
+
+ private final ImmutableList<Artifact> classPathResources;
+
+ private final ImmutableList<Artifact> directJars;
+ private final ImmutableList<Artifact> compileTimeDependencyArtifacts;
+ private final String ruleKind;
+ private final Label targetLabel;
+
+ private final NestedSet<Artifact> excludedArtifacts;
+ private final BuildConfiguration.StrictDepsMode strictJavaDeps;
+
+ /**
+ * Constructor of JavaTargetAttributes.
+ */
+ private JavaTargetAttributes(
+ Set<Artifact> sourceFiles,
+ Set<Artifact> jarFiles,
+ Set<Artifact> compileTimeJarFiles,
+ NestedSetBuilder<Artifact> runtimeClassPath,
+ NestedSetBuilder<Artifact> compileTimeClassPath,
+ List<Artifact> bootClassPath,
+ List<Artifact> nativeLibraries,
+ Set<Artifact> processorPath,
+ Set<String> processorNames,
+ List<Artifact> resources,
+ List<Artifact> messages,
+ List<Artifact> sourceJars,
+ List<Artifact> classPathResources,
+ List<Artifact> directJars,
+ List<Artifact> compileTimeDependencyArtifacts,
+ String ruleKind,
+ Label targetLabel,
+ NestedSetBuilder<Artifact> excludedArtifacts,
+ BuildConfiguration.StrictDepsMode strictJavaDeps) {
+ this.sourceFiles = ImmutableSet.copyOf(sourceFiles);
+ this.jarFiles = ImmutableSet.copyOf(jarFiles);
+ this.compileTimeJarFiles = ImmutableSet.copyOf(compileTimeJarFiles);
+ this.runtimeClassPath = runtimeClassPath.build();
+ this.compileTimeClassPath = compileTimeClassPath.build();
+ this.bootClassPath = ImmutableList.copyOf(bootClassPath);
+ this.nativeLibraries = ImmutableList.copyOf(nativeLibraries);
+ this.processorPath = ImmutableSet.copyOf(processorPath);
+ this.processorNames = ImmutableSet.copyOf(processorNames);
+ this.resources = ImmutableList.copyOf(resources);
+ this.messages = ImmutableList.copyOf(messages);
+ this.sourceJars = ImmutableList.copyOf(sourceJars);
+ this.classPathResources = ImmutableList.copyOf(classPathResources);
+ this.directJars = ImmutableList.copyOf(directJars);
+ this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts);
+ this.ruleKind = ruleKind;
+ this.targetLabel = targetLabel;
+ this.excludedArtifacts = excludedArtifacts.build();
+ this.strictJavaDeps = strictJavaDeps;
+ }
+
+ public List<Artifact> getDirectJars() {
+ return directJars;
+ }
+
+ public List<Artifact> getCompileTimeDependencyArtifacts() {
+ return compileTimeDependencyArtifacts;
+ }
+
+ public List<Artifact> getSourceJars() {
+ return sourceJars;
+ }
+
+ public Collection<Artifact> getResources() {
+ return resources;
+ }
+
+ public List<Artifact> getMessages() {
+ return messages;
+ }
+
+ public ImmutableList<Artifact> getClassPathResources() {
+ return classPathResources;
+ }
+
+ private NestedSet<Artifact> getExcludedArtifacts() {
+ return excludedArtifacts;
+ }
+
+ /**
+ * Returns the artifacts needed on the runtime classpath of this target.
+ *
+ * See also {@link #getRuntimeClassPathForArchive()}.
+ */
+ public NestedSet<Artifact> getRuntimeClassPath() {
+ return runtimeClassPath;
+ }
+
+ /**
+ * Returns the classpath artifacts needed in a deploy jar for this target.
+ *
+ * This excludes the artifacts made available by jars in the deployment
+ * environment.
+ */
+ public Iterable<Artifact> getRuntimeClassPathForArchive() {
+ Iterable<Artifact> runtimeClasspath = getRuntimeClassPath();
+
+ if (getExcludedArtifacts().isEmpty()) {
+ return runtimeClasspath;
+ } else {
+ return Iterables.filter(runtimeClasspath,
+ Predicates.not(Predicates.in(getExcludedArtifacts().toSet())));
+ }
+ }
+
+ public NestedSet<Artifact> getCompileTimeClassPath() {
+ return compileTimeClassPath;
+ }
+
+ public ImmutableList<Artifact> getBootClassPath() {
+ return bootClassPath;
+ }
+
+ public ImmutableSet<Artifact> getProcessorPath() {
+ return processorPath;
+ }
+
+ public Set<Artifact> getSourceFiles() {
+ return sourceFiles;
+ }
+
+ public Set<Artifact> getJarFiles() {
+ return jarFiles;
+ }
+
+ public Set<Artifact> getCompileTimeJarFiles() {
+ return compileTimeJarFiles;
+ }
+
+ public List<Artifact> getNativeLibraries() {
+ return nativeLibraries;
+ }
+
+ public Collection<String> getProcessorNames() {
+ return processorNames;
+ }
+
+ public boolean hasSourceFiles() {
+ return !sourceFiles.isEmpty();
+ }
+
+ public boolean hasSourceJars() {
+ return !sourceJars.isEmpty();
+ }
+
+ public boolean hasJarFiles() {
+ return !jarFiles.isEmpty();
+ }
+
+ public boolean hasResources() {
+ return !resources.isEmpty();
+ }
+
+ public boolean hasMessages() {
+ return !messages.isEmpty();
+ }
+
+ public boolean hasClassPathResources() {
+ return !classPathResources.isEmpty();
+ }
+
+ public Iterable<Artifact> getArchiveInputs(boolean includeClasspath) {
+ IterablesChain.Builder<Artifact> inputs = IterablesChain.builder();
+ if (includeClasspath) {
+ inputs.add(ImmutableList.copyOf(getRuntimeClassPathForArchive()));
+ }
+ inputs.add(getResources());
+ inputs.add(getClassPathResources());
+ if (getExcludedArtifacts().isEmpty()) {
+ return inputs.build();
+ } else {
+ Set<Artifact> excludedJars = Sets.newHashSet(getExcludedArtifacts());
+ return ImmutableList.copyOf(Iterables.filter(
+ inputs.build(), Predicates.not(Predicates.in(excludedJars))));
+ }
+ }
+
+ public String getRuleKind() {
+ return ruleKind;
+ }
+
+ public Label getTargetLabel() {
+ return targetLabel;
+ }
+
+ public BuildConfiguration.StrictDepsMode getStrictJavaDeps() {
+ return strictJavaDeps;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java
new file mode 100644
index 0000000..65ed97a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+import java.util.List;
+
+/**
+ * Implementation for the {@code java_toolchain} rule.
+ */
+public final class JavaToolchain implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ final String source = ruleContext.attributes().get("source_version", Type.STRING);
+ final String target = ruleContext.attributes().get("target_version", Type.STRING);
+ final String encoding = ruleContext.attributes().get("encoding", Type.STRING);
+ final List<String> xlint = ruleContext.attributes().get("xlint", Type.STRING_LIST);
+ final List<String> misc = ruleContext.attributes().get("misc", Type.STRING_LIST);
+ final JavaConfiguration configuration = ruleContext.getFragment(JavaConfiguration.class);
+ JavaToolchainProvider provider = new JavaToolchainProvider(source, target, encoding,
+ ImmutableList.copyOf(xlint), ImmutableList.copyOf(misc),
+ configuration.getDefaultJavacFlags());
+ RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext)
+ .add(JavaToolchainProvider.class, provider)
+ .setFilesToBuild(new NestedSetBuilder<Artifact>(Order.STABLE_ORDER).build())
+ .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY));
+
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java
new file mode 100644
index 0000000..0338fb8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Information about the JDK used by the <code>java_*</code> rules.
+ *
+ * <p>This class contains the data of the {@code java_toolchain} rules, it is a separate object so
+ * it can be shared with other tools.
+ */
+@Immutable
+public class JavaToolchainData {
+ private final ImmutableList<String> options;
+
+ public JavaToolchainData(String source, String target, String encoding,
+ ImmutableList<String> xlint, ImmutableList<String> misc) {
+ Builder<String> builder = ImmutableList.<String>builder();
+ if (!source.isEmpty()) {
+ builder.add("-source", source);
+ }
+ if (!target.isEmpty()) {
+ builder.add("-target", target);
+ }
+ if (!encoding.isEmpty()) {
+ builder.add("-encoding", encoding);
+ }
+ if (!xlint.isEmpty()) {
+ builder.add("-Xlint:" + Joiner.on(",").join(xlint));
+ }
+ this.options = builder.addAll(misc).build();
+ }
+
+ /**
+ * @return the list of options as given by the {@code java_toolchain} rule.
+ */
+ public ImmutableList<String> getJavacOptions() {
+ return options;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java
new file mode 100644
index 0000000..3e210d8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.List;
+
+/**
+ * Information about the JDK used by the <code>java_*</code> rules.
+ */
+@Immutable
+public final class JavaToolchainProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<String> javacOptions;
+
+ public JavaToolchainProvider(String source, String target, String encoding,
+ ImmutableList<String> xlint, ImmutableList<String> misc, List<String> defaultJavacFlags) {
+ super();
+ // merges the defaultJavacFlags from
+ // {@link JavaConfiguration} with the flags from the {@code java_toolchain} rule.
+ JavaToolchainData data = new JavaToolchainData(source, target, encoding, xlint, misc);
+ this.javacOptions = ImmutableList.<String>builder()
+ .addAll(data.getJavacOptions())
+ .addAll(defaultJavacFlags)
+ .build();
+ }
+
+ /**
+ * @return the list of default options for the java compiler
+ */
+ public ImmutableList<String> getJavacOptions() {
+ return javacOptions;
+ }
+
+ /**
+ * An helper method to construct the list of javac options.
+ *
+ * @param ruleContext The rule context of the current rule.
+ * @return the list of flags provided by the {@code java_toolchain} rule merged with the one
+ * provided by the {@link JavaConfiguration} fragment.
+ */
+ public static List<String> getDefaultJavacOptions(RuleContext ruleContext) {
+ JavaToolchainProvider javaToolchain =
+ ruleContext.getPrerequisite(":java_toolchain", Mode.TARGET, JavaToolchainProvider.class);
+ if (javaToolchain == null) {
+ ruleContext.ruleError("No java_toolchain implicit dependency found. This is probably because"
+ + " your java configuration is not up-to-date.");
+ return ImmutableList.of();
+ }
+ return javaToolchain.getJavacOptions();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java
new file mode 100644
index 0000000..16801ee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java
@@ -0,0 +1,92 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.java;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for {@code java_toolchain}
+ */
+@BlazeRule(name = "java_toolchain", ancestors = {BaseRuleClasses.BaseRule.class},
+ factoryClass = JavaToolchain.class)
+public final class JavaToolchainRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder.setUndocumented()
+ /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(source_version) -->
+ The Java source version (e.g., '6' or '7'). It specifies which set of code structures
+ are allowed in the Java source code.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("source_version", STRING).mandatory()) // javac -source flag value.
+ /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(target_version) -->
+ The Java target version (e.g., '6' or '7'). It specifies for which Java runtime the class
+ should be build.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("target_version", STRING).mandatory()) // javac -target flag value.
+ /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(encoding) -->
+ The encoding of the java files (e.g., 'UTF-8').
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("encoding", STRING).mandatory()) // javac -encoding flag value.
+ /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(xlint) -->
+ The list of warning to add or removes from default list. Precedes it with a dash to
+ removes it. Please see the Javac documentation on the -Xlint options for more information.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("xlint", STRING_LIST).value(ImmutableList.<String>of()))
+ /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(xlint) -->
+ The list of extra arguments for the Java compiler. Please refer to the Java compiler
+ documentation for the extensive list of possible Java compiler flags.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("misc", STRING_LIST).value(ImmutableList.<String>of()))
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = java_toolchain, TYPE = OTHER, FAMILY = Java) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>
+Specifies the configuration for the Java compiler. Which toolchain to be used can be changed through
+the --java_toolchain argument. Normally you should not write those kind of rules unless you want to
+tune your Java compiler.
+</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<h4 id="java_binary_examples">Examples</h4>
+
+<p>A simple example would be:
+</p>
+
+<pre class="code">
+java_toolchain(
+ name = "toolchain",
+ source_version = "7",
+ target_version = "7",
+ encoding = "UTF-8",
+ xlint = [ "classfile", "divzero", "empty", "options", "path" ],
+ misc = [ "-g" ],
+)
+</pre>
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java
new file mode 100644
index 0000000..b2a84ec
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java
@@ -0,0 +1,147 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Utility methods for use by Java-related parts of the build system.
+ */
+public abstract class JavaUtil {
+
+ private JavaUtil() {}
+
+ //---------- Java related methods
+
+ /*
+ * TODO(bazel-team): (2009)
+ *
+ * This way of figuring out Java source roots is basically
+ * broken. I think we need to support these two use cases:
+ * (1) A user puts his / her shell into a directory named java.
+ * (2) Someplace in the tree, there's a package named java.
+ *
+ * (1) is more important than (2); and (2) cannot always be guaranteed
+ * due to sloppy implementations in the past; most notably the old
+ * tools/boilerplate_rules.mk code for compiling Java.
+ *
+ * Basically, to implement correct semantics, we will need to configure
+ * Java source roots based on the package path, plus some heuristics to
+ * support legacy code, maybe.
+ *
+ * Roughly:
+ * Given a path, find the source root that applies to it by
+ * - walk over the elements in the package path
+ * - add "java", "javatests" to them
+ * - find the first element that is a maximal prefix to the Java file
+ * - for experimental, some legacy support that basically has some
+ * arbitrary padding before the Java sourceroot.
+ */
+
+ /**
+ * Given the filename of a Java source file, returns the name of the toplevel Java class defined
+ * within it.
+ */
+ public static String getJavaClassName(PathFragment path) {
+ return FileSystemUtils.removeExtension(path.getBaseName());
+ }
+
+ /**
+ * Find the index of the "java" or "javatests" segment in a Java path fragment
+ * that precedes the source root.
+ *
+ * @param path a Java source dir or file path
+ * @return the index of the java segment or -1 iff no java segment was found.
+ */
+ private static int javaSegmentIndex(PathFragment path) {
+ if (path.isAbsolute()) {
+ throw new IllegalArgumentException("path must not be absolute: '" + path + "'");
+ }
+ return path.getFirstSegment(ImmutableSet.of("java", "javatests"));
+ }
+
+ /**
+ * Given the PathFragment of a Java source file, returns the Java package to which it belongs.
+ */
+ public static String getJavaPackageName(PathFragment path) {
+ int index = javaSegmentIndex(path) + 1;
+ path = path.subFragment(index, path.segmentCount() - 1);
+ return path.getPathString().replace('/', '.');
+ }
+
+ /**
+ * Given the PathFragment of a file without extension, returns the
+ * Java fully qualified class name based on the Java root relative path of the
+ * specified path or 'null' if no java root can be determined.
+ * <p>
+ * For example, "java/foo/bar/wiz" and "javatests/foo/bar/wiz" both
+ * result in "foo.bar.wiz".
+ *
+ * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root
+ * of a relative path rather than simply trying to find the "java" or
+ * "javatests" directory.
+ */
+ public static String getJavaFullClassname(PathFragment path) {
+ PathFragment javaPath = getJavaPath(path);
+ if (javaPath != null) {
+ return javaPath.getPathString().replace('/', '.');
+ }
+ return null;
+ }
+
+ /**
+ * Given the PathFragment of a Java source file, returns the Java root relative path or 'null' if
+ * no java root can be determined.
+ *
+ * <p>
+ * For example, "{workspace}/java/foo/bar/wiz" and "{workspace}/javatests/foo/bar/wiz"
+ * both result in "foo/bar/wiz".
+ *
+ * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root
+ * of a relative path rather than simply trying to find the "java" or
+ * "javatests" directory.
+ */
+ public static PathFragment getJavaPath(PathFragment path) {
+ int index = javaSegmentIndex(path);
+ if (index >= 0) {
+ return path.subFragment(index + 1, path.segmentCount());
+ }
+ return null;
+ }
+
+ /**
+ * Given the PathFragment of a Java source file, returns the
+ * Java root of the specified path or 'null' if no java root can be
+ * determined.
+ * <p>
+ * Example 1: "{workspace}/java/foo/bar/wiz" and "{workspace}/javatests/foo/bar/wiz"
+ * result in "{workspace}/java" and "{workspace}/javatests" Example 2:
+ * "java/foo/bar/wiz" and "javatests/foo/bar/wiz" result in "java" and
+ * "javatests"
+ *
+ * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root
+ * of a relative path rather than simply trying to find the "java" or
+ * "javatests" directory.
+ */
+ public static PathFragment getJavaRoot(PathFragment path) {
+ int index = javaSegmentIndex(path);
+ if (index >= 0) {
+ return path.subFragment(0, index + 1);
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java b/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java
new file mode 100644
index 0000000..eda7360
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java
@@ -0,0 +1,120 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkCallable;
+import com.google.devtools.build.lib.syntax.SkylarkModule;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * This class represents a Java virtual machine with a host system and a path.
+ * If the JVM comes from the client, it can optionally also contain a label
+ * pointing to a target that contains all the necessary files.
+ */
+@SkylarkModule(name = "jvm",
+ doc = "A configuration fragment representing the Java virtual machine.")
+@Immutable
+public final class Jvm extends BuildConfiguration.Fragment {
+ private final PathFragment javaHome;
+ private final Label jvmLabel;
+
+ /**
+ * Creates a Jvm instance. Either the {@code javaHome} parameter is absolute,
+ * or the {@code jvmLabel} parameter must be non-null. This restriction might
+ * be lifted in the future. Only the {@code jvmLabel} is optional.
+ */
+ public Jvm(PathFragment javaHome, Label jvmLabel) {
+ Preconditions.checkArgument(javaHome.isAbsolute() ^ (jvmLabel != null));
+ this.javaHome = javaHome;
+ this.jvmLabel = jvmLabel;
+ }
+
+ @Override
+ public String getName() {
+ return "Jvm";
+ }
+
+ @Override
+ public void addImplicitLabels(Multimap<String, Label> implicitLabels) {
+ if (jvmLabel != null) {
+ implicitLabels.put(getName(), jvmLabel);
+ }
+ }
+
+ /**
+ * Returns a path fragment that determines the path to the installation
+ * directory. It is either absolute or relative to the execution root.
+ */
+ public PathFragment getJavaHome() {
+ return javaHome;
+ }
+
+ /**
+ * Returns the path to the javac binary.
+ */
+ public PathFragment getJavacExecutable() {
+ return getJavaHome().getRelative("bin/javac");
+ }
+
+ /**
+ * Returns the path to the jar binary.
+ */
+ public PathFragment getJarExecutable() {
+ return getJavaHome().getRelative("bin/jar");
+ }
+
+ /**
+ * Returns the path to the java binary.
+ */
+ @SkylarkCallable(name = "java_executable", structField = true,
+ doc = "The the java executable, i.e. bin/java relative to the Java home.")
+ public PathFragment getJavaExecutable() {
+ return getJavaHome().getRelative("bin/java");
+ }
+
+ /**
+ * Returns a label. Adding this label to the dependencies of an action that
+ * depends on this JVM is sufficient to ensure that all the required files are
+ * present. Can be <code>null</code>, in which case nothing needs to be added
+ * to the dependencies of an action. We rely on convention to make sure that
+ * this case works, since we can't know which JVMs are installed on the build host.
+ */
+ public Label getJvmLabel() {
+ return jvmLabel;
+ }
+
+ /**
+ * Returns a string that uniquely identifies the JVM for the life time of this
+ * Blaze instance. This value is intended for analysis caching, so it need not
+ * reflect changes in the individual files making up the JVM.
+ */
+ @Override
+ public String cacheKey() {
+ return javaHome.getSafePathString();
+ }
+
+ @Override
+ public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) {
+ globalMakeEnvBuilder.put("JAVABASE", getJavaHome().getPathString());
+ globalMakeEnvBuilder.put("JAVA", getJavaExecutable().getPathString());
+ globalMakeEnvBuilder.put("JAVAC", getJavacExecutable().getPathString());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java
new file mode 100644
index 0000000..7483f88
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java
@@ -0,0 +1,163 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.analysis.RedirectChaser;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * A provider to load jvm configurations from the package path.
+ *
+ * <p>If the given {@code javaHome} is a label, i.e. starts with {@code "//"},
+ * then the loader will look at the target it refers to. If the target is a
+ * filegroup, then the loader will look in it's srcs for a filegroup that ends
+ * with {@code -<cpu>}. It will use that filegroup to construct the actual
+ * {@link Jvm} instance, using the filegroups {@code path} attribute to
+ * construct the new {@code javaHome} path.
+ *
+ * <p>The loader also supports legacy mode, where the JVM can be defined with an abolute path.
+ */
+public final class JvmConfigurationLoader implements ConfigurationFragmentFactory {
+ private final boolean forceLegacy;
+ private final JavaCpuSupplier cpuSupplier;
+
+ public JvmConfigurationLoader(boolean forceLegacy, JavaCpuSupplier cpuSupplier) {
+ this.forceLegacy = forceLegacy;
+ this.cpuSupplier = cpuSupplier;
+ }
+
+ public JvmConfigurationLoader(JavaCpuSupplier cpuSupplier) {
+ this(/*forceLegacy=*/ false, cpuSupplier);
+ }
+
+ @Override
+ public Jvm create(ConfigurationEnvironment env, BuildOptions buildOptions)
+ throws InvalidConfigurationException {
+ JavaOptions javaOptions = buildOptions.get(JavaOptions.class);
+ String javaHome = javaOptions.javaBase;
+ String cpu = cpuSupplier.getJavaCpu(buildOptions, env);
+ if (cpu == null) {
+ return null;
+ }
+
+ if (!forceLegacy && javaHome.startsWith("//")) {
+ return createDefault(env, javaHome, cpu);
+ } else {
+ return createLegacy(javaHome);
+ }
+ }
+
+ @Override
+ public Class<? extends Fragment> creates() {
+ return Jvm.class;
+ }
+
+ @Nullable
+ private Jvm createDefault(ConfigurationEnvironment lookup, String javaHome, String cpu)
+ throws InvalidConfigurationException {
+ try {
+ Label label = Label.parseAbsolute(javaHome);
+ label = RedirectChaser.followRedirects(lookup, label, "jdk");
+ if (label == null) {
+ return null;
+ }
+ Target javaHomeTarget = lookup.getTarget(label);
+ if (javaHomeTarget == null) {
+ return null;
+ }
+ if ((javaHomeTarget instanceof Rule) &&
+ "filegroup".equals(((Rule) javaHomeTarget).getRuleClass())) {
+ RawAttributeMapper javaHomeAttributes = RawAttributeMapper.of((Rule) javaHomeTarget);
+ if (javaHomeAttributes.isConfigurable("srcs", Type.LABEL_LIST)) {
+ throw new InvalidConfigurationException("\"srcs\" in " + javaHome
+ + " is configurable. JAVABASE targets don't support configurable attributes");
+ }
+ List<Label> labels = javaHomeAttributes.get("srcs", Type.LABEL_LIST);
+ for (Label jvmLabel : labels) {
+ if (jvmLabel.getName().endsWith("-" + cpu)) {
+ Target jvmTarget = lookup.getTarget(jvmLabel);
+ if (jvmTarget == null) {
+ return null;
+ }
+ PathFragment javaHomePath = jvmLabel.getPackageFragment();
+ if ((jvmTarget instanceof Rule) &&
+ "filegroup".equals(((Rule) jvmTarget).getRuleClass())) {
+ RawAttributeMapper jvmTargetAttributes = RawAttributeMapper.of((Rule) jvmTarget);
+ if (jvmTargetAttributes.isConfigurable("path", Type.STRING)) {
+ throw new InvalidConfigurationException("\"path\" in " + jvmTarget
+ + " is configurable. JVM targets don't support configurable attributes");
+ }
+ String path = jvmTargetAttributes.get("path", Type.STRING);
+ if (path != null) {
+ javaHomePath = javaHomePath.getRelative(path);
+ }
+ }
+ return new Jvm(javaHomePath, jvmLabel);
+ }
+ }
+ }
+ throw new InvalidConfigurationException("No JVM target found under " + javaHome
+ + " that would work for " + cpu);
+ } catch (NoSuchPackageException e) {
+ throw new InvalidConfigurationException(e.getMessage(), e);
+ } catch (NoSuchTargetException e) {
+ throw new InvalidConfigurationException("No such target: " + e.getMessage(), e);
+ } catch (SyntaxException e) {
+ throw new InvalidConfigurationException(e.getMessage(), e);
+ }
+ }
+
+ private Jvm createLegacy(String javaHome)
+ throws InvalidConfigurationException {
+ if (!javaHome.startsWith("/")) {
+ throw new InvalidConfigurationException("Illegal javabase value '" + javaHome +
+ "', javabase must be an absolute path or label");
+ }
+ return new Jvm(new PathFragment(javaHome), null);
+ }
+
+ /**
+ * Converts the cpu name to a GNU system name. If the cpu is not a known value, it returns
+ * <code>"unknown-unknown-linux-gnu"</code>.
+ */
+ @VisibleForTesting
+ static String convertCpuToGnuSystemName(String cpu) {
+ if ("piii".equals(cpu)) {
+ return "i686-unknown-linux-gnu";
+ } else if ("k8".equals(cpu)) {
+ return "x86_64-unknown-linux-gnu";
+ } else {
+ return "unknown-unknown-linux-gnu";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java
new file mode 100644
index 0000000..f78e386
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Marks configured targets that are able to supply message bundles to their
+ * dependents.
+ */
+@Immutable
+public final class MessageBundleProvider implements TransitiveInfoProvider {
+
+ private final ImmutableList<Artifact> messages;
+
+ public MessageBundleProvider(ImmutableList<Artifact> messages) {
+ this.messages = messages;
+ }
+
+ /**
+ * The set of XML source files containing the message definitions.
+ */
+ public ImmutableList<Artifact> getMessages() {
+ return messages;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java
new file mode 100644
index 0000000..09ac59f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java
@@ -0,0 +1,115 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.cpp.CcNativeLibraryProvider;
+import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
+import com.google.devtools.build.lib.rules.cpp.LinkerInput;
+import com.google.devtools.build.lib.rules.cpp.LinkerInputs;
+import com.google.devtools.build.lib.util.FileType;
+
+/**
+ * A builder that helps construct nested sets of native libraries.
+ */
+public final class NativeLibraryNestedSetBuilder {
+
+ private final NestedSetBuilder<LinkerInput> builder = NestedSetBuilder.linkOrder();
+
+ /**
+ * Build a nested set of native libraries.
+ */
+ public NestedSet<LinkerInput> build() {
+ return builder.build();
+ }
+
+ /**
+ * Include specified artifacts as native libraries in the nested set.
+ */
+ public NativeLibraryNestedSetBuilder addAll(Iterable<Artifact> deps) {
+ for (Artifact dep : deps) {
+ builder.add(new LinkerInputs.SimpleLinkerInput(dep));
+ }
+ return this;
+ }
+
+ /**
+ * Include native libraries of specified dependencies into the nested set.
+ */
+ public NativeLibraryNestedSetBuilder addJavaTargets(
+ Iterable<? extends TransitiveInfoCollection> deps) {
+ for (TransitiveInfoCollection dep : deps) {
+ addJavaTarget(dep);
+ }
+ return this;
+ }
+
+ /**
+ * Include native Java libraries of a specified target into the nested set.
+ */
+ private void addJavaTarget(TransitiveInfoCollection dep) {
+ JavaNativeLibraryProvider javaProvider = dep.getProvider(JavaNativeLibraryProvider.class);
+ if (javaProvider != null) {
+ builder.addTransitive(javaProvider.getTransitiveJavaNativeLibraries());
+ return;
+ }
+
+ CcNativeLibraryProvider ccProvider = dep.getProvider(CcNativeLibraryProvider.class);
+ if (ccProvider != null) {
+ builder.addTransitive(ccProvider.getTransitiveCcNativeLibraries());
+ return;
+ }
+
+ addTarget(dep);
+ }
+
+ /**
+ * Include native C/C++ libraries of specified dependencies into the nested set.
+ */
+ public NativeLibraryNestedSetBuilder addCcTargets(
+ Iterable<? extends TransitiveInfoCollection> deps) {
+ for (TransitiveInfoCollection dep : deps) {
+ addCcTarget(dep);
+ }
+ return this;
+ }
+
+ /**
+ * Include native Java libraries of a specified target into the nested set.
+ */
+ private void addCcTarget(TransitiveInfoCollection dep) {
+ CcNativeLibraryProvider provider = dep.getProvider(CcNativeLibraryProvider.class);
+ if (provider != null) {
+ builder.addTransitive(provider.getTransitiveCcNativeLibraries());
+ } else {
+ addTarget(dep);
+ }
+ }
+
+ /**
+ * Include files and genrule artifacts.
+ */
+ private void addTarget(TransitiveInfoCollection dep) {
+ for (Artifact artifact : FileType.filterList(
+ dep.getProvider(FileProvider.class).getFilesToBuild(),
+ CppFileTypes.SHARED_LIBRARY)) {
+ builder.add(new LinkerInputs.SimpleLinkerInput(artifact));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java
new file mode 100644
index 0000000..ff8507e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * An interface that marks configured targets that can provide Java compilation arguments through
+ * the 'srcs' attribute of Java rules.
+ *
+ * <p>In a perfect world, this would not be necessary for a million reasons, but
+ * this world is far from perfect, thus, we need this.
+ *
+ * <p>Please do not implement this interface with configured target implementations.
+ */
+@Immutable
+public final class SourcesJavaCompilationArgsProvider implements TransitiveInfoProvider {
+ private final JavaCompilationArgs javaCompilationArgs;
+ private final JavaCompilationArgs recursiveJavaCompilationArgs;
+
+ public SourcesJavaCompilationArgsProvider(
+ JavaCompilationArgs javaCompilationArgs,
+ JavaCompilationArgs recursiveJavaCompilationArgs) {
+ this.javaCompilationArgs = javaCompilationArgs;
+ this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs;
+ }
+
+ /**
+ * Returns non-recursively collected Java compilation information for
+ * building this target (called when strict_java_deps = 1).
+ *
+ * <p>Note that some of the parameters are still collected from the complete
+ * transitive closure. The non-recursive collection applies mainly to
+ * compile-time jars.
+ */
+ public JavaCompilationArgs getJavaCompilationArgs() {
+ return javaCompilationArgs;
+ }
+
+ /**
+ * Returns recursively collected Java compilation information for building
+ * this target (called when strict_java_deps = 0).
+ */
+ public JavaCompilationArgs getRecursiveJavaCompilationArgs() {
+ return recursiveJavaCompilationArgs;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java
new file mode 100644
index 0000000..08dcbba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java
@@ -0,0 +1,211 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.java;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.BuildInfoHelper;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Key;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * An action that creates a Java properties file containing the build informations.
+ */
+public class WriteBuildInfoPropertiesAction extends AbstractFileWriteAction {
+ private static final String GUID = "922949ca-1391-4046-a300-74810618dcdc";
+
+ private final ImmutableList<Artifact> valueArtifacts;
+ private final BuildInfoPropertiesTranslator keyTranslations;
+ private final boolean includeVolatile;
+ private final boolean includeNonVolatile;
+
+ private final TimestampFormatter timestampFormatter;
+ /**
+ * An interface to format a timestamp. We are using our custom one to avoid external dependency.
+ */
+ public static interface TimestampFormatter {
+ /**
+ * Return a human readable string for the given {@code timestamp}. {@code timestamp} is given
+ * in milliseconds since 1st of January 1970 at 0am UTC.
+ */
+ public String format(long timestamp);
+ }
+
+ /**
+ * A wrapper around a {@link Writer} that skips the first line assuming the line is pure ASCII. It
+ * can be used to strip the timestamp comment that {@link Properties#store(Writer, String)} adds.
+ */
+ @VisibleForTesting
+ static class StripFirstLineWriter extends Writer {
+ private final Writer writer;
+ private boolean newlineFound = false;
+
+ StripFirstLineWriter(OutputStream out) {
+ this.writer = new OutputStreamWriter(out, UTF_8);
+ }
+
+ @Override
+ public void write(char[] cbuf, int off, int len) throws IOException {
+ if (!newlineFound) {
+ while (len > 0 && cbuf[off] != '\n') {
+ off++;
+ len--;
+ }
+ if (len > 0) {
+ newlineFound = true;
+ off++;
+ len--;
+ }
+ }
+ if (len > 0) {
+ writer.write(cbuf, off, len);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writer.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.close();
+ }
+
+ }
+
+ /**
+ * Creates an action that writes a Java property files with build information.
+ *
+ * <p>It reads the set of build info keys from an action context that is usually contributed to
+ * Blaze by the workspace status module, and the value associated with said keys from the
+ * workspace status files (stable and volatile) written by the workspace status action. The files
+ * generated by this action serve as input to the
+ * {@link com.google.devtools.build.singlejar.SingleJar} program.
+ *
+ * <p>Without input artifacts, this action uses redacted build information.
+ *
+ * @param inputs Artifacts that contain build information, or an empty collection to use redacted
+ * build information
+ * @param output output the properties file Artifact created by this action
+ * @param keyTranslations how to translates available keys. See
+ * {@link BuildInfoPropertiesTranslator}.
+ * @param includeVolatile whether the set of key to write are giving volatile keys or not
+ * @param includeNonVolatile whether the set of key to write are giving non-volatile keys or not
+ * @param timestampFormatter formats dates printed in the properties file
+ */
+ public WriteBuildInfoPropertiesAction(Collection<Artifact> inputs, Artifact output,
+ BuildInfoPropertiesTranslator keyTranslations, boolean includeVolatile,
+ boolean includeNonVolatile, TimestampFormatter timestampFormatter) {
+ super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, inputs, output, /* makeExecutable= */false);
+ this.keyTranslations = keyTranslations;
+ this.includeVolatile = includeVolatile;
+ this.includeNonVolatile = includeNonVolatile;
+ this.timestampFormatter = timestampFormatter;
+ valueArtifacts = ImmutableList.copyOf(inputs);
+
+ if (!inputs.isEmpty()) {
+ // With non-empty inputs we should not generate both volatile and non-volatile data
+ // in the same properties file.
+ Preconditions.checkState(includeVolatile ^ includeNonVolatile);
+ }
+ Preconditions.checkState(
+ output.isConstantMetadata() == (includeVolatile && !inputs.isEmpty()));
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler,
+ final Executor executor) {
+ final long timestamp = System.currentTimeMillis();
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ WorkspaceStatusAction.Context context =
+ executor.getContext(WorkspaceStatusAction.Context.class);
+ Map<String, String> values = new LinkedHashMap<>();
+ for (Artifact valueFile : valueArtifacts) {
+ values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath()));
+ }
+
+ Map<String, String> keys = new HashMap<>();
+ if (includeVolatile) {
+ addValues(keys, values, context.getVolatileKeys());
+ keys.put("BUILD_TIMESTAMP", Long.toString(timestamp / 1000));
+ keys.put("BUILD_TIME", timestampFormatter.format(timestamp));
+ }
+ addValues(keys, values, context.getStableKeys());
+ Properties properties = new Properties();
+ keyTranslations.translate(keys, properties);
+ properties.store(new StripFirstLineWriter(out), null);
+ }
+ };
+ }
+
+ private void addValues(Map<String, String> result, Map<String, String> values,
+ Map<String, Key> keys) {
+ boolean redacted = values.isEmpty();
+ for (Map.Entry<String, WorkspaceStatusAction.Key> key : keys.entrySet()) {
+ if (key.getValue().isInLanguage("Java")) {
+ result.put(key.getKey(), gePropertyValue(values, redacted, key));
+ }
+ }
+ }
+
+ private static String gePropertyValue(Map<String, String> values, boolean redacted,
+ Map.Entry<String, WorkspaceStatusAction.Key> key) {
+ return redacted ? key.getValue().getRedactedValue()
+ : values.containsKey(key.getKey()) ? values.get(key.getKey())
+ : key.getValue().getDefaultValue();
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addString(keyTranslations.computeKey());
+ f.addBoolean(includeVolatile);
+ f.addBoolean(includeNonVolatile);
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public boolean executeUnconditionally() {
+ return isVolatile();
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return includeVolatile && !Iterables.isEmpty(getInputs());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java
new file mode 100644
index 0000000..73554e5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java
@@ -0,0 +1,588 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates;
+import static com.google.devtools.build.xcode.common.TargetDeviceFamily.UI_DEVICE_FAMILY_VALUES;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraActoolArgs;
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.common.InvalidFamilyNameException;
+import com.google.devtools.build.xcode.common.Platform;
+import com.google.devtools.build.xcode.common.RepeatedFamilyNameException;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting;
+
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Support for application-generating ObjC rules. An application is generally composed of a
+ * top-level {@link BundleSupport bundle}, potentially signed, as well as some debug information, if
+ * {@link ObjcConfiguration#generateDebugSymbols() requested}.
+ *
+ * <p>Contains actions, validation logic and provider value generation.
+ *
+ * <p>Methods on this class can be called in any order without impacting the result.
+ */
+public final class ApplicationSupport {
+
+ /**
+ * Template for the containing application folder.
+ */
+ public static final SafeImplicitOutputsFunction IPA = fromTemplates("%{name}.ipa");
+
+ @VisibleForTesting
+ static final String NO_ASSET_CATALOG_ERROR_FORMAT =
+ "a value was specified (%s), but this app does not have any asset catalogs";
+ @VisibleForTesting
+ static final String INVALID_FAMILIES_ERROR =
+ "Expected one or two strings from the list 'iphone', 'ipad'";
+ @VisibleForTesting
+ static final String DEVICE_NO_PROVISIONING_PROFILE =
+ "Provisioning profile must be set for device build";
+
+ @VisibleForTesting
+ static final String PROVISIONING_PROFILE_BUNDLE_FILE = "embedded.mobileprovision";
+
+ private final Attributes attributes;
+ private final BundleSupport bundleSupport;
+ private final RuleContext ruleContext;
+ private final Bundling bundling;
+ private final ObjcProvider objcProvider;
+ private final LinkedBinary linkedBinary;
+ private final ImmutableSet<TargetDeviceFamily> families;
+ private final IntermediateArtifacts intermediateArtifacts;
+
+ /**
+ * Indicator as to whether this rule generates a binary directly or whether only dependencies
+ * should be considered.
+ */
+ enum LinkedBinary {
+ /**
+ * This rule generates its own binary which should be included as well as dependency-generated
+ * binaries.
+ */
+ LOCAL_AND_DEPENDENCIES,
+
+ /**
+ * This rule does not generate its own binary, only consider binaries from dependencies.
+ */
+ DEPENDENCIES_ONLY
+ }
+
+ /**
+ * Creates a new application support within the given rule context.
+ *
+ * @param ruleContext context for the application-generating rule
+ * @param objcProvider provider containing all dependencies' information as well as some of this
+ * rule's
+ * @param optionsProvider provider containing options and plist settings for this rule and its
+ * dependencies
+ * @param linkedBinary whether to look for a linked binary from this rule and dependencies or just
+ * the latter
+ */
+ ApplicationSupport(
+ RuleContext ruleContext, ObjcProvider objcProvider, OptionsProvider optionsProvider,
+ LinkedBinary linkedBinary) {
+ this.linkedBinary = linkedBinary;
+ this.attributes = new Attributes(ruleContext);
+ this.ruleContext = ruleContext;
+ this.objcProvider = objcProvider;
+ this.families = ImmutableSet.copyOf(attributes.families());
+ this.intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ bundling = bundling(ruleContext, objcProvider, optionsProvider);
+ bundleSupport = new BundleSupport(ruleContext, families, bundling, extraActoolArgs());
+ }
+
+ /**
+ * Validates application-related attributes set on this rule and registers any errors with the
+ * rule context.
+ *
+ * @return this application support
+ */
+ ApplicationSupport validateAttributes() {
+ bundleSupport.validateAttributes();
+
+ // No asset catalogs. That means you cannot specify app_icon or
+ // launch_image attributes, since they must not exist. However, we don't
+ // run actool in this case, which means it does not do validity checks,
+ // and we MUST raise our own error somehow...
+ if (!objcProvider.hasAssetCatalogs()) {
+ if (attributes.appIcon() != null) {
+ ruleContext.attributeError("app_icon",
+ String.format(NO_ASSET_CATALOG_ERROR_FORMAT, attributes.appIcon()));
+ }
+ if (attributes.launchImage() != null) {
+ ruleContext.attributeError("launch_image",
+ String.format(NO_ASSET_CATALOG_ERROR_FORMAT, attributes.launchImage()));
+ }
+ }
+
+ if (families.isEmpty()) {
+ ruleContext.attributeError("families", INVALID_FAMILIES_ERROR);
+ }
+
+ return this;
+ }
+
+ /**
+ * Registers actions required to build an application. This includes any
+ * {@link BundleSupport#registerActions(ObjcProvider) bundle} and bundle merge actions, signing
+ * this application if appropriate and combining several single-architecture binaries into one
+ * multi-architecture binary.
+ *
+ * @return this application support
+ */
+ ApplicationSupport registerActions() {
+ bundleSupport.registerActions(objcProvider);
+
+ registerCombineArchitecturesAction();
+
+ ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext);
+ Artifact ipaOutput = ruleContext.getImplicitOutputArtifact(IPA);
+
+ Artifact maybeSignedIpa;
+ if (objcConfiguration.getPlatform() == Platform.SIMULATOR) {
+ maybeSignedIpa = ipaOutput;
+ } else if (attributes.provisioningProfile() == null) {
+ throw new IllegalStateException(DEVICE_NO_PROVISIONING_PROFILE);
+ } else {
+ maybeSignedIpa = registerBundleSigningActions(ipaOutput);
+ }
+
+ BundleMergeControlBytes bundleMergeControlBytes = new BundleMergeControlBytes(
+ bundling, maybeSignedIpa, objcConfiguration, families);
+ registerBundleMergeActions(
+ maybeSignedIpa, bundling.getBundleContentArtifacts(), bundleMergeControlBytes);
+
+ return this;
+ }
+
+ private Artifact registerBundleSigningActions(Artifact ipaOutput) {
+ PathFragment entitlementsDirectory = ruleContext.getUniqueDirectory("entitlements");
+ Artifact teamPrefixFile = ruleContext.getRelatedArtifact(
+ entitlementsDirectory, ".team_prefix_file");
+ registerExtractTeamPrefixAction(teamPrefixFile);
+
+ Artifact entitlementsNeedingSubstitution = attributes.entitlements();
+ if (entitlementsNeedingSubstitution == null) {
+ entitlementsNeedingSubstitution = ruleContext.getRelatedArtifact(
+ entitlementsDirectory, ".entitlements_with_variables");
+ registerExtractEntitlementsAction(entitlementsNeedingSubstitution);
+ }
+ Artifact entitlements = ruleContext.getRelatedArtifact(
+ entitlementsDirectory, ".entitlements");
+ registerEntitlementsVariableSubstitutionAction(
+ entitlementsNeedingSubstitution, entitlements, teamPrefixFile);
+ Artifact ipaUnsigned = ObjcRuleClasses.artifactByAppendingToRootRelativePath(
+ ruleContext, ipaOutput.getExecPath(), ".unsigned");
+ registerSignBundleAction(entitlements, ipaOutput, ipaUnsigned);
+ return ipaUnsigned;
+ }
+
+ /**
+ * Adds bundle- and application-related settings to the given Xcode provider builder.
+ *
+ * @return this application support
+ */
+ ApplicationSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder) {
+ bundleSupport.addXcodeSettings(xcodeProviderBuilder);
+ xcodeProviderBuilder.addXcodeprojBuildSettings(buildSettings());
+
+ return this;
+ }
+
+ /**
+ * Adds any files to the given nested set builder that should be built if this application is the
+ * top level target in a blaze invocation.
+ *
+ * @return this application support
+ */
+ ApplicationSupport addFilesToBuild(NestedSetBuilder<Artifact> filesToBuild) {
+ NestedSetBuilder<Artifact> debugSymbolBuilder = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(objcProvider.get(ObjcProvider.DEBUG_SYMBOLS));
+
+ if (linkedBinary == LinkedBinary.LOCAL_AND_DEPENDENCIES
+ && ObjcRuleClasses.objcConfiguration(ruleContext).generateDebugSymbols()) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ debugSymbolBuilder.add(intermediateArtifacts.dsymPlist())
+ .add(intermediateArtifacts.dsymSymbol())
+ .add(intermediateArtifacts.breakpadSym());
+ }
+
+ filesToBuild.add(ruleContext.getImplicitOutputArtifact(ApplicationSupport.IPA))
+ // TODO(bazel-team): Fat binaries may require some merging of these file rather than just
+ // making them available.
+ .addTransitive(debugSymbolBuilder.build());
+ return this;
+ }
+
+ /**
+ * Creates the {@link XcTestAppProvider} that can be used if this application is used as an
+ * {@code xctest_app}.
+ */
+ XcTestAppProvider xcTestAppProvider() {
+ // We want access to #import-able things from our test rig's dependency graph, but we don't
+ // want to link anything since that stuff is shared automatically by way of the
+ // -bundle_loader linker flag.
+ ObjcProvider partialObjcProvider = new ObjcProvider.Builder()
+ .addTransitiveAndPropagate(ObjcProvider.HEADER, objcProvider)
+ .addTransitiveAndPropagate(ObjcProvider.INCLUDE, objcProvider)
+ .addTransitiveAndPropagate(ObjcProvider.SDK_DYLIB, objcProvider)
+ .addTransitiveAndPropagate(ObjcProvider.SDK_FRAMEWORK, objcProvider)
+ .addTransitiveAndPropagate(ObjcProvider.WEAK_SDK_FRAMEWORK, objcProvider)
+ .addTransitiveAndPropagate(ObjcProvider.FRAMEWORK_DIR, objcProvider)
+ .addTransitiveAndPropagate(ObjcProvider.FRAMEWORK_FILE, objcProvider)
+ .build();
+ // TODO(bazel-team): Handle the FRAMEWORK_DIR key properly. We probably want to add it to
+ // framework search paths, but not actually link it with the -framework flag.
+ return new XcTestAppProvider(intermediateArtifacts.singleArchitectureBinary(),
+ ruleContext.getImplicitOutputArtifact(IPA), partialObjcProvider);
+ }
+
+ private ExtraActoolArgs extraActoolArgs() {
+ ImmutableList.Builder<String> extraArgs = ImmutableList.builder();
+ if (attributes.appIcon() != null) {
+ extraArgs.add("--app-icon", attributes.appIcon());
+ }
+ if (attributes.launchImage() != null) {
+ extraArgs.add("--launch-image", attributes.launchImage());
+ }
+ return new ExtraActoolArgs(extraArgs.build());
+ }
+
+ private Bundling bundling(
+ RuleContext ruleContext, ObjcProvider objcProvider, OptionsProvider optionsProvider) {
+ ImmutableList<BundleableFile> extraBundleFiles;
+ ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext);
+ if (objcConfiguration.getPlatform() == Platform.DEVICE) {
+ extraBundleFiles = ImmutableList.of(new BundleableFile(
+ attributes.provisioningProfile(),
+ PROVISIONING_PROFILE_BUNDLE_FILE));
+ } else {
+ extraBundleFiles = ImmutableList.of();
+ }
+
+ return new Bundling.Builder()
+ .setName(ruleContext.getLabel().getName())
+ .setBundleDirSuffix(".app")
+ .setExtraBundleFiles(extraBundleFiles)
+ .setObjcProvider(objcProvider)
+ .setInfoplistMerging(
+ BundleSupport.infoPlistMerging(ruleContext, objcProvider, optionsProvider))
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .build();
+ }
+
+ private void registerCombineArchitecturesAction() {
+ Artifact resultingLinkedBinary = intermediateArtifacts.combinedArchitectureBinary(".app");
+ NestedSet<Artifact> linkedBinaries = linkedBinaries();
+
+ ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder()
+ .setMnemonic("ObjcCombiningArchitectures")
+ .addTransitiveInputs(linkedBinaries)
+ .addOutput(resultingLinkedBinary)
+ .setExecutable(ObjcActionsBuilder.LIPO)
+ .setCommandLine(CustomCommandLine.builder()
+ .addExecPaths("-create", linkedBinaries)
+ .addExecPath("-o", resultingLinkedBinary)
+ .build())
+ .build(ruleContext));
+ }
+
+ private NestedSet<Artifact> linkedBinaries() {
+ NestedSetBuilder<Artifact> linkedBinariesBuilder = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(attributes.dependentLinkedBinaries());
+ if (linkedBinary == LinkedBinary.LOCAL_AND_DEPENDENCIES) {
+ linkedBinariesBuilder.add(intermediateArtifacts.singleArchitectureBinary());
+ }
+ return linkedBinariesBuilder.build();
+ }
+
+ /** Returns this target's Xcode build settings. */
+ private Iterable<XcodeprojBuildSetting> buildSettings() {
+ ImmutableList.Builder<XcodeprojBuildSetting> buildSettings = new ImmutableList.Builder<>();
+ if (attributes.appIcon() != null) {
+ buildSettings.add(XcodeprojBuildSetting.newBuilder()
+ .setName("ASSETCATALOG_COMPILER_APPICON_NAME")
+ .setValue(attributes.appIcon())
+ .build());
+ }
+ if (attributes.launchImage() != null) {
+ buildSettings.add(XcodeprojBuildSetting.newBuilder()
+ .setName("ASSETCATALOG_COMPILER_LAUNCHIMAGE_NAME")
+ .setValue(attributes.launchImage())
+ .build());
+ }
+
+ // Convert names to a sequence containing "1" and/or "2" for iPhone and iPad, respectively.
+ Iterable<Integer> familyIndexes =
+ families.isEmpty() ? ImmutableList.<Integer>of() : UI_DEVICE_FAMILY_VALUES.get(families);
+ buildSettings.add(XcodeprojBuildSetting.newBuilder()
+ .setName("TARGETED_DEVICE_FAMILY")
+ .setValue(Joiner.on(',').join(familyIndexes))
+ .build());
+
+ Artifact entitlements = attributes.entitlements();
+ if (entitlements != null) {
+ buildSettings.add(XcodeprojBuildSetting.newBuilder()
+ .setName("CODE_SIGN_ENTITLEMENTS")
+ .setValue("$(WORKSPACE_ROOT)/" + entitlements.getExecPathString())
+ .build());
+ }
+
+ return buildSettings.build();
+ }
+
+ private ApplicationSupport registerSignBundleAction(
+ Artifact entitlements, Artifact ipaOutput, Artifact ipaUnsigned) {
+ // TODO(bazel-team): Support variable substitution
+ ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder()
+ .setMnemonic("IosSignBundle")
+ .setProgressMessage("Signing iOS bundle: " + ruleContext.getLabel())
+ .setExecutable(new PathFragment("/bin/bash"))
+ .addArgument("-c")
+ // TODO(bazel-team): Support --resource-rules for resources
+ .addArgument("set -e && "
+ + "t=$(mktemp -d -t signing_intermediate) && "
+ // Get an absolute path since we need to cd into the temp directory for zip.
+ + "signed_ipa=${PWD}/" + ipaOutput.getExecPathString() + " && "
+ + "unzip -qq " + ipaUnsigned.getExecPathString() + " -d ${t} && "
+ + codesignCommand(
+ attributes.provisioningProfile(),
+ entitlements,
+ String.format("${t}/Payload/%s.app", ruleContext.getLabel().getName())) + " && "
+ // Using zip since we need to preserve permissions
+ + "cd \"${t}\" && /usr/bin/zip -q -r \"${signed_ipa}\" .")
+ .addInput(ipaUnsigned)
+ .addInput(attributes.provisioningProfile())
+ .addInput(entitlements)
+ .addOutput(ipaOutput)
+ .build(ruleContext));
+
+ return this;
+ }
+
+ private void registerBundleMergeActions(Artifact ipaUnsigned,
+ NestedSet<Artifact> bundleContentArtifacts, BundleMergeControlBytes controlBytes) {
+ Artifact bundleMergeControlArtifact =
+ ObjcRuleClasses.artifactByAppendingToBaseName(ruleContext, ".ipa-control");
+
+ ruleContext.registerAction(
+ new BinaryFileWriteAction(
+ ruleContext.getActionOwner(), bundleMergeControlArtifact, controlBytes,
+ /*makeExecutable=*/false));
+
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .setMnemonic("IosBundle")
+ .setProgressMessage("Bundling iOS application: " + ruleContext.getLabel())
+ .setExecutable(attributes.bundleMergeExecutable())
+ .addInputArgument(bundleMergeControlArtifact)
+ .addTransitiveInputs(bundleContentArtifacts)
+ .addOutput(ipaUnsigned)
+ .build(ruleContext));
+ }
+
+ private void registerExtractTeamPrefixAction(Artifact teamPrefixFile) {
+ ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder()
+ .setMnemonic("ExtractIosTeamPrefix")
+ .setExecutable(new PathFragment("/bin/bash"))
+ .addArgument("-c")
+ .addArgument("set -e &&"
+ + " PLIST=$(" + extractPlistCommand(attributes.provisioningProfile()) + ") && "
+
+ // We think PlistBuddy uses PRead internally to seek through the file. Or possibly
+ // mmaps the file. Or something similar.
+ //
+ // Pipe FDs do not support PRead or mmap, though.
+ //
+ // <<< however does something magical like write to a temporary file or something
+ // like that internally, which means that this Just Works.
+ + " PREFIX=$(/usr/libexec/PlistBuddy -c 'Print ApplicationIdentifierPrefix:0'"
+ + " /dev/stdin <<< \"${PLIST}\") && "
+ + " echo ${PREFIX} > " + teamPrefixFile.getExecPathString())
+ .addInput(attributes.provisioningProfile())
+ .addOutput(teamPrefixFile)
+ .build(ruleContext));
+ }
+
+ private ApplicationSupport registerExtractEntitlementsAction(Artifact entitlements) {
+ // See Apple Glossary (http://goo.gl/EkhXOb)
+ // An Application Identifier is constructed as: TeamID.BundleID
+ // TeamID is extracted from the provisioning profile.
+ // BundleID consists of a reverse-DNS string to identify the app, where the last component
+ // is the application name, and is specified as an attribute.
+
+ ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder()
+ .setMnemonic("ExtractIosEntitlements")
+ .setProgressMessage("Extracting entitlements: " + ruleContext.getLabel())
+ .setExecutable(new PathFragment("/bin/bash"))
+ .addArgument("-c")
+ .addArgument("set -e && "
+ + "PLIST=$("
+ + extractPlistCommand(attributes.provisioningProfile()) + ") && "
+
+ // We think PlistBuddy uses PRead internally to seek through the file. Or possibly
+ // mmaps the file. Or something similar.
+ //
+ // Pipe FDs do not support PRead or mmap, though.
+ //
+ // <<< however does something magical like write to a temporary file or something
+ // like that internally, which means that this Just Works.
+
+ + "/usr/libexec/PlistBuddy -x -c 'Print Entitlements' /dev/stdin <<< \"${PLIST}\" "
+ + "> " + entitlements.getExecPathString())
+ .addInput(attributes.provisioningProfile())
+ .addOutput(entitlements)
+ .build(ruleContext));
+
+ return this;
+ }
+
+ private void registerEntitlementsVariableSubstitutionAction(Artifact in, Artifact out,
+ Artifact prefix) {
+ String escapedBundleId = ShellUtils.shellEscape(attributes.bundleId());
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .setMnemonic("SubstituteIosEntitlements")
+ .setExecutable(new PathFragment("/bin/bash"))
+ .addArgument("-c")
+ .addArgument("set -e && "
+ + "PREFIX=\"$(cat " + prefix.getExecPathString() + ")\" && "
+ + "sed " + in.getExecPathString() + " "
+ // Replace .* from default entitlements file with bundle ID where suitable.
+ + "-e \"s#${PREFIX}\\.\\*#${PREFIX}." + escapedBundleId + "#g\" "
+
+ // Replace some variables that people put in their own entitlements files
+ + "-e \"s#\\$(AppIdentifierPrefix)#${PREFIX}.#g\" "
+ + "-e \"s#\\$(CFBundleIdentifier)#" + escapedBundleId + "#g\" "
+
+ + "> " + out.getExecPathString())
+ .addInput(in)
+ .addInput(prefix)
+ .addOutput(out)
+ .build(ruleContext));
+ }
+
+
+ private String extractPlistCommand(Artifact provisioningProfile) {
+ return "security cms -D -i " + ShellUtils.shellEscape(provisioningProfile.getExecPathString());
+ }
+
+ private String codesignCommand(
+ Artifact provisioningProfile, Artifact entitlements, String appDir) {
+ String fingerprintCommand =
+ "/usr/libexec/PlistBuddy -c 'Print DeveloperCertificates:0' /dev/stdin <<< "
+ + "$(" + extractPlistCommand(provisioningProfile) + ") | "
+ + "openssl x509 -inform DER -noout -fingerprint | "
+ + "cut -d= -f2 | sed -e 's#:##g'";
+ return String.format(
+ "/usr/bin/codesign --force --sign $(%s) --entitlements %s %s",
+ fingerprintCommand,
+ entitlements.getExecPathString(),
+ appDir);
+ }
+
+ /**
+ * Logic to access attributes required by application support. Attributes are required and
+ * guaranteed to return a value or throw unless they are annotated with {@link Nullable} in which
+ * case they can return {@code null} if no value is defined.
+ */
+ private static class Attributes {
+ private final RuleContext ruleContext;
+
+ private Attributes(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ @Nullable
+ String appIcon() {
+ return stringAttribute("app_icon");
+ }
+
+ @Nullable
+ String launchImage() {
+ return stringAttribute("launch_image");
+ }
+
+ @Nullable
+ Artifact provisioningProfile() {
+ return ruleContext.getPrerequisiteArtifact("provisioning_profile", Mode.TARGET);
+ }
+
+ /**
+ * Returns the value of the {@code families} attribute in a form that is more useful than a list
+ * of strings. Returns an empty set for any invalid {@code families} attribute value, including
+ * an empty list.
+ */
+ Set<TargetDeviceFamily> families() {
+ List<String> rawFamilies = ruleContext.attributes().get("families", Type.STRING_LIST);
+ try {
+ return TargetDeviceFamily.fromNamesInRule(rawFamilies);
+ } catch (InvalidFamilyNameException | RepeatedFamilyNameException e) {
+ return ImmutableSet.of();
+ }
+ }
+
+ @Nullable
+ Artifact entitlements() {
+ return ruleContext.getPrerequisiteArtifact("entitlements", Mode.TARGET);
+ }
+
+ NestedSet<? extends Artifact> dependentLinkedBinaries() {
+ if (ruleContext.attributes().getAttributeDefinition("binary") == null) {
+ return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ }
+
+ return ruleContext.getPrerequisite("binary", Mode.TARGET, ObjcProvider.class)
+ .get(ObjcProvider.LINKED_BINARY);
+ }
+
+ FilesToRunProvider bundleMergeExecutable() {
+ return checkNotNull(ruleContext.getExecutablePrerequisite("$bundlemerge", Mode.HOST));
+ }
+
+ String bundleId() {
+ return checkNotNull(stringAttribute("bundle_id"));
+ }
+
+ @Nullable
+ private String stringAttribute(String attribute) {
+ String value = ruleContext.attributes().get(attribute, Type.STRING);
+ return value.isEmpty() ? null : value;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java
new file mode 100644
index 0000000..d6c993b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+
+import java.util.Locale;
+
+/**
+ * Attributes containing one or more labels.
+ */
+public enum ArtifactListAttribute {
+ BUNDLE_IMPORTS;
+
+ public String attrName() {
+ return name().toLowerCase(Locale.US);
+ }
+
+ /**
+ * The artifacts specified by this attribute on the given rule. Returns an empty sequence if the
+ * attribute is omitted or not available on the rule type.
+ */
+ public Iterable<Artifact> get(RuleContext context) {
+ if (context.attributes().getAttributeDefinition(attrName()) == null) {
+ return ImmutableList.of();
+ } else {
+ return context.getPrerequisiteArtifacts(attrName(), Mode.TARGET).list();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java
new file mode 100644
index 0000000..695dffc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java
@@ -0,0 +1,121 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.ByteSource;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.Control;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.MergeZip;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.VariableSubstitution;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+
+import java.io.InputStream;
+import java.util.Map;
+
+/**
+ * A byte source that can be used to generate a control file for the tool:
+ * {@code //java/com/google/devtools/build/xcode/bundlemerge}. Note that this generates the control
+ * proto and bytes on-the-fly rather than eagerly. This is to prevent a copy of the bundle files and
+ * .xcdatamodels from being stored for each {@code objc_binary} (or any bundle) being built.
+ */
+final class BundleMergeControlBytes extends ByteSource {
+ private final Bundling rootBundling;
+ private final Artifact mergedIpa;
+ private final ObjcConfiguration objcConfiguration;
+ private final ImmutableSet<TargetDeviceFamily> families;
+
+ public BundleMergeControlBytes(
+ Bundling rootBundling, Artifact mergedIpa, ObjcConfiguration objcConfiguration,
+ ImmutableSet<TargetDeviceFamily> families) {
+ this.rootBundling = Preconditions.checkNotNull(rootBundling);
+ this.mergedIpa = Preconditions.checkNotNull(mergedIpa);
+ this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration);
+ this.families = Preconditions.checkNotNull(families);
+ }
+
+ @Override
+ public InputStream openStream() {
+ return control("Payload/", "Payload/", rootBundling)
+ .toByteString()
+ .newInput();
+ }
+
+ private Control control(String mergeZipPrefix, String bundleDirPrefix, Bundling bundling) {
+ ObjcProvider objcProvider = bundling.getObjcProvider();
+ String bundleDir = bundleDirPrefix + bundling.getBundleDir();
+ mergeZipPrefix += bundling.getBundleDir() + "/";
+
+ BundleMergeProtos.Control.Builder control = BundleMergeProtos.Control.newBuilder()
+ .addAllBundleFile(BundleableFile.toBundleFiles(bundling.getExtraBundleFiles()))
+ .addAllBundleFile(BundleableFile.toBundleFiles(objcProvider.get(BUNDLE_FILE)))
+ .addAllSourcePlistFile(Artifact.toExecPaths(
+ bundling.getInfoplistMerging().getPlistWithEverything().asSet()))
+ // TODO(bazel-team): Add rule attribute for specifying targeted device family
+ .setMinimumOsVersion(objcConfiguration.getMinimumOs())
+ .setSdkVersion(objcConfiguration.getIosSdkVersion())
+ .setPlatform(objcConfiguration.getPlatform().name())
+ .setBundleRoot(bundleDir);
+
+ for (Artifact mergeZip : bundling.getMergeZips()) {
+ control.addMergeZip(MergeZip.newBuilder()
+ .setEntryNamePrefix(mergeZipPrefix)
+ .setSourcePath(mergeZip.getExecPathString())
+ .build());
+ }
+
+ for (Xcdatamodel datamodel : objcProvider.get(XCDATAMODEL)) {
+ control.addMergeZip(MergeZip.newBuilder()
+ .setEntryNamePrefix(mergeZipPrefix)
+ .setSourcePath(datamodel.getOutputZip().getExecPathString())
+ .build());
+ }
+ for (TargetDeviceFamily targetDeviceFamily : families) {
+ control.addTargetDeviceFamily(targetDeviceFamily.name());
+ }
+
+ Map<String, String> variableSubstitutions = bundling.variableSubstitutions();
+ for (String variable : variableSubstitutions.keySet()) {
+ control.addVariableSubstitution(VariableSubstitution.newBuilder()
+ .setName(variable)
+ .setValue(variableSubstitutions.get(variable))
+ .build());
+ }
+
+ control.setOutFile(mergedIpa.getExecPathString());
+
+ for (Artifact linkedBinary : bundling.getCombinedArchitectureBinary().asSet()) {
+ control
+ .addBundleFile(BundleMergeProtos.BundleFile.newBuilder()
+ .setSourceFile(linkedBinary.getExecPathString())
+ .setBundlePath(bundling.getName())
+ .setExternalFileAttribute(BundleableFile.EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE)
+ .build())
+ .setExecutableName(bundling.getName());
+ }
+
+ for (Bundling nestedBundling : bundling.getObjcProvider().get(NESTED_BUNDLE)) {
+ control.addNestedBundle(control(mergeZipPrefix, "", nestedBundling));
+ }
+
+ return control.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java
new file mode 100644
index 0000000..5434ff7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java
@@ -0,0 +1,184 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraActoolArgs;
+import com.google.devtools.build.lib.rules.objc.XcodeProvider.Builder;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+
+import java.util.Set;
+
+/**
+ * Support for generating iOS bundles which contain metadata (a plist file), assets, resources and
+ * optionally a binary: registers actions that assemble resources and merge plists, provides data
+ * to providers and validates bundle-related attributes.
+ *
+ * <p>Methods on this class can be called in any order without impacting the result.
+ */
+final class BundleSupport {
+
+ @VisibleForTesting
+ static final String NO_INFOPLIST_ERROR = "An infoplist must be specified either in the "
+ + "'infoplist' attribute or via the 'options' attribute, but none was found";
+
+ private final RuleContext ruleContext;
+ private final Set<TargetDeviceFamily> targetDeviceFamilies;
+ private final ExtraActoolArgs extraActoolArgs;
+ private final Bundling bundling;
+
+ /**
+ * Returns merging instructions for a bundle's {@code Info.plist}.
+ *
+ * @param ruleContext context this bundle is constructed in
+ * @param objcProvider provider containing all dependencies' information as well as some of this
+ * rule's
+ * @param optionsProvider provider containing options and plist settings for this rule and its
+ * dependencies
+ */
+ static InfoplistMerging infoPlistMerging(RuleContext ruleContext,
+ ObjcProvider objcProvider, OptionsProvider optionsProvider) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+
+ return new InfoplistMerging.Builder(ruleContext)
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .setInputPlists(NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(optionsProvider.getInfoplists())
+ .addAll(actoolPartialInfoplist(ruleContext, objcProvider).asSet())
+ .build())
+ .setPlmerge(ruleContext.getExecutablePrerequisite("$plmerge", Mode.HOST))
+ .build();
+ }
+
+ /**
+ * Creates a new bundle support with no special {@code actool} arguments.
+ *
+ * @param ruleContext context this bundle is constructed in
+ * @param targetDeviceFamilies device families used in asset catalogue construction
+ * @param bundling bundle information as configured for this rule
+ */
+ public BundleSupport(
+ RuleContext ruleContext, Set<TargetDeviceFamily> targetDeviceFamilies, Bundling bundling) {
+ this(ruleContext, targetDeviceFamilies, bundling, new ExtraActoolArgs());
+ }
+
+ /**
+ * Creates a new bundle support.
+ *
+ * @param ruleContext context this bundle is constructed in
+ * @param targetDeviceFamilies device families used in asset catalogue construction
+ * @param bundling bundle information as configured for this rule
+ * @param extraActoolArgs any additional parameters to be used for invoking {@code actool}
+ */
+ public BundleSupport(RuleContext ruleContext, Set<TargetDeviceFamily> targetDeviceFamilies,
+ Bundling bundling, ExtraActoolArgs extraActoolArgs) {
+ this.ruleContext = ruleContext;
+ this.targetDeviceFamilies = targetDeviceFamilies;
+ this.extraActoolArgs = extraActoolArgs;
+ this.bundling = bundling;
+ }
+
+ /**
+ * Registers actions required for constructing this bundle, namely merging all involved {@code
+ * Info.plist} files and generating asset catalogues.
+ *
+ * @param objcProvider source of information from this rule's attributes and its dependencies
+ *
+ * @return this bundle support
+ */
+ BundleSupport registerActions(ObjcProvider objcProvider) {
+ registerMergeInfoplistAction();
+ registerActoolActionIfNecessary(objcProvider);
+
+ return this;
+ }
+
+ /**
+ * Adds any Xcode settings related to this bundle to the given provider builder.
+ *
+ * @return this bundle support
+ */
+ BundleSupport addXcodeSettings(Builder xcodeProviderBuilder) {
+ xcodeProviderBuilder.setInfoplistMerging(bundling.getInfoplistMerging());
+ return this;
+ }
+
+ /**
+ * Validates any rule attributes and dependencies related to this bundle.
+ *
+ * @return this bundle support
+ */
+ BundleSupport validateAttributes() {
+ if (bundling.getInfoplistMerging().getInputPlists().isEmpty()) {
+ ruleContext.ruleError(NO_INFOPLIST_ERROR);
+ }
+ return this;
+ }
+
+ private void registerMergeInfoplistAction() {
+ // TODO(bazel-team): Move action implementation from InfoplistMerging to this class.
+ ruleContext.registerAction(bundling.getInfoplistMerging().getMergeAction());
+ }
+
+ private void registerActoolActionIfNecessary(ObjcProvider objcProvider) {
+ Optional<Artifact> actoolzipOutput = bundling.getActoolzipOutput();
+ if (!actoolzipOutput.isPresent()) {
+ return;
+ }
+
+ ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext);
+
+ Artifact actoolPartialInfoplist = actoolPartialInfoplist(ruleContext, objcProvider).get();
+ actionsBuilder.registerActoolzipAction(
+ new ObjcRuleClasses.Tools(ruleContext),
+ objcProvider,
+ actoolzipOutput.get(),
+ new ObjcActionsBuilder.ExtraActoolOutputs(actoolPartialInfoplist),
+ new ExtraActoolArgs(
+ new ImmutableList.Builder<String>()
+ .addAll(extraActoolArgs)
+ .add("--output-partial-info-plist", actoolPartialInfoplist.getExecPathString())
+ .build()),
+ targetDeviceFamilies);
+ }
+
+ /**
+ * Returns the artifact that is a plist file generated by an invocation of {@code actool} or
+ * {@link Optional#absent()} if no asset catalogues are present in this target and its
+ * dependencies.
+ *
+ * <p>All invocations of {@code actool} generate this kind of plist file, which contains metadata
+ * about the {@code app_icon} and {@code launch_image} if supplied. If neither an app icon or a
+ * launch image was supplied, the plist file generated is empty.
+ */
+ private static Optional<Artifact> actoolPartialInfoplist(
+ RuleContext ruleContext, ObjcProvider objcProvider) {
+ if (objcProvider.hasAssetCatalogs()) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ return Optional.of(intermediateArtifacts.actoolPartialInfoplist());
+ } else {
+ return Optional.absent();
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java
new file mode 100644
index 0000000..eea7bf0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java
@@ -0,0 +1,149 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ArtifactListAttribute.BUNDLE_IMPORTS;
+import static com.google.devtools.build.lib.rules.objc.ObjcCommon.BUNDLE_CONTAINER_TYPE;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.BundleFile;
+import com.google.devtools.build.xcode.util.Value;
+
+/**
+ * Represents a file which is processed to another file and bundled. It contains the
+ * {@code Artifact} corresponding to the original file as well as the {@code Artifact} for the file
+ * converted to its bundled form. Examples of files that fit this pattern are .strings and .xib
+ * files.
+ */
+public final class BundleableFile extends Value<BundleableFile> {
+ static final int EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE = 0100755 << 16;
+ static final int DEFAULT_EXTERNAL_FILE_ATTRIBUTE = 0100644 << 16;
+
+ private final Artifact bundled;
+ private final String bundlePath;
+ private final int zipExternalFileAttribute;
+
+ /**
+ * Creates an instance whose {@code zipExternalFileAttribute} value is
+ * {@link #DEFAULT_EXTERNAL_FILE_ATTRIBUTE}.
+ */
+ BundleableFile(Artifact bundled, String bundlePath) {
+ this(bundled, bundlePath, DEFAULT_EXTERNAL_FILE_ATTRIBUTE);
+ }
+
+ /**
+ * @param bundled the {@link Artifact} whose data is placed in the bundle
+ * @param bundlePath the path of the file in the bundle
+ * @param the external file attribute of the file in the central directory of the bundle (zip
+ * file). The lower 16 bits contain the MS-DOS file attributes. The upper 16 bits contain the
+ * Unix file attributes, for instance 0100755 (octal) for a regular file with permissions
+ * {@code rwxr-xr-x}.
+ */
+ BundleableFile(Artifact bundled, String bundlePath, int zipExternalFileAttribute) {
+ super(new ImmutableMap.Builder<String, Object>()
+ .put("bundled", bundled)
+ .put("bundlePath", bundlePath)
+ .put("zipExternalFileAttribute", zipExternalFileAttribute)
+ .build());
+ this.bundled = bundled;
+ this.bundlePath = bundlePath;
+ this.zipExternalFileAttribute = zipExternalFileAttribute;
+ }
+
+ static String bundlePath(PathFragment path) {
+ String containingDir = path.getParentDirectory().getBaseName();
+ return (containingDir.endsWith(".lproj") ? (containingDir + "/") : "") + path.getBaseName();
+ }
+
+ /**
+ * Given a sequence of non-compiled resource files, returns a sequence of the same length of
+ * instances of this class. Non-compiled resource files are resources which are not processed
+ * before placing them in the final bundle. This is different from (for example) {@code .strings}
+ * and {@code .xib} files, which must be converted to binary plist form or compiled.
+ *
+ * @param files a sequence of artifacts corresponding to non-compiled resource files
+ */
+ public static Iterable<BundleableFile> nonCompiledResourceFiles(Iterable<Artifact> files) {
+ ImmutableList.Builder<BundleableFile> result = new ImmutableList.Builder<>();
+ for (Artifact file : files) {
+ result.add(new BundleableFile(file, bundlePath(file.getExecPath())));
+ }
+ return result.build();
+ }
+
+ /**
+ * Returns an instance for every file in a bundle directory.
+ * <p>
+ * This uses the parent-most container matching {@code *.bundle} as the bundle root.
+ * TODO(bazel-team): add something like an import_root attribute to specify this explicitly, which
+ * will be helpful if a bundle that appears to be nested needs to be imported alone.
+ */
+ public static Iterable<BundleableFile> bundleImportsFromRule(RuleContext context) {
+ ImmutableList.Builder<BundleableFile> result = new ImmutableList.Builder<>();
+ for (Artifact artifact : BUNDLE_IMPORTS.get(context)) {
+ for (PathFragment container :
+ ObjcCommon.farthestContainerMatching(BUNDLE_CONTAINER_TYPE, artifact).asSet()) {
+ // TODO(bazel-team): Figure out if we need to remove symbols of architectures we aren't
+ // building for from the binary in the bundle.
+ result.add(new BundleableFile(
+ artifact,
+ // The path from the artifact's container (including the container), to the artifact
+ // itself. For instance, if artifact is foo/bar.bundle/baz, then this value
+ // is bar.bundle/baz.
+ artifact.getExecPath().relativeTo(container.getParentDirectory()).getSafePathString()));
+ }
+ }
+ return result.build();
+ }
+
+ /**
+ * The artifact that is ultimately bundled.
+ */
+ public Artifact getBundled() {
+ return bundled;
+ }
+
+ /**
+ * Returns bundle files for each given strings file. These are used to merge the strings files to
+ * the final application bundle.
+ */
+ public static Iterable<BundleFile> toBundleFiles(Iterable<BundleableFile> files) {
+ ImmutableList.Builder<BundleFile> result = new ImmutableList.Builder<>();
+ for (BundleableFile file : files) {
+ result.add(BundleFile.newBuilder()
+ .setBundlePath(file.bundlePath)
+ .setSourceFile(file.bundled.getExecPathString())
+ .setExternalFileAttribute(file.zipExternalFileAttribute)
+ .build());
+ }
+ return result.build();
+ }
+
+ /**
+ * Returns the artifacts for the bundled files. These can be used, for instance, as the input
+ * files to add to the bundlemerge action for a bundle that contains all the given files.
+ */
+ public static Iterable<Artifact> toArtifacts(Iterable<BundleableFile> files) {
+ ImmutableList.Builder<Artifact> result = new ImmutableList.Builder<>();
+ for (BundleableFile file : files) {
+ result.add(file.bundled);
+ }
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java
new file mode 100644
index 0000000..484c553
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java
@@ -0,0 +1,254 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.MERGE_ZIP;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.xcode.util.Value;
+
+import java.util.Map;
+
+/**
+ * Contains information regarding the creation of an iOS bundle.
+ */
+@Immutable
+final class Bundling extends Value<Bundling> {
+ static final class Builder {
+ private String name;
+ private String bundleDirSuffix;
+ private ImmutableList<BundleableFile> extraBundleFiles = ImmutableList.of();
+ private ObjcProvider objcProvider;
+ private InfoplistMerging infoplistMerging;
+ private IntermediateArtifacts intermediateArtifacts;
+
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder setBundleDirSuffix(String bundleDirSuffix) {
+ this.bundleDirSuffix = bundleDirSuffix;
+ return this;
+ }
+
+ public Builder setExtraBundleFiles(ImmutableList<BundleableFile> extraBundleFiles) {
+ this.extraBundleFiles = extraBundleFiles;
+ return this;
+ }
+
+ public Builder setObjcProvider(ObjcProvider objcProvider) {
+ this.objcProvider = objcProvider;
+ return this;
+ }
+
+ public Builder setInfoplistMerging(InfoplistMerging infoplistMerging) {
+ this.infoplistMerging = infoplistMerging;
+ return this;
+ }
+
+ public Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) {
+ this.intermediateArtifacts = intermediateArtifacts;
+ return this;
+ }
+
+ private static NestedSet<Artifact> nestedBundleContentArtifacts(Iterable<Bundling> bundles) {
+ NestedSetBuilder<Artifact> artifacts = NestedSetBuilder.stableOrder();
+ for (Bundling bundle : bundles) {
+ artifacts.addTransitive(bundle.getBundleContentArtifacts());
+ }
+ return artifacts.build();
+ }
+
+ public Bundling build() {
+ Preconditions.checkNotNull(intermediateArtifacts, "intermediateArtifacts");
+
+ Optional<Artifact> actoolzipOutput = Optional.absent();
+ if (!Iterables.isEmpty(objcProvider.get(ASSET_CATALOG))) {
+ actoolzipOutput = Optional.of(intermediateArtifacts.actoolzipOutput());
+ }
+
+ Optional<Artifact> combinedArchitectureBinary = Optional.absent();
+ if (!Iterables.isEmpty(objcProvider.get(LIBRARY))
+ || !Iterables.isEmpty(objcProvider.get(IMPORTED_LIBRARY))) {
+ combinedArchitectureBinary =
+ Optional.of(intermediateArtifacts.combinedArchitectureBinary(bundleDirSuffix));
+ }
+
+ NestedSet<Artifact> mergeZips = NestedSetBuilder.<Artifact>stableOrder()
+ .addAll(actoolzipOutput.asSet())
+ .addTransitive(objcProvider.get(MERGE_ZIP))
+ .build();
+ NestedSet<Artifact> bundleContentArtifacts = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(nestedBundleContentArtifacts(objcProvider.get(NESTED_BUNDLE)))
+ .addAll(combinedArchitectureBinary.asSet())
+ .addAll(infoplistMerging.getPlistWithEverything().asSet())
+ .addTransitive(mergeZips)
+ .addAll(BundleableFile.toArtifacts(extraBundleFiles))
+ .addAll(BundleableFile.toArtifacts(objcProvider.get(BUNDLE_FILE)))
+ .addAll(Xcdatamodel.outputZips(objcProvider.get(XCDATAMODEL)))
+ .build();
+
+ return new Bundling(name, bundleDirSuffix, combinedArchitectureBinary, extraBundleFiles,
+ objcProvider, infoplistMerging, actoolzipOutput, bundleContentArtifacts, mergeZips);
+ }
+ }
+
+ private final String name;
+ private final String bundleDirSuffix;
+ private final Optional<Artifact> combinedArchitectureBinary;
+ private final ImmutableList<BundleableFile> extraBundleFiles;
+ private final ObjcProvider objcProvider;
+ private final InfoplistMerging infoplistMerging;
+ private final Optional<Artifact> actoolzipOutput;
+ private final NestedSet<Artifact> bundleContentArtifacts;
+ private final NestedSet<Artifact> mergeZips;
+
+ private Bundling(
+ String name,
+ String bundleDirSuffix,
+ Optional<Artifact> combinedArchitectureBinary,
+ ImmutableList<BundleableFile> extraBundleFiles,
+ ObjcProvider objcProvider,
+ InfoplistMerging infoplistMerging,
+ Optional<Artifact> actoolzipOutput,
+ NestedSet<Artifact> bundleContentArtifacts,
+ NestedSet<Artifact> mergeZips) {
+ super(new ImmutableMap.Builder<String, Object>()
+ .put("name", name)
+ .put("bundleDirSuffix", bundleDirSuffix)
+ .put("combinedArchitectureBinary", combinedArchitectureBinary)
+ .put("extraBundleFiles", extraBundleFiles)
+ .put("objcProvider", objcProvider)
+ .put("infoplistMerging", infoplistMerging)
+ .put("actoolzipOutput", actoolzipOutput)
+ .put("bundleContentArtifacts", bundleContentArtifacts)
+ .put("mergeZips", mergeZips)
+ .build());
+ this.name = name;
+ this.bundleDirSuffix = bundleDirSuffix;
+ this.combinedArchitectureBinary = combinedArchitectureBinary;
+ this.extraBundleFiles = extraBundleFiles;
+ this.objcProvider = objcProvider;
+ this.infoplistMerging = infoplistMerging;
+ this.actoolzipOutput = actoolzipOutput;
+ this.bundleContentArtifacts = bundleContentArtifacts;
+ this.mergeZips = mergeZips;
+ }
+
+ /**
+ * The bundle directory. For apps, {@code "Payload/" + bundleDir} is the directory in the bundle
+ * zip archive in which every file is found including the linked binary, nested bundles, and
+ * everything returned by {@link #getExtraBundleFiles()}. In an application bundle, for instance,
+ * this function returns {@code "(name).app"}.
+ */
+ public String getBundleDir() {
+ return name + bundleDirSuffix;
+ }
+
+ /**
+ * The name of the bundle, from which the bundle root and the path of the linked binary in the
+ * bundle archive are derived.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * An {@link Optional} with the linked binary artifact, or {@link Optional#absent()} if it is
+ * empty and should not be included in the bundle.
+ */
+ public Optional<Artifact> getCombinedArchitectureBinary() {
+ return combinedArchitectureBinary;
+ }
+
+ /**
+ * Extra bundle files to include in the bundle which are not automatically deduced by the contents
+ * of the provider. These files are placed under the bundle root (possibly nested, of course,
+ * depending on the bundle path of the files).
+ */
+ public ImmutableList<BundleableFile> getExtraBundleFiles() {
+ return extraBundleFiles;
+ }
+
+ /**
+ * The {@link ObjcProvider} for this bundle.
+ */
+ public ObjcProvider getObjcProvider() {
+ return objcProvider;
+ }
+
+ /**
+ * Information on the Info.plist and its merge inputs for this bundle. Note that an infoplist is
+ * only included in the bundle if it has one or more merge inputs.
+ */
+ public InfoplistMerging getInfoplistMerging() {
+ return infoplistMerging;
+ }
+
+ /**
+ * The location of the actoolzip output for this bundle. This is non-absent only included in the
+ * bundle if there is at least one asset catalog artifact supplied by
+ * {@link ObjcProvider#ASSET_CATALOG}.
+ */
+ public Optional<Artifact> getActoolzipOutput() {
+ return actoolzipOutput;
+ }
+
+ /**
+ * Returns all zip files whose contents should be merged into this bundle under the main bundle
+ * directory. For instance, if a merge zip contains files a/b and c/d, then the resulting bundling
+ * would have additional files at:
+ * <ul>
+ * <li>{bundleDir}/a/b
+ * <li>{bundleDir}/c/d
+ * </ul>
+ */
+ public NestedSet<Artifact> getMergeZips() {
+ return mergeZips;
+ }
+
+ /**
+ * Returns the variable substitutions that should be used when merging the plist info file of
+ * this bundle.
+ */
+ public Map<String, String> variableSubstitutions() {
+ return ImmutableMap.of(
+ "EXECUTABLE_NAME", name,
+ "BUNDLE_NAME", name + bundleDirSuffix,
+ "PRODUCT_NAME", name);
+ }
+
+ /**
+ * Returns the artifacts that are required to generate this bundle.
+ */
+ public NestedSet<Artifact> getBundleContentArtifacts() {
+ return bundleContentArtifacts;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java
new file mode 100644
index 0000000..5aac139
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java
@@ -0,0 +1,98 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+
+/**
+ * Artifacts related to compilation. Any rule containing compilable sources will create an instance
+ * of this class.
+ */
+final class CompilationArtifacts {
+ static class Builder {
+ private Iterable<Artifact> srcs = ImmutableList.of();
+ private Iterable<Artifact> nonArcSrcs = ImmutableList.of();
+ private Optional<Artifact> pchFile;
+ private IntermediateArtifacts intermediateArtifacts;
+
+ Builder addSrcs(Iterable<Artifact> srcs) {
+ this.srcs = Iterables.concat(this.srcs, srcs);
+ return this;
+ }
+
+ Builder addNonArcSrcs(Iterable<Artifact> nonArcSrcs) {
+ this.nonArcSrcs = Iterables.concat(this.nonArcSrcs, nonArcSrcs);
+ return this;
+ }
+
+ Builder setPchFile(Optional<Artifact> pchFile) {
+ Preconditions.checkState(this.pchFile == null,
+ "pchFile is already set to: %s", this.pchFile);
+ this.pchFile = Preconditions.checkNotNull(pchFile);
+ return this;
+ }
+
+ Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) {
+ Preconditions.checkState(this.intermediateArtifacts == null,
+ "intermediateArtifacts is already set to: %s", this.intermediateArtifacts);
+ this.intermediateArtifacts = intermediateArtifacts;
+ return this;
+ }
+
+ CompilationArtifacts build() {
+ Optional<Artifact> archive = Optional.absent();
+ if (!Iterables.isEmpty(srcs) || !Iterables.isEmpty(nonArcSrcs)) {
+ archive = Optional.of(intermediateArtifacts.archive());
+ }
+ return new CompilationArtifacts(srcs, nonArcSrcs, archive, pchFile);
+ }
+ }
+
+ private final Iterable<Artifact> srcs;
+ private final Iterable<Artifact> nonArcSrcs;
+ private final Optional<Artifact> archive;
+ private final Optional<Artifact> pchFile;
+
+ private CompilationArtifacts(
+ Iterable<Artifact> srcs,
+ Iterable<Artifact> nonArcSrcs,
+ Optional<Artifact> archive,
+ Optional<Artifact> pchFile) {
+ this.srcs = Preconditions.checkNotNull(srcs);
+ this.nonArcSrcs = Preconditions.checkNotNull(nonArcSrcs);
+ this.archive = Preconditions.checkNotNull(archive);
+ this.pchFile = Preconditions.checkNotNull(pchFile);
+ }
+
+ public Iterable<Artifact> getSrcs() {
+ return srcs;
+ }
+
+ public Iterable<Artifact> getNonArcSrcs() {
+ return nonArcSrcs;
+ }
+
+ public Optional<Artifact> getArchive() {
+ return archive;
+ }
+
+ public Optional<Artifact> getPchFile() {
+ return pchFile;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java
new file mode 100644
index 0000000..1e23798
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java
@@ -0,0 +1,235 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.NON_ARC_SRCS_TYPE;
+import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.SRCS_TYPE;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes;
+import com.google.devtools.build.lib.rules.objc.XcodeProvider.Builder;
+import com.google.devtools.build.lib.shell.ShellUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Support for rules that compile sources. Provides ways to determine files that should be output,
+ * registering Xcode settings and generating the various actions that might be needed for
+ * compilation.
+ *
+ * <p>Methods on this class can be called in any order without impacting the result.
+ */
+final class CompilationSupport {
+
+ @VisibleForTesting
+ static final String ABSOLUTE_INCLUDES_PATH_FORMAT =
+ "The path '%s' is absolute, but only relative paths are allowed.";
+
+ /**
+ * Returns information about the given rule's compilation artifacts.
+ */
+ // TODO(bazel-team): Remove this information from ObjcCommon and move it internal to this class.
+ static CompilationArtifacts compilationArtifacts(RuleContext ruleContext) {
+ return new CompilationArtifacts.Builder()
+ .addSrcs(ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET)
+ .errorsForNonMatching(SRCS_TYPE)
+ .list())
+ .addNonArcSrcs(ruleContext.getPrerequisiteArtifacts("non_arc_srcs", Mode.TARGET)
+ .errorsForNonMatching(NON_ARC_SRCS_TYPE)
+ .list())
+ .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext))
+ .setPchFile(Optional.fromNullable(ruleContext.getPrerequisiteArtifact("pch", Mode.TARGET)))
+ .build();
+ }
+
+ private final RuleContext ruleContext;
+ private final CompilationAttributes attributes;
+
+ /**
+ * Creates a new compilation support for the given rule.
+ */
+ CompilationSupport(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ this.attributes = new CompilationAttributes(ruleContext);
+ }
+
+ /**
+ * Registers all actions necessary to compile this rule's sources and archive them.
+ *
+ * @param common common information about this rule and its dependencies
+ * @param optionsProvider option and plist information about this rule and its dependencies
+ *
+ * @return this compilation support
+ */
+ CompilationSupport registerCompileAndArchiveActions(
+ ObjcCommon common, OptionsProvider optionsProvider) {
+ if (common.getCompilationArtifacts().isPresent()) {
+ ObjcRuleClasses.actionsBuilder(ruleContext).registerCompileAndArchiveActions(
+ common.getCompilationArtifacts().get(), common.getObjcProvider(), optionsProvider);
+ }
+ return this;
+ }
+
+ /**
+ * Registers any actions necessary to link this rule and its dependencies. Debug symbols are
+ * generated if {@link ObjcConfiguration#generateDebugSymbols()} is set.
+ *
+ * @param objcProvider common information about this rule's attributes and its dependencies
+ * @param extraLinkArgs any additional arguments to pass to the linker
+ * @param extraLinkInputs any additional input artifacts to pass to the link action
+ *
+ * @return this compilation support
+ */
+ CompilationSupport registerLinkActions(ObjcProvider objcProvider, ExtraLinkArgs extraLinkArgs,
+ ExtraLinkInputs extraLinkInputs) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ Optional<Artifact> dsymBundle;
+ if (ObjcRuleClasses.objcConfiguration(ruleContext).generateDebugSymbols()) {
+ registerDsymActions();
+ dsymBundle = Optional.of(intermediateArtifacts.dsymBundle());
+ } else {
+ dsymBundle = Optional.absent();
+ }
+
+ ObjcRuleClasses.actionsBuilder(ruleContext).registerLinkAction(
+ intermediateArtifacts.singleArchitectureBinary(), objcProvider, extraLinkArgs,
+ extraLinkInputs, dsymBundle);
+ return this;
+ }
+
+ /**
+ * Registers actions that compile and archive j2Objc dependencies of this rule.
+ *
+ * @param optionsProvider option and plist information about this rule and its dependencies
+ * @param objcProvider common information about this rule's attributes and its dependencies
+ *
+ * @return this compilation support
+ */
+ CompilationSupport registerJ2ObjcCompileAndArchiveActions(
+ OptionsProvider optionsProvider, ObjcProvider objcProvider) {
+ for (J2ObjcSource j2ObjcSource : ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext).getSrcs()) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.j2objcIntermediateArtifacts(ruleContext, j2ObjcSource);
+ CompilationArtifacts compilationArtifact = new CompilationArtifacts.Builder()
+ .addNonArcSrcs(j2ObjcSource.getObjcSrcs())
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .setPchFile(Optional.<Artifact>absent())
+ .build();
+ ObjcActionsBuilder actionBuilder = new ObjcActionsBuilder(
+ ruleContext,
+ intermediateArtifacts,
+ ObjcRuleClasses.objcConfiguration(ruleContext),
+ ruleContext.getConfiguration(),
+ ruleContext);
+ actionBuilder
+ .registerCompileAndArchiveActions(compilationArtifact, objcProvider, optionsProvider);
+ }
+
+ return this;
+ }
+
+ /**
+ * Sets compilation-related Xcode project information on the given provider builder.
+ *
+ * @param common common information about this rule's attributes and its dependencies
+ * @param optionsProvider option and plist information about this rule and its dependencies
+ * @return this compilation support
+ */
+ CompilationSupport addXcodeSettings(Builder xcodeProviderBuilder,
+ ObjcCommon common, OptionsProvider optionsProvider) {
+ ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext);
+ for (CompilationArtifacts artifacts : common.getCompilationArtifacts().asSet()) {
+ xcodeProviderBuilder.setCompilationArtifacts(artifacts);
+ }
+ xcodeProviderBuilder
+ .addHeaders(attributes.hdrs())
+ .addUserHeaderSearchPaths(ObjcCommon.userHeaderSearchPaths(ruleContext.getConfiguration()))
+ .addHeaderSearchPaths("$(WORKSPACE_ROOT)", attributes.headerSearchPaths())
+ .addHeaderSearchPaths("$(SDKROOT)/usr/include", attributes.sdkIncludes())
+ .addCompilationModeCopts(objcConfiguration.getCoptsForCompilationMode())
+ .addCopts(objcConfiguration.getCopts())
+ .addCopts(optionsProvider.getCopts());
+ return this;
+ }
+
+ /**
+ * Validates compilation-related attributes on this rule.
+ *
+ * @return this compilation support
+ */
+ CompilationSupport validateAttributes() {
+ for (PathFragment absoluteInclude :
+ Iterables.filter(attributes.includes(), PathFragment.IS_ABSOLUTE)) {
+ ruleContext.attributeError(
+ "includes", String.format(ABSOLUTE_INCLUDES_PATH_FORMAT, absoluteInclude));
+ }
+
+ return this;
+ }
+
+ private CompilationSupport registerDsymActions() {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ Artifact dsymBundle = intermediateArtifacts.dsymBundle();
+ Artifact debugSymbolFile = intermediateArtifacts.dsymSymbol();
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .setMnemonic("UnzipDsym")
+ .setProgressMessage("Unzipping dSYM file: " + ruleContext.getLabel())
+ .setExecutable(new PathFragment("/usr/bin/unzip"))
+ .addInput(dsymBundle)
+ .setCommandLine(CustomCommandLine.builder()
+ .add(dsymBundle.getExecPathString())
+ .add("-d")
+ .add(stripSuffix(dsymBundle.getExecPathString(),
+ IntermediateArtifacts.TMP_DSYM_BUNDLE_SUFFIX) + ".app.dSYM")
+ .build())
+ .addOutput(intermediateArtifacts.dsymPlist())
+ .addOutput(debugSymbolFile)
+ .build(ruleContext));
+
+ Artifact dumpsyms = ruleContext.getPrerequisiteArtifact("$dumpsyms", Mode.HOST);
+ Artifact breakpadFile = intermediateArtifacts.breakpadSym();
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .setMnemonic("GenBreakpad")
+ .setProgressMessage("Generating breakpad file: " + ruleContext.getLabel())
+ .setShellCommand(ImmutableList.of("/bin/bash", "-c"))
+ .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, ""))
+ .addInput(dumpsyms)
+ .addInput(debugSymbolFile)
+ .addArgument(String.format("%s %s > %s",
+ ShellUtils.shellEscape(dumpsyms.getExecPathString()),
+ ShellUtils.shellEscape(debugSymbolFile.getExecPathString()),
+ ShellUtils.shellEscape(breakpadFile.getExecPathString())))
+ .addOutput(breakpadFile)
+ .build(ruleContext));
+ return this;
+ }
+
+ private String stripSuffix(String str, String suffix) {
+ // TODO(bazel-team): Throw instead of returning null?
+ return str.endsWith(suffix) ? str.substring(0, str.length() - suffix.length()) : null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java
new file mode 100644
index 0000000..cd84710
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+
+/**
+ * Represents a strings file.
+ */
+public class CompiledResourceFile {
+ private final Artifact original;
+ private final BundleableFile bundled;
+
+ private CompiledResourceFile(Artifact original, BundleableFile bundled) {
+ this.original = Preconditions.checkNotNull(original);
+ this.bundled = Preconditions.checkNotNull(bundled);
+ }
+
+ /**
+ * The checked-in version of the bundled file.
+ */
+ public Artifact getOriginal() {
+ return original;
+ }
+
+ public BundleableFile getBundled() {
+ return bundled;
+ }
+
+ public static final Function<CompiledResourceFile, BundleableFile> TO_BUNDLED =
+ new Function<CompiledResourceFile, BundleableFile>() {
+ @Override
+ public BundleableFile apply(CompiledResourceFile input) {
+ return input.bundled;
+ }
+ };
+
+ /**
+ * Given a sequence of artifacts corresponding to {@code .strings} files, returns a sequence of
+ * the same length of instances of this class. The value returned by {@link #getBundled()} of each
+ * instance will be the plist file in binary form.
+ */
+ public static Iterable<CompiledResourceFile> fromStringsFiles(
+ IntermediateArtifacts intermediateArtifacts, Iterable<Artifact> strings) {
+ ImmutableList.Builder<CompiledResourceFile> result = new ImmutableList.Builder<>();
+ for (Artifact originalFile : strings) {
+ Artifact binaryFile = intermediateArtifacts.convertedStringsFile(originalFile);
+ result.add(new CompiledResourceFile(
+ originalFile,
+ new BundleableFile(binaryFile, BundleableFile.bundlePath(originalFile.getExecPath()))));
+ }
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java
new file mode 100644
index 0000000..2257136
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java
@@ -0,0 +1,23 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+/**
+ * Strings used to express requirements on action execution environments.
+ */
+public class ExecutionRequirements {
+ /** If an action would not successfully run other than on Darwin. */
+ public static final String REQUIRES_DARWIN = "requires-darwin";
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java b/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java
new file mode 100644
index 0000000..58369ad
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java
@@ -0,0 +1,142 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.xcode.util.Interspersing;
+
+/**
+ * Supplies information regarding Infoplist merging for a particular binary. This includes:
+ * <ul>
+ * <li>the Info.plist which contains the fields from every source. If there is only one source
+ * plist, this is that plist.
+ * <li>the action to merge all the Infoplists into a single one. This is present even if there is
+ * only one Infoplist, to prevent a Bazel error when an Artifact does not have a generating
+ * action.
+ * </ul>
+ */
+class InfoplistMerging {
+ static class Builder {
+ private final ActionConstructionContext context;
+ private NestedSet<Artifact> inputPlists;
+ private FilesToRunProvider plmerge;
+ private IntermediateArtifacts intermediateArtifacts;
+
+ public Builder(ActionConstructionContext context) {
+ this.context = Preconditions.checkNotNull(context);
+ }
+
+ public Builder setInputPlists(NestedSet<Artifact> inputPlists) {
+ Preconditions.checkState(this.inputPlists == null);
+ this.inputPlists = inputPlists;
+ return this;
+ }
+
+ public Builder setPlmerge(FilesToRunProvider plmerge) {
+ Preconditions.checkState(this.plmerge == null);
+ this.plmerge = plmerge;
+ return this;
+ }
+
+ public Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) {
+ this.intermediateArtifacts = intermediateArtifacts;
+ return this;
+ }
+
+ /**
+ * This static factory method prevents retention of the outer {@link Builder} class reference by
+ * the anonymous {@link CommandLine} instance.
+ */
+ private static CommandLine mergeCommandLine(
+ final NestedSet<Artifact> inputPlists, final Artifact mergedInfoplist) {
+ return new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return new ImmutableList.Builder<String>()
+ .addAll(Interspersing.beforeEach(
+ "--source_file", Artifact.toExecPaths(inputPlists)))
+ .add("--out_file", mergedInfoplist.getExecPathString())
+ .build();
+ }
+ };
+ }
+
+ public InfoplistMerging build() {
+ Preconditions.checkNotNull(intermediateArtifacts, "intermediateArtifacts");
+
+ Optional<Artifact> plistWithEverything = Optional.absent();
+ Action[] mergeActions = new Action[0];
+
+ int inputs = Iterables.size(inputPlists);
+ if (inputs == 1) {
+ plistWithEverything = Optional.of(Iterables.getOnlyElement(inputPlists));
+ } else if (inputs > 1) {
+ Artifact merged = intermediateArtifacts.mergedInfoplist();
+
+ plistWithEverything = Optional.of(merged);
+ mergeActions = new SpawnAction.Builder()
+ .setMnemonic("MergeInfoPlistFiles")
+ .setExecutable(plmerge)
+ .setCommandLine(mergeCommandLine(inputPlists, merged))
+ .addTransitiveInputs(inputPlists)
+ .addOutput(merged)
+ .build(context);
+ }
+
+ return new InfoplistMerging(plistWithEverything, mergeActions, inputPlists);
+ }
+ }
+
+ private final Optional<Artifact> plistWithEverything;
+ private final Action[] mergeActions;
+ private final NestedSet<Artifact> inputPlists;
+
+ private InfoplistMerging(Optional<Artifact> plistWithEverything, Action[] mergeActions,
+ NestedSet<Artifact> inputPlists) {
+ this.plistWithEverything = plistWithEverything;
+ this.mergeActions = mergeActions;
+ this.inputPlists = inputPlists;
+ }
+
+ /**
+ * Creates action to merge multiple Info.plist files of a binary into a single Info.plist. No
+ * action is necessary if there is only one source.
+ */
+ public Action[] getMergeAction() {
+ return mergeActions;
+ }
+
+ /**
+ * An {@link Optional} with the merged infoplist, or {@link Optional#absent()} if there are no
+ * merge inputs and it should not be included in the bundle.
+ */
+ public Optional<Artifact> getPlistWithEverything() {
+ return plistWithEverything;
+ }
+
+ public NestedSet<Artifact> getInputPlists() {
+ return inputPlists;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java
new file mode 100644
index 0000000..b84bf03
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java
@@ -0,0 +1,220 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Factory class for generating artifacts which are used as intermediate output.
+ */
+// TODO(bazel-team): This should really be named DerivedArtifacts as it contains methods for
+// final as well as intermediate artifacts.
+final class IntermediateArtifacts {
+
+ /**
+ * Extension used on the temporary dsym bundle location. Must end in {@code .dSYM} for dsymutil
+ * to generate a plist file.
+ */
+ static final String TMP_DSYM_BUNDLE_SUFFIX = ".temp.app.dSYM";
+
+ private final AnalysisEnvironment analysisEnvironment;
+ private final Root binDirectory;
+ private final Label ownerLabel;
+ private final String archiveFileNameSuffix;
+
+ IntermediateArtifacts(
+ AnalysisEnvironment analysisEnvironment, Root binDirectory, Label ownerLabel,
+ String archiveFileNameSuffix) {
+ this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment);
+ this.binDirectory = Preconditions.checkNotNull(binDirectory);
+ this.ownerLabel = Preconditions.checkNotNull(ownerLabel);
+ this.archiveFileNameSuffix = Preconditions.checkNotNull(archiveFileNameSuffix);
+ }
+
+ /**
+ * Returns a derived artifact in the bin directory obtained by appending some extension to the end
+ * of the given {@link PathFragment}.
+ */
+ private Artifact appendExtension(PathFragment original, String extension) {
+ return analysisEnvironment.getDerivedArtifact(
+ FileSystemUtils.appendExtension(original, extension), binDirectory);
+ }
+
+ /**
+ * Returns a derived artifact in the bin directory obtained by appending some extension to the end
+ * of the {@link PathFragment} corresponding to the owner {@link Label}.
+ */
+ private Artifact appendExtension(String extension) {
+ return appendExtension(ownerLabel.toPathFragment(), extension);
+ }
+
+ /**
+ * The output of using {@code actooloribtoolzip} to run {@code actool} for a given bundle which is
+ * merged under the {@code .app} or {@code .bundle} directory root.
+ */
+ public Artifact actoolzipOutput() {
+ return appendExtension(".actool.zip");
+ }
+
+ /**
+ * Output of the partial infoplist generated by {@code actool} when given the
+ * {@code --output-partial-info-plist [path]} flag.
+ */
+ public Artifact actoolPartialInfoplist() {
+ return appendExtension(".actool-PartialInfo.plist");
+ }
+
+ /**
+ * The Info.plist file for a bundle which is comprised of more than one originating plist file.
+ * This is not needed for a bundle which has no source Info.plist files, or only one Info.plist
+ * file, since no merging occurs in that case.
+ */
+ public Artifact mergedInfoplist() {
+ return appendExtension("-MergedInfo.plist");
+ }
+
+ /**
+ * The .objlist file, which contains a list of paths of object files to archive and is read by
+ * libtool in the archive action.
+ */
+ public Artifact objList() {
+ return appendExtension(".objlist");
+ }
+
+ /**
+ * The artifact which is the binary (or library) which is comprised of one or more .a files linked
+ * together.
+ */
+ public Artifact singleArchitectureBinary() {
+ return appendExtension("_bin");
+ }
+
+ /**
+ * Lipo binary generated by combining one or more linked binaries. This binary is the one included
+ * in generated bundles and invoked as entry point to the application.
+ *
+ * @param bundleDirSuffix suffix of the bundle containing this binary
+ */
+ public Artifact combinedArchitectureBinary(String bundleDirSuffix) {
+ String baseName = ownerLabel.toPathFragment().getBaseName();
+ return appendExtension(bundleDirSuffix + "/" + baseName);
+ }
+
+ /**
+ * The {@code .a} file which contains all the compiled sources for a rule.
+ */
+ public Artifact archive() {
+ PathFragment labelPath = ownerLabel.toPathFragment();
+ PathFragment rootRelative = labelPath
+ .getParentDirectory()
+ .getRelative(String.format("lib%s%s.a", labelPath.getBaseName(), archiveFileNameSuffix));
+ return analysisEnvironment.getDerivedArtifact(rootRelative, binDirectory);
+ }
+
+ /**
+ * The debug symbol bundle file which contains debug symbols generated by dsymutil.
+ */
+ public Artifact dsymBundle() {
+ return appendExtension(TMP_DSYM_BUNDLE_SUFFIX);
+ }
+
+ /**
+ * The artifact for the .o file that should be generated when compiling the {@code source}
+ * artifact.
+ */
+ public Artifact objFile(Artifact source) {
+ return analysisEnvironment.getDerivedArtifact(
+ FileSystemUtils.replaceExtension(
+ AnalysisUtils.getUniqueDirectory(ownerLabel, new PathFragment("_objs"))
+ .getRelative(source.getRootRelativePath()),
+ ".o"),
+ binDirectory);
+ }
+
+ /**
+ * Returns the artifact corresponding to the pbxproj control file, which specifies the information
+ * required to generate the Xcode project file.
+ */
+ public Artifact pbxprojControlArtifact() {
+ return appendExtension(".xcodeproj-control");
+ }
+
+ /**
+ * The artifact which contains the zipped-up results of compiling the storyboard. This is merged
+ * into the final bundle under the {@code .app} or {@code .bundle} directory root.
+ */
+ public Artifact compiledStoryboardZip(Artifact input) {
+ return appendExtension("/" + BundleableFile.bundlePath(input.getExecPath()) + ".zip");
+ }
+
+ /**
+ * Returns the artifact which is the output of building an entire xcdatamodel[d] made of artifacts
+ * specified by a single rule.
+ *
+ * @param containerDir the containing *.xcdatamodeld or *.xcdatamodel directory
+ * @return the artifact for the zipped up compilation results.
+ */
+ public Artifact compiledMomZipArtifact(PathFragment containerDir) {
+ return appendExtension(
+ "/" + FileSystemUtils.replaceExtension(containerDir, ".zip").getBaseName());
+ }
+
+ /**
+ * Returns the compiled (i.e. converted to binary plist format) artifact corresponding to the
+ * given {@code .strings} file.
+ */
+ public Artifact convertedStringsFile(Artifact originalFile) {
+ return appendExtension(originalFile.getExecPath(), ".binary");
+ }
+
+ /**
+ * Returns the artifact corresponding to the zipped-up compiled form of the given {@code .xib}
+ * file.
+ */
+ public Artifact compiledXibFileZip(Artifact originalFile) {
+ return analysisEnvironment.getDerivedArtifact(
+ FileSystemUtils.replaceExtension(originalFile.getExecPath(), ".nib.zip"),
+ binDirectory);
+ }
+
+ /**
+ * Debug symbol plist generated for a linked binary.
+ */
+ public Artifact dsymPlist() {
+ return appendExtension(".app.dSYM/Contents/Info.plist");
+ }
+
+ /**
+ * Debug symbol file generated for a linked binary.
+ */
+ public Artifact dsymSymbol() {
+ return appendExtension(
+ String.format(".app.dSYM/Contents/Resources/DWARF/%s_bin", ownerLabel.getName()));
+ }
+
+ /**
+ * Breakpad debug symbol representation.
+ */
+ public Artifact breakpadSym() {
+ return appendExtension(".breakpad");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java
new file mode 100644
index 0000000..b81d9b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java
@@ -0,0 +1,117 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+
+/**
+ * Rule definition for ios_application.
+ */
+@BlazeRule(name = "$ios_application",
+ ancestors = { BaseRuleClasses.BaseRule.class,
+ ObjcRuleClasses.ObjcBaseResourcesRule.class,
+ ObjcRuleClasses.ObjcHasInfoplistRule.class,
+ ObjcRuleClasses.ObjcHasEntitlementsRule.class },
+ type = RuleClassType.ABSTRACT) // TODO(bazel-team): Add factory once this becomes a real rule.
+public class IosApplicationRule implements RuleDefinition {
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(app_icon) -->
+ The name of the application icon, which should be in one of the asset
+ catalogs of this target or a (transitive) dependency. In a new project,
+ this is initialized to "AppIcon" by Xcode.
+ <p>
+ If the application icon is not in an asset catalog, do not use this
+ attribute. Instead, add a CFBundleIcons entry to the Info.plist file.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("app_icon", STRING))
+ /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(launch_image) -->
+ The name of the launch image, which should be in one of the asset
+ catalogs of this target or a (transitive) dependency. In a new project,
+ this is initialized to "LaunchImage" by Xcode.
+ <p>
+ If the launch image is not in an asset catalog, do not use this
+ attribute. Instead, add an appropriately-named image resource to the
+ bundle.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("launch_image", STRING))
+ /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(bundle_id) -->
+ The bundle ID (reverse-DNS path followed by app name) of the binary. If none is specified, a
+ junk value will be used.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("bundle_id", STRING)
+ .value(new Attribute.ComputedDefault() {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ // For tests and similar, we don't want to force people to explicitly specify
+ // throw-away data.
+ return "example." + rule.getName();
+ }
+ }))
+ /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(families) -->
+ The device families to which this binary is targeted. This is known as
+ the <code>TARGETED_DEVICE_FAMILY</code> build setting in Xcode project
+ files. It is a list of one or more of the strings <code>"iphone"</code>
+ and <code>"ipad"</code>.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("families", STRING_LIST)
+ .value(ImmutableList.of(TargetDeviceFamily.IPHONE.getNameInRule())))
+ /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(provisioning_profile) -->
+ The provisioning profile (.mobileprovision file) to use when bundling
+ the application.
+ <p>
+ This is only used for non-simulator builds.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("provisioning_profile", LABEL)
+ .value(env.getLabel("//tools/objc:default_provisioning_profile"))
+ .allowedFileTypes(FileType.of(".mobileprovision")))
+ // TODO(bazel-team): Consider ways to trim dependencies so that changes to deps of these
+ // tools don't trigger all objc_* targets. Right now we check-in deploy jars, but we
+ // need a less painful and error-prone way.
+ /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(binary) -->
+ The binary target included in the final bundle.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("binary", LABEL)
+ .allowedRuleClasses("objc_binary")
+ .allowedFileTypes()
+ .mandatory()
+ .direct_compile_time_input())
+ .add(attr("$bundlemerge", LABEL).cfg(HOST).exec()
+ .value(env.getLabel("//tools/objc:bundlemerge")))
+ .add(attr("$dumpsyms", LABEL).cfg(HOST).exec()
+ .value(env.getLabel("//tools/objc:dump_syms")))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java
new file mode 100644
index 0000000..2f01633
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java
@@ -0,0 +1,42 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for the "ios_device" rule.
+ */
+public final class IosDevice implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext context) throws InterruptedException {
+ IosDeviceProvider provider = new IosDeviceProvider.Builder()
+ .setType(context.attributes().get("type", STRING))
+ .setIosVersion(context.attributes().get("ios_version", STRING))
+ .setLocale(context.attributes().get("locale", STRING))
+ .build();
+
+ return new RuleConfiguredTargetBuilder(context)
+ .add(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .add(IosDeviceProvider.class, provider)
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java
new file mode 100644
index 0000000..6f01e11
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java
@@ -0,0 +1,73 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Provider that describes a simulator device.
+ */
+@Immutable
+public final class IosDeviceProvider implements TransitiveInfoProvider {
+ /** A builder of {@link IosDeviceProvider}s. */
+ public static final class Builder {
+ private String type;
+ private String iosVersion;
+ private String locale;
+
+ public Builder setType(String type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder setIosVersion(String iosVersion) {
+ this.iosVersion = iosVersion;
+ return this;
+ }
+
+ public Builder setLocale(String locale) {
+ this.locale = locale;
+ return this;
+ }
+
+ public IosDeviceProvider build() {
+ return new IosDeviceProvider(this);
+ }
+ }
+
+ private final String type;
+ private final String iosVersion;
+ private final String locale;
+
+ private IosDeviceProvider(Builder builder) {
+ this.type = Preconditions.checkNotNull(builder.type);
+ this.iosVersion = Preconditions.checkNotNull(builder.iosVersion);
+ this.locale = Preconditions.checkNotNull(builder.locale);
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getIosVersion() {
+ return iosVersion;
+ }
+
+ public String getLocale() {
+ return locale;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java
new file mode 100644
index 0000000..e028e70
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java
@@ -0,0 +1,67 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for ios_device.
+ */
+@BlazeRule(name = "ios_device",
+ factoryClass = IosDevice.class,
+ ancestors = { BaseRuleClasses.RuleBase.class })
+public final class IosDeviceRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE(ios_device).ATTRIBUTE(ios_version) -->
+ The operating system version of the device. This corresponds to the
+ <code>simctl</code> runtime.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("ios_version", STRING)
+ .mandatory())
+ /* <!-- #BLAZE_RULE(ios_device).ATTRIBUTE(type) -->
+ The hardware type. This corresponds to the <code>simctl</code> device
+ type.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("type", STRING)
+ .mandatory())
+ .add(attr("locale", STRING)
+ .undocumented("this is not yet supported by any test runner")
+ .value("en"))
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = ios_device, TYPE = BINARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule defines an iOS device profile that defines a simulator against
+which to run tests</p>.
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java
new file mode 100644
index 0000000..0a9fb7b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java
@@ -0,0 +1,155 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.common.Platform;
+import com.google.devtools.build.xcode.util.Interspersing;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting;
+
+import java.util.List;
+
+/**
+ * Utility code for use when generating iOS SDK commands.
+ */
+public class IosSdkCommands {
+ public static final String DEVELOPER_DIR = "/Applications/Xcode.app/Contents/Developer";
+ public static final String BIN_DIR =
+ DEVELOPER_DIR + "/Toolchains/XcodeDefault.xctoolchain/usr/bin";
+ public static final String ACTOOL_PATH = DEVELOPER_DIR + "/usr/bin/actool";
+ public static final String IBTOOL_PATH = DEVELOPER_DIR + "/usr/bin/ibtool";
+ public static final String MOMC_PATH = DEVELOPER_DIR + "/usr/bin/momc";
+
+ // There is a handy reference to many clang warning flags at
+ // http://nshipster.com/clang-diagnostics/
+ // There is also a useful narrative for many Xcode settings at
+ // http://www.xs-labs.com/en/blog/2011/02/04/xcode-build-settings/
+ @VisibleForTesting
+ static final ImmutableMap<String, String> DEFAULT_WARNINGS =
+ new ImmutableMap.Builder<String, String>()
+ .put("GCC_WARN_64_TO_32_BIT_CONVERSION", "-Wshorten-64-to-32")
+ .put("CLANG_WARN_BOOL_CONVERSION", "-Wbool-conversion")
+ .put("CLANG_WARN_CONSTANT_CONVERSION", "-Wconstant-conversion")
+ // Double-underscores are intentional - thanks Xcode.
+ .put("CLANG_WARN__DUPLICATE_METHOD_MATCH", "-Wduplicate-method-match")
+ .put("CLANG_WARN_EMPTY_BODY", "-Wempty-body")
+ .put("CLANG_WARN_ENUM_CONVERSION", "-Wenum-conversion")
+ .put("CLANG_WARN_INT_CONVERSION", "-Wint-conversion")
+ .put("CLANG_WARN_UNREACHABLE_CODE", "-Wunreachable-code")
+ .put("GCC_WARN_ABOUT_RETURN_TYPE", "-Wmismatched-return-types")
+ .put("GCC_WARN_UNDECLARED_SELECTOR", "-Wundeclared-selector")
+ .put("GCC_WARN_UNINITIALIZED_AUTOS", "-Wuninitialized")
+ .put("GCC_WARN_UNUSED_FUNCTION", "-Wunused-function")
+ .put("GCC_WARN_UNUSED_VARIABLE", "-Wunused-variable")
+ .build();
+
+ static final ImmutableList<String> DEFAULT_LINKER_FLAGS = ImmutableList.of("-ObjC");
+
+ private IosSdkCommands() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ private static String platformDir(ObjcConfiguration configuration) {
+ return DEVELOPER_DIR + "/Platforms/" + configuration.getPlatform().getNameInPlist()
+ + ".platform";
+ }
+
+ public static String sdkDir(ObjcConfiguration configuration) {
+ return platformDir(configuration) + "/Developer/SDKs/"
+ + configuration.getPlatform().getNameInPlist() + configuration.getIosSdkVersion() + ".sdk";
+ }
+
+ public static String frameworkDir(ObjcConfiguration configuration) {
+ return platformDir(configuration) + "/Developer/Library/Frameworks";
+ }
+
+ private static Iterable<PathFragment> uniqueParentDirectories(Iterable<PathFragment> paths) {
+ ImmutableSet.Builder<PathFragment> parents = new ImmutableSet.Builder<>();
+ for (PathFragment path : paths) {
+ parents.add(path.getParentDirectory());
+ }
+ return parents.build();
+ }
+
+ public static List<String> commonLinkAndCompileArgsForClang(
+ ObjcProvider provider, ObjcConfiguration configuration) {
+ ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
+ if (configuration.getPlatform() == Platform.SIMULATOR) {
+ builder.add("-mios-simulator-version-min=" + configuration.getMinimumOs());
+ } else {
+ builder.add("-miphoneos-version-min=" + configuration.getMinimumOs());
+ }
+
+ if (configuration.generateDebugSymbols()) {
+ builder.add("-g");
+ }
+
+ return builder
+ .add("-arch", configuration.getIosCpu())
+ .add("-isysroot", sdkDir(configuration))
+ // TODO(bazel-team): Pass framework search paths to Xcodegen.
+ .add("-F", sdkDir(configuration) + "/Developer/Library/Frameworks")
+ // As of sdk8.1, XCTest is in a base Framework dir
+ .add("-F", frameworkDir(configuration))
+ // Add custom (non-SDK) framework search paths. For each framework foo/bar.framework,
+ // include "foo" as a search path.
+ .addAll(Interspersing.beforeEach(
+ "-F",
+ PathFragment.safePathStrings(uniqueParentDirectories(provider.get(FRAMEWORK_DIR)))))
+ .build();
+ }
+
+ public static Iterable<String> compileArgsForClang(ObjcConfiguration configuration) {
+ return Iterables.concat(
+ DEFAULT_WARNINGS.values(),
+ platformSpecificCompileArgsForClang(configuration)
+ );
+ }
+
+ private static List<String> platformSpecificCompileArgsForClang(ObjcConfiguration configuration) {
+ switch (configuration.getPlatform()) {
+ case DEVICE:
+ return ImmutableList.of();
+ case SIMULATOR:
+ // These are added by Xcode when building, because the simulator is built on OSX
+ // frameworks so we aim compile to match the OSX objc runtime.
+ return ImmutableList.of(
+ "-fexceptions",
+ "-fasm-blocks",
+ "-fobjc-abi-version=2",
+ "-fobjc-legacy-dispatch");
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ public static Iterable<? extends XcodeprojBuildSetting> defaultWarningsForXcode() {
+ return Iterables.transform(DEFAULT_WARNINGS.keySet(),
+ new Function<String, XcodeprojBuildSetting>() {
+ @Override
+ public XcodeprojBuildSetting apply(String key) {
+ return XcodeprojBuildSetting.newBuilder().setName(key).setValue("YES").build();
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java
new file mode 100644
index 0000000..9e9ddc4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java
@@ -0,0 +1,163 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.objc.ApplicationSupport.LinkedBinary;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Contains information needed to create a {@link RuleConfiguredTarget} and invoke test runners
+ * for some instantiation of this rule.
+ */
+// TODO(bazel-team): Extract a TestSupport class that takes on most of the logic in this class.
+public abstract class IosTest implements RuleConfiguredTargetFactory {
+ private static final ImmutableList<SdkFramework> AUTOMATIC_SDK_FRAMEWORKS_FOR_XCTEST =
+ ImmutableList.of(new SdkFramework("XCTest"));
+
+ public static final String TARGET_DEVICE = "target_device";
+ public static final String IS_XCTEST = "xctest";
+ public static final String XCTEST_APP = "xctest_app";
+
+ @VisibleForTesting
+ public static final String REQUIRES_SOURCE_ERROR =
+ "ios_test requires at least one source file in srcs or non_arc_srcs";
+
+ /**
+ * Creates a target, including registering actions, just as {@link #create(RuleContext)} does.
+ * The difference between {@link #create(RuleContext)} and this method is that this method does
+ * only what is needed to support tests on the environment besides generate the Xcodeproj file
+ * and build the app and test {@code .ipa}s. The {@link #create(RuleContext)} method delegates
+ * to this method.
+ */
+ protected abstract ConfiguredTarget create(RuleContext ruleContext, ObjcCommon common,
+ XcodeProvider xcodeProvider, NestedSet<Artifact> filesToBuild) throws InterruptedException;
+
+ @Override
+ public final ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ ObjcCommon common = common(ruleContext);
+ OptionsProvider optionsProvider = optionsProvider(ruleContext);
+
+ if (!common.getCompilationArtifacts().get().getArchive().isPresent()) {
+ ruleContext.ruleError(REQUIRES_SOURCE_ERROR);
+ }
+
+ XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder();
+ NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(common.getStoryboards().getOutputZips())
+ .addAll(Xcdatamodel.outputZips(common.getDatamodels()));
+
+ XcodeProductType productType;
+ ExtraLinkArgs extraLinkArgs;
+ ExtraLinkInputs extraLinkInputs;
+ if (!isXcTest(ruleContext)) {
+ productType = XcodeProductType.APPLICATION;
+ extraLinkArgs = new ExtraLinkArgs();
+ extraLinkInputs = new ExtraLinkInputs();
+ } else {
+ productType = XcodeProductType.UNIT_TEST;
+ XcodeProvider appIpaXcodeProvider =
+ ruleContext.getPrerequisite(XCTEST_APP, Mode.TARGET, XcodeProvider.class);
+ xcodeProviderBuilder
+ .setTestHost(appIpaXcodeProvider)
+ .setProductType(productType);
+
+ Artifact bundleLoader = xcTestAppProvider(ruleContext).getBundleLoader();
+
+ // -bundle causes this binary to be linked as a bundle and not require an entry point
+ // (i.e. main())
+ // -bundle_loader causes the code in this test to have access to the symbols in the test rig,
+ // or more specifically, the flag causes ld to consider the given binary when checking for
+ // missing symbols.
+ extraLinkArgs = new ExtraLinkArgs(
+ "-bundle",
+ "-bundle_loader", bundleLoader.getExecPathString());
+ extraLinkInputs = new ExtraLinkInputs(bundleLoader);
+ }
+
+ new CompilationSupport(ruleContext)
+ .registerLinkActions(
+ common.getObjcProvider(), extraLinkArgs, extraLinkInputs)
+ .registerJ2ObjcCompileAndArchiveActions(optionsProvider, common.getObjcProvider())
+ .registerCompileAndArchiveActions(common, optionsProvider)
+ .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider)
+ .validateAttributes();
+
+ new ApplicationSupport(
+ ruleContext, common.getObjcProvider(), optionsProvider, LinkedBinary.LOCAL_AND_DEPENDENCIES)
+ .registerActions()
+ .addXcodeSettings(xcodeProviderBuilder)
+ .addFilesToBuild(filesToBuild)
+ .validateAttributes();
+
+ new ResourceSupport(ruleContext)
+ .registerActions(common.getStoryboards())
+ .validateAttributes()
+ .addXcodeSettings(xcodeProviderBuilder);
+
+ new XcodeSupport(ruleContext)
+ .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), productType)
+ .addDependencies(xcodeProviderBuilder)
+ .addFilesToBuild(filesToBuild)
+ .registerActions(xcodeProviderBuilder.build());
+
+ return create(ruleContext, common, xcodeProviderBuilder.build(), filesToBuild.build());
+ }
+
+ protected static boolean isXcTest(RuleContext ruleContext) {
+ return ruleContext.attributes().get(IS_XCTEST, Type.BOOLEAN);
+ }
+
+ private OptionsProvider optionsProvider(RuleContext ruleContext) {
+ return new OptionsProvider.Builder()
+ .addCopts(ruleContext.getTokenizedStringListAttr("copts"))
+ .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list())
+ .addTransitive(Optional.fromNullable(
+ ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class)))
+ .build();
+ }
+
+ /** Returns the {@link XcTestAppProvider} of the {@code xctest_app} attribute. */
+ private static XcTestAppProvider xcTestAppProvider(RuleContext ruleContext) {
+ return ruleContext.getPrerequisite(XCTEST_APP, Mode.TARGET, XcTestAppProvider.class);
+ }
+
+ private ObjcCommon common(RuleContext ruleContext) {
+ ImmutableList<SdkFramework> extraSdkFrameworks = isXcTest(ruleContext)
+ ? AUTOMATIC_SDK_FRAMEWORKS_FOR_XCTEST : ImmutableList.<SdkFramework>of();
+ List<ObjcProvider> extraDepObjcProviders = new ArrayList<>();
+ if (isXcTest(ruleContext)) {
+ extraDepObjcProviders.add(xcTestAppProvider(ruleContext).getObjcProvider());
+ }
+
+ return ObjcLibrary.common(ruleContext, extraSdkFrameworks, /*alwayslink=*/false,
+ new ObjcLibrary.ExtraImportLibraries(), new ObjcLibrary.Defines(), extraDepObjcProviders);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java
new file mode 100644
index 0000000..4bd7b0c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.Iterator;
+
+/**
+ * Base class for tiny container types that encapsulate an iterable.
+ */
+abstract class IterableWrapper<E> implements Iterable<E> {
+ private final Iterable<E> contents;
+
+ IterableWrapper(Iterable<E> contents) {
+ this.contents = Preconditions.checkNotNull(contents);
+ }
+
+ IterableWrapper(E... contents) {
+ this.contents = ImmutableList.copyOf(contents);
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return contents.iterator();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java
new file mode 100644
index 0000000..93ff980
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * This provider is exported by java_library rules to supply ObjC header to Java type mapping files
+ * for J2ObjC translation. J2ObjC needs the mapping files to be able to output translated files with
+ * correct header import paths in the same directories of the Java source files.
+ */
+@Immutable
+public final class J2ObjcHeaderMappingFileProvider implements TransitiveInfoProvider {
+ private final NestedSet<Artifact> mappingFiles;
+
+ public J2ObjcHeaderMappingFileProvider(NestedSet<Artifact> mappingFiles) {
+ this.mappingFiles = mappingFiles;
+ }
+
+ public NestedSet<Artifact> getMappingFiles() {
+ return mappingFiles;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java
new file mode 100644
index 0000000..81ba292
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java
@@ -0,0 +1,124 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Iterators;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * An object that captures information of ObjC files generated by J2ObjC in a single target.
+ */
+public class J2ObjcSource {
+
+ /**
+ * Indicates the type of files from which the ObjC files included in {@link J2ObjcSource} are
+ * generated.
+ */
+ public enum SourceType {
+ /**
+ * Indicates the original file type is java source file.
+ */
+ JAVA,
+
+ /**
+ * Indicates the original file type is proto file.
+ */
+ PROTO;
+ }
+
+ private final Label targetLabel;
+ private final Iterable<Artifact> objcSrcs;
+ private final Iterable<Artifact> objcHdrs;
+ private final PathFragment objcFilePath;
+ private final SourceType sourceType;
+
+ /**
+ * Constructs a J2ObjcSource containing target information for j2objc transpilation.
+ *
+ * @param targetLabel the @{code Label} of the associated target.
+ * @param objcSrcs the {@code Iterable} containing objc source files generated by J2ObjC
+ * @param objcHdrs the {@code Iterable} containing objc header files generated by J2ObjC
+ * @param objcFilePath the {@code PathFragment} under which all the generated objc files are. It
+ * can be used as header search path for objc compilations.
+ * @param sourceType the type of files from which the ObjC files are generated.
+ */
+ public J2ObjcSource(Label targetLabel, Iterable<Artifact> objcSrcs,
+ Iterable<Artifact> objcHdrs, PathFragment objcFilePath, SourceType sourceType) {
+ this.targetLabel = targetLabel;
+ this.objcSrcs = objcSrcs;
+ this.objcHdrs = objcHdrs;
+ this.objcFilePath = objcFilePath;
+ this.sourceType = sourceType;
+ }
+
+ /**
+ * Returns the label of the associated target.
+ */
+ public Label getTargetLabel() {
+ return targetLabel;
+ }
+
+ /**
+ * Returns the objc source files generated by J2ObjC.
+ */
+ public Iterable<Artifact> getObjcSrcs() {
+ return objcSrcs;
+ }
+
+ /*
+ * Returns the objc header files generated by J2ObjC
+ */
+ public Iterable<Artifact> getObjcHdrs() {
+ return objcHdrs;
+ }
+
+ /**
+ * Returns the {@code PathFragment} which represents a directory where the generated ObjC files
+ * reside and which can also be used as header search path in ObjC compilation.
+ */
+ public PathFragment getObjcFilePath() {
+ return objcFilePath;
+ }
+
+ /**
+ * Returns the type of files from which the ObjC files inside this object are generated.
+ */
+ public SourceType getSourceType() {
+ return sourceType;
+ }
+
+ @Override
+ public final boolean equals(Object other) {
+ if (!(other instanceof J2ObjcSource)) {
+ return false;
+ }
+
+ J2ObjcSource that = (J2ObjcSource) other;
+ return Objects.equal(this.targetLabel, that.targetLabel)
+ && Iterators.elementsEqual(this.objcSrcs.iterator(), that.objcSrcs.iterator())
+ && Iterators.elementsEqual(this.objcHdrs.iterator(), that.objcHdrs.iterator())
+ && Objects.equal(this.objcFilePath, that.objcFilePath)
+ && this.sourceType == that.sourceType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(targetLabel, objcSrcs, objcHdrs, objcFilePath, sourceType);
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java
new file mode 100644
index 0000000..1e577c5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * This provider is exported by java_library rules to supply J2ObjC-translated ObjC sources to
+ * objc_binary for compilation and linking.
+ */
+@Immutable
+public final class J2ObjcSrcsProvider implements TransitiveInfoProvider {
+ private final NestedSet<J2ObjcSource> srcs;
+ private final boolean hasProtos;
+
+ public J2ObjcSrcsProvider(NestedSet<J2ObjcSource> srcs, boolean hasProtos) {
+ this.srcs = srcs;
+ this.hasProtos = hasProtos;
+ }
+
+ public NestedSet<J2ObjcSource> getSrcs() {
+ return srcs;
+ }
+
+ /**
+ * Returns whether the translated source files in the provider has proto files.
+ */
+ public boolean hasProtos() {
+ return hasProtos;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java
new file mode 100644
index 0000000..26742d9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java
@@ -0,0 +1,599 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.IosSdkCommands.BIN_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag.USES_CPP;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.HEADER;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.INCLUDE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.io.ByteSource;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionRegistry;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext;
+import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.CommandLine;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.util.LazyString;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+import com.google.devtools.build.xcode.util.Interspersing;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+/**
+ * Object that creates actions used by Objective-C rules.
+ */
+final class ObjcActionsBuilder {
+ private final ActionConstructionContext context;
+ private final IntermediateArtifacts intermediateArtifacts;
+ private final ObjcConfiguration objcConfiguration;
+ private final BuildConfiguration buildConfiguration;
+ private final ActionRegistry actionRegistry;
+
+ ObjcActionsBuilder(ActionConstructionContext context, IntermediateArtifacts intermediateArtifacts,
+ ObjcConfiguration objcConfiguration, BuildConfiguration buildConfiguration,
+ ActionRegistry actionRegistry) {
+ this.context = Preconditions.checkNotNull(context);
+ this.intermediateArtifacts = Preconditions.checkNotNull(intermediateArtifacts);
+ this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration);
+ this.buildConfiguration = Preconditions.checkNotNull(buildConfiguration);
+ this.actionRegistry = Preconditions.checkNotNull(actionRegistry);
+ }
+
+ /**
+ * Creates a new spawn action builder that requires a darwin architecture to run.
+ */
+ // TODO(bazel-team): Use everywhere we currently set the execution info manually.
+ static SpawnAction.Builder spawnOnDarwinActionBuilder() {
+ return new SpawnAction.Builder()
+ .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, ""));
+ }
+
+ static final PathFragment JAVA = new PathFragment("/usr/bin/java");
+ static final PathFragment CLANG = new PathFragment(BIN_DIR + "/clang");
+ static final PathFragment CLANG_PLUSPLUS = new PathFragment(BIN_DIR + "/clang++");
+ static final PathFragment LIBTOOL = new PathFragment(BIN_DIR + "/libtool");
+ static final PathFragment IBTOOL = new PathFragment(IosSdkCommands.IBTOOL_PATH);
+ static final PathFragment DSYMUTIL = new PathFragment(BIN_DIR + "/dsymutil");
+ static final PathFragment LIPO = new PathFragment(BIN_DIR + "/lipo");
+
+ // TODO(bazel-team): Reference a rule target rather than a jar file when Darwin runfiles work
+ // better.
+ private static SpawnAction.Builder spawnJavaOnDarwinActionBuilder(Artifact deployJarArtifact) {
+ return spawnOnDarwinActionBuilder()
+ .setExecutable(JAVA)
+ .addExecutableArguments("-jar", deployJarArtifact.getExecPathString())
+ .addInput(deployJarArtifact);
+ }
+
+ private void registerCompileAction(
+ Artifact sourceFile,
+ Artifact objFile,
+ Optional<Artifact> pchFile,
+ ObjcProvider objcProvider,
+ Iterable<String> otherFlags,
+ OptionsProvider optionsProvider) {
+ CustomCommandLine.Builder commandLine = new CustomCommandLine.Builder();
+ if (ObjcRuleClasses.CPP_SOURCES.matches(sourceFile.getExecPath())) {
+ commandLine.add("-stdlib=libc++");
+ }
+ commandLine
+ .add(IosSdkCommands.compileArgsForClang(objcConfiguration))
+ .add(IosSdkCommands.commonLinkAndCompileArgsForClang(
+ objcProvider, objcConfiguration))
+ .add(objcConfiguration.getCoptsForCompilationMode())
+ .addBeforeEachPath("-iquote", ObjcCommon.userHeaderSearchPaths(buildConfiguration))
+ .addBeforeEachExecPath("-include", pchFile.asSet())
+ .addBeforeEachPath("-I", objcProvider.get(INCLUDE))
+ .add(otherFlags)
+ .addFormatEach("-D%s", objcProvider.get(DEFINE))
+ .add(objcConfiguration.getCopts())
+ .add(optionsProvider.getCopts())
+ .addExecPath("-c", sourceFile)
+ .addExecPath("-o", objFile);
+
+ register(spawnOnDarwinActionBuilder()
+ .setMnemonic("ObjcCompile")
+ .setExecutable(CLANG)
+ .setCommandLine(commandLine.build())
+ .addInput(sourceFile)
+ .addOutput(objFile)
+ .addTransitiveInputs(objcProvider.get(HEADER))
+ .addTransitiveInputs(objcProvider.get(FRAMEWORK_FILE))
+ .addInputs(pchFile.asSet())
+ .build(context));
+ }
+
+ private static final ImmutableList<String> ARC_ARGS = ImmutableList.of("-fobjc-arc");
+ private static final ImmutableList<String> NON_ARC_ARGS = ImmutableList.of("-fno-objc-arc");
+
+ /**
+ * Creates actions to compile each source file individually, and link all the compiled object
+ * files into a single archive library.
+ */
+ void registerCompileAndArchiveActions(CompilationArtifacts compilationArtifacts,
+ ObjcProvider objcProvider, OptionsProvider optionsProvider) {
+ ImmutableList.Builder<Artifact> objFiles = new ImmutableList.Builder<>();
+ for (Artifact sourceFile : compilationArtifacts.getSrcs()) {
+ Artifact objFile = intermediateArtifacts.objFile(sourceFile);
+ objFiles.add(objFile);
+ registerCompileAction(sourceFile, objFile, compilationArtifacts.getPchFile(),
+ objcProvider, ARC_ARGS, optionsProvider);
+ }
+ for (Artifact nonArcSourceFile : compilationArtifacts.getNonArcSrcs()) {
+ Artifact objFile = intermediateArtifacts.objFile(nonArcSourceFile);
+ objFiles.add(objFile);
+ registerCompileAction(nonArcSourceFile, objFile, compilationArtifacts.getPchFile(),
+ objcProvider, NON_ARC_ARGS, optionsProvider);
+ }
+ for (Artifact archive : compilationArtifacts.getArchive().asSet()) {
+ registerAll(archiveActions(context, objFiles.build(), archive, objcConfiguration,
+ intermediateArtifacts.objList()));
+ }
+ }
+
+ private static Iterable<Action> archiveActions(
+ ActionConstructionContext context,
+ final Iterable<Artifact> objFiles,
+ final Artifact archive,
+ final ObjcConfiguration objcConfiguration,
+ final Artifact objList) {
+
+ ImmutableList.Builder<Action> actions = new ImmutableList.Builder<>();
+
+ actions.add(new FileWriteAction(
+ context.getActionOwner(), objList, joinExecPaths(objFiles), /*makeExecutable=*/ false));
+
+ actions.add(spawnOnDarwinActionBuilder()
+ .setMnemonic("ObjcLink")
+ .setExecutable(LIBTOOL)
+ .setCommandLine(new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return new ImmutableList.Builder<String>()
+ .add("-static")
+ .add("-filelist").add(objList.getExecPathString())
+ .add("-arch_only").add(objcConfiguration.getIosCpu())
+ .add("-syslibroot").add(IosSdkCommands.sdkDir(objcConfiguration))
+ .add("-o").add(archive.getExecPathString())
+ .build();
+ }
+ })
+ .addInputs(objFiles)
+ .addInput(objList)
+ .addOutput(archive)
+ .build(context));
+
+ return actions.build();
+ }
+
+ private void register(Action... action) {
+ actionRegistry.registerAction(action);
+ }
+
+ private void registerAll(Iterable<? extends Action> actions) {
+ for (Action action : actions) {
+ actionRegistry.registerAction(action);
+ }
+ }
+
+ private static ByteSource xcodegenControlFileBytes(
+ final Artifact pbxproj, final XcodeProvider.Project project, final String minimumOs) {
+ return new ByteSource() {
+ @Override
+ public InputStream openStream() {
+ return XcodeGenProtos.Control.newBuilder()
+ .setPbxproj(pbxproj.getExecPathString())
+ .addAllTarget(project.targets())
+ .addBuildSetting(XcodeGenProtos.XcodeprojBuildSetting.newBuilder()
+ .setName("IPHONEOS_DEPLOYMENT_TARGET")
+ .setValue(minimumOs)
+ .build())
+ .build()
+ .toByteString()
+ .newInput();
+ }
+ };
+ }
+
+ /**
+ * Generates actions needed to create an Xcode project file.
+ */
+ void registerXcodegenActions(
+ ObjcRuleClasses.Tools baseTools, Artifact pbxproj, XcodeProvider.Project project) {
+ Artifact controlFile = intermediateArtifacts.pbxprojControlArtifact();
+ register(new BinaryFileWriteAction(
+ context.getActionOwner(),
+ controlFile,
+ xcodegenControlFileBytes(pbxproj, project, objcConfiguration.getMinimumOs()),
+ /*makeExecutable=*/false));
+ register(new SpawnAction.Builder()
+ .setMnemonic("GenerateXcodeproj")
+ .setExecutable(baseTools.xcodegen())
+ .addArgument("--control")
+ .addInputArgument(controlFile)
+ .addOutput(pbxproj)
+ .addTransitiveInputs(project.getInputsToXcodegen())
+ .build(context));
+ }
+
+ /**
+ * Creates actions to convert all files specified by the strings attribute into binary format.
+ */
+ private static Iterable<Action> convertStringsActions(
+ ActionConstructionContext context,
+ ObjcRuleClasses.Tools baseTools,
+ StringsFiles stringsFiles) {
+ ImmutableList.Builder<Action> result = new ImmutableList.Builder<>();
+ for (CompiledResourceFile stringsFile : stringsFiles) {
+ final Artifact original = stringsFile.getOriginal();
+ final Artifact bundled = stringsFile.getBundled().getBundled();
+ result.add(new SpawnAction.Builder()
+ .setMnemonic("ConvertStringsPlist")
+ .setExecutable(baseTools.plmerge())
+ .setCommandLine(new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return ImmutableList.of("--source_file", original.getExecPathString(),
+ "--out_file", bundled.getExecPathString());
+ }
+ })
+ .addInput(original)
+ .addOutput(bundled)
+ .build(context));
+ }
+ return result.build();
+ }
+
+ private Action[] ibtoolzipAction(ObjcRuleClasses.Tools baseTools, String mnemonic, Artifact input,
+ Artifact zipOutput, String archiveRoot) {
+ return spawnJavaOnDarwinActionBuilder(baseTools.actooloribtoolzipDeployJar())
+ .setMnemonic(mnemonic)
+ .setCommandLine(new CustomCommandLine.Builder()
+ // The next three arguments are positional, i.e. they don't have flags before them.
+ .addPath(zipOutput.getExecPath())
+ .add(archiveRoot)
+ .addPath(IBTOOL)
+
+ .add("--minimum-deployment-target").add(objcConfiguration.getMinimumOs())
+ .addPath(input.getExecPath())
+ .build())
+ .addOutput(zipOutput)
+ .addInput(input)
+ .build(context);
+ }
+
+ /**
+ * Creates actions to convert all files specified by the xibs attribute into nib format.
+ */
+ private Iterable<Action> convertXibsActions(ObjcRuleClasses.Tools baseTools, XibFiles xibFiles) {
+ ImmutableList.Builder<Action> result = new ImmutableList.Builder<>();
+ for (Artifact original : xibFiles) {
+ Artifact zipOutput = intermediateArtifacts.compiledXibFileZip(original);
+ String archiveRoot = BundleableFile.bundlePath(
+ FileSystemUtils.replaceExtension(original.getExecPath(), ".nib"));
+ result.add(ibtoolzipAction(baseTools, "XibCompile", original, zipOutput, archiveRoot));
+ }
+ return result.build();
+ }
+
+ /**
+ * Outputs of an {@code actool} action besides the zip file.
+ */
+ static final class ExtraActoolOutputs extends IterableWrapper<Artifact> {
+ ExtraActoolOutputs(Artifact... extraActoolOutputs) {
+ super(extraActoolOutputs);
+ }
+ }
+
+ static final class ExtraActoolArgs extends IterableWrapper<String> {
+ ExtraActoolArgs(Iterable<String> args) {
+ super(args);
+ }
+
+ ExtraActoolArgs(String... args) {
+ super(args);
+ }
+ }
+
+ void registerActoolzipAction(
+ ObjcRuleClasses.Tools tools,
+ ObjcProvider provider,
+ Artifact zipOutput,
+ ExtraActoolOutputs extraActoolOutputs,
+ ExtraActoolArgs extraActoolArgs,
+ Set<TargetDeviceFamily> families) {
+ // TODO(bazel-team): Do not use the deploy jar explicitly here. There is currently a bug where
+ // we cannot .setExecutable({java_binary target}) and set REQUIRES_DARWIN in the execution info.
+ // Note that below we set the archive root to the empty string. This means that the generated
+ // zip file will be rooted at the bundle root, and we have to prepend the bundle root to each
+ // entry when merging it with the final .ipa file.
+ register(spawnJavaOnDarwinActionBuilder(tools.actooloribtoolzipDeployJar())
+ .setMnemonic("AssetCatalogCompile")
+ .addTransitiveInputs(provider.get(ASSET_CATALOG))
+ .addOutput(zipOutput)
+ .addOutputs(extraActoolOutputs)
+ .setCommandLine(actoolzipCommandLine(
+ objcConfiguration,
+ provider,
+ zipOutput,
+ extraActoolArgs,
+ ImmutableSet.copyOf(families)))
+ .build(context));
+ }
+
+ private static CommandLine actoolzipCommandLine(
+ final ObjcConfiguration objcConfiguration,
+ final ObjcProvider provider,
+ final Artifact zipOutput,
+ final ExtraActoolArgs extraActoolArgs,
+ final ImmutableSet<TargetDeviceFamily> families) {
+ return new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ ImmutableList.Builder<String> args = new ImmutableList.Builder<String>()
+ // The next three arguments are positional, i.e. they don't have flags before them.
+ .add(zipOutput.getExecPathString())
+ .add("") // archive root
+ .add(IosSdkCommands.ACTOOL_PATH)
+ .add("--platform")
+ .add(objcConfiguration.getPlatform().getLowerCaseNameInPlist())
+ .add("--minimum-deployment-target").add(objcConfiguration.getMinimumOs());
+ for (TargetDeviceFamily targetDeviceFamily : families) {
+ args.add("--target-device").add(targetDeviceFamily.name().toLowerCase(Locale.US));
+ }
+ return args
+ .addAll(PathFragment.safePathStrings(provider.get(XCASSETS_DIR)))
+ .addAll(extraActoolArgs)
+ .build();
+ }
+ };
+ }
+
+ void registerIbtoolzipAction(ObjcRuleClasses.Tools tools, Artifact input, Artifact outputZip) {
+ String archiveRoot = BundleableFile.bundlePath(input.getExecPath()) + "c";
+ register(ibtoolzipAction(tools, "StoryboardCompile", input, outputZip, archiveRoot));
+ }
+
+ @VisibleForTesting
+ static Iterable<String> commonMomczipArguments(ObjcConfiguration configuration) {
+ return ImmutableList.of(
+ "-XD_MOMC_SDKROOT=" + IosSdkCommands.sdkDir(configuration),
+ "-XD_MOMC_IOS_TARGET_VERSION=" + configuration.getMinimumOs(),
+ "-MOMC_PLATFORMS", configuration.getPlatform().getLowerCaseNameInPlist(),
+ "-XD_MOMC_TARGET_VERSION=10.6");
+ }
+
+ private static Iterable<Action> momczipActions(ActionConstructionContext context,
+ ObjcRuleClasses.Tools baseTools, final ObjcConfiguration objcConfiguration,
+ Iterable<Xcdatamodel> datamodels) {
+ ImmutableList.Builder<Action> result = new ImmutableList.Builder<>();
+ for (Xcdatamodel datamodel : datamodels) {
+ final Artifact outputZip = datamodel.getOutputZip();
+ final String archiveRoot = datamodel.archiveRootForMomczip();
+ final String container = datamodel.getContainer().getSafePathString();
+ result.add(spawnJavaOnDarwinActionBuilder(baseTools.momczipDeployJar())
+ .setMnemonic("MomCompile")
+ .addOutput(outputZip)
+ .addInputs(datamodel.getInputs())
+ .setCommandLine(new CommandLine() {
+ @Override
+ public Iterable<String> arguments() {
+ return new ImmutableList.Builder<String>()
+ .add(outputZip.getExecPathString())
+ .add(archiveRoot)
+ .add(IosSdkCommands.MOMC_PATH)
+ .addAll(commonMomczipArguments(objcConfiguration))
+ .add(container)
+ .build();
+ }
+ })
+ .build(context));
+ }
+ return result.build();
+ }
+
+ private static final String FRAMEWORK_SUFFIX = ".framework";
+
+ /**
+ * All framework names to pass to the linker using {@code -framework} flags. For a framework in
+ * the directory foo/bar.framework, the name is "bar". Each framework is found without using the
+ * full path by means of the framework search paths. The search paths are added by
+ * {@link IosSdkCommands#commonLinkAndCompileArgsForClang(ObjcProvider, ObjcConfiguration)}).
+ *
+ * <p>It's awful that we can't pass the full path to the framework and avoid framework search
+ * paths, but this is imposed on us by clang. clang does not support passing the full path to the
+ * framework, so Bazel cannot do it either.
+ */
+ private static Iterable<String> frameworkNames(ObjcProvider provider) {
+ List<String> names = new ArrayList<>();
+ Iterables.addAll(names, SdkFramework.names(provider.get(SDK_FRAMEWORK)));
+ for (PathFragment frameworkDir : provider.get(FRAMEWORK_DIR)) {
+ String segment = frameworkDir.getBaseName();
+ Preconditions.checkState(segment.endsWith(FRAMEWORK_SUFFIX),
+ "expect %s to end with %s, but it does not", segment, FRAMEWORK_SUFFIX);
+ names.add(segment.substring(0, segment.length() - FRAMEWORK_SUFFIX.length()));
+ }
+ return names;
+ }
+
+ static final class ExtraLinkArgs extends IterableWrapper<String> {
+ ExtraLinkArgs(Iterable<String> args) {
+ super(args);
+ }
+
+ ExtraLinkArgs(String... args) {
+ super(args);
+ }
+ }
+
+ static final class ExtraLinkInputs extends IterableWrapper<Artifact> {
+ ExtraLinkInputs(Iterable<Artifact> inputs) {
+ super(inputs);
+ }
+
+ ExtraLinkInputs(Artifact... inputs) {
+ super(inputs);
+ }
+ }
+
+ private static final class LinkCommandLine extends CommandLine {
+ private static final Joiner commandJoiner = Joiner.on(' ');
+ private final ObjcProvider objcProvider;
+ private final ObjcConfiguration objcConfiguration;
+ private final Artifact linkedBinary;
+ private final Optional<Artifact> dsymBundle;
+ private final ExtraLinkArgs extraLinkArgs;
+
+ LinkCommandLine(ObjcConfiguration objcConfiguration, ExtraLinkArgs extraLinkArgs,
+ ObjcProvider objcProvider, Artifact linkedBinary, Optional<Artifact> dsymBundle) {
+ this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration);
+ this.extraLinkArgs = Preconditions.checkNotNull(extraLinkArgs);
+ this.objcProvider = Preconditions.checkNotNull(objcProvider);
+ this.linkedBinary = Preconditions.checkNotNull(linkedBinary);
+ this.dsymBundle = Preconditions.checkNotNull(dsymBundle);
+ }
+
+ Iterable<String> dylibPaths() {
+ ImmutableList.Builder<String> args = new ImmutableList.Builder<>();
+ for (String dylib : objcProvider.get(SDK_DYLIB)) {
+ args.add(String.format(
+ "%s/usr/lib/%s.dylib", IosSdkCommands.sdkDir(objcConfiguration), dylib));
+ }
+ return args.build();
+ }
+
+ @Override
+ public Iterable<String> arguments() {
+ StringBuilder argumentStringBuilder = new StringBuilder();
+
+ Iterable<String> archiveExecPaths = Artifact.toExecPaths(
+ Iterables.concat(objcProvider.get(LIBRARY), objcProvider.get(IMPORTED_LIBRARY)));
+ commandJoiner.appendTo(argumentStringBuilder, new ImmutableList.Builder<String>()
+ .add(objcProvider.is(USES_CPP) ? CLANG_PLUSPLUS.toString() : CLANG.toString())
+ .addAll(objcProvider.is(USES_CPP)
+ ? ImmutableList.of("-stdlib=libc++") : ImmutableList.<String>of())
+ .addAll(IosSdkCommands.commonLinkAndCompileArgsForClang(objcProvider, objcConfiguration))
+ .add("-Xlinker", "-objc_abi_version")
+ .add("-Xlinker", "2")
+ .add("-fobjc-link-runtime")
+ .addAll(IosSdkCommands.DEFAULT_LINKER_FLAGS)
+ .addAll(Interspersing.beforeEach("-framework", frameworkNames(objcProvider)))
+ .addAll(Interspersing.beforeEach(
+ "-weak_framework", SdkFramework.names(objcProvider.get(WEAK_SDK_FRAMEWORK))))
+ .add("-o", linkedBinary.getExecPathString())
+ .addAll(archiveExecPaths)
+ .addAll(dylibPaths())
+ .addAll(extraLinkArgs)
+ .build());
+
+ // Call to dsymutil for debug symbol generation must happen in the link action.
+ // All debug symbol information is encoded in object files inside archive files. To generate
+ // the debug symbol bundle, dsymutil will look inside the linked binary for the encoded
+ // absolute paths to archive files, which are only valid in the link action.
+ for (Artifact justDsymBundle : dsymBundle.asSet()) {
+ argumentStringBuilder.append(" ");
+ commandJoiner.appendTo(argumentStringBuilder, new ImmutableList.Builder<String>()
+ .add("&&")
+ .add(DSYMUTIL.toString())
+ .add(linkedBinary.getExecPathString())
+ .add("-o").add(justDsymBundle.getExecPathString())
+ .build());
+ }
+
+ return ImmutableList.of(argumentStringBuilder.toString());
+ }
+ }
+
+ /**
+ * Generates an action to link a binary.
+ */
+ void registerLinkAction(Artifact linkedBinary, ObjcProvider objcProvider,
+ ExtraLinkArgs extraLinkArgs, ExtraLinkInputs extraLinkInputs, Optional<Artifact> dsymBundle) {
+ extraLinkArgs = new ExtraLinkArgs(Iterables.concat(
+ Interspersing.beforeEach(
+ "-force_load", Artifact.toExecPaths(objcProvider.get(FORCE_LOAD_LIBRARY))),
+ extraLinkArgs));
+ register(spawnOnDarwinActionBuilder()
+ .setMnemonic("ObjcLink")
+ .setShellCommand(ImmutableList.of("/bin/bash", "-c"))
+ .setCommandLine(
+ new LinkCommandLine(objcConfiguration, extraLinkArgs, objcProvider, linkedBinary,
+ dsymBundle))
+ .addOutput(linkedBinary)
+ .addOutputs(dsymBundle.asSet())
+ .addTransitiveInputs(objcProvider.get(LIBRARY))
+ .addTransitiveInputs(objcProvider.get(IMPORTED_LIBRARY))
+ .addTransitiveInputs(objcProvider.get(FRAMEWORK_FILE))
+ .addInputs(extraLinkInputs)
+ .build(context));
+ }
+
+ static final class StringsFiles extends IterableWrapper<CompiledResourceFile> {
+ StringsFiles(Iterable<CompiledResourceFile> files) {
+ super(files);
+ }
+ }
+
+ /**
+ * Registers actions for resource conversion that are needed by all rules that inherit from
+ * {@link ObjcBase}.
+ */
+ void registerResourceActions(ObjcRuleClasses.Tools baseTools, StringsFiles stringsFiles,
+ XibFiles xibFiles, Iterable<Xcdatamodel> datamodels) {
+ registerAll(convertStringsActions(context, baseTools, stringsFiles));
+ registerAll(convertXibsActions(baseTools, xibFiles));
+ registerAll(momczipActions(context, baseTools, objcConfiguration, datamodels));
+ }
+
+ static LazyString joinExecPaths(final Iterable<Artifact> artifacts) {
+ return new LazyString() {
+ @Override
+ public String toString() {
+ return Artifact.joinExecPaths("\n", artifacts);
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java
new file mode 100644
index 0000000..52c897f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java
@@ -0,0 +1,147 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.XcodeProductType.APPLICATION;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.objc.ApplicationSupport.LinkedBinary;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs;
+import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes;
+
+/**
+ * Implementation for the "objc_binary" rule.
+ */
+public class ObjcBinary implements RuleConfiguredTargetFactory {
+
+ @VisibleForTesting
+ static final String REQUIRES_AT_LEAST_ONE_LIBRARY_OR_SOURCE_FILE =
+ "At least one library dependency or source file is required.";
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ ObjcCommon common = common(ruleContext);
+ OptionsProvider optionsProvider = optionsProvider(ruleContext);
+
+ ObjcProvider objcProvider = common.getObjcProvider();
+ if (!hasLibraryOrSources(objcProvider)) {
+ ruleContext.ruleError(REQUIRES_AT_LEAST_ONE_LIBRARY_OR_SOURCE_FILE);
+ return null;
+ }
+
+ XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder();
+ NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder();
+
+ new CompilationSupport(ruleContext)
+ .registerJ2ObjcCompileAndArchiveActions(optionsProvider, common.getObjcProvider())
+ .registerCompileAndArchiveActions(common, optionsProvider)
+ .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider)
+ .registerLinkActions(common.getObjcProvider(), new ExtraLinkArgs(), new ExtraLinkInputs())
+ .validateAttributes();
+
+ // TODO(bazel-team): Remove once all bundle users are migrated to ios_application.
+ ApplicationSupport applicationSupport = new ApplicationSupport(
+ ruleContext, common.getObjcProvider(), optionsProvider, LinkedBinary.LOCAL_AND_DEPENDENCIES)
+ .registerActions()
+ .addXcodeSettings(xcodeProviderBuilder)
+ .addFilesToBuild(filesToBuild)
+ .validateAttributes();
+
+ new ResourceSupport(ruleContext)
+ .registerActions(common.getStoryboards())
+ .validateAttributes()
+ .addXcodeSettings(xcodeProviderBuilder);
+
+ XcodeSupport xcodeSupport = new XcodeSupport(ruleContext)
+ // TODO(bazel-team): Use LIBRARY_STATIC as parameter instead of APPLICATION once objc_binary
+ // no longer creates an application bundle
+ .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), APPLICATION)
+ .addDependencies(xcodeProviderBuilder)
+ .addFilesToBuild(filesToBuild);
+ XcodeProvider xcodeProvider = xcodeProviderBuilder.build();
+ xcodeSupport.registerActions(xcodeProvider);
+
+ // TODO(bazel-team): Stop exporting an XcTestAppProvider once objc_binary no longer creates an
+ // application bundle.
+ return common.configuredTarget(
+ filesToBuild.build(),
+ Optional.of(xcodeProvider),
+ Optional.<ObjcProvider>absent(),
+ Optional.of(applicationSupport.xcTestAppProvider()),
+ Optional.<J2ObjcSrcsProvider>absent());
+ }
+
+ private OptionsProvider optionsProvider(RuleContext ruleContext) {
+ return new OptionsProvider.Builder()
+ .addCopts(ruleContext.getTokenizedStringListAttr("copts"))
+ .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list())
+ .addTransitive(Optional.fromNullable(
+ ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class)))
+ .build();
+ }
+
+ private boolean hasLibraryOrSources(ObjcProvider objcProvider) {
+ return !Iterables.isEmpty(objcProvider.get(LIBRARY)) // Includes sources from this target.
+ || !Iterables.isEmpty(objcProvider.get(IMPORTED_LIBRARY));
+ }
+
+ private ObjcCommon common(RuleContext ruleContext) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ CompilationArtifacts compilationArtifacts =
+ CompilationSupport.compilationArtifacts(ruleContext);
+
+ return new ObjcCommon.Builder(ruleContext)
+ .setCompilationAttributes(new CompilationAttributes(ruleContext))
+ .setResourceAttributes(new ResourceAttributes(ruleContext))
+ .setCompilationArtifacts(compilationArtifacts)
+ .addDepObjcProviders(ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProvider.class))
+ .addDepObjcProviders(
+ ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class))
+ .addNonPropagatedDepObjcProviders(
+ ruleContext.getPrerequisites("non_propagated_deps", Mode.TARGET, ObjcProvider.class))
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .setAlwayslink(false)
+ .addExtraImportLibraries(j2ObjcLibraries(ruleContext))
+ .setLinkedBinary(intermediateArtifacts.singleArchitectureBinary())
+ .build();
+ }
+
+ private Iterable<Artifact> j2ObjcLibraries(RuleContext ruleContext) {
+ J2ObjcSrcsProvider j2ObjcSrcsProvider = ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext);
+ ImmutableList.Builder<Artifact> j2objcLibraries = ImmutableList.builder();
+
+ // TODO(bazel-team): Refactor the code to stop flattening the nested set here.
+ for (J2ObjcSource j2ObjcSource : j2ObjcSrcsProvider.getSrcs()) {
+ j2objcLibraries.add(
+ ObjcRuleClasses.j2objcIntermediateArtifacts(ruleContext, j2ObjcSource).archive());
+ }
+
+ return j2objcLibraries.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java
new file mode 100644
index 0000000..c9fc57e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for objc_binary.
+ */
+@BlazeRule(name = "objc_binary",
+ factoryClass = ObjcBinary.class,
+ ancestors = { ObjcLibraryRule.class, IosApplicationRule.class })
+public class ObjcBinaryRule implements RuleDefinition {
+
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ // TODO(bazel-team): Remove bundling functionality (dependency on IosApplicationRule).
+ /*<!-- #BLAZE_RULE(objc_binary).IMPLICIT_OUTPUTS -->
+ <ul>
+ <li><code><var>name</var>.ipa</code>: the application bundle as an <code>.ipa</code>
+ file</li>
+ <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which
+ can be used to develop or build on a Mac.</li>
+ </ul>
+ <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/
+ .setImplicitOutputsFunction(
+ ImplicitOutputsFunction.fromFunctions(ApplicationSupport.IPA, XcodeSupport.PBXPROJ))
+ .removeAttribute("binary")
+ .removeAttribute("alwayslink")
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_binary, TYPE = BINARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule produces an application bundle by linking one or more Objective-C libraries.</p>
+
+${IMPLICIT_OUTPUTS}
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java
new file mode 100644
index 0000000..6585cba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for {@code objc_bundle}.
+ */
+public class ObjcBundle implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ ObjcCommon common = new ObjcCommon.Builder(ruleContext).build();
+
+ ImmutableList<Artifact> bundleImports = ruleContext
+ .getPrerequisiteArtifacts("bundle_imports", Mode.TARGET).list();
+ Iterable<String> bundleImportErrors =
+ ObjcCommon.notInContainerErrors(bundleImports, ObjcCommon.BUNDLE_CONTAINER_TYPE);
+ for (String error : bundleImportErrors) {
+ ruleContext.attributeError("bundle_imports", error);
+ }
+
+ return common.configuredTarget(
+ /*filesToBuild=*/NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER),
+ Optional.<XcodeProvider>absent(),
+ Optional.of(common.getObjcProvider()),
+ Optional.<XcTestAppProvider>absent(),
+ Optional.<J2ObjcSrcsProvider>absent());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java
new file mode 100644
index 0000000..0e7f6b0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE;
+import static com.google.devtools.build.lib.rules.objc.XcodeProductType.BUNDLE;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+
+/**
+ * Implementation for {@code objc_bundle_library}.
+ */
+public class ObjcBundleLibrary implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ ObjcCommon common = common(ruleContext);
+ OptionsProvider optionsProvider = optionsProvider(ruleContext);
+
+ Bundling bundling = bundling(ruleContext, common, optionsProvider);
+
+ XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder();
+ NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder();
+
+ // TODO(bazel-team): Figure out if the target device is important, and what to set it to. It may
+ // have to inherit this from the binary being built. As of this writing, this is only used for
+ // asset catalogs compilation (actool).
+ new BundleSupport(ruleContext, ImmutableSet.of(TargetDeviceFamily.IPHONE), bundling)
+ .registerActions(common.getObjcProvider())
+ .addXcodeSettings(xcodeProviderBuilder);
+
+ new ResourceSupport(ruleContext)
+ .validateAttributes()
+ .registerActions(common.getStoryboards())
+ .addXcodeSettings(xcodeProviderBuilder);
+
+ new XcodeSupport(ruleContext)
+ .addFilesToBuild(filesToBuild)
+ .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), BUNDLE)
+ .registerActions(xcodeProviderBuilder.build());
+
+ ObjcProvider nestedBundleProvider = new ObjcProvider.Builder()
+ .add(NESTED_BUNDLE, bundling)
+ .build();
+
+ return common.configuredTarget(
+ filesToBuild.build(),
+ Optional.of(xcodeProviderBuilder.build()),
+ Optional.of(nestedBundleProvider),
+ Optional.<XcTestAppProvider>absent(),
+ Optional.<J2ObjcSrcsProvider>absent());
+ }
+
+ private OptionsProvider optionsProvider(RuleContext ruleContext) {
+ return new OptionsProvider.Builder()
+ .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list())
+ .build();
+ }
+
+ private Bundling bundling(
+ RuleContext ruleContext, ObjcCommon common, OptionsProvider optionsProvider) {
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ return new Bundling.Builder()
+ .setName(ruleContext.getLabel().getName())
+ .setBundleDirSuffix(".bundle")
+ .setObjcProvider(common.getObjcProvider())
+ .setInfoplistMerging(
+ BundleSupport.infoPlistMerging(ruleContext, common.getObjcProvider(), optionsProvider))
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .build();
+ }
+
+ private ObjcCommon common(RuleContext ruleContext) {
+ return new ObjcCommon.Builder(ruleContext)
+ .setResourceAttributes(new ResourceAttributes(ruleContext))
+ .addDepObjcProviders(
+ ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class))
+ .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java
new file mode 100644
index 0000000..b9405a6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for objc_bundle_library.
+ */
+@BlazeRule(name = "objc_bundle_library",
+ factoryClass = ObjcBundleLibrary.class,
+ ancestors = { ObjcRuleClasses.ObjcBaseResourcesRule.class,
+ ObjcRuleClasses.ObjcHasInfoplistRule.class })
+public class ObjcBundleLibraryRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /*<!-- #BLAZE_RULE(objc_bundle_library).IMPLICIT_OUTPUTS -->
+ <ul>
+ <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which
+ can be used to develop or build on a Mac.</li>
+ </ul>
+ <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/
+ .setImplicitOutputsFunction(ImplicitOutputsFunction.fromFunctions(XcodeSupport.PBXPROJ))
+ /* <!-- #BLAZE_RULE(objc_bundle_library).ATTRIBUTE(bundles) -->
+ The list of bundle targets that this target requires to be included in the final bundle.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("bundles", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedRuleClasses("objc_bundle", "objc_bundle_library")
+ .allowedFileTypes())
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_bundle_library, TYPE = LIBRARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule encapsulates a library which is provided to dependers as a bundle.
+A <code>objc_bundle_library</code>'s resources are put in a nested bundle in
+the final iOS application.
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java
new file mode 100644
index 0000000..9be0d04
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+/**
+ * Rule definition for objc_bundle.
+ */
+@BlazeRule(name = "objc_bundle",
+ factoryClass = ObjcBundle.class,
+ ancestors = { BaseRuleClasses.BaseRule.class })
+public class ObjcBundleRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(objc_bundle).ATTRIBUTE(bundle_imports) -->
+ The list of files under a <code>.bundle</code> directory which are
+ provided to Objective-C targets that depend on this target.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("bundle_imports", LABEL_LIST)
+ .allowedFileTypes(FileTypeSet.ANY_FILE)
+ .nonEmpty())
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_bundle, TYPE = LIBRARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule encapsulates an already-built bundle. It is defined by a list of
+files in one or more <code>.bundle</code> directories.
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java
new file mode 100644
index 0000000..f85609a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.xcode.common.BuildOptionsUtil.DEFAULT_OPTIONS_NAME;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.analysis.config.FragmentOptions;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.common.options.Option;
+
+import java.util.List;
+
+/**
+ * Command-line options for building Objective-C targets.
+ */
+public class
+ ObjcCommandLineOptions extends FragmentOptions {
+ @Option(name = "ios_sdk_version",
+ defaultValue = DEFAULT_SDK_VERSION,
+ category = "undocumented",
+ help = "Specifies the version of the iOS SDK to use to build iOS applications."
+ )
+ public String iosSdkVersion;
+
+ @VisibleForTesting static final String DEFAULT_SDK_VERSION = "8.1";
+
+ @Option(name = "ios_simulator_version",
+ defaultValue = "7.1",
+ category = "undocumented",
+ help = "The version of iOS to run on the simulator when running tests. This is ignored if the"
+ + " ios_test rule specifies the target device.",
+ deprecationWarning = "This flag is deprecated in favor of the target_device attribute and"
+ + " will eventually removed.")
+ public String iosSimulatorVersion;
+
+ @Option(name = "ios_cpu",
+ defaultValue = "i386",
+ category = "undocumented",
+ help = "Specifies to target CPU of iOS compilation.")
+ public String iosCpu;
+
+ @Option(name = "xcode_options",
+ defaultValue = DEFAULT_OPTIONS_NAME,
+ category = "undocumented",
+ help = "Specifies the name of the build settings to use.")
+ public String xcodeOptions;
+
+ @Option(name = "objc_generate_debug_symbols",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "Specifies whether to generate debug symbol(.dSYM) file.")
+ public boolean generateDebugSymbols;
+
+ @Option(name = "objccopt",
+ allowMultiple = true,
+ defaultValue = "",
+ category = "flags",
+ help = "Additional options to pass to Objective C compilation.")
+ public List<String> copts;
+
+ @Option(name = "ios_minimum_os",
+ defaultValue = DEFAULT_MINIMUM_IOS,
+ category = "flags",
+ help = "Minimum compatible iOS version for target simulators and devices.")
+ public String iosMinimumOs;
+
+ @VisibleForTesting static final String DEFAULT_MINIMUM_IOS = "7.0";
+
+ @Override
+ public void addAllLabels(Multimap<String, Label> labelMap) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java
new file mode 100644
index 0000000..ade86ee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java
@@ -0,0 +1,620 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_IMPORT_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FLAG;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_FOR_XCODEGEN;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag.USES_CPP;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.GENERAL_RESOURCE_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.HEADER;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.INCLUDE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LINKED_BINARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.MERGE_ZIP;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL;
+import static com.google.devtools.build.lib.vfs.PathFragment.TO_PATH_FRAGMENT;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.cpp.CcCommon;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.util.Interspersing;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Contains information common to multiple objc_* rules, and provides a unified API for extracting
+ * and accessing it.
+ */
+// TODO(bazel-team): Decompose and subsume area-specific logic and data into the various *Support
+// classes. Make sure to distinguish rule output (providers, runfiles, ...) from intermediate,
+// rule-internal information. Any provider created by a rule should not be read, only published.
+public final class ObjcCommon {
+ /**
+ * Provides a way to access attributes that are common to all compilation rules that inherit from
+ * {@link ObjcRuleClasses.ObjcCompilationRule}.
+ */
+ // TODO(bazel-team): Delete and move into support-specific attributes classes once ObjcCommon is
+ // gone.
+ static final class CompilationAttributes {
+ private final RuleContext ruleContext;
+ private final ObjcSdkFrameworks.Attributes sdkFrameworkAttributes;
+
+ CompilationAttributes(RuleContext ruleContext) {
+ this.ruleContext = Preconditions.checkNotNull(ruleContext);
+ this.sdkFrameworkAttributes = new ObjcSdkFrameworks.Attributes(ruleContext);
+ }
+
+ ImmutableList<Artifact> hdrs() {
+ return ImmutableList.copyOf(CcCommon.getHeaders(ruleContext));
+ }
+
+ Iterable<PathFragment> includes() {
+ return Iterables.transform(
+ ruleContext.attributes().get("includes", Type.STRING_LIST),
+ PathFragment.TO_PATH_FRAGMENT);
+ }
+
+ Iterable<PathFragment> sdkIncludes() {
+ return Iterables.transform(
+ ruleContext.attributes().get("sdk_includes", Type.STRING_LIST),
+ PathFragment.TO_PATH_FRAGMENT);
+ }
+
+ /**
+ * Returns the value of the sdk_frameworks attribute plus frameworks that are included
+ * automatically.
+ */
+ ImmutableSet<SdkFramework> sdkFrameworks() {
+ return sdkFrameworkAttributes.sdkFrameworks();
+ }
+
+ /**
+ * Returns the value of the weak_sdk_frameworks attribute.
+ */
+ ImmutableSet<SdkFramework> weakSdkFrameworks() {
+ return sdkFrameworkAttributes.weakSdkFrameworks();
+ }
+
+ /**
+ * Returns the value of the sdk_dylibs attribute.
+ */
+ ImmutableSet<String> sdkDylibs() {
+ return sdkFrameworkAttributes.sdkDylibs();
+ }
+
+ /**
+ * Returns the exec paths of all header search paths that should be added to this target and
+ * dependers on this target, obtained from the {@code includes} attribute.
+ */
+ ImmutableList<PathFragment> headerSearchPaths() {
+ ImmutableList.Builder<PathFragment> paths = new ImmutableList.Builder<>();
+ PathFragment packageFragment = ruleContext.getLabel().getPackageFragment();
+ List<PathFragment> rootFragments = ImmutableList.of(
+ packageFragment,
+ ruleContext.getConfiguration().getGenfilesFragment().getRelative(packageFragment));
+
+ Iterable<PathFragment> relativeIncludes =
+ Iterables.filter(includes(), Predicates.not(PathFragment.IS_ABSOLUTE));
+ for (PathFragment include : relativeIncludes) {
+ for (PathFragment rootFragment : rootFragments) {
+ paths.add(rootFragment.getRelative(include).normalize());
+ }
+ }
+ return paths.build();
+ }
+ }
+
+ /**
+ * Provides a way to access attributes that are common to all resources rules that inherit from
+ * {@link ObjcRuleClasses.ObjcBaseResourcesRule}.
+ */
+ // TODO(bazel-team): Delete and move into support-specific attributes classes once ObjcCommon is
+ // gone.
+ static final class ResourceAttributes {
+ private final RuleContext ruleContext;
+
+ ResourceAttributes(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ ImmutableList<Artifact> strings() {
+ return ruleContext.getPrerequisiteArtifacts("strings", Mode.TARGET).list();
+ }
+
+ ImmutableList<Artifact> xibs() {
+ return ruleContext.getPrerequisiteArtifacts("xibs", Mode.TARGET)
+ .errorsForNonMatching(ObjcRuleClasses.XIB_TYPE)
+ .list();
+ }
+
+ ImmutableList<Artifact> storyboards() {
+ return ruleContext.getPrerequisiteArtifacts("storyboards", Mode.TARGET).list();
+ }
+
+ ImmutableList<Artifact> resources() {
+ return ruleContext.getPrerequisiteArtifacts("resources", Mode.TARGET).list();
+ }
+
+ ImmutableList<Artifact> datamodels() {
+ return ruleContext.getPrerequisiteArtifacts("datamodels", Mode.TARGET).list();
+ }
+
+ ImmutableList<Artifact> assetCatalogs() {
+ return ruleContext.getPrerequisiteArtifacts("asset_catalogs", Mode.TARGET).list();
+ }
+ }
+
+ static class Builder {
+ private RuleContext context;
+ private Optional<CompilationAttributes> compilationAttributes = Optional.absent();
+ private Optional<ResourceAttributes> resourceAttributes = Optional.absent();
+ private Iterable<SdkFramework> extraSdkFrameworks = ImmutableList.of();
+ private Iterable<SdkFramework> extraWeakSdkFrameworks = ImmutableList.of();
+ private Iterable<String> extraSdkDylibs = ImmutableList.of();
+ private Iterable<Artifact> frameworkImports = ImmutableList.of();
+ private Optional<CompilationArtifacts> compilationArtifacts = Optional.absent();
+ private Iterable<ObjcProvider> depObjcProviders = ImmutableList.of();
+ private Iterable<ObjcProvider> directDepObjcProviders = ImmutableList.of();
+ private Iterable<String> defines = ImmutableList.of();
+ private Iterable<PathFragment> userHeaderSearchPaths = ImmutableList.of();
+ private Iterable<Artifact> headers = ImmutableList.of();
+ private IntermediateArtifacts intermediateArtifacts;
+ private boolean alwayslink;
+ private Iterable<Artifact> extraImportLibraries = ImmutableList.of();
+ private Optional<Artifact> linkedBinary = Optional.absent();
+
+ Builder(RuleContext context) {
+ this.context = Preconditions.checkNotNull(context);
+ }
+
+ public Builder setCompilationAttributes(CompilationAttributes baseCompilationAttributes) {
+ Preconditions.checkState(!this.compilationAttributes.isPresent(),
+ "compilationAttributes is already set to: %s", this.compilationAttributes);
+ this.compilationAttributes = Optional.of(baseCompilationAttributes);
+ return this;
+ }
+
+ public Builder setResourceAttributes(ResourceAttributes baseResourceAttributes) {
+ Preconditions.checkState(!this.resourceAttributes.isPresent(),
+ "resourceAttributes is already set to: %s", this.resourceAttributes);
+ this.resourceAttributes = Optional.of(baseResourceAttributes);
+ return this;
+ }
+
+ Builder addExtraSdkFrameworks(Iterable<SdkFramework> extraSdkFrameworks) {
+ this.extraSdkFrameworks = Iterables.concat(this.extraSdkFrameworks, extraSdkFrameworks);
+ return this;
+ }
+
+ Builder addExtraWeakSdkFrameworks(Iterable<SdkFramework> extraWeakSdkFrameworks) {
+ this.extraWeakSdkFrameworks =
+ Iterables.concat(this.extraWeakSdkFrameworks, extraWeakSdkFrameworks);
+ return this;
+ }
+
+ Builder addExtraSdkDylibs(Iterable<String> extraSdkDylibs) {
+ this.extraSdkDylibs = Iterables.concat(this.extraSdkDylibs, extraSdkDylibs);
+ return this;
+ }
+
+ Builder addFrameworkImports(Iterable<Artifact> frameworkImports) {
+ this.frameworkImports = Iterables.concat(this.frameworkImports, frameworkImports);
+ return this;
+ }
+
+ Builder setCompilationArtifacts(CompilationArtifacts compilationArtifacts) {
+ Preconditions.checkState(!this.compilationArtifacts.isPresent(),
+ "compilationArtifacts is already set to: %s", this.compilationArtifacts);
+ this.compilationArtifacts = Optional.of(compilationArtifacts);
+ return this;
+ }
+
+ /**
+ * Add providers which will be exposed both to the declaring rule and to any dependers on the
+ * declaring rule.
+ */
+ Builder addDepObjcProviders(Iterable<ObjcProvider> depObjcProviders) {
+ this.depObjcProviders = Iterables.concat(this.depObjcProviders, depObjcProviders);
+ return this;
+ }
+
+ /**
+ * Add providers which will only be used by the declaring rule, and won't be propagated to any
+ * dependers on the declaring rule.
+ */
+ Builder addNonPropagatedDepObjcProviders(Iterable<ObjcProvider> directDepObjcProviders) {
+ this.directDepObjcProviders = Iterables.concat(
+ this.directDepObjcProviders, directDepObjcProviders);
+ return this;
+ }
+
+ public Builder addUserHeaderSearchPaths(Iterable<PathFragment> userHeaderSearchPaths) {
+ this.userHeaderSearchPaths =
+ Iterables.concat(this.userHeaderSearchPaths, userHeaderSearchPaths);
+ return this;
+ }
+
+ public Builder addDefines(Iterable<String> defines) {
+ this.defines = Iterables.concat(this.defines, defines);
+ return this;
+ }
+
+ public Builder addHeaders(Iterable<Artifact> headers) {
+ this.headers = Iterables.concat(this.headers, headers);
+ return this;
+ }
+
+ Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) {
+ this.intermediateArtifacts = intermediateArtifacts;
+ return this;
+ }
+
+ Builder setAlwayslink(boolean alwayslink) {
+ this.alwayslink = alwayslink;
+ return this;
+ }
+
+ /**
+ * Adds additional static libraries to be linked into the final ObjC application bundle.
+ */
+ Builder addExtraImportLibraries(Iterable<Artifact> extraImportLibraries) {
+ this.extraImportLibraries = Iterables.concat(this.extraImportLibraries, extraImportLibraries);
+ return this;
+ }
+
+ /**
+ * Sets a linked binary generated by this rule to be propagated to dependers.
+ */
+ Builder setLinkedBinary(Artifact linkedBinary) {
+ this.linkedBinary = Optional.of(linkedBinary);
+ return this;
+ }
+
+ ObjcCommon build() {
+ Iterable<BundleableFile> bundleImports = BundleableFile.bundleImportsFromRule(context);
+
+ ObjcProvider.Builder objcProvider = new ObjcProvider.Builder()
+ .addAll(IMPORTED_LIBRARY, extraImportLibraries)
+ .addAll(BUNDLE_FILE, bundleImports)
+ .addAll(BUNDLE_IMPORT_DIR,
+ uniqueContainers(BundleableFile.toArtifacts(bundleImports), BUNDLE_CONTAINER_TYPE))
+ .addAll(SDK_FRAMEWORK, extraSdkFrameworks)
+ .addAll(WEAK_SDK_FRAMEWORK, extraWeakSdkFrameworks)
+ .addAll(SDK_DYLIB, extraSdkDylibs)
+ .addAll(FRAMEWORK_FILE, frameworkImports)
+ .addAll(FRAMEWORK_DIR, uniqueContainers(frameworkImports, FRAMEWORK_CONTAINER_TYPE))
+ .addAll(INCLUDE, userHeaderSearchPaths)
+ .addAll(DEFINE, defines)
+ .addAll(HEADER, headers)
+ .addTransitiveAndPropagate(depObjcProviders)
+ .addTransitiveWithoutPropagating(directDepObjcProviders);
+
+ Storyboards storyboards;
+ Iterable<Xcdatamodel> datamodels;
+ if (compilationAttributes.isPresent()) {
+ CompilationAttributes attributes = compilationAttributes.get();
+ ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(context);
+ Iterable<PathFragment> sdkIncludes = Iterables.transform(
+ Interspersing.prependEach(
+ IosSdkCommands.sdkDir(objcConfiguration) + "/usr/include/",
+ PathFragment.safePathStrings(attributes.sdkIncludes())),
+ TO_PATH_FRAGMENT);
+ objcProvider
+ .addAll(HEADER, attributes.hdrs())
+ .addAll(INCLUDE, attributes.headerSearchPaths())
+ .addAll(INCLUDE, sdkIncludes)
+ .addAll(SDK_FRAMEWORK, attributes.sdkFrameworks())
+ .addAll(WEAK_SDK_FRAMEWORK, attributes.weakSdkFrameworks())
+ .addAll(SDK_DYLIB, attributes.sdkDylibs());
+ }
+
+ if (resourceAttributes.isPresent()) {
+ ResourceAttributes attributes = resourceAttributes.get();
+ storyboards = Storyboards.fromInputs(attributes.storyboards(), intermediateArtifacts);
+ datamodels = Xcdatamodels.xcdatamodels(intermediateArtifacts, attributes.datamodels());
+ Iterable<CompiledResourceFile> compiledResources =
+ CompiledResourceFile.fromStringsFiles(intermediateArtifacts, attributes.strings());
+ XibFiles xibFiles = new XibFiles(attributes.xibs());
+
+ objcProvider
+ .addTransitiveAndPropagate(MERGE_ZIP, storyboards.getOutputZips())
+ .addAll(MERGE_ZIP, xibFiles.compiledZips(intermediateArtifacts))
+ .addAll(GENERAL_RESOURCE_FILE, storyboards.getInputs())
+ .addAll(GENERAL_RESOURCE_FILE, attributes.resources())
+ .addAll(GENERAL_RESOURCE_FILE, attributes.strings())
+ .addAll(GENERAL_RESOURCE_FILE, attributes.xibs())
+ .addAll(BUNDLE_FILE, BundleableFile.nonCompiledResourceFiles(attributes.resources()))
+ .addAll(BUNDLE_FILE,
+ Iterables.transform(compiledResources, CompiledResourceFile.TO_BUNDLED))
+ .addAll(XCASSETS_DIR,
+ uniqueContainers(attributes.assetCatalogs(), ASSET_CATALOG_CONTAINER_TYPE))
+ .addAll(ASSET_CATALOG, attributes.assetCatalogs())
+ .addAll(XCDATAMODEL, datamodels);
+ } else {
+ storyboards = Storyboards.empty();
+ datamodels = ImmutableList.of();
+ }
+
+ for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) {
+ objcProvider.addAll(LIBRARY, artifacts.getArchive().asSet());
+
+ boolean usesCpp = false;
+ for (Artifact sourceFile :
+ Iterables.concat(artifacts.getSrcs(), artifacts.getNonArcSrcs())) {
+ usesCpp = usesCpp || ObjcRuleClasses.CPP_SOURCES.matches(sourceFile.getExecPath());
+ }
+ if (usesCpp) {
+ objcProvider.add(FLAG, USES_CPP);
+ }
+ }
+
+ if (alwayslink) {
+ for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) {
+ for (Artifact archive : artifacts.getArchive().asSet()) {
+ objcProvider.add(FORCE_LOAD_LIBRARY, archive);
+ objcProvider.add(FORCE_LOAD_FOR_XCODEGEN,
+ "$(BUILT_PRODUCTS_DIR)/" + archive.getExecPath().getBaseName());
+ }
+ }
+ for (Artifact archive : extraImportLibraries) {
+ objcProvider.add(FORCE_LOAD_LIBRARY, archive);
+ objcProvider.add(FORCE_LOAD_FOR_XCODEGEN,
+ "$(WORKSPACE_ROOT)/" + archive.getExecPath().getSafePathString());
+ }
+ }
+
+ objcProvider.addAll(LINKED_BINARY, linkedBinary.asSet());
+
+ return new ObjcCommon(
+ context, objcProvider.build(), storyboards, datamodels, compilationArtifacts);
+ }
+
+ }
+
+ static final FileType BUNDLE_CONTAINER_TYPE = FileType.of(".bundle");
+
+ static final FileType ASSET_CATALOG_CONTAINER_TYPE = FileType.of(".xcassets");
+
+ static final FileType FRAMEWORK_CONTAINER_TYPE = FileType.of(".framework");
+ private final RuleContext context;
+ private final ObjcProvider objcProvider;
+ private final Storyboards storyboards;
+ private final Iterable<Xcdatamodel> datamodels;
+
+ private final Optional<CompilationArtifacts> compilationArtifacts;
+
+ private ObjcCommon(
+ RuleContext context,
+ ObjcProvider objcProvider,
+ Storyboards storyboards,
+ Iterable<Xcdatamodel> datamodels,
+ Optional<CompilationArtifacts> compilationArtifacts) {
+ this.context = Preconditions.checkNotNull(context);
+ this.objcProvider = Preconditions.checkNotNull(objcProvider);
+ this.storyboards = Preconditions.checkNotNull(storyboards);
+ this.datamodels = Preconditions.checkNotNull(datamodels);
+ this.compilationArtifacts = Preconditions.checkNotNull(compilationArtifacts);
+ }
+
+ public ObjcProvider getObjcProvider() {
+ return objcProvider;
+ }
+
+ public Optional<CompilationArtifacts> getCompilationArtifacts() {
+ return compilationArtifacts;
+ }
+
+ /**
+ * Returns all storyboards declared in this rule (not including others in the transitive
+ * dependency tree).
+ */
+ public Storyboards getStoryboards() {
+ return storyboards;
+ }
+
+ /**
+ * Returns all datamodels declared in this rule (not including others in the transitive
+ * dependency tree).
+ */
+ public Iterable<Xcdatamodel> getDatamodels() {
+ return datamodels;
+ }
+
+ /**
+ * Returns an {@link Optional} containing the compiled {@code .a} file, or
+ * {@link Optional#absent()} if this object contains no {@link CompilationArtifacts} or the
+ * compilation information has no sources.
+ */
+ public Optional<Artifact> getCompiledArchive() {
+ for (CompilationArtifacts justCompilationArtifacts : compilationArtifacts.asSet()) {
+ return justCompilationArtifacts.getArchive();
+ }
+ return Optional.absent();
+ }
+
+ /**
+ * Reports any known errors to the {@link RuleContext}. This should be called exactly once for
+ * a target.
+ */
+ public void reportErrors() {
+
+ // TODO(bazel-team): Report errors for rules that are not actually useful (i.e. objc_library
+ // without sources or resources, empty objc_bundles)
+ }
+
+ static ImmutableList<PathFragment> userHeaderSearchPaths(BuildConfiguration configuration) {
+ return ImmutableList.of(
+ new PathFragment("."),
+ configuration.getGenfilesFragment());
+ }
+
+ /**
+ * Returns the first directory in the sequence of parents of the exec path of the given artifact
+ * that matches {@code type}. For instance, if {@code type} is FileType.of(".foo") and the exec
+ * path of {@code artifact} is {@code a/b/c/bar.foo/d/e}, then the return value is
+ * {@code a/b/c/bar.foo}.
+ */
+ static Optional<PathFragment> nearestContainerMatching(FileType type, Artifact artifact) {
+ PathFragment container = artifact.getExecPath();
+ do {
+ if (type.matches(container)) {
+ return Optional.of(container);
+ }
+ container = container.getParentDirectory();
+ } while (container != null);
+ return Optional.absent();
+ }
+
+ /**
+ * Similar to {@link #nearestContainerMatching(FileType, Artifact)}, but tries matching several
+ * file types in {@code types}, and returns a path for the first match in the sequence.
+ */
+ static Optional<PathFragment> nearestContainerMatching(
+ Iterable<FileType> types, Artifact artifact) {
+ for (FileType type : types) {
+ for (PathFragment container : nearestContainerMatching(type, artifact).asSet()) {
+ return Optional.of(container);
+ }
+ }
+ return Optional.absent();
+ }
+
+ /**
+ * Returns all directories matching {@code containerType} that contain the items in
+ * {@code artifacts}. This function ignores artifacts that are not in any directory matching
+ * {@code containerType}.
+ */
+ static Iterable<PathFragment> uniqueContainers(
+ Iterable<Artifact> artifacts, FileType containerType) {
+ ImmutableSet.Builder<PathFragment> containers = new ImmutableSet.Builder<>();
+ for (Artifact artifact : artifacts) {
+ containers.addAll(ObjcCommon.nearestContainerMatching(containerType, artifact).asSet());
+ }
+ return containers.build();
+ }
+
+ /**
+ * Similar to {@link #nearestContainerMatching(FileType, Artifact)}, but returns the container
+ * closest to the root that matches the given type.
+ */
+ static Optional<PathFragment> farthestContainerMatching(FileType type, Artifact artifact) {
+ PathFragment container = artifact.getExecPath();
+ Optional<PathFragment> lastMatch = Optional.absent();
+ do {
+ if (type.matches(container)) {
+ lastMatch = Optional.of(container);
+ }
+ container = container.getParentDirectory();
+ } while (container != null);
+ return lastMatch;
+ }
+
+ static Iterable<String> notInContainerErrors(
+ Iterable<Artifact> artifacts, FileType containerType) {
+ return notInContainerErrors(artifacts, ImmutableList.of(containerType));
+ }
+
+ @VisibleForTesting
+ static final String NOT_IN_CONTAINER_ERROR_FORMAT =
+ "File '%s' is not in a directory of one of these type(s): %s";
+
+ static Iterable<String> notInContainerErrors(
+ Iterable<Artifact> artifacts, Iterable<FileType> containerTypes) {
+ Set<String> errors = new HashSet<>();
+ for (Artifact artifact : artifacts) {
+ boolean inContainer = nearestContainerMatching(containerTypes, artifact).isPresent();
+ if (!inContainer) {
+ errors.add(String.format(NOT_IN_CONTAINER_ERROR_FORMAT,
+ artifact.getExecPath(), Iterables.toString(containerTypes)));
+ }
+ }
+ return errors;
+ }
+
+ /**
+ * @param filesToBuild files to build for this target. These also become the data runfiles. Note
+ * that this method may add more files to create the complete list of files to build for this
+ * target.
+ * @param maybeTargetProvider the provider for this target.
+ * @param maybeExportedProvider the {@link ObjcProvider} for this target. This should generally be
+ * present whenever {@code objc_} rules may depend on this target.
+ * @param maybeJ2ObjcSrcsProvider the {@link J2ObjcSrcsProvider} for this target.
+ */
+ public ConfiguredTarget configuredTarget(NestedSet<Artifact> filesToBuild,
+ Optional<XcodeProvider> maybeTargetProvider, Optional<ObjcProvider> maybeExportedProvider,
+ Optional<XcTestAppProvider> maybeXcTestAppProvider,
+ Optional<J2ObjcSrcsProvider> maybeJ2ObjcSrcsProvider) {
+ NestedSet<Artifact> allFilesToBuild = NestedSetBuilder.<Artifact>stableOrder()
+ .addTransitive(filesToBuild)
+ .addTransitive(storyboards.getOutputZips())
+ .addAll(Xcdatamodel.outputZips(datamodels))
+ .build();
+
+ RunfilesProvider runfilesProvider = RunfilesProvider.withData(
+ new Runfiles.Builder()
+ .addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES)
+ .build(),
+ new Runfiles.Builder().addTransitiveArtifacts(allFilesToBuild).build());
+
+ RuleConfiguredTargetBuilder target = new RuleConfiguredTargetBuilder(context)
+ .setFilesToBuild(allFilesToBuild)
+ .add(RunfilesProvider.class, runfilesProvider);
+ for (ObjcProvider exportedProvider : maybeExportedProvider.asSet()) {
+ target.addProvider(ObjcProvider.class, exportedProvider);
+ }
+ for (XcTestAppProvider xcTestAppProvider : maybeXcTestAppProvider.asSet()) {
+ target.addProvider(XcTestAppProvider.class, xcTestAppProvider);
+ }
+ for (XcodeProvider targetProvider : maybeTargetProvider.asSet()) {
+ target.addProvider(XcodeProvider.class, targetProvider);
+ }
+ for (J2ObjcSrcsProvider j2ObjcSrcsProvider : maybeJ2ObjcSrcsProvider.asSet()) {
+ target.addProvider(J2ObjcSrcsProvider.class, j2ObjcSrcsProvider);
+ }
+ return target.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java
new file mode 100644
index 0000000..3f2e073
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java
@@ -0,0 +1,136 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.CompilationMode;
+import com.google.devtools.build.xcode.common.Platform;
+
+import java.util.List;
+
+/**
+ * A compiler configuration containing flags required for Objective-C compilation.
+ */
+public class ObjcConfiguration extends BuildConfiguration.Fragment {
+ @VisibleForTesting
+ static final ImmutableList<String> DBG_COPTS = ImmutableList.of("-O0", "-DDEBUG=1",
+ "-fstack-protector", "-fstack-protector-all", "-D_GLIBCXX_DEBUG_PEDANTIC", "-D_GLIBCXX_DEBUG",
+ "-D_GLIBCPP_CONCEPT_CHECKS");
+
+ @VisibleForTesting
+ static final ImmutableList<String> FASTBUILD_COPTS = ImmutableList.of("-O0", "-DDEBUG=1");
+
+ @VisibleForTesting
+ static final ImmutableList<String> OPT_COPTS =
+ ImmutableList.of("-Os", "-DNDEBUG=1", "-Wno-unused-variable", "-Winit-self", "-Wno-extra");
+
+ private final String iosSdkVersion;
+ private final String iosMinimumOs;
+ private final String iosSimulatorVersion;
+ private final String iosCpu;
+ private final String xcodeOptions;
+ private final boolean generateDebugSymbols;
+ private final List<String> copts;
+ private final CompilationMode compilationMode;
+
+ ObjcConfiguration(ObjcCommandLineOptions objcOptions, BuildConfiguration.Options options) {
+ this.iosSdkVersion = Preconditions.checkNotNull(objcOptions.iosSdkVersion, "iosSdkVersion");
+ this.iosMinimumOs = Preconditions.checkNotNull(objcOptions.iosMinimumOs, "iosMinimumOs");
+ this.iosSimulatorVersion =
+ Preconditions.checkNotNull(objcOptions.iosSimulatorVersion, "iosSimulatorVersion");
+ this.iosCpu = Preconditions.checkNotNull(objcOptions.iosCpu, "iosCpu");
+ this.xcodeOptions = Preconditions.checkNotNull(objcOptions.xcodeOptions, "xcodeOptions");
+ this.generateDebugSymbols = objcOptions.generateDebugSymbols;
+ this.copts = ImmutableList.copyOf(objcOptions.copts);
+ this.compilationMode = Preconditions.checkNotNull(options.compilationMode, "compilationMode");
+ }
+
+ public String getIosSdkVersion() {
+ return iosSdkVersion;
+ }
+
+ /**
+ * Returns the minimum iOS version supported by binaries and libraries. Any dependencies on newer
+ * iOS version features or libraries will become weak dependencies which are only loaded if the
+ * runtime OS supports them.
+ */
+ public String getMinimumOs() {
+ return iosMinimumOs;
+ }
+
+ public String getIosSimulatorVersion() {
+ return iosSimulatorVersion;
+ }
+
+ public String getIosCpu() {
+ return iosCpu;
+ }
+
+ public Platform getPlatform() {
+ return Platform.forArch(getIosCpu());
+ }
+
+ public String getXcodeOptions() {
+ return xcodeOptions;
+ }
+
+ public boolean generateDebugSymbols() {
+ return generateDebugSymbols;
+ }
+
+ /**
+ * Returns the current compilation mode.
+ */
+ public CompilationMode getCompilationMode() {
+ return compilationMode;
+ }
+
+ /**
+ * Returns the default set of clang options for the current compilation mode.
+ */
+ public List<String> getCoptsForCompilationMode() {
+ switch (compilationMode) {
+ case DBG:
+ return DBG_COPTS;
+ case FASTBUILD:
+ return FASTBUILD_COPTS;
+ case OPT:
+ return OPT_COPTS;
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /**
+ * Returns options passed to (Apple) clang when compiling Objective C. These options should be
+ * applied after any default options but before options specified in the attributes of the rule.
+ */
+ public List<String> getCopts() {
+ return copts;
+ }
+
+ @Override
+ public String getName() {
+ return "Objective-C";
+ }
+
+ @Override
+ public String cacheKey() {
+ return iosSdkVersion;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java
new file mode 100644
index 0000000..19713a3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+
+/**
+ * A loader that creates ObjcConfiguration instances based on Objective-C configurations and
+ * command-line options.
+ */
+public class ObjcConfigurationLoader implements ConfigurationFragmentFactory {
+ @Override
+ public ObjcConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions)
+ throws InvalidConfigurationException {
+ return new ObjcConfiguration(buildOptions.get(ObjcCommandLineOptions.class),
+ buildOptions.get(BuildConfiguration.Options.class));
+ }
+
+ @Override
+ public Class<? extends BuildConfiguration.Fragment> creates() {
+ return ObjcConfiguration.class;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java
new file mode 100644
index 0000000..c6a0037
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.objc.ObjcSdkFrameworks.Attributes;
+
+/**
+ * Implementation for the {@code objc_framework} rule.
+ */
+public class ObjcFramework implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ Attributes sdkFrameworkAttributes = new Attributes(ruleContext);
+
+ ImmutableList<Artifact> frameworkImports =
+ ruleContext.getPrerequisiteArtifacts("framework_imports", Mode.TARGET).list();
+ ObjcCommon common = new ObjcCommon.Builder(ruleContext)
+ .addFrameworkImports(
+ frameworkImports)
+ .addExtraSdkFrameworks(sdkFrameworkAttributes.sdkFrameworks())
+ .addExtraWeakSdkFrameworks(sdkFrameworkAttributes.weakSdkFrameworks())
+ .addExtraSdkDylibs(sdkFrameworkAttributes.sdkDylibs())
+ .build();
+
+ Iterable<String> containerErrors =
+ ObjcCommon.notInContainerErrors(frameworkImports, ObjcCommon.FRAMEWORK_CONTAINER_TYPE);
+ for (String error : containerErrors) {
+ ruleContext.attributeError("framework_imports", error);
+ }
+
+ return common.configuredTarget(
+ NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER) /* filesToBuild */,
+ Optional.<XcodeProvider>absent(),
+ Optional.of(common.getObjcProvider()),
+ Optional.<XcTestAppProvider>absent(),
+ Optional.<J2ObjcSrcsProvider>absent());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java
new file mode 100644
index 0000000..7fcfdd3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcSdkFrameworksRule;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+/**
+ * Rule definition for objc_framework.
+ */
+@BlazeRule(name = "objc_framework",
+ factoryClass = ObjcFramework.class,
+ ancestors = { BaseRuleClasses.BaseRule.class, ObjcSdkFrameworksRule.class})
+public class ObjcFrameworkRule implements RuleDefinition {
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(objc_framework).ATTRIBUTE(framework_imports) -->
+ The list of files under a <code>.framework</code> directory which are
+ provided to Objective-C targets that depend on this target.
+ <i>(List of <a href="build-ref.html#labels">labels</a>; required)</i>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("framework_imports", LABEL_LIST)
+ .allowedFileTypes(FileTypeSet.ANY_FILE)
+ .mandatory()
+ .nonEmpty())
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_framework, TYPE = LIBRARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule encapsulates an already-built framework. It is defined by a list
+of files in one or more <code>.framework</code> directories.
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java
new file mode 100644
index 0000000..70743ed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC;
+
+import com.google.common.base.Optional;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes;
+
+/**
+ * Implementation for {@code objc_import}.
+ */
+public class ObjcImport implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ ObjcCommon common = new ObjcCommon.Builder(ruleContext)
+ .setCompilationAttributes(new CompilationAttributes(ruleContext))
+ .setResourceAttributes(new ResourceAttributes(ruleContext))
+ .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext))
+ .setAlwayslink(ruleContext.attributes().get("alwayslink", Type.BOOLEAN))
+ .addExtraImportLibraries(
+ ruleContext.getPrerequisiteArtifacts("archives", Mode.TARGET).list())
+ .build();
+
+ XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder();
+ NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder();
+
+ new CompilationSupport(ruleContext)
+ .addXcodeSettings(xcodeProviderBuilder, common, OptionsProvider.DEFAULT)
+ .validateAttributes();
+
+ new ResourceSupport(ruleContext)
+ .registerActions(common.getStoryboards())
+ .validateAttributes()
+ .addXcodeSettings(xcodeProviderBuilder);
+
+ new XcodeSupport(ruleContext)
+ .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC)
+ .registerActions(xcodeProviderBuilder.build())
+ .addFilesToBuild(filesToBuild);
+
+ return common.configuredTarget(
+ filesToBuild.build(),
+ Optional.of(xcodeProviderBuilder.build()),
+ Optional.of(common.getObjcProvider()),
+ Optional.<XcTestAppProvider>absent(),
+ Optional.<J2ObjcSrcsProvider>absent());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java
new file mode 100644
index 0000000..24e9412
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcCompilationRule;
+import com.google.devtools.build.lib.util.FileType;
+
+/**
+ * Rule definition for {@code objc_import}.
+ */
+@BlazeRule(name = "objc_import",
+ factoryClass = ObjcImport.class,
+ ancestors = { ObjcCompilationRule.class })
+public class ObjcImportRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /*<!-- #BLAZE_RULE(objc_import).IMPLICIT_OUTPUTS -->
+ <ul>
+ <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which
+ can be used to develop or build on a Mac.</li>
+ </ul>
+ <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/
+ .setImplicitOutputsFunction(XcodeSupport.PBXPROJ)
+ /* <!-- #BLAZE_RULE(objc_import).ATTRIBUTE(archives) -->
+ The list of <code>.a</code> files provided to Objective-C targets that
+ depend on this target.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("archives", LABEL_LIST)
+ .mandatory()
+ .nonEmpty()
+ .allowedFileTypes(FileType.of(".a")))
+ /* <!-- #BLAZE_RULE(objc_import).ATTRIBUTE(alwayslink) -->
+ If 1, any bundle or binary that depends (directly or indirectly) on this
+ library will link in all the archive files listed in
+ <code>archives</code>, even if some contain no symbols referenced by the
+ binary.
+ ${SYNOPSIS}
+ This is useful if your code isn't explicitly called by code in
+ the binary, e.g., if your code registers to receive some callback
+ provided by some service.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("alwayslink", BOOLEAN))
+ .removeAttribute("deps")
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_import, TYPE = LIBRARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule encapsulates an already-compiled static library in the form of an
+<code>.a</code> file. It also allows exporting headers and resources using the same
+attributes supported by <code>objc_library</code>.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java
new file mode 100644
index 0000000..4136ffe
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes;
+import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes;
+
+/**
+ * Implementation for {@code objc_library}.
+ */
+public class ObjcLibrary implements RuleConfiguredTargetFactory {
+
+ /**
+ * An {@link IterableWrapper} containing extra library {@link Artifact}s to be linked into the
+ * final ObjC application bundle.
+ */
+ static final class ExtraImportLibraries extends IterableWrapper<Artifact> {
+ ExtraImportLibraries(Artifact... extraImportLibraries) {
+ super(extraImportLibraries);
+ }
+ }
+
+ /**
+ * An {@link IterableWrapper} containing defines as specified in the {@code defines} attribute to
+ * be applied to this target and all depending targets' compilation actions.
+ */
+ static final class Defines extends IterableWrapper<String> {
+ Defines(Iterable<String> defines) {
+ super(defines);
+ }
+
+ Defines(String... defines) {
+ super(defines);
+ }
+ }
+
+ /**
+ * Constructs an {@link ObjcCommon} instance based on the attributes of the given rule. The rule
+ * should inherit from {@link ObjcLibraryRule}..
+ */
+ static ObjcCommon common(RuleContext ruleContext, Iterable<SdkFramework> extraSdkFrameworks,
+ boolean alwayslink, ExtraImportLibraries extraImportLibraries, Defines defines,
+ Iterable<ObjcProvider> extraDepObjcProviders) {
+ CompilationArtifacts compilationArtifacts =
+ CompilationSupport.compilationArtifacts(ruleContext);
+
+ return new ObjcCommon.Builder(ruleContext)
+ .setCompilationAttributes(new CompilationAttributes(ruleContext))
+ .setResourceAttributes(new ResourceAttributes(ruleContext))
+ .addExtraSdkFrameworks(extraSdkFrameworks)
+ .addDefines(defines)
+ .setCompilationArtifacts(compilationArtifacts)
+ .addDepObjcProviders(ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProvider.class))
+ .addDepObjcProviders(
+ ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class))
+ .addNonPropagatedDepObjcProviders(ruleContext.getPrerequisites("non_propagated_deps",
+ Mode.TARGET, ObjcProvider.class))
+ .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext))
+ .setAlwayslink(alwayslink)
+ .addExtraImportLibraries(extraImportLibraries)
+ .addDepObjcProviders(extraDepObjcProviders)
+ .build();
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ ObjcCommon common = common(
+ ruleContext, ImmutableList.<SdkFramework>of(),
+ ruleContext.attributes().get("alwayslink", Type.BOOLEAN), new ExtraImportLibraries(),
+ new Defines(ruleContext.getTokenizedStringListAttr("defines")),
+ ImmutableList.<ObjcProvider>of());
+ OptionsProvider optionsProvider = optionsProvider(ruleContext);
+
+ XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder();
+ NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder()
+ .addAll(common.getCompiledArchive().asSet());
+
+ new CompilationSupport(ruleContext)
+ .registerCompileAndArchiveActions(common, optionsProvider)
+ .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider)
+ .validateAttributes();
+
+ new ResourceSupport(ruleContext)
+ .registerActions(common.getStoryboards())
+ .validateAttributes()
+ .addXcodeSettings(xcodeProviderBuilder);
+
+ new XcodeSupport(ruleContext)
+ .addFilesToBuild(filesToBuild)
+ .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC)
+ .addDependencies(xcodeProviderBuilder)
+ .registerActions(xcodeProviderBuilder.build());
+
+ return common.configuredTarget(
+ filesToBuild.build(),
+ Optional.of(xcodeProviderBuilder.build()),
+ Optional.of(common.getObjcProvider()),
+ Optional.<XcTestAppProvider>absent(),
+ Optional.of(ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext)));
+ }
+
+ private OptionsProvider optionsProvider(RuleContext ruleContext) {
+ return new OptionsProvider.Builder()
+ .addCopts(ruleContext.getTokenizedStringListAttr("copts"))
+ .addTransitive(Optional.fromNullable(
+ ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class)))
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java
new file mode 100644
index 0000000..f721492
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java
@@ -0,0 +1,157 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.NON_ARC_SRCS_TYPE;
+import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.SRCS_TYPE;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcCompilationRule;
+import com.google.devtools.build.lib.util.FileType;
+
+/**
+ * Rule definition for objc_library.
+ */
+@BlazeRule(name = "objc_library",
+ factoryClass = ObjcLibrary.class,
+ ancestors = { ObjcCompilationRule.class,
+ ObjcRuleClasses.ObjcOptsRule.class })
+public class ObjcLibraryRule implements RuleDefinition {
+ private static final Iterable<String> ALLOWED_DEPS_RULE_CLASSES = ImmutableSet.of(
+ "objc_library",
+ "objc_import",
+ "objc_bundle",
+ "objc_framework",
+ "objc_bundle_library",
+ "objc_proto_library",
+ "j2objc_library");
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /*<!-- #BLAZE_RULE(objc_library).IMPLICIT_OUTPUTS -->
+ <ul>
+ <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which
+ can be used to develop or build on a Mac.</li>
+ </ul>
+ <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/
+ .setImplicitOutputsFunction(XcodeSupport.PBXPROJ)
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(srcs) -->
+ The list of C, C++, Objective-C, and Objective-C++ files that are
+ processed to create the library target.
+ ${SYNOPSIS}
+ These are your checked-in source files, plus any generated files.
+ These are compiled into .o files with Clang, so headers should not go
+ here (see the hdrs attribute).
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("srcs", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedFileTypes(SRCS_TYPE))
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(non_arc_srcs) -->
+ The list of Objective-C files that are processed to create the
+ library target that DO NOT use ARC.
+ ${SYNOPSIS}
+ The files in this attribute are treated very similar to those in the
+ srcs attribute, but are compiled without ARC enabled.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("non_arc_srcs", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedFileTypes(NON_ARC_SRCS_TYPE))
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(pch) -->
+ Header file to prepend to every source file being compiled (both arc
+ and non-arc). Note that the file will not be precompiled - this is
+ simply a convenience, not a build-speed enhancement.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("pch", LABEL)
+ .direct_compile_time_input()
+ .allowedFileTypes(FileType.of(".pch")))
+ .add(attr("options", LABEL)
+ .undocumented("objc_options will probably be removed")
+ .allowedFileTypes()
+ .allowedRuleClasses("objc_options"))
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(alwayslink) -->
+ If 1, any bundle or binary that depends (directly or indirectly) on this
+ library will link in all the object files for the files listed in
+ <code>srcs</code> and <code>non_arc_srcs</code>, even if some contain no
+ symbols referenced by the binary.
+ ${SYNOPSIS}
+ This is useful if your code isn't explicitly called by code in
+ the binary, e.g., if your code registers to receive some callback
+ provided by some service.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("alwayslink", BOOLEAN))
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(deps) -->
+ The list of targets that are linked together to form the final bundle.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .override(attr("deps", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedRuleClasses(ALLOWED_DEPS_RULE_CLASSES)
+ .allowedFileTypes())
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(bundles) -->
+ The list of bundle targets that this target requires to be included in the final bundle.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("bundles", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedRuleClasses("objc_bundle", "objc_bundle_library")
+ .allowedFileTypes())
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(non_propagated_deps) -->
+ The list of targets that are required in order to build this target,
+ but which are not included in the final bundle.
+ <br />
+ This attribute should only rarely be used, and probably only for proto
+ dependencies.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("non_propagated_deps", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedRuleClasses(ALLOWED_DEPS_RULE_CLASSES)
+ .allowedFileTypes())
+ /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(defines) -->
+ Extra <code>-D</code> flags to pass to the compiler. They should be in
+ the form <code>KEY=VALUE</code> or simply <code>KEY</code> and are
+ passed not only the compiler for this target (as <code>copts</code>
+ are) but also to all <code>objc_</code> dependers of this target.
+ ${SYNOPSIS}
+ Subject to <a href="#make_variables">"Make variable"</a> substitution and
+ <a href="#sh-tokenization">Bourne shell tokenization</a>.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("defines", STRING_LIST))
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_library, TYPE = LIBRARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule produces a static library from the given Objective-C source files.</p>
+
+${IMPLICIT_OUTPUTS}
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java
new file mode 100644
index 0000000..a7e2b8f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for the {@code objc_options} rule.
+ */
+public class ObjcOptions implements RuleConfiguredTargetFactory {
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .add(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .add(OptionsProvider.class,
+ new OptionsProvider.Builder()
+ .addCopts(ruleContext.getTokenizedStringListAttr("copts"))
+ .addInfoplists(
+ ruleContext.getPrerequisiteArtifacts("infoplists", Mode.TARGET).list())
+ .build())
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java
new file mode 100644
index 0000000..7f26bfe
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.PLIST_TYPE;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcOptsRule;
+
+/**
+ * Rule definition for {@code objc_options}.
+ */
+@BlazeRule(name = "objc_options",
+ factoryClass = ObjcOptions.class,
+ ancestors = { BaseRule.class, ObjcOptsRule.class })
+public class ObjcOptionsRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ // TODO(bazel-team): Figure out if we really need objc_options, and if
+ // we don't, delete it.
+ .setUndocumented()
+ /* <!-- #BLAZE_RULE(objc_options).ATTRIBUTE(xcode_name)[DEPRECATED] -->
+ This attribute is ignored and will be removed.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("xcode_name", Type.STRING))
+ /* <!-- #BLAZE_RULE(objc_options).ATTRIBUTE(infoplists) -->
+ infoplist files to merge with the final binary's infoplist. This
+ corresponds to a single file <i>appname</i>-Info.plist in Xcode
+ projects.
+ <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("infoplists", Type.LABEL_LIST)
+ .allowedFileTypes(PLIST_TYPE))
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_options, TYPE = OTHER, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule provides a nameable set of build settings to use when building
+Objective-C targets.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java
new file mode 100644
index 0000000..647b221
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java
@@ -0,0 +1,242 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE;
+import static com.google.common.base.CaseFormat.UPPER_CAMEL;
+import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisUtils;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.SpawnAction;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.rules.proto.ProtoSourcesProvider;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import javax.annotation.Nullable;
+
+/**
+ * Implementation for the "objc_proto_library" rule.
+ */
+public class ObjcProtoLibrary implements RuleConfiguredTargetFactory {
+ private static final Function<Artifact, PathFragment> PARENT_PATHFRAGMENT =
+ new Function<Artifact, PathFragment>() {
+ @Override
+ public PathFragment apply(Artifact input) {
+ return input.getExecPath().getParentDirectory();
+ }
+ };
+
+ @VisibleForTesting
+ static final String NO_PROTOS_ERROR =
+ "no protos to compile - a non-empty deps attribute is required";
+
+ @Override
+ public ConfiguredTarget create(final RuleContext ruleContext) throws InterruptedException {
+ Artifact compileProtos = ruleContext.getPrerequisiteArtifact(
+ ObjcRuleClasses.ObjcProtoRule.COMPILE_PROTOS_ATTR, Mode.HOST);
+ Optional<Artifact> optionsFile = Optional.fromNullable(
+ ruleContext.getPrerequisiteArtifact(ObjcProtoLibraryRule.OPTIONS_FILE_ATTR, Mode.HOST));
+ NestedSet<Artifact> protos = NestedSetBuilder.<Artifact>stableOrder()
+ .addAll(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET)
+ .filter(FileType.of(".proto"))
+ .list())
+ .addTransitive(maybeGetProtoSources(ruleContext))
+ .build();
+
+ if (Iterables.isEmpty(protos)) {
+ ruleContext.ruleError(NO_PROTOS_ERROR);
+ }
+
+ ImmutableList<Artifact> libProtobuf = ruleContext
+ .getPrerequisiteArtifacts(ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET)
+ .list();
+ ImmutableList<Artifact> protoSupport = ruleContext
+ .getPrerequisiteArtifacts(ObjcRuleClasses.ObjcProtoRule.PROTO_SUPPORT_ATTR, Mode.HOST)
+ .list();
+
+ // Generate sources in a package-and-rule-scoped directory; adds both the
+ // package-and-rule-scoped directory and the header-containing-directory to the include path of
+ // dependers.
+ PathFragment rootRelativeOutputDir = new PathFragment(
+ ruleContext.getLabel().getPackageFragment(),
+ new PathFragment("_generated_protos_" + ruleContext.getLabel().getName()));
+ PathFragment workspaceRelativeOutputDir = new PathFragment(
+ ruleContext.getBinOrGenfilesDirectory().getExecPath(), rootRelativeOutputDir);
+ PathFragment generatedProtoDir =
+ new PathFragment(workspaceRelativeOutputDir, ruleContext.getLabel().getPackageFragment());
+
+ boolean outputCpp =
+ ruleContext.attributes().get(ObjcProtoLibraryRule.OUTPUT_CPP_ATTR, Type.BOOLEAN);
+
+ ImmutableList<Artifact> protoGeneratedSources = outputArtifacts(
+ ruleContext, rootRelativeOutputDir, protos, FileType.of(".pb." + (outputCpp ? "cc" : "m")),
+ outputCpp);
+ ImmutableList<Artifact> protoGeneratedHeaders = outputArtifacts(
+ ruleContext, rootRelativeOutputDir, protos, FileType.of(".pb.h"), outputCpp);
+
+ Artifact inputFileList = ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ AnalysisUtils.getUniqueDirectory(ruleContext.getLabel(), new PathFragment("_protos"))
+ .getRelative("_proto_input_files"),
+ ruleContext.getConfiguration().getGenfilesDirectory());
+
+ ruleContext.registerAction(new FileWriteAction(
+ ruleContext.getActionOwner(),
+ inputFileList,
+ ObjcActionsBuilder.joinExecPaths(protos),
+ false));
+
+ CustomCommandLine.Builder commandLineBuilder = new CustomCommandLine.Builder()
+ .add(compileProtos.getExecPathString())
+ .add("--input-file-list").add(inputFileList.getExecPathString())
+ .add("--output-dir").add(workspaceRelativeOutputDir.getSafePathString());
+ if (optionsFile.isPresent()) {
+ commandLineBuilder
+ .add("--compiler-options-path")
+ .add(optionsFile.get().getExecPathString());
+ }
+ if (outputCpp) {
+ commandLineBuilder.add("--generate-cpp");
+ }
+
+ if (!Iterables.isEmpty(protos)) {
+ ruleContext.registerAction(new SpawnAction.Builder()
+ .setMnemonic("GenObjcProtos")
+ .addInput(compileProtos)
+ .addInputs(optionsFile.asSet())
+ .addInputs(protos)
+ .addInput(inputFileList)
+ .addInputs(libProtobuf)
+ .addInputs(protoSupport)
+ .addOutputs(Iterables.concat(protoGeneratedSources, protoGeneratedHeaders))
+ .setExecutable(new PathFragment("/usr/bin/python"))
+ .setCommandLine(commandLineBuilder.build())
+ .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, ""))
+ .build(ruleContext));
+ }
+
+ IntermediateArtifacts intermediateArtifacts =
+ ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ CompilationArtifacts compilationArtifacts = new CompilationArtifacts.Builder()
+ .addNonArcSrcs(protoGeneratedSources)
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .setPchFile(Optional.<Artifact>absent())
+ .build();
+
+ ImmutableSet<PathFragment> searchPathEntries = new ImmutableSet.Builder<PathFragment>()
+ .add(workspaceRelativeOutputDir)
+ .add(generatedProtoDir)
+ .addAll(Iterables.transform(protoGeneratedHeaders, PARENT_PATHFRAGMENT))
+ .build();
+ ObjcCommon common = new ObjcCommon.Builder(ruleContext)
+ .setCompilationArtifacts(compilationArtifacts)
+ .addUserHeaderSearchPaths(searchPathEntries)
+ .addDepObjcProviders(ruleContext.getPrerequisites(
+ ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET, ObjcProvider.class))
+ .setIntermediateArtifacts(intermediateArtifacts)
+ .addHeaders(protoGeneratedHeaders)
+ .addHeaders(protoGeneratedSources)
+ .build();
+
+ XcodeProvider xcodeProvider = new XcodeProvider.Builder()
+ .setLabel(ruleContext.getLabel())
+ .addUserHeaderSearchPaths(searchPathEntries)
+ .addDependencies(ruleContext.getPrerequisites(
+ ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET, XcodeProvider.class))
+ .addCopts(ObjcRuleClasses.objcConfiguration(ruleContext).getCopts())
+ .setProductType(LIBRARY_STATIC)
+ .addHeaders(protoGeneratedHeaders)
+ .setCompilationArtifacts(common.getCompilationArtifacts().get())
+ .setObjcProvider(common.getObjcProvider())
+ .build();
+
+ ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext);
+ actionsBuilder
+ .registerCompileAndArchiveActions(
+ compilationArtifacts, common.getObjcProvider(), OptionsProvider.DEFAULT);
+ actionsBuilder.registerXcodegenActions(
+ new ObjcRuleClasses.Tools(ruleContext),
+ ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ),
+ XcodeProvider.Project.fromTopLevelTarget(xcodeProvider));
+
+ return common.configuredTarget(
+ NestedSetBuilder.<Artifact>stableOrder()
+ .addAll(common.getCompiledArchive().asSet())
+ .addAll(protoGeneratedSources)
+ .addAll(protoGeneratedHeaders)
+ .add(ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ))
+ .build(),
+ Optional.of(xcodeProvider),
+ Optional.of(common.getObjcProvider()),
+ Optional.<XcTestAppProvider>absent(),
+ Optional.<J2ObjcSrcsProvider>absent());
+ }
+
+ private NestedSet<Artifact> maybeGetProtoSources(RuleContext ruleContext) {
+ NestedSetBuilder<Artifact> artifacts = new NestedSetBuilder<>(Order.STABLE_ORDER);
+ Iterable<ProtoSourcesProvider> providers =
+ ruleContext.getPrerequisites("deps", Mode.TARGET, ProtoSourcesProvider.class);
+ for (ProtoSourcesProvider provider : providers) {
+ artifacts.addTransitive(provider.getTransitiveProtoSources());
+ }
+ return artifacts.build();
+ }
+
+ private ImmutableList<Artifact> outputArtifacts(RuleContext ruleContext,
+ PathFragment rootRelativeOutputDir, Iterable<Artifact> protos, FileType newFileType,
+ boolean outputCpp) {
+ ImmutableList.Builder<Artifact> builder = new ImmutableList.Builder<>();
+ for (Artifact proto : protos) {
+ String protoOutputName;
+ if (outputCpp) {
+ protoOutputName = proto.getFilename();
+ } else {
+ String lowerUnderscoreBaseName = proto.getFilename().replace('-', '_').toLowerCase();
+ protoOutputName = LOWER_UNDERSCORE.to(UPPER_CAMEL, lowerUnderscoreBaseName);
+ }
+ PathFragment rawFragment = new PathFragment(
+ rootRelativeOutputDir,
+ proto.getExecPath().getParentDirectory(),
+ new PathFragment(protoOutputName));
+ @Nullable PathFragment outputFile = FileSystemUtils.replaceExtension(
+ rawFragment,
+ newFileType.getExtensions().get(0),
+ ".proto");
+ if (outputFile != null) {
+ builder.add(ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ outputFile, ruleContext.getBinOrGenfilesDirectory()));
+ }
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java
new file mode 100644
index 0000000..a25f96e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java
@@ -0,0 +1,80 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for objc_proto_library.
+ *
+ * This is a temporary rule until it is better known how to support proto_library rules.
+ */
+@BlazeRule(name = "objc_proto_library",
+ factoryClass = ObjcProtoLibrary.class,
+ ancestors = { BaseRuleClasses.RuleBase.class, ObjcRuleClasses.ObjcProtoRule.class })
+public class ObjcProtoLibraryRule implements RuleDefinition {
+ static final String OPTIONS_FILE_ATTR = "options_file";
+ static final String OUTPUT_CPP_ATTR = "output_cpp";
+ static final String LIBPROTOBUF_ATTR = "$lib_protobuf";
+
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(deps) -->
+ The directly depended upon proto_library rules.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .override(attr("deps", LABEL_LIST)
+ .allowedRuleClasses("proto_library", "filegroup")
+ .legacyAllowAnyFileType())
+ /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(options_file) -->
+ Optional options file to apply to protos which affects compilation (e.g. class
+ whitelist/blacklist settings).
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr(OPTIONS_FILE_ATTR, LABEL).legacyAllowAnyFileType().singleArtifact().cfg(HOST))
+ /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(output_cpp) -->
+ If true, output C++ rather than ObjC.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr(OUTPUT_CPP_ATTR, BOOLEAN).value(false))
+ // TODO(bazel-team): Use //external:objc_proto_lib when bind() support is a little better
+ .add(attr(LIBPROTOBUF_ATTR, LABEL).allowedRuleClasses("objc_library")
+ .value(env.getLabel(
+ "//googlemac/ThirdParty/ProtocolBuffers2/objectivec:ProtocolBuffers_lib")))
+ .add(attr("$xcodegen", LABEL).cfg(HOST).exec()
+ .value(env.getLabel("//tools/objc:xcodegen")))
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_proto_library, TYPE = LIBRARY, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule produces a static library from the given proto_library dependencies, after applying an
+options file.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java
new file mode 100644
index 0000000..c48710e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java
@@ -0,0 +1,313 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.collect.nestedset.Order.LINK_ORDER;
+import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A provider that provides all compiling and linking information in the transitive closure of its
+ * deps that are needed for building Objective-C rules.
+ */
+@Immutable
+public final class ObjcProvider implements TransitiveInfoProvider {
+ /**
+ * Represents one of the things this provider can provide transitively. Things are provided as
+ * {@link NestedSet}s of type E.
+ */
+ public static class Key<E> {
+ private final Order order;
+
+ private Key(Order order) {
+ this.order = Preconditions.checkNotNull(order);
+ }
+ }
+
+ public static final Key<Artifact> LIBRARY = new Key<>(LINK_ORDER);
+ public static final Key<Artifact> IMPORTED_LIBRARY = new Key<>(LINK_ORDER);
+
+ /**
+ * Single-architecture linked binaries to be combined for the final multi-architecture binary.
+ */
+ public static final Key<Artifact> LINKED_BINARY = new Key<>(STABLE_ORDER);
+
+ /**
+ * Indicates which libraries to load with {@code -force_load}. This is a subset of the union of
+ * the {@link #LIBRARY} and {@link #IMPORTED_LIBRARY} sets.
+ */
+ public static final Key<Artifact> FORCE_LOAD_LIBRARY = new Key<>(LINK_ORDER);
+
+ /**
+ * Libraries to pass with -force_load flags when setting the linkopts in Xcodegen. This is needed
+ * in addition to {@link #FORCE_LOAD_LIBRARY} because that one, contains a mixture of import
+ * archives (which are not built by Xcode) and built-from-source library archives (which are built
+ * by Xcode). Archives that are built by Xcode are placed directly under
+ * {@code BUILT_PRODUCTS_DIR} while those not built by Xcode appear somewhere in the Bazel
+ * workspace under {@code WORKSPACE_ROOT}.
+ */
+ public static final Key<String> FORCE_LOAD_FOR_XCODEGEN = new Key<>(LINK_ORDER);
+
+ public static final Key<Artifact> HEADER = new Key<>(STABLE_ORDER);
+
+ /**
+ * Include search paths specified with {@code -I} on the command line. Also known as header search
+ * paths (and distinct from <em>user</em> header search paths).
+ */
+ public static final Key<PathFragment> INCLUDE = new Key<>(LINK_ORDER);
+
+ /**
+ * Key for values in {@code defines} attributes. These are passed as {@code -D} flags to all
+ * invocations of the compiler for this target and all depending targets.
+ */
+ public static final Key<String> DEFINE = new Key<>(STABLE_ORDER);
+
+ public static final Key<Artifact> ASSET_CATALOG = new Key<>(STABLE_ORDER);
+
+ /**
+ * Added to {@link TargetControl#getGeneralResourceFileList()} when running Xcodegen.
+ */
+ public static final Key<Artifact> GENERAL_RESOURCE_FILE = new Key<>(STABLE_ORDER);
+
+ /**
+ * Exec paths of {@code .bundle} directories corresponding to imported bundles to link.
+ * These are passed to Xcodegen.
+ */
+ public static final Key<PathFragment> BUNDLE_IMPORT_DIR = new Key<>(STABLE_ORDER);
+
+ /**
+ * Files that are plopped into the final bundle at some arbitrary bundle path. Note that these are
+ * not passed to Xcodegen, and these don't include information about where the file originated
+ * from.
+ */
+ public static final Key<BundleableFile> BUNDLE_FILE = new Key<>(STABLE_ORDER);
+
+ public static final Key<PathFragment> XCASSETS_DIR = new Key<>(STABLE_ORDER);
+ public static final Key<String> SDK_DYLIB = new Key<>(STABLE_ORDER);
+ public static final Key<SdkFramework> SDK_FRAMEWORK = new Key<>(STABLE_ORDER);
+ public static final Key<SdkFramework> WEAK_SDK_FRAMEWORK = new Key<>(STABLE_ORDER);
+ public static final Key<Xcdatamodel> XCDATAMODEL = new Key<>(STABLE_ORDER);
+ public static final Key<Flag> FLAG = new Key<>(STABLE_ORDER);
+
+ /**
+ * Merge zips to include in the bundle. The entries of these zip files are included in the final
+ * bundle with the same path. The entries in the merge zips should not include the bundle root
+ * path (e.g. {@code Foo.app}).
+ */
+ public static final Key<Artifact> MERGE_ZIP = new Key<>(STABLE_ORDER);
+
+ /**
+ * Exec paths of {@code .framework} directories corresponding to frameworks to link. These cause
+ * -F arguments (framework search paths) to be added to each compile action, and -framework (link
+ * framework) arguments to be added to each link action.
+ */
+ public static final Key<PathFragment> FRAMEWORK_DIR = new Key<>(LINK_ORDER);
+
+ /**
+ * Files in {@code .framework} directories that should be included as inputs when compiling and
+ * linking.
+ */
+ public static final Key<Artifact> FRAMEWORK_FILE = new Key<>(STABLE_ORDER);
+
+ /**
+ * Bundles which should be linked in as a nested bundle to the final application.
+ */
+ public static final Key<Bundling> NESTED_BUNDLE = new Key<>(STABLE_ORDER);
+
+ /**
+ * Artifact containing information on debug symbols
+ */
+ public static final Key<Artifact> DEBUG_SYMBOLS = new Key<>(STABLE_ORDER);
+
+ /**
+ * Flags that apply to a transitive build dependency tree. Each item in the enum corresponds to a
+ * flag. If the item is included in the key {@link #FLAG}, then the flag is considered set.
+ */
+ public enum Flag {
+ /**
+ * Indicates that C++ (or Objective-C++) is used in any source file. This affects how the linker
+ * is invoked.
+ */
+ USES_CPP;
+ }
+
+ private final ImmutableMap<Key<?>, NestedSet<?>> items;
+
+ // Items which should be passed to direct dependers, but not transitive dependers.
+ private final ImmutableMap<Key<?>, NestedSet<?>> nonPropagatedItems;
+
+ private ObjcProvider(
+ ImmutableMap<Key<?>, NestedSet<?>> items,
+ ImmutableMap<Key<?>, NestedSet<?>> nonPropagatedItems) {
+ this.items = Preconditions.checkNotNull(items);
+ this.nonPropagatedItems = Preconditions.checkNotNull(nonPropagatedItems);
+ }
+
+ /**
+ * All artifacts, bundleable files, etc. of the type specified by {@code key}.
+ */
+ @SuppressWarnings("unchecked")
+ public <E> NestedSet<E> get(Key<E> key) {
+ Preconditions.checkNotNull(key);
+ NestedSetBuilder<E> builder = new NestedSetBuilder<>(key.order);
+ if (nonPropagatedItems.containsKey(key)) {
+ builder.addTransitive((NestedSet<E>) nonPropagatedItems.get(key));
+ }
+ if (items.containsKey(key)) {
+ builder.addTransitive((NestedSet<E>) items.get(key));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Indicates whether {@code flag} is set on this provider.
+ */
+ public boolean is(Flag flag) {
+ return Iterables.contains(get(FLAG), flag);
+ }
+
+ /**
+ * Indicates whether this provider has any asset catalogs. This is true whenever some target in
+ * its transitive dependency tree specifies a non-empty {@code asset_catalogs} attribute.
+ */
+ public boolean hasAssetCatalogs() {
+ return !get(XCASSETS_DIR).isEmpty();
+ }
+
+ /**
+ * A builder for this context with an API that is optimized for collecting information from
+ * several transitive dependencies.
+ */
+ public static final class Builder {
+ private final Map<Key<?>, NestedSetBuilder<?>> items = new HashMap<>();
+ private final Map<Key<?>, NestedSetBuilder<?>> nonPropagatedItems = new HashMap<>();
+
+ private static void maybeAddEmptyBuilder(Map<Key<?>, NestedSetBuilder<?>> set, Key<?> key) {
+ if (!set.containsKey(key)) {
+ set.put(key, new NestedSetBuilder<>(key.order));
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private void uncheckedAddAll(Key key, Iterable toAdd) {
+ maybeAddEmptyBuilder(items, key);
+ items.get(key).addAll(toAdd);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private void uncheckedAddTransitive(Key key, NestedSet toAdd, boolean propagate) {
+ Map<Key<?>, NestedSetBuilder<?>> set = propagate ? items : nonPropagatedItems;
+ maybeAddEmptyBuilder(set, key);
+ set.get(key).addTransitive(toAdd);
+ }
+
+ /**
+ * Adds elements in items, and propagate them to any (transitive) dependers on this
+ * ObjcProvider.
+ */
+ public <E> Builder addTransitiveAndPropagate(Key<E> key, NestedSet<E> items) {
+ uncheckedAddTransitive(key, items, true);
+ return this;
+ }
+
+ /**
+ * Add all elements from provider, and propagate them to any (transitive) dependers on this
+ * ObjcProvider.
+ */
+ public Builder addTransitiveAndPropagate(ObjcProvider provider) {
+ for (Map.Entry<Key<?>, NestedSet<?>> typeEntry : provider.items.entrySet()) {
+ uncheckedAddTransitive(typeEntry.getKey(), typeEntry.getValue(), true);
+ }
+ return this;
+ }
+
+ /**
+ * Add all elements from a single key of the given provider, and propagate them to any
+ * (transitive) dependers on this ObjcProvider.
+ */
+ public <E> Builder addTransitiveAndPropagate(Key<E> key, ObjcProvider provider) {
+ addTransitiveAndPropagate(key, provider.get(key));
+ return this;
+ }
+
+ /**
+ * Add all elements from providers, and propagate them to any (transitive) dependers on this
+ * ObjcProvider.
+ */
+ public Builder addTransitiveAndPropagate(Iterable<ObjcProvider> providers) {
+ for (ObjcProvider provider : providers) {
+ addTransitiveAndPropagate(provider);
+ }
+ return this;
+ }
+
+ /**
+ * Add elements from providers, but don't propagate them to any dependers on this ObjcProvider.
+ * These elements will be exposed to {@link #get(Key)} calls, but not to any ObjcProviders
+ * which add this provider to themself.
+ */
+ public Builder addTransitiveWithoutPropagating(Iterable<ObjcProvider> providers) {
+ for (ObjcProvider provider : providers) {
+ for (Map.Entry<Key<?>, NestedSet<?>> typeEntry : provider.items.entrySet()) {
+ uncheckedAddTransitive(typeEntry.getKey(), typeEntry.getValue(), false);
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Add element, and propagate it to any (transitive) dependers on this ObjcProvider.
+ */
+ public <E> Builder add(Key<E> key, E toAdd) {
+ uncheckedAddAll(key, ImmutableList.of(toAdd));
+ return this;
+ }
+
+ /**
+ * Add elements in toAdd, and propagate them to any (transitive) dependers on this ObjcProvider.
+ */
+ public <E> Builder addAll(Key<E> key, Iterable<? extends E> toAdd) {
+ uncheckedAddAll(key, toAdd);
+ return this;
+ }
+
+ public ObjcProvider build() {
+ ImmutableMap.Builder<Key<?>, NestedSet<?>> propagated = new ImmutableMap.Builder<>();
+ for (Map.Entry<Key<?>, NestedSetBuilder<?>> typeEntry : items.entrySet()) {
+ propagated.put(typeEntry.getKey(), typeEntry.getValue().build());
+ }
+ ImmutableMap.Builder<Key<?>, NestedSet<?>> nonPropagated = new ImmutableMap.Builder<>();
+ for (Map.Entry<Key<?>, NestedSetBuilder<?>> typeEntry : nonPropagatedItems.entrySet()) {
+ nonPropagated.put(typeEntry.getKey(), typeEntry.getValue().build());
+ }
+ return new ObjcProvider(propagated.build(), nonPropagated.build());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java
new file mode 100644
index 0000000..ad1e4dc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java
@@ -0,0 +1,531 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.AttributeMap;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Shared utility code for Objective-C rules.
+ */
+public class ObjcRuleClasses {
+
+ private ObjcRuleClasses() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Returns a derived Artifact by appending a String to a root-relative path. This is similar to
+ * {@link RuleContext#getRelatedArtifact(PathFragment, String)}, except the existing extension is
+ * not removed.
+ */
+ static Artifact artifactByAppendingToRootRelativePath(
+ RuleContext ruleContext, PathFragment path, String suffix) {
+ return ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ path.replaceName(path.getBaseName() + suffix),
+ ruleContext.getBinOrGenfilesDirectory());
+ }
+
+ static IntermediateArtifacts intermediateArtifacts(RuleContext ruleContext) {
+ return new IntermediateArtifacts(
+ ruleContext.getAnalysisEnvironment(), ruleContext.getBinOrGenfilesDirectory(),
+ ruleContext.getLabel(), /*archiveFileNameSuffix=*/"");
+ }
+
+ /**
+ * Returns a {@link IntermediateArtifacts} to be used to compile and link the ObjC source files
+ * in {@code j2ObjcSource}.
+ */
+ static IntermediateArtifacts j2objcIntermediateArtifacts(RuleContext ruleContext,
+ J2ObjcSource j2ObjcSource) {
+ // We need to append "_j2objc" to the name of the generated archive file to distinguish it from
+ // the C/C++ archive file created by proto_library targets with attribute cc_api_version
+ // specified.
+ return new IntermediateArtifacts(
+ ruleContext.getAnalysisEnvironment(),
+ ruleContext.getConfiguration().getBinDirectory(),
+ j2ObjcSource.getTargetLabel(),
+ /*archiveFileNameSuffix=*/"_j2objc");
+ }
+
+ /**
+ * Returns a {@link J2ObjcSrcsProvider} with J2ObjC-generated ObjC file information from the
+ * current rule, and from rules that can be reached transitively through the "deps" attribute.
+ *
+ * @param ruleContext the rule context of the current rule
+ * @param currentSource J2ObjC-generated ObjC file information from the current rule to contribute
+ * to the returned provider
+ * @return a {@link J2ObjcSrcsProvider} containing {@code currentSources} and source information
+ * from the transitive closure.
+ */
+ public static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext,
+ J2ObjcSource currentSource) {
+ return j2ObjcSrcsProvider(ruleContext, Optional.of(currentSource));
+ }
+
+ /**
+ * Returns a {@link J2ObjcSrcsProvider} with J2ObjC-generated ObjC file information from rules
+ * that can be reached transitively through the "deps" attribute.
+ *
+ * @param ruleContext the rule context of the current rule
+ * @return a {@link J2ObjcSrcsProvider} containing source information from the transitive closure.
+ */
+ public static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext) {
+ return j2ObjcSrcsProvider(ruleContext, Optional.<J2ObjcSource>absent());
+ }
+
+ private static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext,
+ Optional<J2ObjcSource> currentSource) {
+ NestedSetBuilder<J2ObjcSource> builder = NestedSetBuilder.stableOrder();
+ builder.addAll(currentSource.asSet());
+ boolean hasProtos = currentSource.isPresent()
+ && currentSource.get().getSourceType() == J2ObjcSource.SourceType.PROTO;
+
+ for (J2ObjcSrcsProvider provider :
+ ruleContext.getPrerequisites("deps", Mode.TARGET, J2ObjcSrcsProvider.class)) {
+ builder.addTransitive(provider.getSrcs());
+ hasProtos |= provider.hasProtos();
+ }
+
+ return new J2ObjcSrcsProvider(builder.build(), hasProtos);
+ }
+
+ public static Artifact artifactByAppendingToBaseName(RuleContext context, String suffix) {
+ return artifactByAppendingToRootRelativePath(
+ context, context.getLabel().toPathFragment(), suffix);
+ }
+
+ static ObjcActionsBuilder actionsBuilder(RuleContext ruleContext) {
+ return new ObjcActionsBuilder(
+ ruleContext,
+ intermediateArtifacts(ruleContext),
+ ObjcRuleClasses.objcConfiguration(ruleContext),
+ ruleContext.getConfiguration(),
+ ruleContext);
+ }
+
+ public static ObjcConfiguration objcConfiguration(RuleContext ruleContext) {
+ return ruleContext.getFragment(ObjcConfiguration.class);
+ }
+
+ @VisibleForTesting
+ static final Iterable<SdkFramework> AUTOMATIC_SDK_FRAMEWORKS = ImmutableList.of(
+ new SdkFramework("Foundation"), new SdkFramework("UIKit"));
+
+ /**
+ * Attributes for {@code objc_*} rules that have compiler (and in the future, possibly linker)
+ * options
+ */
+ @BlazeRule(name = "$objc_opts_rule",
+ type = RuleClassType.ABSTRACT)
+ public static class ObjcOptsRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE($objc_opts_rule).ATTRIBUTE(copts) -->
+ Extra flags to pass to the compiler.
+ ${SYNOPSIS}
+ Subject to <a href="#make_variables">"Make variable"</a> substitution and
+ <a href="#sh-tokenization">Bourne shell tokenization</a>.
+ These flags will only apply to this target, and not those upon which
+ it depends, or those which depend on it.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("copts", STRING_LIST))
+ .build();
+ }
+ }
+
+ /**
+ * Attributes for {@code objc_*} rules that can link in SDK frameworks.
+ */
+ @BlazeRule(name = "$objc_sdk_frameworks_rule",
+ type = RuleClassType.ABSTRACT)
+ public static class ObjcSdkFrameworksRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(sdk_frameworks) -->
+ Names of SDK frameworks to link with. For instance, "XCTest" or
+ "Cocoa". "UIKit" and "Foundation" are always included and do not mean
+ anything if you include them.
+ When linking a library, only those frameworks named in that library's
+ sdk_frameworks attribute are linked in. When linking a binary, all
+ SDK frameworks named in that binary's transitive dependency graph are
+ used.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("sdk_frameworks", STRING_LIST))
+ /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(weak_sdk_frameworks) -->
+ Names of SDK frameworks to weakly link with. For instance,
+ "MediaAccessibility". In difference to regularly linked SDK
+ frameworks, symbols from weakly linked frameworks do not cause an
+ error if they are not present at runtime.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("weak_sdk_frameworks", STRING_LIST))
+ /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(sdk_dylibs) -->
+ Names of SDK .dylib libraries to link with. For instance, "libz" or
+ "libarchive". "libc++" is included automatically if the binary has
+ any C++ or Objective-C++ sources in its dependency tree. When linking
+ a binary, all libraries named in that binary's transitive dependency
+ graph are used.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("sdk_dylibs", STRING_LIST))
+ .build();
+ }
+ }
+
+ /**
+ * Iff a file matches this type, it is considered to use C++.
+ */
+ static final FileType CPP_SOURCES = FileType.of(".cc", ".cpp", ".mm", ".cxx", ".C");
+
+ private static final FileType NON_CPP_SOURCES = FileType.of(".m", ".c");
+
+ static final FileTypeSet SRCS_TYPE = FileTypeSet.of(NON_CPP_SOURCES, CPP_SOURCES);
+
+ static final FileTypeSet NON_ARC_SRCS_TYPE = FileTypeSet.of(FileType.of(".m", ".mm"));
+
+ static final FileTypeSet PLIST_TYPE = FileTypeSet.of(FileType.of(".plist"));
+
+ static final FileTypeSet STORYBOARD_TYPE = FileTypeSet.of(FileType.of(".storyboard"));
+
+ static final FileType XIB_TYPE = FileType.of(".xib");
+
+ /**
+ * Common attributes for {@code objc_*} rules that allow the definition of resources such as
+ * storyboards.
+ */
+ @BlazeRule(name = "$objc_base_resources_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRule.class })
+ public static class ObjcBaseResourcesRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(strings) -->
+ Files which are plists of strings, often localizable. These files
+ are converted to binary plists (if they are not already) and placed
+ in the bundle root of the final package. If this file's immediate
+ containing directory is named *.lproj (e.g. en.lproj, Base.lproj), it
+ will be placed under a directory of that name in the final bundle.
+ This allows for localizable strings.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("strings", LABEL_LIST).legacyAllowAnyFileType()
+ .direct_compile_time_input())
+ /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(xibs) -->
+ Files which are .xib resources, possibly localizable. These files are
+ compiled to .nib files and placed the bundle root of the final
+ package. If this file's immediate containing directory is named
+ *.lproj (e.g. en.lproj, Base.lproj), it will be placed under a
+ directory of that name in the final bundle. This allows for
+ localizable UI.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("xibs", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedFileTypes(XIB_TYPE))
+ /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(storyboards) -->
+ Files which are .storyboard resources, possibly localizable. These
+ files are compiled to .storyboardc directories, which are placed in
+ the bundle root of the final package. If the storyboards's immediate
+ containing directory is named *.lproj (e.g. en.lproj, Base.lproj), it
+ will be placed under a directory of that name in the final bundle.
+ This allows for localizable UI.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("storyboards", LABEL_LIST)
+ .allowedFileTypes(STORYBOARD_TYPE))
+ /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(resources) -->
+ Files to include in the final application bundle. They are not
+ processed or compiled in any way besides the processing done by the
+ rules that actually generate them. These files are placed in the root
+ of the bundle (e.g. Payload/foo.app/...) in most cases. However, if
+ they appear to be localized (i.e. are contained in a directory called
+ *.lproj), they will be placed in a directory of the same name in the
+ app bundle.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("resources", LABEL_LIST).legacyAllowAnyFileType().direct_compile_time_input())
+ /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(datamodels) -->
+ Files that comprise the data models of the final linked binary.
+ Each file must have a containing directory named *.xcdatamodel, which
+ is usually contained by another *.xcdatamodeld (note the added d)
+ directory.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("datamodels", LABEL_LIST).legacyAllowAnyFileType()
+ .direct_compile_time_input())
+ /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(asset_catalogs) -->
+ Files that comprise the asset catalogs of the final linked binary.
+ Each file must have a containing directory named *.xcassets. This
+ containing directory becomes the root of one of the asset catalogs
+ linked with any binary that depends directly or indirectly on this
+ target.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("asset_catalogs", LABEL_LIST).legacyAllowAnyFileType()
+ .direct_compile_time_input())
+ .add(attr("$xcodegen", LABEL).cfg(HOST).exec()
+ .value(env.getLabel("//tools/objc:xcodegen")))
+ .add(attr("$plmerge", LABEL).cfg(HOST).exec()
+ .value(env.getLabel("//tools/objc:plmerge")))
+ .add(attr("$momczip_deploy", LABEL).cfg(HOST)
+ .value(env.getLabel("//tools/objc:momczip_deploy.jar")))
+ .add(attr("$actooloribtoolzip_deploy", LABEL).cfg(HOST)
+ .value(env.getLabel("//tools/objc:actooloribtoolzip_deploy.jar")))
+ .build();
+ }
+ }
+
+ /**
+ * Common attributes for {@code objc_*} rules that contain compilable content.
+ */
+ @BlazeRule(name = "$objc_compilation_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { BaseRuleClasses.RuleBase.class, ObjcSdkFrameworksRule.class,
+ ObjcBaseResourcesRule.class })
+ public static class ObjcCompilationRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(hdrs) -->
+ The list of Objective-C files that are included as headers by source
+ files in this rule or by users of this library.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("hdrs", LABEL_LIST)
+ .direct_compile_time_input()
+ .allowedFileTypes(FileTypeSet.ANY_FILE))
+ /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(includes) -->
+ List of <code>#include/#import</code> search paths to add to this target
+ and all depending targets. This is to support third party and
+ open-sourced libraries that do not specify the entire workspace path in
+ their <code>#import/#include</code> statements.
+ <p>
+ The paths are interpreted relative to the package directory, and the
+ genfiles and bin roots (e.g. <code>blaze-genfiles/pkg/includedir</code>
+ and <code>blaze-out/pkg/includedir</code>) are included in addition to the
+ actual client root.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("includes", Type.STRING_LIST))
+ /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(sdk_includes) -->
+ List of <code>#include/#import</code> search paths to add to this target
+ and all depending targets, where each path is relative to
+ <code>$(SDKROOT)/usr/include</code>.
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("sdk_includes", Type.STRING_LIST))
+ .build();
+ }
+ }
+
+ /**
+ * Common attributes for rules that uses ObjC proto compiler.
+ */
+ @BlazeRule(name = "$objc_proto_rule",
+ type = RuleClassType.ABSTRACT)
+ public static class ObjcProtoRule implements RuleDefinition {
+
+ /**
+ * A Predicate that returns true if the ObjC proto compiler and its support deps are needed by
+ * the current rule.
+ *
+ * <p>For proto_library rules, this will return true if they have a j2objc_api_version
+ * attribute, and it is greater than 0. For other rules, this will return true by default.
+ */
+ public static final Predicate<AttributeMap> USE_PROTO_COMPILER = new Predicate<AttributeMap>() {
+ @Override
+ public boolean apply(AttributeMap rule) {
+ return rule.getAttributeDefinition("j2objc_api_version") == null
+ || rule.get("j2objc_api_version", Type.INTEGER) != 0;
+ }
+ };
+
+ public static final String COMPILE_PROTOS_ATTR = "$googlemac_proto_compiler";
+ public static final String PROTO_SUPPORT_ATTR = "$googlemac_proto_compiler_support";
+
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .add(attr(COMPILE_PROTOS_ATTR, LABEL)
+ .allowedFileTypes(FileType.of(".py"))
+ .cfg(HOST)
+ .singleArtifact()
+ .condition(USE_PROTO_COMPILER)
+ .value(env.getLabel("//tools/objc:compile_protos")))
+ .add(attr(PROTO_SUPPORT_ATTR, LABEL)
+ .legacyAllowAnyFileType()
+ .cfg(HOST)
+ .condition(USE_PROTO_COMPILER)
+ .value(env.getLabel("//tools/objc:proto_support")))
+ .build();
+ }
+ }
+
+ /**
+ * Base rule definition for iOS test rules.
+ */
+ @BlazeRule(name = "$ios_test_base_rule",
+ type = RuleClassType.ABSTRACT,
+ ancestors = { ObjcBinaryRule.class })
+ public static class IosTestBaseRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(target_device) -->
+ The device against which to run the test.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr(IosTest.TARGET_DEVICE, LABEL)
+ .allowedFileTypes()
+ .allowedRuleClasses("ios_device"))
+ /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(xctest) -->
+ Whether this target contains tests using the XCTest testing framework.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr(IosTest.IS_XCTEST, BOOLEAN))
+ /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(xctest_app) -->
+ A <code>objc_binary</code> target that contains the app bundle to test against in XCTest.
+ This attribute is only valid if <code>xctest</code> is true.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr(IosTest.XCTEST_APP, LABEL)
+ .value(new Attribute.ComputedDefault(IosTest.IS_XCTEST) {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.get(IosTest.IS_XCTEST, Type.BOOLEAN)
+ ? env.getLabel("//tools/objc:xctest_app")
+ : null;
+ }
+ })
+ .allowedFileTypes()
+ .allowedRuleClasses("objc_binary"))
+ .override(attr("infoplist", LABEL)
+ .value(new Attribute.ComputedDefault(IosTest.IS_XCTEST) {
+ @Override
+ public Object getDefault(AttributeMap rule) {
+ return rule.get(IosTest.IS_XCTEST, Type.BOOLEAN)
+ ? env.getLabel("//tools/objc:xctest_infoplist")
+ : null;
+ }
+ })
+ .allowedFileTypes(PLIST_TYPE))
+ .build();
+ }
+ }
+
+ /**
+ * Abstract rule type with the {@code infoplist} attribute.
+ */
+ @BlazeRule(name = "$objc_has_infoplist_rule",
+ type = RuleClassType.ABSTRACT)
+ public static class ObjcHasInfoplistRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE($objc_has_infoplist_rule).ATTRIBUTE(infoplist) -->
+ The infoplist file. This corresponds to <i>appname</i>-Info.plist in Xcode projects.
+ ${SYNOPSIS}
+ Blaze will perform variable substitution on the plist file for the following values:
+ <ul>
+ <li><code>${EXECUTABLE_NAME}</code>: The name of the executable generated and included
+ in the bundle by blaze, which can be used as the value for
+ <code>CFBundleExecutable</code> within the plist.
+ <li><code>${BUNDLE_NAME}</code>: This target's name and bundle suffix (.bundle or .app)
+ in the form<code><var>name</var></code>.<code>suffix</code>.
+ <li><code>${PRODUCT_NAME}</code>: This target's name.
+ </ul>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("infoplist", LABEL)
+ .allowedFileTypes(PLIST_TYPE))
+ .build();
+ }
+ }
+
+ /**
+ * Abstract rule type with the {@code entitlements} attribute.
+ */
+ @BlazeRule(name = "$objc_has_entitlements_rule",
+ type = RuleClassType.ABSTRACT)
+ public static class ObjcHasEntitlementsRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) {
+ return builder
+ /* <!-- #BLAZE_RULE($objc_has_entitlements_rule).ATTRIBUTE(entitlements) -->
+ The entitlements file required for device builds of this application. See
+ <a href="https://developer.apple.com/library/mac/documentation/Miscellaneous/Reference/EntitlementKeyReference/Chapters/AboutEntitlements.html">the apple documentation</a>
+ for more information. If absent, the default entitlements from the
+ provisioning profile will be used.
+ <p>
+ The following variables are substituted as per
+ <a href="https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html">their definitions in Apple's documentation</a>:
+ $(AppIdentifierPrefix) and $(CFBundleIdentifier).
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .add(attr("entitlements", LABEL).legacyAllowAnyFileType())
+ .build();
+ }
+ }
+
+ /**
+ * Object that supplies tools used by all rules which have the helper tools common to most rule
+ * implementations.
+ */
+ static final class Tools {
+ private final RuleContext ruleContext;
+
+ Tools(RuleContext ruleContext) {
+ this.ruleContext = Preconditions.checkNotNull(ruleContext);
+ }
+
+ Artifact actooloribtoolzipDeployJar() {
+ return ruleContext.getPrerequisiteArtifact("$actooloribtoolzip_deploy", Mode.HOST);
+ }
+
+ Artifact momczipDeployJar() {
+ return ruleContext.getPrerequisiteArtifact("$momczip_deploy", Mode.HOST);
+ }
+
+ FilesToRunProvider xcodegen() {
+ return ruleContext.getExecutablePrerequisite("$xcodegen", Mode.HOST);
+ }
+
+ FilesToRunProvider plmerge() {
+ return ruleContext.getExecutablePrerequisite("$plmerge", Mode.HOST);
+ }
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java
new file mode 100644
index 0000000..2ff3a7c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcSdkFrameworksRule;
+
+/**
+ * Common logic for rules that inherit from {@link ObjcSdkFrameworksRule}.
+ */
+public class ObjcSdkFrameworks {
+
+ /**
+ * Class that handles extraction and processing of attributes common to inheritors of {@link
+ * ObjcSdkFrameworksRule}.
+ */
+ public static class Attributes {
+
+ private final RuleContext ruleContext;
+
+ public Attributes(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ /**
+ * Returns the SDK frameworks defined on the rule's {@code sdk_frameworks} attribute as well as
+ * base frameworks defined in {@link ObjcRuleClasses#AUTOMATIC_SDK_FRAMEWORKS}.
+ */
+ ImmutableSet<SdkFramework> sdkFrameworks() {
+ ImmutableSet.Builder<SdkFramework> result = new ImmutableSet.Builder<>();
+ result.addAll(ObjcRuleClasses.AUTOMATIC_SDK_FRAMEWORKS);
+ for (String explicit : ruleContext.attributes().get("sdk_frameworks", Type.STRING_LIST)) {
+ result.add(new SdkFramework(explicit));
+ }
+ return result.build();
+ }
+
+ /**
+ * Returns all SDK frameworks defined on the rule's {@code weak_sdk_frameworks} attribute.
+ */
+ ImmutableSet<SdkFramework> weakSdkFrameworks() {
+ ImmutableSet.Builder<SdkFramework> result = new ImmutableSet.Builder<>();
+ for (String frameworkName :
+ ruleContext.attributes().get("weak_sdk_frameworks", Type.STRING_LIST)) {
+ result.add(new SdkFramework(frameworkName));
+ }
+ return result.build();
+ }
+
+ /**
+ * Returns all SDK dylibs defined on the rule's {@code sdk_dylibs} attribute.
+ */
+ ImmutableSet<String> sdkDylibs() {
+ return ImmutableSet.copyOf(ruleContext.attributes().get("sdk_dylibs", Type.STRING_LIST));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java
new file mode 100644
index 0000000..e167f89
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java
@@ -0,0 +1,47 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * Implementation for {@code objc_xcodeproj}.
+ */
+public class ObjcXcodeproj implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ XcodeProvider.Project project = XcodeProvider.Project.fromTopLevelTargets(
+ ruleContext.getPrerequisites("deps", Mode.TARGET, XcodeProvider.class));
+ Artifact pbxproj = ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ);
+
+ ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext);
+ actionsBuilder.registerXcodegenActions(
+ new ObjcRuleClasses.Tools(ruleContext), pbxproj, project);
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .setFilesToBuild(NestedSetBuilder.create(Order.STABLE_ORDER, pbxproj))
+ .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY)
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java
new file mode 100644
index 0000000..0a6c4ba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java
@@ -0,0 +1,78 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST;
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.BOOLEAN;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+
+/**
+ * Rule definition for {@code objc_xcodeproj}.
+ */
+@BlazeRule(name = "objc_xcodeproj",
+ factoryClass = ObjcXcodeproj.class,
+ ancestors = { BaseRuleClasses.RuleBase.class })
+public class ObjcXcodeprojRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ /*<!-- #BLAZE_RULE(objc_xcodeproj).IMPLICIT_OUTPUTS -->
+ <ul>
+ <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: A combined Xcode project file
+ containing all the included targets which can be used to develop or build on a Mac.</li>
+ </ul>
+ <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/
+ .setImplicitOutputsFunction(XcodeSupport.PBXPROJ)
+ /* <!-- #BLAZE_RULE(objc_xcodeproj).ATTRIBUTE(deps) -->
+ The list of targets to include in the combined Xcode project file.
+ ${SYNOPSIS}
+ <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/
+ .override(builder.copy("deps")
+ .nonEmpty()
+ .allowedRuleClasses(
+ "objc_binary",
+ "ios_test",
+ "objc_bundle_library",
+ "objc_import",
+ "objc_library"))
+ .override(attr("testonly", BOOLEAN)
+ .nonconfigurable("Must support test deps.")
+ .value(true))
+ .add(attr("$xcodegen", LABEL)
+ .cfg(HOST)
+ .exec()
+ .value(env.getLabel("//tools/objc:xcodegen")))
+ .build();
+ }
+}
+
+/*<!-- #BLAZE_RULE (NAME = objc_xcodeproj, TYPE = OTHER, FAMILY = Objective-C) -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>This rule combines build information about several objc targets (and all their transitive
+dependencies) into a single Xcode project file, for use in developing on a Mac.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java
new file mode 100644
index 0000000..f87c96a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.xcode.util.Value;
+
+/**
+ * Provides information contained in a {@code objc_options} target.
+ */
+@Immutable
+final class OptionsProvider
+ extends Value<OptionsProvider>
+ implements TransitiveInfoProvider {
+ static final class Builder {
+ private Iterable<String> copts = ImmutableList.of();
+ private final NestedSetBuilder<Artifact> infoplists = NestedSetBuilder.stableOrder();
+
+ /**
+ * Adds copts to the end of the copts sequence.
+ */
+ public Builder addCopts(Iterable<String> copts) {
+ this.copts = Iterables.concat(this.copts, copts);
+ return this;
+ }
+
+ public Builder addInfoplists(Iterable<Artifact> infoplists) {
+ this.infoplists.addAll(infoplists);
+ return this;
+ }
+
+ /**
+ * Adds infoplists and copts from the given provider, if present. copts are added to the end of
+ * the sequence.
+ */
+ public Builder addTransitive(Optional<OptionsProvider> maybeProvider) {
+ for (OptionsProvider provider : maybeProvider.asSet()) {
+ this.copts = Iterables.concat(this.copts, provider.copts);
+ this.infoplists.addTransitive(provider.infoplists);
+ }
+ return this;
+ }
+
+ public OptionsProvider build() {
+ return new OptionsProvider(ImmutableList.copyOf(copts), infoplists.build());
+ }
+ }
+
+ public static final OptionsProvider DEFAULT = new Builder().build();
+
+ private final ImmutableList<String> copts;
+ private final NestedSet<Artifact> infoplists;
+
+ private OptionsProvider(ImmutableList<String> copts, NestedSet<Artifact> infoplists) {
+ super(copts, infoplists);
+ this.copts = Preconditions.checkNotNull(copts);
+ this.infoplists = Preconditions.checkNotNull(infoplists);
+ }
+
+ public ImmutableList<String> getCopts() {
+ return copts;
+ }
+
+ public NestedSet<Artifact> getInfoplists() {
+ return infoplists;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java
new file mode 100644
index 0000000..d1a717c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java
@@ -0,0 +1,123 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+
+/**
+ * Support for resource processing on Objc rules.
+ *
+ * <p>Methods on this class can be called in any order without impacting the result.
+ */
+final class ResourceSupport {
+ private final RuleContext ruleContext;
+ private final Attributes attributes;
+ private final IntermediateArtifacts intermediateArtifacts;
+ private final Iterable<Xcdatamodel> datamodels;
+
+ /**
+ * Creates a new resource support for the given context.
+ */
+ ResourceSupport(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ this.attributes = new Attributes(ruleContext);
+ this.intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext);
+ this.datamodels = Xcdatamodels.xcdatamodels(intermediateArtifacts, attributes.datamodels());
+ }
+
+ /**
+ * Registers resource generating actions (strings, storyboards, ...).
+ *
+ * @param storyboards storyboards defined by this rule
+ *
+ * @return this resource support
+ */
+ ResourceSupport registerActions(Storyboards storyboards) {
+ ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext);
+
+ ObjcRuleClasses.Tools tools = new ObjcRuleClasses.Tools(ruleContext);
+ actionsBuilder.registerResourceActions(
+ tools,
+ new ObjcActionsBuilder.StringsFiles(
+ CompiledResourceFile.fromStringsFiles(intermediateArtifacts, attributes.strings())),
+ new XibFiles(attributes.xibs()),
+ datamodels);
+ for (Artifact storyboardInput : storyboards.getInputs()) {
+ actionsBuilder.registerIbtoolzipAction(
+ tools, storyboardInput, intermediateArtifacts.compiledStoryboardZip(storyboardInput));
+ }
+ return this;
+ }
+
+ /**
+ * Adds common xcode settings to the given provider builder.
+ *
+ * @return this resource support
+ */
+ ResourceSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder) {
+ xcodeProviderBuilder.addInputsToXcodegen(Xcdatamodel.inputsToXcodegen(datamodels));
+ return this;
+ }
+
+ /**
+ * Validates resource attributes on this rule.
+ *
+ * @return this resource support
+ */
+ ResourceSupport validateAttributes() {
+ Iterable<String> assetCatalogErrors = ObjcCommon.notInContainerErrors(
+ attributes.assetCatalogs(), ObjcCommon.ASSET_CATALOG_CONTAINER_TYPE);
+ for (String error : assetCatalogErrors) {
+ ruleContext.attributeError("asset_catalogs", error);
+ }
+
+ Iterable<String> dataModelErrors =
+ ObjcCommon.notInContainerErrors(attributes.datamodels(), Xcdatamodels.CONTAINER_TYPES);
+ for (String error : dataModelErrors) {
+ ruleContext.attributeError("datamodels", error);
+ }
+
+ return this;
+ }
+
+ private static class Attributes {
+ private final RuleContext ruleContext;
+
+ Attributes(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ ImmutableList<Artifact> datamodels() {
+ return ruleContext.getPrerequisiteArtifacts("datamodels", Mode.TARGET).list();
+ }
+
+ ImmutableList<Artifact> xibs() {
+ return ruleContext.getPrerequisiteArtifacts("xibs", Mode.TARGET)
+ .errorsForNonMatching(ObjcRuleClasses.XIB_TYPE)
+ .list();
+ }
+
+ ImmutableList<Artifact> strings() {
+ return ruleContext.getPrerequisiteArtifacts("strings", Mode.TARGET).list();
+ }
+
+ ImmutableList<Artifact> assetCatalogs() {
+ return ruleContext.getPrerequisiteArtifacts("asset_catalogs", Mode.TARGET).list();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java b/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java
new file mode 100644
index 0000000..c692fcd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.util.Value;
+
+/**
+ * Represents the name of an SDK framework.
+ * <p>
+ * Besides being a glorified String, this class prevents you from adding framework names to an
+ * argument list without explicitly specifying how to prefix them.
+ */
+final class SdkFramework extends Value<SdkFramework> {
+ private final String name;
+
+ public SdkFramework(String name) {
+ super(name);
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns an iterable which contains the name of each given framework in the same order.
+ */
+ static Iterable<String> names(Iterable<SdkFramework> frameworks) {
+ ImmutableList.Builder<String> result = new ImmutableList.Builder<>();
+ for (SdkFramework framework : frameworks) {
+ result.add(framework.getName());
+ }
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java
new file mode 100644
index 0000000..204c22d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+
+/**
+ * Contains information about storyboards for a single target. This does not include information
+ * about the transitive closure. A storyboard:
+ * <ul>
+ * <li>Is a single file with an extension of {@code .storyboard} in its uncompiled, checked-in
+ * form.
+ * <li>Can be in a localized {@code .lproj} directory, including {@code Base.lproj}.
+ * <li>Compiles with {@code ibtool} to a directory with extension {@code .storyboardc} (note the
+ * added "c")
+ * </ul>
+ *
+ * <p>The {@link NestedSet}s stored in this class are only one level deep, and do not include the
+ * storyboards in the transitive closure. This is to facilitate structural sharing between copies
+ * of the sequences - the output zips can be added transitively to the inputs of the merge bundle
+ * action, as well as to the files to build set, and only one instance of the sequence exists for
+ * each set.
+ */
+final class Storyboards {
+ private final NestedSet<Artifact> outputZips;
+ private final NestedSet<Artifact> inputs;
+
+ private Storyboards(NestedSet<Artifact> outputZips, NestedSet<Artifact> inputs) {
+ this.outputZips = outputZips;
+ this.inputs = inputs;
+ }
+
+ public NestedSet<Artifact> getOutputZips() {
+ return outputZips;
+ }
+
+ public NestedSet<Artifact> getInputs() {
+ return inputs;
+ }
+
+ static Storyboards empty() {
+ return new Storyboards(
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER),
+ NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER));
+ }
+
+ /**
+ * Generates a set of new instances given the raw storyboard inputs.
+ * @param inputs the {@code .storyboard} files.
+ * @param intermediateArtifacts the object used to determine the output zip {@link Artifact}s.
+ */
+ static Storyboards fromInputs(
+ Iterable<Artifact> inputs, IntermediateArtifacts intermediateArtifacts) {
+ NestedSetBuilder<Artifact> outputZips = NestedSetBuilder.stableOrder();
+ for (Artifact input : inputs) {
+ outputZips.add(intermediateArtifacts.compiledStoryboardZip(input));
+ }
+ return new Storyboards(outputZips.build(), NestedSetBuilder.wrap(Order.STABLE_ORDER, inputs));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java
new file mode 100644
index 0000000..1fc5d5d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java
@@ -0,0 +1,57 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Supplies information needed when a dependency serves as an {@code xctest_app}.
+ */
+@Immutable
+final class XcTestAppProvider implements TransitiveInfoProvider {
+ private final Artifact bundleLoader;
+ private final Artifact ipa;
+ private final ObjcProvider objcProvider;
+
+ XcTestAppProvider(Artifact bundleLoader, Artifact ipa, ObjcProvider objcProvider) {
+ this.bundleLoader = Preconditions.checkNotNull(bundleLoader);
+ this.ipa = Preconditions.checkNotNull(ipa);
+ this.objcProvider = Preconditions.checkNotNull(objcProvider);
+ }
+
+ /**
+ * The bundle loader, which corresponds to the test app's binary.
+ */
+ public Artifact getBundleLoader() {
+ return bundleLoader;
+ }
+
+ public Artifact getIpa() {
+ return ipa;
+ }
+
+ /**
+ * An {@link ObjcProvider} that should be included by any test target that uses this app as its
+ * {@code xctest_app}. This is <strong>not</strong> a typical {@link ObjcProvider} - it has
+ * certain linker-releated keys omitted, such as {@link ObjcProvider#LIBRARY}, since XcTests have
+ * access to symbols in their test rig without linking them into the main test binary.
+ */
+ public ObjcProvider getObjcProvider() {
+ return objcProvider;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java
new file mode 100644
index 0000000..5b29435
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java
@@ -0,0 +1,136 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.util.Value;
+
+/**
+ * Represents an .xcdatamodel[d] directory - knowing all {@code Artifact}s contained therein - and
+ * the .zip file that it is compiled to which should be merged with the final application bundle.
+ * <p>
+ * An .xcdatamodel (here and below note that lack or presence of a d) directory contains the schema
+ * for a managed object, or a managed object model. It typically has two files: {@code layout} and
+ * {@code contents}, although this detail isn't addressed in Bazel code. Directories of this
+ * sort are compiled into a single .mom file. If the .xcdatamodel directory is inside a
+ * .xcdatamodeld directory, then the .mom file is placed inside a .momd directory. The .momd
+ * directory or .mom file is placed in the bundle root of the final bundle.
+ * <p>
+ * An .xcdatamodeld directory contains several .xcdatamodel directories, each corresponding to a
+ * different version. In addition the .xcdatamodeld directory contains a {@code .xccurrentversion}
+ * file which identifies the current version. (this file is also not handled explicitly by Bazel
+ * code).
+ * <p>
+ * When processing artifacts referenced by a {@code datamodels} attribute, we must determine if it
+ * is in a .xcdatamodeld directory or only a .xcdatamodel directory. We also must group the
+ * artifacts by their container, the container being an .xcdatamodeld directory if possible, and a
+ * .xcdatamodel directory otherwise. Every container is compiled with a single invocation of the
+ * Managed Object Model Compiler (momc) and corresponds to exactly one instance of this class. We
+ * invoke momc indirectly through the momczip tool (part of Bazel) which runs momc and zips the
+ * output. The files in this zip are placed in the bundle root of the final application, not unlike
+ * the zips generated by {@code actooloribtoolzip}.
+ */
+class Xcdatamodel extends Value<Xcdatamodel> {
+ private final Artifact outputZip;
+ private final ImmutableSet<Artifact> inputs;
+ private final PathFragment container;
+
+ Xcdatamodel(Artifact outputZip, ImmutableSet<Artifact> inputs, PathFragment container) {
+ super(ImmutableMap.of(
+ "outputZip", outputZip,
+ "inputs", inputs,
+ "container", container));
+ this.outputZip = outputZip;
+ this.inputs = inputs;
+ this.container = container;
+ }
+
+ /**
+ * Returns the files that should be supplied to Xcodegen when generating a project that includes
+ * all of the given xcdatamodels.
+ */
+ public static Iterable<Artifact> inputsToXcodegen(Iterable<Xcdatamodel> datamodels) {
+ ImmutableSet.Builder<Artifact> inputs = new ImmutableSet.Builder<>();
+ for (Xcdatamodel datamodel : datamodels) {
+ for (Artifact generalInput : datamodel.inputs) {
+ if (generalInput.getExecPath().getBaseName().equals(".xccurrentversion")) {
+ inputs.add(generalInput);
+ }
+ }
+ }
+ return inputs.build();
+ }
+
+ public Artifact getOutputZip() {
+ return outputZip;
+ }
+
+ /**
+ * Returns every known file in the container. This is every input file that is processed by momc.
+ */
+ public ImmutableSet<Artifact> getInputs() {
+ return inputs;
+ }
+
+ public PathFragment getContainer() {
+ return container;
+ }
+
+ /**
+ * The ARCHIVE_ROOT passed to momczip. The archive root is the name of the .mom file
+ * unversioned object models, and the name of the .momd directory for versioned object models.
+ */
+ public String archiveRootForMomczip() {
+ return name() + (container.getBaseName().endsWith(".xcdatamodeld") ? ".momd" : ".mom");
+ }
+
+ /**
+ * The name of the data model. This is the name of the container without the extension. For
+ * instance, if the container is "foo/Information.xcdatamodel" or "bar/Information.xcdatamodeld",
+ * then the name is "Information".
+ */
+ public String name() {
+ String baseContainerName = container.getBaseName();
+ int lastDot = baseContainerName.lastIndexOf('.');
+ return baseContainerName.substring(0, lastDot);
+ }
+
+ public static Iterable<Artifact> outputZips(Iterable<Xcdatamodel> models) {
+ return Iterables.transform(models, new Function<Xcdatamodel, Artifact>() {
+ @Override
+ public Artifact apply(Xcdatamodel model) {
+ return model.getOutputZip();
+ }
+ });
+ }
+
+ /**
+ * Returns a sequence of all unique *.xcdatamodel directories that contain all the artifacts of
+ * the given models. Note that this does not return any *.xcdatamodeld directories.
+ */
+ static Iterable<PathFragment> xcdatamodelDirs(Iterable<Xcdatamodel> models) {
+ ImmutableSet.Builder<PathFragment> result = new ImmutableSet.Builder<>();
+ for (Xcdatamodel model : models) {
+ result.addAll(ObjcCommon.uniqueContainers(model.getInputs(), FileType.of(".xcdatamodel")));
+ }
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java
new file mode 100644
index 0000000..32d48aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Utility code for getting information specific to xcdatamodels for a single rule.
+ */
+class Xcdatamodels {
+ private Xcdatamodels() {}
+
+ static final ImmutableList<FileType> CONTAINER_TYPES =
+ ImmutableList.of(FileType.of(".xcdatamodeld"), FileType.of(".xcdatamodel"));
+
+ static Iterable<Xcdatamodel> xcdatamodels(
+ IntermediateArtifacts intermediateArtifacts, Iterable<Artifact> xcdatamodels) {
+ ImmutableSet.Builder<Xcdatamodel> result = new ImmutableSet.Builder<>();
+ Multimap<PathFragment, Artifact> artifactsByContainer = byContainer(xcdatamodels);
+
+ for (Map.Entry<PathFragment, Collection<Artifact>> modelDirEntry :
+ artifactsByContainer.asMap().entrySet()) {
+ PathFragment container = modelDirEntry.getKey();
+ Artifact outputZip = intermediateArtifacts.compiledMomZipArtifact(container);
+ result.add(
+ new Xcdatamodel(outputZip, ImmutableSet.copyOf(modelDirEntry.getValue()), container));
+ }
+
+ return result.build();
+ }
+
+
+ /**
+ * Arrange a sequence of artifacts into entries of a multimap by their nearest container
+ * directory, preferring {@code .xcdatamodeld} over {@code .xcdatamodel}.
+ * If an artifact is not inside any containing directory, then it is not present in the returned
+ * map.
+ */
+ static Multimap<PathFragment, Artifact> byContainer(Iterable<Artifact> artifacts) {
+ ImmutableSetMultimap.Builder<PathFragment, Artifact> result =
+ new ImmutableSetMultimap.Builder<>();
+ for (Artifact artifact : artifacts) {
+ for (PathFragment modelDir :
+ ObjcCommon.nearestContainerMatching(CONTAINER_TYPES, artifact).asSet()) {
+ result.put(modelDir, artifact);
+ }
+ }
+ return result.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java
new file mode 100644
index 0000000..1a68206
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+/**
+ * Possible values that {@code objc_*} rules care about for what Xcode project files refer to as
+ * "product type."
+ */
+enum XcodeProductType {
+ LIBRARY_STATIC("com.apple.product-type.library.static"),
+ BUNDLE("com.apple.product-type.bundle"),
+ APPLICATION("com.apple.product-type.application"),
+ UNIT_TEST("com.apple.product-type.bundle.unit-test"),
+ EXTENSION("com.apple.product-type.app-extension");
+
+ private final String identifier;
+
+ XcodeProductType(String identifier) {
+ this.identifier = identifier;
+ }
+
+ /**
+ * Returns the string used to identify this product type in the {@code productType} field of
+ * {@code PBXNativeTarget} objects in Xcode project files.
+ */
+ public String getIdentifier() {
+ return identifier;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java
new file mode 100644
index 0000000..b244cd9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java
@@ -0,0 +1,452 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_IMPORT_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_FOR_XCODEGEN;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.GENERAL_RESOURCE_FILE;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR;
+import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.xcode.util.Interspersing;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.DependencyControl;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting;
+
+import java.util.EnumSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Provider which provides transitive dependency information that is specific to Xcodegen. In
+ * particular, it provides a sequence of targets which can be used to create a self-contained
+ * {@code .xcodeproj} file.
+ */
+@Immutable
+public final class XcodeProvider implements TransitiveInfoProvider {
+ /**
+ * A builder for instances of {@link XcodeProvider}.
+ */
+ public static final class Builder {
+ private Label label;
+ private final NestedSetBuilder<String> userHeaderSearchPaths = NestedSetBuilder.stableOrder();
+ private final NestedSetBuilder<String> headerSearchPaths = NestedSetBuilder.stableOrder();
+ private Optional<InfoplistMerging> infoplistMerging = Optional.absent();
+ private final NestedSetBuilder<XcodeProvider> dependencies = NestedSetBuilder.stableOrder();
+ private final ImmutableList.Builder<XcodeprojBuildSetting> xcodeprojBuildSettings =
+ new ImmutableList.Builder<>();
+ private final ImmutableList.Builder<String> copts = new ImmutableList.Builder<>();
+ private final ImmutableList.Builder<String> compilationModeCopts =
+ new ImmutableList.Builder<>();
+ private XcodeProductType productType;
+ private final ImmutableList.Builder<Artifact> headers = new ImmutableList.Builder<>();
+ private Optional<CompilationArtifacts> compilationArtifacts = Optional.absent();
+ private ObjcProvider objcProvider;
+ private Optional<XcodeProvider> testHost = Optional.absent();
+ private final NestedSetBuilder<Artifact> inputsToXcodegen = NestedSetBuilder.stableOrder();
+
+ /**
+ * Sets the label of the build target which corresponds to this Xcode target.
+ */
+ public Builder setLabel(Label label) {
+ this.label = label;
+ return this;
+ }
+
+ /**
+ * Adds user header search paths for this target.
+ */
+ public Builder addUserHeaderSearchPaths(Iterable<PathFragment> userHeaderSearchPaths) {
+ this.userHeaderSearchPaths.addAll(rootEach("$(WORKSPACE_ROOT)", userHeaderSearchPaths));
+ return this;
+ }
+
+ /**
+ * Adds header search paths for this target. Each path is interpreted relative to the given
+ * root, such as {@code "$(WORKSPACE_ROOT)"}.
+ */
+ public Builder addHeaderSearchPaths(String root, Iterable<PathFragment> paths) {
+ this.headerSearchPaths.addAll(rootEach(root, paths));
+ return this;
+ }
+
+ /**
+ * Sets the Info.plist merging information. Used for applications. May be
+ * absent for other bundles.
+ */
+ public Builder setInfoplistMerging(InfoplistMerging infoplistMerging) {
+ this.infoplistMerging = Optional.of(infoplistMerging);
+ return this;
+ }
+
+ /**
+ * Adds items in the {@link NestedSet}s of the given target to the corresponding sets in this
+ * builder. This is useful if the given target is a dependency or like a dependency
+ * (e.g. a test host). The given provider is not registered as a dependency with this provider.
+ */
+ private void addTransitiveSets(XcodeProvider dependencyish) {
+ inputsToXcodegen.addTransitive(dependencyish.inputsToXcodegen);
+ userHeaderSearchPaths.addTransitive(dependencyish.userHeaderSearchPaths);
+ headerSearchPaths.addTransitive(dependencyish.headerSearchPaths);
+ }
+
+ /**
+ * Adds {@link XcodeProvider}s corresponding to direct dependencies of this target which should
+ * be added in the {@code .xcodeproj} file.
+ */
+ public Builder addDependencies(Iterable<XcodeProvider> dependencies) {
+ for (XcodeProvider dependency : dependencies) {
+ this.dependencies.add(dependency);
+ this.dependencies.addTransitive(dependency.dependencies);
+ this.addTransitiveSets(dependency);
+ }
+ return this;
+ }
+
+ /**
+ * Adds additional build settings of this target.
+ */
+ public Builder addXcodeprojBuildSettings(
+ Iterable<XcodeprojBuildSetting> xcodeprojBuildSettings) {
+ this.xcodeprojBuildSettings.addAll(xcodeprojBuildSettings);
+ return this;
+ }
+
+ /**
+ * Sets the copts to use when compiling the Xcode target.
+ */
+ public Builder addCopts(Iterable<String> copts) {
+ this.copts.addAll(copts);
+ return this;
+ }
+
+ /**
+ * Sets the copts derived from compilation mode to use when compiling the Xcode target. These
+ * will be included before the DEFINE options.
+ */
+ public Builder addCompilationModeCopts(Iterable<String> copts) {
+ this.compilationModeCopts.addAll(copts);
+ return this;
+ }
+
+ /**
+ * Sets the product type for the PBXTarget in the .xcodeproj file.
+ */
+ public Builder setProductType(XcodeProductType productType) {
+ this.productType = productType;
+ return this;
+ }
+
+ /**
+ * Adds to the header files of this target. It needs not to include the header files of
+ * dependencies.
+ */
+ public Builder addHeaders(Iterable<Artifact> headers) {
+ this.headers.addAll(headers);
+ return this;
+ }
+
+ /**
+ * The compilation artifacts for this target.
+ */
+ public Builder setCompilationArtifacts(CompilationArtifacts compilationArtifacts) {
+ this.compilationArtifacts = Optional.of(compilationArtifacts);
+ return this;
+ }
+
+ /**
+ * Sets the {@link ObjcProvider} corresponding to this target.
+ */
+ public Builder setObjcProvider(ObjcProvider objcProvider) {
+ this.objcProvider = objcProvider;
+ return this;
+ }
+
+ /**
+ * Sets the test host. This is used for xctest targets.
+ */
+ public Builder setTestHost(XcodeProvider testHost) {
+ Preconditions.checkState(!this.testHost.isPresent());
+ this.testHost = Optional.of(testHost);
+ this.addTransitiveSets(testHost);
+ return this;
+ }
+
+ /**
+ * Adds inputs that are passed to Xcodegen when generating the project file.
+ */
+ public Builder addInputsToXcodegen(Iterable<Artifact> inputsToXcodegen) {
+ this.inputsToXcodegen.addAll(inputsToXcodegen);
+ return this;
+ }
+
+ public XcodeProvider build() {
+ Preconditions.checkArgument(
+ !testHost.isPresent() || (productType == XcodeProductType.UNIT_TEST),
+ "%s product types cannot have a test host (test host: %s).", productType, testHost);
+ return new XcodeProvider(this);
+ }
+ }
+
+ /**
+ * A collection of top-level targets that can be used to create a complete project.
+ */
+ public static final class Project {
+ private final NestedSet<Artifact> inputsToXcodegen;
+ private final ImmutableList<XcodeProvider> topLevelTargets;
+
+ private Project(
+ NestedSet<Artifact> inputsToXcodegen, ImmutableList<XcodeProvider> topLevelTargets) {
+ this.inputsToXcodegen = inputsToXcodegen;
+ this.topLevelTargets = topLevelTargets;
+ }
+
+ public static Project fromTopLevelTarget(XcodeProvider topLevelTarget) {
+ return fromTopLevelTargets(ImmutableList.of(topLevelTarget));
+ }
+
+ public static Project fromTopLevelTargets(Iterable<XcodeProvider> topLevelTargets) {
+ NestedSetBuilder<Artifact> inputsToXcodegen = NestedSetBuilder.stableOrder();
+ for (XcodeProvider target : topLevelTargets) {
+ inputsToXcodegen.addTransitive(target.inputsToXcodegen);
+ }
+ return new Project(inputsToXcodegen.build(), ImmutableList.copyOf(topLevelTargets));
+ }
+
+ /**
+ * Returns artifacts that are passed to the Xcodegen action when generating a project file that
+ * contains all of the given targets.
+ */
+ public NestedSet<Artifact> getInputsToXcodegen() {
+ return inputsToXcodegen;
+ }
+
+ public ImmutableList<XcodeProvider> getTopLevelTargets() {
+ return topLevelTargets;
+ }
+
+ /**
+ * Returns all the target controls that must be added to the xcodegen control. No other target
+ * controls are needed to generate a functional project file. This method creates a new list
+ * whenever it is called.
+ */
+ public ImmutableList<TargetControl> targets() {
+ // Collect all the dependencies of all the providers, filtering out duplicates.
+ Set<XcodeProvider> providerSet = new LinkedHashSet<>();
+ for (XcodeProvider target : topLevelTargets) {
+ Iterables.addAll(providerSet, target.providers());
+ }
+
+ ImmutableList.Builder<TargetControl> controls = new ImmutableList.Builder<>();
+ for (XcodeProvider provider : providerSet) {
+ controls.add(provider.targetControl());
+ }
+ return controls.build();
+ }
+ }
+
+ private final Label label;
+ private final NestedSet<String> userHeaderSearchPaths;
+ private final NestedSet<String> headerSearchPaths;
+ private final Optional<InfoplistMerging> infoplistMerging;
+ private final NestedSet<XcodeProvider> dependencies;
+ private final ImmutableList<XcodeprojBuildSetting> xcodeprojBuildSettings;
+ private final ImmutableList<String> copts;
+ private final ImmutableList<String> compilationModeCopts;
+ private final XcodeProductType productType;
+ private final ImmutableList<Artifact> headers;
+ private final Optional<CompilationArtifacts> compilationArtifacts;
+ private final ObjcProvider objcProvider;
+ private final Optional<XcodeProvider> testHost;
+ private final NestedSet<Artifact> inputsToXcodegen;
+
+ private XcodeProvider(Builder builder) {
+ this.label = Preconditions.checkNotNull(builder.label);
+ this.userHeaderSearchPaths = builder.userHeaderSearchPaths.build();
+ this.headerSearchPaths = builder.headerSearchPaths.build();
+ this.infoplistMerging = builder.infoplistMerging;
+ this.dependencies = builder.dependencies.build();
+ this.xcodeprojBuildSettings = builder.xcodeprojBuildSettings.build();
+ this.copts = builder.copts.build();
+ this.compilationModeCopts = builder.compilationModeCopts.build();
+ this.productType = Preconditions.checkNotNull(builder.productType);
+ this.headers = builder.headers.build();
+ this.compilationArtifacts = builder.compilationArtifacts;
+ this.objcProvider = Preconditions.checkNotNull(builder.objcProvider);
+ this.testHost = Preconditions.checkNotNull(builder.testHost);
+ this.inputsToXcodegen = builder.inputsToXcodegen.build();
+ }
+
+ /**
+ * Creates a builder whose values are all initialized to this provider.
+ */
+ public Builder toBuilder() {
+ Builder builder = new Builder();
+ builder.label = label;
+ builder.userHeaderSearchPaths.addAll(userHeaderSearchPaths);
+ builder.headerSearchPaths.addTransitive(headerSearchPaths);
+ builder.infoplistMerging = infoplistMerging;
+ builder.dependencies.addTransitive(dependencies);
+ builder.xcodeprojBuildSettings.addAll(xcodeprojBuildSettings);
+ builder.copts.addAll(copts);
+ builder.productType = productType;
+ builder.headers.addAll(headers);
+ builder.compilationArtifacts = compilationArtifacts;
+ builder.objcProvider = objcProvider;
+ builder.testHost = testHost;
+ builder.inputsToXcodegen.addTransitive(inputsToXcodegen);
+ return builder;
+ }
+
+ /**
+ * Returns a list of this provider and all its transitive dependencies.
+ */
+ private Iterable<XcodeProvider> providers() {
+ Set<XcodeProvider> providers = new LinkedHashSet<>();
+ providers.add(this);
+ Iterables.addAll(providers, dependencies);
+ for (XcodeProvider justTestHost : testHost.asSet()) {
+ providers.add(justTestHost);
+ Iterables.addAll(providers, justTestHost.dependencies);
+ }
+ return ImmutableList.copyOf(providers);
+ }
+
+ private static final EnumSet<XcodeProductType> CAN_LINK_PRODUCT_TYPES = EnumSet.of(
+ XcodeProductType.APPLICATION, XcodeProductType.BUNDLE, XcodeProductType.UNIT_TEST);
+
+ private TargetControl targetControl() {
+ String buildFilePath = label.getPackageFragment().getSafePathString() + "/BUILD";
+ // TODO(bazel-team): Add provisioning profile information when Xcodegen supports it.
+ TargetControl.Builder targetControl = TargetControl.newBuilder()
+ .setName(label.getName())
+ .setLabel(label.toString())
+ .setProductType(productType.getIdentifier())
+ .addAllImportedLibrary(Artifact.toExecPaths(objcProvider.get(IMPORTED_LIBRARY)))
+ .addAllUserHeaderSearchPath(userHeaderSearchPaths)
+ .addAllHeaderSearchPath(headerSearchPaths)
+ .addAllSupportFile(Artifact.toExecPaths(headers))
+ .addAllCopt(compilationModeCopts)
+ .addAllCopt(Interspersing.prependEach("-D", objcProvider.get(DEFINE)))
+ .addAllCopt(copts)
+ .addAllLinkopt(
+ Interspersing.beforeEach("-force_load", objcProvider.get(FORCE_LOAD_FOR_XCODEGEN)))
+ .addAllLinkopt(IosSdkCommands.DEFAULT_LINKER_FLAGS)
+ .addAllLinkopt(Interspersing.beforeEach(
+ "-weak_framework", SdkFramework.names(objcProvider.get(WEAK_SDK_FRAMEWORK))))
+ .addAllBuildSetting(xcodeprojBuildSettings)
+ .addAllBuildSetting(IosSdkCommands.defaultWarningsForXcode())
+ .addAllSdkFramework(SdkFramework.names(objcProvider.get(SDK_FRAMEWORK)))
+ .addAllFramework(PathFragment.safePathStrings(objcProvider.get(FRAMEWORK_DIR)))
+ .addAllXcassetsDir(PathFragment.safePathStrings(objcProvider.get(XCASSETS_DIR)))
+ .addAllXcdatamodel(PathFragment.safePathStrings(
+ Xcdatamodel.xcdatamodelDirs(objcProvider.get(XCDATAMODEL))))
+ .addAllBundleImport(PathFragment.safePathStrings(objcProvider.get(BUNDLE_IMPORT_DIR)))
+ .addAllSdkDylib(objcProvider.get(SDK_DYLIB))
+ .addAllGeneralResourceFile(Artifact.toExecPaths(objcProvider.get(GENERAL_RESOURCE_FILE)))
+ .addSupportFile(buildFilePath);
+
+ if (CAN_LINK_PRODUCT_TYPES.contains(productType)) {
+ for (XcodeProvider dependency : dependencies) {
+ // Only add a library target to a binary's dependencies if it has source files to compile.
+ // Xcode cannot build targets without a source file in the PBXSourceFilesBuildPhase, so if
+ // such a target is present in the control file, it is only to get Xcodegen to put headers
+ // and resources not used by the final binary in the Project Navigator.
+ //
+ // The exception to this rule is the objc_bundle_library target. Bundles are generally used
+ // for resources and can lack a PBXSourceFilesBuildPhase in the project file and still be
+ // considered valid by Xcode.
+ boolean hasSources = dependency.compilationArtifacts.isPresent()
+ && dependency.compilationArtifacts.get().getArchive().isPresent();
+ if (hasSources || (dependency.productType == XcodeProductType.BUNDLE)) {
+ targetControl.addDependency(DependencyControl.newBuilder()
+ .setTargetLabel(dependency.label.toString())
+ .build());
+ }
+ }
+ for (XcodeProvider justTestHost : testHost.asSet()) {
+ targetControl.addDependency(DependencyControl.newBuilder()
+ .setTargetLabel(justTestHost.label.toString())
+ .setTestHost(true)
+ .build());
+ }
+ }
+
+ for (InfoplistMerging merging : infoplistMerging.asSet()) {
+ for (Artifact infoplist : merging.getPlistWithEverything().asSet()) {
+ targetControl.setInfoplist(infoplist.getExecPathString());
+ }
+ }
+ for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) {
+ targetControl
+ .addAllSourceFile(Artifact.toExecPaths(artifacts.getSrcs()))
+ .addAllNonArcSourceFile(Artifact.toExecPaths(artifacts.getNonArcSrcs()));
+
+ for (Artifact pchFile : artifacts.getPchFile().asSet()) {
+ targetControl
+ .setPchPath(pchFile.getExecPathString())
+ .addSupportFile(pchFile.getExecPathString());
+ }
+ }
+
+ if (objcProvider.is(Flag.USES_CPP)) {
+ targetControl.addSdkDylib("libc++");
+ }
+
+ return targetControl.build();
+ }
+
+ /**
+ * Prepends the given path to each path in {@code paths}. Empty paths are
+ * transformed to the value of {@code variable} rather than {@code variable + "/."}
+ */
+ @VisibleForTesting
+ static Iterable<String> rootEach(final String prefix, Iterable<PathFragment> paths) {
+ Preconditions.checkArgument(prefix.startsWith("$"),
+ "prefix should start with a build setting variable like '$(NAME)': %s", prefix);
+ Preconditions.checkArgument(!prefix.endsWith("/"),
+ "prefix should not end with '/': %s", prefix);
+ return Iterables.transform(paths, new Function<PathFragment, String>() {
+ @Override
+ public String apply(PathFragment input) {
+ if (input.getSafePathString().equals(".")) {
+ return prefix;
+ } else {
+ return prefix + "/" + input.getSafePathString();
+ }
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java
new file mode 100644
index 0000000..f64c6bd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java
@@ -0,0 +1,102 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction;
+
+/**
+ * Support for Objc rule types that export an Xcode provider or generate xcode project files.
+ *
+ * <p>Methods on this class can be called in any order without impacting the result.
+ */
+public final class XcodeSupport {
+
+ /**
+ * Template for a target's xcode project.
+ */
+ public static final SafeImplicitOutputsFunction PBXPROJ =
+ fromTemplates("%{name}.xcodeproj/project.pbxproj");
+
+ private final RuleContext ruleContext;
+
+ /**
+ * Creates a new xcode support for the given context.
+ */
+ XcodeSupport(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ /**
+ * Adds xcode project files to the given builder.
+ *
+ * @return this xcode support
+ */
+ XcodeSupport addFilesToBuild(NestedSetBuilder<Artifact> filesToBuild) {
+ filesToBuild.add(ruleContext.getImplicitOutputArtifact(PBXPROJ));
+ return this;
+ }
+
+ /**
+ * Registers actions that generate the rule's Xcode project.
+ *
+ * @param xcodeProvider information about this rule's xcode settings and that of its dependencies
+ * @return this xcode support
+ */
+ XcodeSupport registerActions(XcodeProvider xcodeProvider) {
+ ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext);
+ actionsBuilder.registerXcodegenActions(
+ new ObjcRuleClasses.Tools(ruleContext),
+ ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ),
+ XcodeProvider.Project.fromTopLevelTarget(xcodeProvider));
+ return this;
+ }
+
+ /**
+ * Adds common xcode settings to the given provider builder.
+ *
+ * @param objcProvider provider containing all dependencies' information as well as some of this
+ * rule's
+ * @param productType type of this rule's Xcode target
+ *
+ * @return this xcode support
+ */
+ XcodeSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder,
+ ObjcProvider objcProvider, XcodeProductType productType) {
+ xcodeProviderBuilder
+ .setLabel(ruleContext.getLabel())
+ .setObjcProvider(objcProvider)
+ .setProductType(productType);
+ return this;
+ }
+
+ /**
+ * Adds dependencies to the given provider builder from the {@code deps} and {@code bundles}
+ * attributes.
+ *
+ * @return this xcode support
+ */
+ XcodeSupport addDependencies(XcodeProvider.Builder xcodeProviderBuilder) {
+ xcodeProviderBuilder
+ .addDependencies(ruleContext.getPrerequisites("deps", Mode.TARGET, XcodeProvider.class))
+ .addDependencies(ruleContext.getPrerequisites("bundles", Mode.TARGET, XcodeProvider.class));
+ return this;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java
new file mode 100644
index 0000000..9be1d06
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.objc;
+
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+
+/**
+ * A sequence of xib source files. Each {@code .xib} file can be compiled to a {@code .nib} file or
+ * directory. Because it might be a directory, we always use zip files to store the output and use
+ * the {@code actooloribtoolzip} utility to run ibtool and zip the output.
+ */
+public final class XibFiles extends IterableWrapper<Artifact> {
+ public XibFiles(Iterable<Artifact> artifacts) {
+ super(artifacts);
+ }
+
+ /**
+ * Returns a sequence where each element of this sequence is converted to the file which contains
+ * the compiled contents of the xib.
+ */
+ public ImmutableList<Artifact> compiledZips(IntermediateArtifacts intermediateArtifacts) {
+ ImmutableList.Builder<Artifact> zips = new ImmutableList.Builder<>();
+ for (Artifact xib : this) {
+ zips.add(intermediateArtifacts.compiledXibFileZip(xib));
+ }
+ return zips.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java
new file mode 100644
index 0000000..0663f62
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.proto;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Configured target classes that implement this class can contribute .proto files to the
+ * compilation of proto_library rules.
+ */
+@Immutable
+public final class ProtoSourcesProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<Artifact> transitiveImports;
+ private final NestedSet<Artifact> transitiveProtoSources;
+ private final ImmutableList<Artifact> protoSources;
+
+ public ProtoSourcesProvider(NestedSet<Artifact> transitiveImports,
+ NestedSet<Artifact> transitiveProtoSources,
+ ImmutableList<Artifact> protoSources) {
+ this.transitiveImports = transitiveImports;
+ this.transitiveProtoSources = transitiveProtoSources;
+ this.protoSources = protoSources;
+ }
+
+ /**
+ * Transitive imports including weak dependencies
+ * This determines the order of "-I" arguments to the protocol compiler, and
+ * that is probably important
+ */
+ public NestedSet<Artifact> getTransitiveImports() {
+ return transitiveImports;
+ }
+
+ /**
+ * Returns the proto sources for this rule and all its dependent protocol
+ * buffer rules.
+ */
+ public NestedSet<Artifact> getTransitiveProtoSources() {
+ return transitiveProtoSources;
+ }
+
+ /**
+ * Returns the proto sources from the 'srcs' attribute. If the library is a proxy library
+ * that has no sources, return the sources from the direct deps.
+ */
+ public ImmutableList<Artifact> getProtoSources() {
+ return protoSources;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java
new file mode 100644
index 0000000..6a19f92
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Util;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Generates baseline (empty) coverage for the given non-test target.
+ */
+public class BaselineCoverageAction extends AbstractFileWriteAction
+ implements NotifyOnActionCacheHit {
+ // TODO(bazel-team): Remove this list of languages by separately collecting offline and online
+ // instrumented files.
+ private static final List<String> OFFLINE_INSTRUMENTATION_SUFFIXES = ImmutableList.of(
+ ".c", ".cc", ".cpp", ".dart", ".go", ".h", ".java", ".py");
+ private final Iterable<Artifact> instrumentedFiles;
+
+ private BaselineCoverageAction(
+ ActionOwner owner, Iterable<Artifact> instrumentedFiles, Artifact output) {
+ super(owner, ImmutableList.<Artifact>of(), output, false);
+ this.instrumentedFiles = instrumentedFiles;
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "BaselineCoverage";
+ }
+
+ @Override
+ public String computeKey() {
+ return new Fingerprint()
+ .addStrings(getInstrumentedFilePathStrings())
+ .hexDigestAndReset();
+ }
+
+ private Iterable<String> getInstrumentedFilePathStrings() {
+ List<String> result = new ArrayList<>();
+ for (Artifact instrumentedFile : instrumentedFiles) {
+ String pathString = instrumentedFile.getExecPathString();
+ for (String suffix : OFFLINE_INSTRUMENTATION_SUFFIXES) {
+ if (pathString.endsWith(suffix)) {
+ result.add(pathString);
+ break;
+ }
+ }
+ }
+
+ return result;
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler,
+ Executor executor) {
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ PrintWriter writer = new PrintWriter(out);
+ for (String execPath : getInstrumentedFilePathStrings()) {
+ writer.write("SF:" + execPath + "\n");
+ writer.write("end_of_record\n");
+ }
+ writer.flush();
+ }
+ };
+ }
+
+ @Override
+ protected void afterWrite(Executor executor) {
+ notifyAboutBaselineCoverage(executor.getEventBus());
+ }
+
+ @Override
+ public void actionCacheHit(Executor executor) {
+ notifyAboutBaselineCoverage(executor.getEventBus());
+ }
+
+ /**
+ * Notify interested parties about new baseline coverage data.
+ */
+ private void notifyAboutBaselineCoverage(EventBus eventBus) {
+ Artifact output = Iterables.getOnlyElement(getOutputs());
+ String ownerString = Label.print(getOwner().getLabel());
+ eventBus.post(new BaselineCoverageResult(output, ownerString));
+ }
+
+ /**
+ * Returns collection of baseline coverage artifacts associated with the given target.
+ * Will always return 0 or 1 elements.
+ */
+ public static ImmutableList<Artifact> getBaselineCoverageArtifacts(RuleContext ruleContext,
+ Iterable<Artifact> instrumentedFiles) {
+ // Baseline coverage artifacts will still go into "testlogs" directory.
+ Artifact coverageData = ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ Util.getWorkspaceRelativePath(ruleContext.getTarget()).getChild("baseline_coverage.dat"),
+ ruleContext.getConfiguration().getTestLogsDirectory());
+ ruleContext.registerAction(new BaselineCoverageAction(
+ ruleContext.getActionOwner(), instrumentedFiles, coverageData));
+
+ return ImmutableList.of(coverageData);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java
new file mode 100644
index 0000000..4af2df0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+
+/**
+ * This event is used to notify about a successfully built baseline coverage artifact.
+ */
+public class BaselineCoverageResult {
+
+ private final Artifact baselineCoverageData;
+ private final String ownerString;
+
+ public BaselineCoverageResult(Artifact baselineCoverageData, String ownerString) {
+ this.baselineCoverageData = Preconditions.checkNotNull(baselineCoverageData);
+ this.ownerString = Preconditions.checkNotNull(ownerString);
+ }
+
+ public Artifact getArtifact() {
+ return baselineCoverageData;
+ }
+
+ public String getOwnerString() {
+ return ownerString;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java b/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java
new file mode 100644
index 0000000..5f7571a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A factory class to create coverage report actions.
+ */
+public interface CoverageReportActionFactory {
+
+ /**
+ * Returns a coverage report Action. May return null if it's not necessary to create
+ * such an Action based on the input parameters and some other data available to
+ * the factory implementation, such as command line arguments.
+ */
+ @Nullable
+ public Action createCoverageReportAction(Iterable<ConfiguredTarget> targetsToTest,
+ Set<Artifact> baselineCoverageArtifacts,
+ ArtifactFactory artifactFactory, ArtifactOwner artifactOwner);
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java
new file mode 100644
index 0000000..3cb5750
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
+
+import java.io.IOException;
+
+/**
+ * Test strategy wrapper called 'exclusive'. It should delegate to a test strategy for local
+ * execution. The name 'exclusive' triggers behavior it triggers behavior in
+ * SkyframeExecutor to schedule test execution sequentially after non-test actions. This
+ * ensures streamed test output is not polluted by other action output.
+ */
+@ExecutionStrategy(contextType = TestActionContext.class,
+ name = { "exclusive" })
+public class ExclusiveTestStrategy implements TestActionContext {
+ private TestActionContext parent;
+
+ public ExclusiveTestStrategy(TestActionContext parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ public void exec(TestRunnerAction action,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
+ parent.exec(action, actionExecutionContext);
+ }
+
+ @Override
+ public TestResult newCachedTestResult(
+ Path execRoot, TestRunnerAction action, TestResultData cached) throws IOException {
+ return parent.newCachedTestResult(execRoot, action, cached);
+ }
+
+ @Override
+ public String strategyLocality(TestRunnerAction testRunnerAction) {
+ return "exclusive";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java
new file mode 100644
index 0000000..6c0d73c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.util.Map;
+
+/**
+ * This provider can be implemented by rules which need special environments to run in (especially
+ * tests).
+ */
+@Immutable
+public final class ExecutionInfoProvider implements TransitiveInfoProvider {
+
+ private final ImmutableMap<String, String> executionInfo;
+
+ public ExecutionInfoProvider(Map<String, String> requirements) {
+ this.executionInfo = ImmutableMap.copyOf(requirements);
+ }
+
+ /**
+ * Returns a map to indicate special execution requirements, such as hardware
+ * platforms, web browsers, etc. Rule tags, such as "requires-XXX", may also be added
+ * as keys to the map.
+ */
+ public ImmutableMap<String, String> getExecutionInfo() {
+ return executionInfo;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java
new file mode 100644
index 0000000..e5ab219
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Util;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Creates instrumented file manifest to list instrumented source files.
+ */
+class InstrumentedFileManifestAction extends AbstractFileWriteAction {
+
+ private static final String GUID = "d9ddb800-f9a1-01Da-238d-988311a8475b";
+
+ private final Collection<Artifact> collectedSourceFiles;
+ private final Collection<Artifact> metadataFiles;
+ private final RegexFilter instrumentationFilter;
+
+ private InstrumentedFileManifestAction(ActionOwner owner, Collection<Artifact> inputs,
+ Collection<Artifact> additionalSourceFiles, Collection<Artifact> gcnoFiles,
+ Artifact output, RegexFilter instrumentationFilter) {
+ super(owner, inputs, output, false);
+ this.collectedSourceFiles = additionalSourceFiles;
+ this.metadataFiles = gcnoFiles;
+ this.instrumentationFilter = instrumentationFilter;
+ }
+
+ @Override
+ public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) {
+ return new DeterministicWriter() {
+ @Override
+ public void writeOutputFile(OutputStream out) throws IOException {
+ Writer writer = null;
+ try {
+ // Save exec paths for both instrumented source files and gcno files in the manifest
+ // in the naturally sorted order.
+ String[] fileNames = Iterables.toArray(Iterables.transform(
+ Iterables.concat(collectedSourceFiles, metadataFiles),
+ new Function<Artifact, String> () {
+ @Override
+ public String apply(Artifact artifact) { return artifact.getExecPathString(); }
+ }), String.class);
+ Arrays.sort(fileNames);
+ writer = new OutputStreamWriter(out, ISO_8859_1);
+ for (String name : fileNames) {
+ writer.write(name);
+ writer.write('\n');
+ }
+ } finally {
+ if (writer != null) {
+ writer.close();
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addString(instrumentationFilter.toString());
+ return f.hexDigestAndReset();
+ }
+
+ /**
+ * Instantiates instrumented file manifest for the given target.
+ *
+ * @param ruleContext context of the executable configured target
+ * @param additionalSourceFiles additional instrumented source files, as
+ * collected by the {@link InstrumentedFilesCollector}
+ * @param metadataFiles *.gcno/*.em files collected by the {@link InstrumentedFilesCollector}
+ * @return instrumented file manifest artifact
+ */
+ public static Artifact getInstrumentedFileManifest(final RuleContext ruleContext,
+ final Collection<Artifact> additionalSourceFiles, final Collection<Artifact> metadataFiles) {
+ // Instrumented manifest makes sense only for rules with binary output.
+ Preconditions.checkState(ruleContext.getRule().hasBinaryOutput());
+ final Artifact instrumentedFileManifest =
+ ruleContext.getAnalysisEnvironment().getDerivedArtifact(
+ // Do not use replaceExtension(), as we may get name conflicts (two target-names have the
+ // same base name and only differ by extension).
+ FileSystemUtils.appendExtension(
+ Util.getWorkspaceRelativePath(ruleContext.getTarget()), ".instrumented_files"),
+ ruleContext.getConfiguration().getBinDirectory());
+
+ // Instrumented manifest artifact might already exist in case when multiple test
+ // actions that use slightly different subsets of runfiles set are generated for the same rule.
+ // So check whether we need to create a new action instance.
+ ImmutableList<Artifact> inputs = ImmutableList.<Artifact>builder()
+ .addAll(additionalSourceFiles)
+ .addAll(metadataFiles)
+ .build();
+ ruleContext.registerAction(new InstrumentedFileManifestAction(
+ ruleContext.getActionOwner(), inputs, additionalSourceFiles, metadataFiles,
+ instrumentedFileManifest, ruleContext.getConfiguration().getInstrumentationFilter()));
+
+ return instrumentedFileManifest;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java
new file mode 100644
index 0000000..e62a3b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java
@@ -0,0 +1,211 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A helper class for collecting instrumented files and metadata for a target.
+ */
+public final class InstrumentedFilesCollector {
+
+ /**
+ * The set of file types and attributes to visit to collect instrumented files for a certain rule
+ * type. The class is intentionally immutable, so that a single instance is sufficient for all
+ * rules of the same type (and in some cases all rules of related types, such as all {@code foo_*}
+ * rules).
+ */
+ @Immutable
+ public static final class InstrumentationSpec {
+ private final FileTypeSet instrumentedFileTypes;
+ private final Collection<String> instrumentedAttributes;
+
+ public InstrumentationSpec(FileTypeSet instrumentedFileTypes,
+ Collection<String> instrumentedAttributes) {
+ this.instrumentedFileTypes = instrumentedFileTypes;
+ this.instrumentedAttributes = ImmutableList.copyOf(instrumentedAttributes);
+ }
+
+ public InstrumentationSpec(FileTypeSet instrumentedFileTypes,
+ String... instrumentedAttributes) {
+ this(instrumentedFileTypes, ImmutableList.copyOf(instrumentedAttributes));
+ }
+
+ /**
+ * Returns a new instrumentation spec with the given attribute names replacing the ones
+ * stored in this object.
+ */
+ public InstrumentationSpec withAttributes(String... instrumentedAttributes) {
+ return new InstrumentationSpec(instrumentedFileTypes, instrumentedAttributes);
+ }
+ }
+
+ /**
+ * The implementation for the local metadata collection. The intention is that implementations
+ * recurse over the locally (i.e., for that configured target) created actions and collect
+ * metadata files.
+ */
+ public abstract static class LocalMetadataCollector {
+ /**
+ * Recursively runs over the local actions and add metadata files to the metadataFilesBuilder.
+ */
+ public abstract void collectMetadataArtifacts(
+ Iterable<Artifact> artifacts, AnalysisEnvironment analysisEnvironment,
+ NestedSetBuilder<Artifact> metadataFilesBuilder);
+
+ /**
+ * Adds action output of a particular type to metadata files.
+ *
+ * <p>Only adds the first output that matches the given file type.
+ *
+ * @param metadataFilesBuilder builder to collect metadata files
+ * @param action the action whose outputs to scan
+ * @param fileType the filetype of outputs which should be collected
+ */
+ protected void addOutputs(NestedSetBuilder<Artifact> metadataFilesBuilder,
+ Action action, FileType fileType) {
+ for (Artifact output : action.getOutputs()) {
+ if (fileType.matches(output.getFilename())) {
+ metadataFilesBuilder.add(output);
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Only collects files transitively from srcs, deps, and data attributes.
+ */
+ public static final InstrumentationSpec TRANSITIVE_COLLECTION_SPEC = new InstrumentationSpec(
+ FileTypeSet.NO_FILE,
+ "srcs", "deps", "data");
+
+ /**
+ * An explicit constant for a {@link LocalMetadataCollector} that doesn't collect anything.
+ */
+ public static final LocalMetadataCollector NO_METADATA_COLLECTOR = null;
+
+ private final RuleContext ruleContext;
+ private final InstrumentationSpec spec;
+ private final LocalMetadataCollector localMetadataCollector;
+ private final NestedSet<Artifact> instrumentationMetadataFiles;
+ private final NestedSet<Artifact> instrumentedFiles;
+
+ public InstrumentedFilesCollector(RuleContext ruleContext, InstrumentationSpec spec,
+ LocalMetadataCollector localMetadataCollector, Iterable<Artifact> rootFiles) {
+ this.ruleContext = ruleContext;
+ this.spec = spec;
+ this.localMetadataCollector = localMetadataCollector;
+ Preconditions.checkNotNull(ruleContext, "RuleContext already cleared. That means that the"
+ + " collector data was already memoized. You do not have to call it again.");
+ if (!ruleContext.getConfiguration().isCodeCoverageEnabled()) {
+ instrumentedFiles = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ instrumentationMetadataFiles = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ } else {
+ NestedSetBuilder<Artifact> instrumentedFilesBuilder =
+ NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Artifact> metadataFilesBuilder = NestedSetBuilder.stableOrder();
+ collect(ruleContext.getAnalysisEnvironment(), instrumentedFilesBuilder, metadataFilesBuilder,
+ rootFiles);
+ instrumentedFiles = instrumentedFilesBuilder.build();
+ instrumentationMetadataFiles = metadataFilesBuilder.build();
+ }
+ }
+
+ /**
+ * Returns instrumented source files for the target provided during construction.
+ */
+ public final NestedSet<Artifact> getInstrumentedFiles() {
+ return instrumentedFiles;
+ }
+
+ /**
+ * Returns instrumentation metadata files for the target provided during construction.
+ */
+ public final NestedSet<Artifact> getInstrumentationMetadataFiles() {
+ return instrumentationMetadataFiles;
+ }
+
+ /**
+ * Collects instrumented files and metadata files.
+ */
+ private void collect(AnalysisEnvironment analysisEnvironment,
+ NestedSetBuilder<Artifact> instrumentedFilesBuilder,
+ NestedSetBuilder<Artifact> metadataFilesBuilder,
+ Iterable<Artifact> rootFiles) {
+ for (TransitiveInfoCollection dep : getAllPrerequisites()) {
+ InstrumentedFilesProvider provider = dep.getProvider(InstrumentedFilesProvider.class);
+ if (provider != null) {
+ instrumentedFilesBuilder.addTransitive(provider.getInstrumentedFiles());
+ metadataFilesBuilder.addTransitive(provider.getInstrumentationMetadataFiles());
+ } else if (shouldIncludeLocalSources()) {
+ for (Artifact artifact : dep.getProvider(FileProvider.class).getFilesToBuild()) {
+ if (artifact.isSourceArtifact() &&
+ spec.instrumentedFileTypes.matches(artifact.getFilename())) {
+ instrumentedFilesBuilder.add(artifact);
+ }
+ }
+ }
+ }
+
+ if (localMetadataCollector != null) {
+ localMetadataCollector.collectMetadataArtifacts(rootFiles,
+ analysisEnvironment, metadataFilesBuilder);
+ }
+ }
+
+ /**
+ * Returns the list of attributes which should be (transitively) checked for sources and
+ * instrumentation metadata.
+ */
+ private Collection<String> getSourceAttributes() {
+ return spec.instrumentedAttributes;
+ }
+
+ private boolean shouldIncludeLocalSources() {
+ return ruleContext.getConfiguration().getInstrumentationFilter().isIncluded(
+ ruleContext.getLabel().toString());
+ }
+
+ private Iterable<TransitiveInfoCollection> getAllPrerequisites() {
+ List<TransitiveInfoCollection> prerequisites = new ArrayList<>();
+ for (String attr : getSourceAttributes()) {
+ if (ruleContext.getRule().isAttrDefined(attr, Type.LABEL_LIST) ||
+ ruleContext.getRule().isAttrDefined(attr, Type.LABEL)) {
+ Iterables.addAll(prerequisites, ruleContext.getPrerequisites(attr, Mode.DONT_CHECK));
+ }
+ }
+ return prerequisites;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java
new file mode 100644
index 0000000..b1f956c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+
+/**
+ * A provider of instrumented file sources and instrumentation metadata.
+ */
+public interface InstrumentedFilesProvider extends TransitiveInfoProvider {
+
+ /**
+ * Returns a collection of source files for instrumented binaries.
+ */
+ NestedSet<Artifact> getInstrumentedFiles();
+
+ /**
+ * Returns a collection of instrumentation metadata files.
+ */
+ NestedSet<Artifact> getInstrumentationMetadataFiles();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java
new file mode 100644
index 0000000..1452e2d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+
+/**
+ * An implementation class for the InstrumentedFilesProvider interface.
+ */
+public final class InstrumentedFilesProviderImpl implements InstrumentedFilesProvider {
+ public static final InstrumentedFilesProvider EMPTY = new InstrumentedFilesProvider() {
+ @Override
+ public NestedSet<Artifact> getInstrumentedFiles() {
+ return NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER);
+ }
+ @Override
+ public NestedSet<Artifact> getInstrumentationMetadataFiles() {
+ return NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER);
+ }
+ };
+
+ private final NestedSet<Artifact> instrumentedFiles;
+ private final NestedSet<Artifact> instrumentationMetadataFiles;
+
+ public InstrumentedFilesProviderImpl(InstrumentedFilesCollector collector) {
+ this.instrumentedFiles = collector.getInstrumentedFiles();
+ this.instrumentationMetadataFiles = collector.getInstrumentationMetadataFiles();
+ }
+
+ @Override
+ public NestedSet<Artifact> getInstrumentedFiles() {
+ return instrumentedFiles;
+ }
+
+ @Override
+ public NestedSet<Artifact> getInstrumentationMetadataFiles() {
+ return instrumentationMetadataFiles;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java
new file mode 100644
index 0000000..006f789
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java
@@ -0,0 +1,224 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
+import com.google.devtools.common.options.OptionsClassProvider;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Runs TestRunnerAction actions.
+ */
+@ExecutionStrategy(contextType = TestActionContext.class,
+ name = { "standalone" })
+public class StandaloneTestStrategy extends TestStrategy {
+ /*
+ TODO(bazel-team):
+
+ * tests
+ * It would be nice to get rid of (cd $TEST_SRCDIR) in the test-setup script.
+ * test timeouts.
+ * parsing XML output.
+
+ */
+ protected final PathFragment runfilesPrefix;
+
+ public StandaloneTestStrategy(OptionsClassProvider requestOptions,
+ OptionsClassProvider startupOptions, BinTools binTools, PathFragment runfilesPrefix) {
+ super(requestOptions, startupOptions, binTools);
+
+ this.runfilesPrefix = runfilesPrefix;
+ }
+
+ private static final String TEST_SETUP = "tools/test/test-setup.sh";
+
+ @Override
+ public void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException {
+ Path runfilesDir = null;
+ try {
+ runfilesDir = TestStrategy.getLocalRunfilesDirectory(
+ action, actionExecutionContext, binTools);
+ } catch (ExecException e) {
+ throw new TestExecException(e.getMessage());
+ }
+
+ Path workingDirectory = runfilesDir.getRelative(runfilesPrefix);
+ Map<String, String> env = getEnv(action, runfilesDir);
+ Spawn spawn = new BaseSpawn(getArgs(action), env,
+ action.getTestProperties().getExecutionInfo(),
+ action,
+ action.getTestProperties().getLocalResourceUsage());
+
+ Executor executor = actionExecutionContext.getExecutor();
+ try {
+ FileSystemUtils.createDirectoryAndParents(workingDirectory);
+ FileOutErr fileOutErr = new FileOutErr(action.getTestLog().getPath(),
+ action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getTestStderr());
+ TestResultData data = execute(
+ actionExecutionContext.withFileOutErr(fileOutErr), spawn, action);
+ appendStderr(fileOutErr.getOutputFile(), fileOutErr.getErrorFile());
+ finalizeTest(actionExecutionContext, action, data);
+ } catch (IOException e) {
+ executor.getEventHandler().handle(Event.error("Caught I/O exception: " + e));
+ throw new EnvironmentalExecException("unexpected I/O exception", e);
+ }
+ }
+
+ private Map<String, String> getEnv(TestRunnerAction action, Path runfilesDir) {
+ Map<String, String> vars = getDefaultTestEnvironment(action);
+ BuildConfiguration config = action.getConfiguration();
+
+ vars.putAll(config.getDefaultShellEnvironment());
+ vars.putAll(config.getTestEnv());
+ vars.put("TEST_SRCDIR", runfilesDir.getRelative(runfilesPrefix).getPathString());
+
+ // TODO(bazel-team): set TEST_TMPDIR.
+
+ return vars;
+ }
+
+ private TestResultData execute(
+ ActionExecutionContext actionExecutionContext, Spawn spawn, TestRunnerAction action)
+ throws TestExecException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ Closeable streamed = null;
+ Path testLogPath = action.getTestLog().getPath();
+ TestResultData.Builder builder = TestResultData.newBuilder();
+
+ try {
+ try {
+ if (executionOptions.testOutput.equals(TestOutputFormat.STREAMED)) {
+ streamed = new StreamedTestOutput(
+ Reporter.outErrForReporter(
+ actionExecutionContext.getExecutor().getEventHandler()), testLogPath);
+ }
+ executor.getSpawnActionContext(action.getMnemonic()).exec(spawn, actionExecutionContext);
+
+ builder.setTestPassed(true)
+ .setStatus(BlazeTestStatus.PASSED)
+ .setCachable(true);
+ } catch (ExecException e) {
+ // Execution failed, which we consider a test failure.
+
+ // TODO(bazel-team): set cachable==true for relevant statuses (failure, but not for
+ // timeout, etc.)
+ builder.setTestPassed(false)
+ .setStatus(BlazeTestStatus.FAILED);
+ } finally {
+ if (streamed != null) {
+ streamed.close();
+ }
+ }
+
+ TestCase details = parseTestResult(
+ action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getXmlOutputPath());
+ if (details != null) {
+ builder.setTestCase(details);
+ }
+
+ return builder.build();
+ } catch (IOException e) {
+ throw new TestExecException(e.getMessage());
+ }
+ }
+
+ /**
+ * Outputs test result to the stdout after test has finished (e.g. for --test_output=all or
+ * --test_output=errors). Will also try to group output lines together (up to 10000 lines) so
+ * parallel test outputs will not get interleaved.
+ */
+ protected void processTestOutput(Executor executor, FileOutErr outErr, TestResult result)
+ throws IOException {
+ Path testOutput = executor.getExecRoot().getRelative(result.getTestLogPath().asFragment());
+ boolean isPassed = result.getData().getTestPassed();
+ try {
+ if (TestLogHelper.shouldOutputTestLog(executionOptions.testOutput, isPassed)) {
+ TestLogHelper.writeTestLog(testOutput, result.getTestName(), outErr.getOutputStream());
+ }
+ } finally {
+ if (isPassed) {
+ executor.getEventHandler().handle(new Event(EventKind.PASS, null, result.getTestName()));
+ } else {
+ if (result.getData().getStatus() == BlazeTestStatus.TIMEOUT) {
+ executor.getEventHandler().handle(
+ new Event(EventKind.TIMEOUT, null, result.getTestName()
+ + " (see " + testOutput + ")"));
+ } else {
+ executor.getEventHandler().handle(
+ new Event(EventKind.FAIL, null, result.getTestName() + " (see " + testOutput + ")"));
+ }
+ }
+ }
+ }
+
+ private final void finalizeTest(ActionExecutionContext actionExecutionContext,
+ TestRunnerAction action, TestResultData data) throws IOException, ExecException {
+ TestResult result = new TestResult(action, data, false);
+ postTestResult(actionExecutionContext.getExecutor(), result);
+
+ processTestOutput(actionExecutionContext.getExecutor(),
+ actionExecutionContext.getFileOutErr(), result);
+ // TODO(bazel-team): handle --test_output=errors, --test_output=all.
+
+ if (!executionOptions.testKeepGoing && data.getStatus() != BlazeTestStatus.PASSED) {
+ throw new TestExecException("Test failed: aborting");
+ }
+ }
+
+ private List<String> getArgs(TestRunnerAction action) {
+ List<String> args = Lists.newArrayList(TEST_SETUP);
+ TestTargetExecutionSettings execSettings = action.getExecutionSettings();
+
+ // Execute the test using the alias in the runfiles tree.
+ args.add(execSettings.getExecutable().getRootRelativePath().getPathString());
+ args.addAll(execSettings.getArgs());
+
+ return args;
+ }
+
+ @Override
+ public String strategyLocality(TestRunnerAction action) { return "standalone"; }
+
+ @Override
+ public TestResult newCachedTestResult(
+ Path execRoot, TestRunnerAction action, TestResultData data) {
+ return new TestResult(action, data, /*cached*/ true);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java
new file mode 100644
index 0000000..2ac9a0f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java
@@ -0,0 +1,270 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.Util;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.rules.test.TestProvider.TestParams;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.EnumConverter;
+
+import java.util.Collection;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class to create test actions.
+ */
+public final class TestActionBuilder {
+
+ private final RuleContext ruleContext;
+ private RunfilesSupport runfilesSupport;
+ private Artifact executable;
+ private ExecutionInfoProvider executionRequirements;
+ private InstrumentedFilesProvider instrumentedFiles;
+ private int explicitShardCount;
+
+ public TestActionBuilder(RuleContext ruleContext) {
+ this.ruleContext = ruleContext;
+ }
+
+ /**
+ * Creates the test actions and artifacts using the previously set parameters.
+ *
+ * @return ordered list of test status artifacts
+ */
+ public TestParams build() {
+ Preconditions.checkState(runfilesSupport != null);
+ boolean local = TargetUtils.isTestRuleAndRunsLocally(ruleContext.getRule());
+ TestShardingStrategy strategy = ruleContext.getConfiguration().testShardingStrategy();
+ int shards = strategy.getNumberOfShards(
+ local, explicitShardCount, isTestShardingCompliant(),
+ TestSize.getTestSize(ruleContext.getRule()));
+ Preconditions.checkState(shards >= 0);
+ return createTestAction(Util.getWorkspaceRelativePath(ruleContext.getLabel()), shards);
+ }
+
+ private boolean isTestShardingCompliant() {
+ // See if it has a data dependency on the special target
+ // //tools:test_sharding_compliant. Test runners add this dependency
+ // to show they speak the sharding protocol.
+ // There are certain cases where this heuristic may fail, giving
+ // a "false positive" (where we shard the test even though the
+ // it isn't supported). We may want to refine this logic, but
+ // heuristically sharding is currently experimental. Also, we do detect
+ // false-positive cases and return an error.
+ return runfilesSupport.getRunfilesSymlinkNames().contains(
+ new PathFragment("tools/test_sharding_compliant"));
+ }
+
+ /**
+ * Set the runfiles and executable to be run as a test.
+ */
+ public TestActionBuilder setFilesToRunProvider(FilesToRunProvider provider) {
+ Preconditions.checkNotNull(provider.getRunfilesSupport());
+ Preconditions.checkNotNull(provider.getExecutable());
+ this.runfilesSupport = provider.getRunfilesSupport();
+ this.executable = provider.getExecutable();
+ return this;
+ }
+
+ public TestActionBuilder setInstrumentedFiles(
+ @Nullable InstrumentedFilesProvider instrumentedFiles) {
+ this.instrumentedFiles = instrumentedFiles;
+ return this;
+ }
+
+ public TestActionBuilder setExecutionRequirements(
+ @Nullable ExecutionInfoProvider executionRequirements) {
+ this.executionRequirements = executionRequirements;
+ return this;
+ }
+
+ /**
+ * Set the explicit shard count. Note that this may be overridden by the sharding strategy.
+ */
+ public TestActionBuilder setShardCount(int explicitShardCount) {
+ this.explicitShardCount = explicitShardCount;
+ return this;
+ }
+
+ /**
+ * Converts to {@link TestActionBuilder.TestShardingStrategy}.
+ */
+ public static class ShardingStrategyConverter extends EnumConverter<TestShardingStrategy> {
+ public ShardingStrategyConverter() {
+ super(TestShardingStrategy.class, "test sharding strategy");
+ }
+ }
+
+ /**
+ * A strategy for running the same tests in many processes.
+ */
+ public static enum TestShardingStrategy {
+ EXPLICIT {
+ @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr,
+ boolean testShardingCompliant, TestSize testSize) {
+ return Math.max(shardCountFromAttr, 0);
+ }
+ },
+
+ EXPERIMENTAL_HEURISTIC {
+ @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr,
+ boolean testShardingCompliant, TestSize testSize) {
+ if (shardCountFromAttr >= 0) {
+ return shardCountFromAttr;
+ }
+ if (isLocal || !testShardingCompliant) {
+ return 0;
+ }
+ return testSize.getDefaultShards();
+ }
+ },
+
+ DISABLED {
+ @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr,
+ boolean testShardingCompliant, TestSize testSize) {
+ return 0;
+ }
+ };
+
+ public abstract int getNumberOfShards(boolean isLocal, int shardCountFromAttr,
+ boolean testShardingCompliant, TestSize testSize);
+ }
+
+ /**
+ * Creates a test action and artifacts for the given rule. The test action will
+ * use the specified executable and runfiles.
+ *
+ * @param targetName the relative path of the target to run
+ * @return ordered list of test artifacts, one per action. These are used to drive
+ * execution in Skyframe, and by AggregatingTestListener and
+ * TestResultAnalyzer to keep track of completed and pending test runs.
+ */
+ private TestParams createTestAction(PathFragment targetName, int shards) {
+ BuildConfiguration config = ruleContext.getConfiguration();
+ AnalysisEnvironment env = ruleContext.getAnalysisEnvironment();
+ Root root = config.getTestLogsDirectory();
+
+ NestedSetBuilder<Artifact> inputsBuilder = NestedSetBuilder.stableOrder();
+ inputsBuilder.addTransitive(
+ NestedSetBuilder.create(Order.STABLE_ORDER, runfilesSupport.getRunfilesMiddleman()));
+ for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("$test_runtime", Mode.HOST)) {
+ inputsBuilder.addTransitive(dep.getProvider(FileProvider.class).getFilesToBuild());
+ }
+ TestTargetProperties testProperties = new TestTargetProperties(
+ ruleContext, executionRequirements);
+
+ // If the test rule does not provide InstrumentedFilesProvider, there's not much that we can do.
+ final boolean collectCodeCoverage = config.isCodeCoverageEnabled()
+ && instrumentedFiles != null;
+
+ TestTargetExecutionSettings executionSettings;
+ if (collectCodeCoverage) {
+ // Add instrumented file manifest artifact to the list of inputs. This file will contain
+ // exec paths of all source files that should be included into the code coverage output.
+ Collection<Artifact> metadataFiles =
+ ImmutableList.copyOf(instrumentedFiles.getInstrumentationMetadataFiles());
+ inputsBuilder.addTransitive(NestedSetBuilder.wrap(Order.STABLE_ORDER, metadataFiles));
+ for (TransitiveInfoCollection dep :
+ ruleContext.getPrerequisites(":coverage_support", Mode.HOST)) {
+ inputsBuilder.addTransitive(dep.getProvider(FileProvider.class).getFilesToBuild());
+ }
+ Artifact instrumentedFileManifest =
+ InstrumentedFileManifestAction.getInstrumentedFileManifest(ruleContext,
+ ImmutableList.copyOf(instrumentedFiles.getInstrumentedFiles()),
+ metadataFiles);
+ executionSettings = new TestTargetExecutionSettings(ruleContext, runfilesSupport,
+ executable, instrumentedFileManifest, shards);
+ inputsBuilder.add(instrumentedFileManifest);
+ } else {
+ executionSettings = new TestTargetExecutionSettings(ruleContext, runfilesSupport,
+ executable, null, shards);
+ }
+
+ if (config.getRunUnder() != null) {
+ Artifact runUnderExecutable = executionSettings.getRunUnderExecutable();
+ if (runUnderExecutable != null) {
+ inputsBuilder.add(runUnderExecutable);
+ }
+ }
+
+ int runsPerTest = config.getRunsPerTestForLabel(ruleContext.getLabel());
+
+ Iterable<Artifact> inputs = inputsBuilder.build();
+ int shardRuns = (shards > 0 ? shards : 1);
+ List<Artifact> results = Lists.newArrayListWithCapacity(runsPerTest * shardRuns);
+ ImmutableList.Builder<Artifact> coverageArtifacts = ImmutableList.builder();
+
+ for (int run = 0; run < runsPerTest; run++) {
+ // Use a 1-based index for user friendliness.
+ String runSuffix =
+ runsPerTest > 1 ? String.format("_run_%d_of_%d", run + 1, runsPerTest) : "";
+ for (int shard = 0; shard < shardRuns; shard++) {
+ String suffix = (shardRuns > 1 ? String.format("_shard_%d_of_%d", shard + 1, shards) : "")
+ + runSuffix;
+ Artifact testLog = env.getDerivedArtifact(
+ targetName.getChild("test" + suffix + ".log"), root);
+ Artifact cacheStatus = env.getDerivedArtifact(
+ targetName.getChild("test" + suffix + ".cache_status"), root);
+
+ Artifact coverageArtifact = null;
+ if (collectCodeCoverage) {
+ coverageArtifact =
+ env.getDerivedArtifact(targetName.getChild("coverage" + suffix + ".dat"), root);
+ coverageArtifacts.add(coverageArtifact);
+ }
+
+ Artifact microCoverageArtifact = null;
+ if (collectCodeCoverage && config.isMicroCoverageEnabled()) {
+ microCoverageArtifact =
+ env.getDerivedArtifact(targetName.getChild("coverage" + suffix + ".micro.dat"), root);
+ }
+
+ env.registerAction(new TestRunnerAction(
+ ruleContext.getActionOwner(), inputs,
+ testLog, cacheStatus,
+ coverageArtifact, microCoverageArtifact,
+ testProperties, executionSettings,
+ shard, run, config));
+ results.add(cacheStatus);
+ }
+ }
+ // TODO(bazel-team): Passing the reportGenerator to every TestParams is a bit strange.
+ Artifact reportGenerator = collectCodeCoverage
+ ? ruleContext.getPrerequisiteArtifact(":coverage_report_generator", Mode.HOST) : null;
+ return new TestParams(runsPerTest, shards, TestTimeout.getTestTimeout(ruleContext.getRule()),
+ ruleContext.getRule().getRuleClass(), ImmutableList.copyOf(results),
+ coverageArtifacts.build(), reportGenerator);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java
new file mode 100644
index 0000000..7f7a916
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
+
+import java.io.IOException;
+
+/**
+ * A context for the execution of test actions ({@link TestRunnerAction}).
+ */
+public interface TestActionContext extends ActionContext {
+
+ /**
+ * Executes the test command, directing standard out / err to {@code outErr}. The status of
+ * the test should be communicated by posting a {@link TestResult} object to the eventbus.
+ */
+ void exec(TestRunnerAction action,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException;
+
+ /**
+ * String describing where the action will run.
+ */
+ String strategyLocality(TestRunnerAction action);
+
+ /**
+ * Creates a cached test result.
+ */
+ TestResult newCachedTestResult(Path execRoot, TestRunnerAction action, TestResultData cached)
+ throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java
new file mode 100644
index 0000000..462c24c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.BufferedOutputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+
+/**
+ * A helper class for test log handling. It determines whether the test log should
+ * be output and formats the test log for console display.
+ */
+public class TestLogHelper {
+
+ public static final String HEADER_DELIMITER =
+ "-----------------------------------------------------------------------------";
+
+ /**
+ * Determines whether the test log should be output from the current outputMode
+ * and whether the test has passed or not.
+ */
+ public static boolean shouldOutputTestLog(TestOutputFormat outputMode, boolean hasPassed) {
+ return (outputMode == TestOutputFormat.ALL) ||
+ (!hasPassed && (outputMode == TestOutputFormat.ERRORS));
+ }
+
+ /**
+ * Reads the contents of the test log from the provided testOutput file, adds
+ * header and footer and returns the result.
+ * This method also looks for a header delimiter and cuts off the text before it,
+ * except if the header is 50 lines or longer.
+ */
+ public static void writeTestLog(Path testOutput, String testName, OutputStream out)
+ throws IOException {
+ InputStream input = null;
+ PrintStream printOut = new PrintStream(new BufferedOutputStream(out));
+ try {
+ final String outputHeader =
+ "==================== Test output for " + testName + ":";
+ final String outputFooter =
+ "================================================================================";
+
+ printOut.println(outputHeader);
+ printOut.flush();
+
+ input = testOutput.getInputStream();
+ FilterTestHeaderOutputStream filteringOutputStream = getHeaderFilteringOutputStream(printOut);
+ ByteStreams.copy(input, filteringOutputStream);
+
+ if (!filteringOutputStream.foundHeader()) {
+ InputStream inputAgain = testOutput.getInputStream();
+ try {
+ ByteStreams.copy(inputAgain, out);
+ } finally {
+ inputAgain.close();
+ }
+ }
+
+ printOut.println(outputFooter);
+ } finally {
+ printOut.flush();
+ if (input != null) {
+ input.close();
+ }
+ }
+ }
+
+ /**
+ * Returns an output stream that doesn't write to original until it
+ * sees HEADER_DELIMITER by itself on a line.
+ */
+ public static FilterTestHeaderOutputStream getHeaderFilteringOutputStream(OutputStream original) {
+ return new FilterTestHeaderOutputStream(original);
+ }
+
+ private TestLogHelper() {
+ // Prevent Java from creating a public constructor.
+ }
+
+ /**
+ * Use this class to filter the streaming output of a test until we see the
+ * header delimiter.
+ */
+ public static class FilterTestHeaderOutputStream extends FilterOutputStream {
+
+ private boolean seenDelimiter = false;
+ private StringBuilder lineBuilder = new StringBuilder();
+
+ private static final int NEWLINE = '\n';
+
+ public FilterTestHeaderOutputStream(OutputStream out) {
+ super(out);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (seenDelimiter) {
+ out.write(b);
+ } else if (b == NEWLINE) {
+ String line = lineBuilder.toString();
+ lineBuilder = new StringBuilder();
+ if (line.equals(TestLogHelper.HEADER_DELIMITER)) {
+ seenDelimiter = true;
+ }
+ } else if (lineBuilder.length() <= TestLogHelper.HEADER_DELIMITER.length()) {
+ lineBuilder.append((char) b);
+ }
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ if (seenDelimiter) {
+ out.write(b, off, len);
+ } else {
+ super.write(b, off, len);
+ }
+ }
+
+ public boolean foundHeader() {
+ return seenDelimiter;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java
new file mode 100644
index 0000000..d24fe6b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.packages.TestTimeout;
+
+import java.util.List;
+
+/**
+ * A {@link TransitiveInfoProvider} for configured targets that implement test rules.
+ */
+@Immutable
+public final class TestProvider implements TransitiveInfoProvider {
+ private final TestParams testParams;
+ private final ImmutableList<String> testTags;
+
+ public TestProvider(TestParams testParams, ImmutableList<String> testTags) {
+ this.testParams = testParams;
+ this.testTags = testTags;
+ }
+
+ /**
+ * Returns the {@link TestParams} object for the test represented by the corresponding configured
+ * target.
+ */
+ public TestParams getTestParams() {
+ return testParams;
+ }
+
+ /**
+ * Temporary hack to allow dependencies on test_suite targets to continue to work for the time
+ * being.
+ */
+ public List<String> getTestTags() {
+ return testTags;
+ }
+
+ /**
+ * Returns the test status artifacts for a specified configured target
+ *
+ * @param target the configured target. Should belong to a test rule.
+ * @return the test status artifacts
+ */
+ public static ImmutableList<Artifact> getTestStatusArtifacts(TransitiveInfoCollection target) {
+ return target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts();
+ }
+
+ /**
+ * A value class describing the properties of a test.
+ */
+ public static class TestParams {
+ private final int runs;
+ private final int shards;
+ private final TestTimeout timeout;
+ private final String testRuleClass;
+ private final ImmutableList<Artifact> testStatusArtifacts;
+ private final ImmutableList<Artifact> coverageArtifacts;
+ private final Artifact coverageReportGenerator;
+
+ /**
+ * Don't call this directly. Instead use {@link TestActionBuilder}.
+ */
+ TestParams(int runs, int shards, TestTimeout timeout, String testRuleClass,
+ ImmutableList<Artifact> testStatusArtifacts,
+ ImmutableList<Artifact> coverageArtifacts,
+ Artifact coverageReportGenerator) {
+ this.runs = runs;
+ this.shards = shards;
+ this.timeout = timeout;
+ this.testRuleClass = testRuleClass;
+ this.testStatusArtifacts = testStatusArtifacts;
+ this.coverageArtifacts = coverageArtifacts;
+ this.coverageReportGenerator = coverageReportGenerator;
+ }
+
+ /**
+ * Returns the number of times this test should be run.
+ */
+ public int getRuns() {
+ return runs;
+ }
+
+ /**
+ * Returns the number of shards for this test.
+ */
+ public int getShards() {
+ return shards;
+ }
+
+ /**
+ * Returns the timeout of this test.
+ */
+ public TestTimeout getTimeout() {
+ return timeout;
+ }
+
+ /**
+ * Returns the test rule class.
+ */
+ public String getTestRuleClass() {
+ return testRuleClass;
+ }
+
+ /**
+ * Returns a list of test status artifacts that represent serialized test status protobuffers
+ * produced by testing this target.
+ */
+ public ImmutableList<Artifact> getTestStatusArtifacts() {
+ return testStatusArtifacts;
+ }
+
+ /**
+ * Returns the coverageArtifacts
+ */
+ public ImmutableList<Artifact> getCoverageArtifacts() {
+ return coverageArtifacts;
+ }
+
+ /**
+ * Returns the coverage report generator tool.
+ */
+ public Artifact getCoverageReportGenerator() {
+ return coverageReportGenerator;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java
new file mode 100644
index 0000000..b0de6cd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
+
+/**
+ * This is the event passed from the various test strategies to the {@code RecordingTestListener}
+ * upon test completion.
+ */
+@ThreadSafe
+@Immutable
+public class TestResult {
+
+ private final TestRunnerAction testAction;
+ private final TestResultData data;
+ private final boolean cached;
+
+ /**
+ * Construct the TestResult for the given test / status.
+ *
+ * @param testAction The test that was run.
+ * @param data test result protobuffer.
+ * @param cached true if this is a cached test result.
+ */
+ public TestResult(TestRunnerAction testAction, TestResultData data, boolean cached) {
+ this.testAction = Preconditions.checkNotNull(testAction);
+ this.data = data;
+ this.cached = cached;
+ }
+
+ public static boolean isBlazeTestStatusPassed(BlazeTestStatus status) {
+ return status == BlazeTestStatus.PASSED || status == BlazeTestStatus.FLAKY;
+ }
+
+ /**
+ * @return The test action.
+ */
+ public TestRunnerAction getTestAction() {
+ return testAction;
+ }
+
+ /**
+ * @return The test log path. Note, that actual log file may no longer
+ * correspond to this artifact - use getActualLogPath() method if
+ * you need log location.
+ */
+ public Path getTestLogPath() {
+ return testAction.getTestLog().getPath();
+ }
+
+ /**
+ * Return if result was loaded from local action cache.
+ */
+ public final boolean isCached() {
+ return cached;
+ }
+
+ /**
+ * @return Coverage data artifact, if available and null otherwise.
+ */
+ public PathFragment getCoverageData() {
+ if (data.getHasCoverage()) {
+ return testAction.getCoverageData().getExecPath();
+ }
+ return null;
+ }
+
+ /**
+ * @return The test status artifact.
+ */
+ public Artifact getTestStatusArtifact() {
+ // these artifacts are used to keep track of the number of pending and completed tests.
+ return testAction.getCacheStatusArtifact();
+ }
+
+
+ /**
+ * Gets the test name in a user-friendly format.
+ * Will generally include the target name and shard number, if applicable.
+ *
+ * @return The test name.
+ */
+ public String getTestName() {
+ return testAction.getTestName();
+ }
+
+ /**
+ * @return The test label.
+ */
+ public String getLabel() {
+ return Label.print(testAction.getOwner().getLabel());
+ }
+
+ /**
+ * @return The test shard number.
+ */
+ public int getShardNum() {
+ return testAction.getShardNum();
+ }
+
+ /**
+ * @return Total number of test shards. 0 means
+ * no sharding, whereas 1 means degenerate sharding.
+ */
+ public int getTotalShards() {
+ return testAction.getExecutionSettings().getTotalShards();
+ }
+
+ public TestResultData getData() {
+ return data;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java
new file mode 100644
index 0000000..28500a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java
@@ -0,0 +1,607 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
+import com.google.devtools.common.options.TriState;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.logging.Level;
+
+import javax.annotation.Nullable;
+
+/**
+ * An Action representing a test with the associated environment (runfiles,
+ * environment variables, test result, etc). It consumes test executable and
+ * runfiles artifacts and produces test result and test status artifacts.
+ */
+// Not final so that we can mock it in tests.
+public class TestRunnerAction extends AbstractAction implements NotifyOnActionCacheHit {
+
+ private static final String GUID = "94857c93-f11c-4cbc-8c1b-e0a281633f9e";
+
+ private final BuildConfiguration configuration;
+ private final Artifact testLog;
+ private final Artifact cacheStatus;
+ private final PathFragment testWarningsPath;
+ private final PathFragment splitLogsPath;
+ private final PathFragment splitLogsDir;
+ private final PathFragment undeclaredOutputsDir;
+ private final PathFragment undeclaredOutputsZipPath;
+ private final PathFragment undeclaredOutputsAnnotationsDir;
+ private final PathFragment undeclaredOutputsManifestPath;
+ private final PathFragment undeclaredOutputsAnnotationsPath;
+ private final PathFragment xmlOutputPath;
+ @Nullable
+ private final PathFragment testShard;
+ private final PathFragment testExitSafe;
+ private final PathFragment testStderr;
+ private final PathFragment testInfrastructureFailure;
+ private final PathFragment baseDir;
+ private final String namePrefix;
+ private final Artifact coverageData;
+ private final Artifact microCoverageData;
+ private final TestTargetProperties testProperties;
+ private final TestTargetExecutionSettings executionSettings;
+ private final int shardNum;
+ private final int runNumber;
+
+ // Mutable state related to test caching.
+ private boolean checkedCaching = false;
+ private boolean unconditionalExecution = false;
+
+ private static ImmutableList<Artifact> list(Artifact... artifacts) {
+ ImmutableList.Builder<Artifact> builder = ImmutableList.builder();
+ for (Artifact artifact : artifacts) {
+ if (artifact != null) {
+ builder.add(artifact);
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Create new TestRunnerAction instance. Should not be called directly.
+ * Use {@link TestActionBuilder} instead.
+ *
+ * @param shardNum The shard number. Must be 0 if totalShards == 0
+ * (no sharding). Otherwise, must be >= 0 and < totalShards.
+ * @param runNumber test run number
+ */
+ TestRunnerAction(ActionOwner owner,
+ Iterable<Artifact> inputs,
+ Artifact testLog,
+ Artifact cacheStatus,
+ Artifact coverageArtifact,
+ Artifact microCoverageArtifact,
+ TestTargetProperties testProperties,
+ TestTargetExecutionSettings executionSettings,
+ int shardNum,
+ int runNumber,
+ BuildConfiguration configuration) {
+ super(owner, inputs, list(testLog, cacheStatus, coverageArtifact, microCoverageArtifact));
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.testLog = testLog;
+ this.cacheStatus = cacheStatus;
+ this.coverageData = coverageArtifact;
+ this.microCoverageData = microCoverageArtifact;
+ this.shardNum = shardNum;
+ this.runNumber = runNumber;
+ this.testProperties = Preconditions.checkNotNull(testProperties);
+ this.executionSettings = Preconditions.checkNotNull(executionSettings);
+
+ this.baseDir = cacheStatus.getExecPath().getParentDirectory();
+ this.namePrefix = FileSystemUtils.removeExtension(cacheStatus.getExecPath().getBaseName());
+
+ int totalShards = executionSettings.getTotalShards();
+ Preconditions.checkState((totalShards == 0 && shardNum == 0) ||
+ (totalShards > 0 && 0 <= shardNum && shardNum < totalShards));
+ this.testExitSafe = baseDir.getChild(namePrefix + ".exited_prematurely");
+ // testShard Path should be set only if sharding is enabled.
+ this.testShard = totalShards > 1
+ ? baseDir.getChild(namePrefix + ".shard")
+ : null;
+ this.xmlOutputPath = baseDir.getChild(namePrefix + ".xml");
+ this.testWarningsPath = baseDir.getChild(namePrefix + ".warnings");
+ this.testStderr = baseDir.getChild(namePrefix + ".err");
+ this.splitLogsDir = baseDir.getChild(namePrefix + ".raw_splitlogs");
+ // See note in {@link #getSplitLogsPath} on the choice of file name.
+ this.splitLogsPath = splitLogsDir.getChild("test.splitlogs");
+ this.undeclaredOutputsDir = baseDir.getChild(namePrefix + ".outputs");
+ this.undeclaredOutputsZipPath = undeclaredOutputsDir.getChild("outputs.zip");
+ this.undeclaredOutputsAnnotationsDir = baseDir.getChild(namePrefix + ".outputs_manifest");
+ this.undeclaredOutputsManifestPath = undeclaredOutputsAnnotationsDir.getChild("MANIFEST");
+ this.undeclaredOutputsAnnotationsPath = undeclaredOutputsAnnotationsDir.getChild("ANNOTATIONS");
+ this.testInfrastructureFailure = baseDir.getChild(namePrefix + ".infrastructure_failure");
+ }
+
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public final PathFragment getBaseDir() {
+ return baseDir;
+ }
+
+ public final String getNamePrefix() {
+ return namePrefix;
+ }
+
+ @Override
+ public boolean showsOutputUnconditionally() {
+ return true;
+ }
+
+ @Override
+ public int getInputCount() {
+ return Iterables.size(getInputs());
+ }
+
+ @Override
+ protected String computeKey() {
+ Fingerprint f = new Fingerprint();
+ f.addString(GUID);
+ f.addStrings(executionSettings.getArgs());
+ f.addString(executionSettings.getTestFilter() == null ? "" : executionSettings.getTestFilter());
+ RunUnder runUnder = executionSettings.getRunUnder();
+ f.addString(runUnder == null ? "" : runUnder.getValue());
+ f.addStringMap(configuration.getTestEnv());
+ f.addString(testProperties.getSize().toString());
+ f.addString(testProperties.getTimeout().toString());
+ f.addStrings(testProperties.getTags());
+ f.addInt(testProperties.isLocal() ? 1 : 0);
+ f.addInt(shardNum);
+ f.addInt(executionSettings.getTotalShards());
+ f.addInt(runNumber);
+ f.addInt(configuration.getRunsPerTestForLabel(getOwner().getLabel()));
+ f.addInt(configuration.isCodeCoverageEnabled() ? 1 : 0);
+ return f.hexDigestAndReset();
+ }
+
+ @Override
+ public boolean executeUnconditionally() {
+ // Note: isVolatile must return true if executeUnconditionally can ever return true
+ // for this instance.
+ unconditionalExecution = updateExecuteUnconditionallyFromTestStatus();
+ checkedCaching = true;
+ return unconditionalExecution;
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return true;
+ }
+
+ /**
+ * Saves cache status to disk.
+ */
+ public void saveCacheStatus(TestResultData data) throws IOException {
+ try (OutputStream out = cacheStatus.getPath().getOutputStream()) {
+ data.writeTo(out);
+ }
+ }
+
+ /**
+ * Returns the cache from disk, or null if there is an error.
+ */
+ @Nullable
+ private TestResultData readCacheStatus() {
+ try (InputStream in = cacheStatus.getPath().getInputStream()) {
+ return TestResultData.parseFrom(in);
+ } catch (IOException expected) {
+
+ }
+ return null;
+ }
+
+ private boolean updateExecuteUnconditionallyFromTestStatus() {
+ if (configuration.cacheTestResults() == TriState.NO || testProperties.isExternal()
+ || (configuration.cacheTestResults() == TriState.AUTO
+ && configuration.getRunsPerTestForLabel(getOwner().getLabel()) > 1)) {
+ return true;
+ }
+
+ // Test will not be executed unconditionally - check whether test result exists and is
+ // valid. If it is, method will return false and we will rely on the dependency checker
+ // to make a decision about test execution.
+ TestResultData status = readCacheStatus();
+ if (status != null) {
+ if (!status.getCachable()) {
+ return true;
+ }
+
+ return (configuration.cacheTestResults() == TriState.AUTO
+ && !status.getTestPassed());
+ }
+
+ // CacheStatus is an artifact, so if it does not exist, the dependency checker will rebuild
+ // it. We can't return "true" here, as it also signals to not accept cached remote results.
+ return false;
+ }
+
+ /**
+ * May only be called after the dependency checked called executeUnconditionally().
+ * Returns whether caching has been deemed safe by looking at the previous test run
+ * (for local caching). If the previous run is not present, return "true" here, as
+ * remote execution caching should be safe.
+ */
+ public boolean shouldCacheResult() {
+ Preconditions.checkState(checkedCaching);
+ return !unconditionalExecution;
+ }
+
+ @Override
+ public void actionCacheHit(Executor executor) {
+ checkedCaching = false;
+ try {
+ executor.getEventBus().post(
+ executor.getContext(TestActionContext.class).newCachedTestResult(
+ executor.getExecRoot(), this, readCacheStatus()));
+ } catch (IOException e) {
+ LoggingUtil.logToRemote(Level.WARNING, "Failed creating cached protocol buffer", e);
+ }
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ // return null here to indicate that resources would be managed manually
+ // during action execution.
+ return null;
+ }
+
+ @Override
+ protected String getRawProgressMessage() {
+ return "Testing " + getTestName();
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return executor.getContext(TestActionContext.class).strategyLocality(this);
+ }
+
+ /**
+ * Deletes <b>all</b> possible test outputs.
+ *
+ * TestRunnerAction potentially can create many more non-declared outputs - xml output,
+ * coverage data file and logs for failed attempts. All those outputs are uniquely
+ * identified by the test log base name with arbitrary prefix and extension.
+ */
+ @Override
+ protected void deleteOutputs(Path execRoot) throws IOException {
+ super.deleteOutputs(execRoot);
+
+ // We do not rely on globs, as it causes quadratic behavior in --runs_per_test and test
+ // shard count.
+
+ // We also need to remove *.(xml|data|shard|warnings|zip) files if they are present.
+ execRoot.getRelative(xmlOutputPath).delete();
+ execRoot.getRelative(testWarningsPath).delete();
+ // Note that splitLogsPath points to a file inside the splitLogsDir so
+ // it's not necessary to delete it explicitly.
+ FileSystemUtils.deleteTree(execRoot.getRelative(splitLogsDir));
+ FileSystemUtils.deleteTree(execRoot.getRelative(undeclaredOutputsDir));
+ FileSystemUtils.deleteTree(execRoot.getRelative(undeclaredOutputsAnnotationsDir));
+ execRoot.getRelative(testStderr).delete();
+ execRoot.getRelative(testExitSafe).delete();
+ if (testShard != null) {
+ execRoot.getRelative(testShard).delete();
+ }
+ execRoot.getRelative(testInfrastructureFailure).delete();
+
+ // Coverage files use "coverage" instead of "test".
+ String coveragePrefix = "coverage" + namePrefix.substring(4);
+
+ // We cannot use coverageData artifact since it may be null. Generate coverage name instead.
+ execRoot.getRelative(baseDir.getChild(coveragePrefix + ".dat")).delete();
+ // We cannot use microcoverageData artifact since it may be null. Generate filename instead.
+ execRoot.getRelative(baseDir.getChild(coveragePrefix + ".micro.dat")).delete();
+
+ // Delete files fetched from remote execution.
+ execRoot.getRelative(baseDir.getChild(namePrefix + ".zip")).delete();
+ deleteTestAttemptsDirMaybe(execRoot.getRelative(baseDir), namePrefix);
+ }
+
+ private void deleteTestAttemptsDirMaybe(Path outputDir, String namePrefix) throws IOException {
+ Path testAttemptsDir = outputDir.getChild(namePrefix + "_attempts");
+ if (testAttemptsDir.exists()) {
+ // Normally we should have used deleteTree(testAttemptsDir). However, if test output is
+ // in a FUSE filesystem implemented with the high-level API, there may be .fuse???????
+ // entries, which prevent removing the directory. As a workaround, code below will throw
+ // IOException if it will fail to remove something inside testAttemptsDir, but will
+ // silently suppress any exceptions when deleting testAttemptsDir itself.
+ FileSystemUtils.deleteTreesBelow(testAttemptsDir);
+ try {
+ testAttemptsDir.delete();
+ } catch (IOException e) {
+ // Do nothing.
+ }
+ }
+ }
+
+ /**
+ * Gets the test name in a user-friendly format.
+ * Will generally include the target name and run/shard numbers, if applicable.
+ */
+ public String getTestName() {
+ String suffix = getTestSuffix();
+ String label = Label.print(getOwner().getLabel());
+ return suffix.isEmpty() ? label : label + " " + suffix;
+ }
+
+ /**
+ * Gets the test suffix in a user-friendly format, eg "(shard 1 of 7)".
+ * Will include the target name and run/shard numbers, if applicable.
+ */
+ public String getTestSuffix() {
+ int totalShards = executionSettings.getTotalShards();
+ // Use a 1-based index for user friendliness.
+ int runsPerTest = configuration.getRunsPerTestForLabel(getOwner().getLabel());
+ if (totalShards > 1 && runsPerTest > 1) {
+ return String.format("(shard %d of %d, run %d of %d)", shardNum + 1, totalShards,
+ runNumber + 1, runsPerTest);
+ } else if (totalShards > 1) {
+ return String.format("(shard %d of %d)", shardNum + 1, totalShards);
+ } else if (runsPerTest > 1) {
+ return String.format("(run %d of %d)", runNumber + 1, runsPerTest);
+ } else {
+ return "";
+ }
+ }
+
+ public Artifact getTestLog() {
+ return testLog;
+ }
+
+ public ResolvedPaths resolve(Path execRoot) {
+ return new ResolvedPaths(execRoot);
+ }
+
+ public Artifact getCacheStatusArtifact() {
+ return cacheStatus;
+ }
+
+ public PathFragment getTestWarningsPath() {
+ return testWarningsPath;
+ }
+
+ public PathFragment getSplitLogsPath() {
+ return splitLogsPath;
+ }
+
+ /**
+ * @return path to the optional zip file of undeclared test outputs.
+ */
+ public PathFragment getUndeclaredOutputsZipPath() {
+ return undeclaredOutputsZipPath;
+ }
+
+ /**
+ * @return path to the undeclared output manifest file.
+ */
+ public PathFragment getUndeclaredOutputsManifestPath() {
+ return undeclaredOutputsManifestPath;
+ }
+
+ /**
+ * @return path to the undeclared output annotations file.
+ */
+ public PathFragment getUndeclaredOutputsAnnotationsPath() {
+ return undeclaredOutputsAnnotationsPath;
+ }
+
+ public PathFragment getTestShard() {
+ return testShard;
+ }
+
+ public PathFragment getExitSafeFile() {
+ return testExitSafe;
+ }
+
+ public PathFragment getInfrastructureFailureFile() {
+ return testInfrastructureFailure;
+ }
+
+ /**
+ * @return path to the optionally created XML output file created by the test.
+ */
+ public PathFragment getXmlOutputPath() {
+ return xmlOutputPath;
+ }
+
+ /**
+ * @return coverage data artifact or null if code coverage was not requested.
+ */
+ @Nullable public Artifact getCoverageData() {
+ return coverageData;
+ }
+
+ /**
+ * @return microcoverage data artifact or null if code coverage was not requested.
+ */
+ @Nullable public Artifact getMicroCoverageData() {
+ return microCoverageData;
+ }
+
+ public TestTargetProperties getTestProperties() {
+ return testProperties;
+ }
+
+ public TestTargetExecutionSettings getExecutionSettings() {
+ return executionSettings;
+ }
+
+ public boolean isSharded() {
+ return testShard != null;
+ }
+
+ /**
+ * @return the shard number for this action.
+ * If getTotalShards() > 0, must be >= 0 and < getTotalShards().
+ * Otherwise, must be 0.
+ */
+ public int getShardNum() {
+ return shardNum;
+ }
+
+ /**
+ * @return run number.
+ */
+ public int getRunNumber() {
+ return runNumber;
+ }
+
+ @Override
+ public Artifact getPrimaryOutput() {
+ return testLog;
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ TestActionContext context =
+ actionExecutionContext.getExecutor().getContext(TestActionContext.class);
+ try {
+ context.exec(this, actionExecutionContext);
+ } catch (ExecException e) {
+ throw e.toActionExecutionException(this);
+ } finally {
+ checkedCaching = false;
+ }
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "TestRunner";
+ }
+
+ /**
+ * The same set of paths as the parent test action, resolved against a given exec root.
+ */
+ public final class ResolvedPaths {
+ private final Path execRoot;
+
+ ResolvedPaths(Path execRoot) {
+ this.execRoot = Preconditions.checkNotNull(execRoot);
+ }
+
+ private Path getPath(PathFragment relativePath) {
+ return execRoot.getRelative(relativePath);
+ }
+
+ public final Path getBaseDir() {
+ return getPath(baseDir);
+ }
+
+ /**
+ * In rare cases, error messages will be printed to stderr instead of stdout. The test action is
+ * responsible for appending anything in the stderr file to the real test.log.
+ */
+ public Path getTestStderr() {
+ return getPath(testStderr);
+ }
+
+ public Path getTestWarningsPath() {
+ return getPath(testWarningsPath);
+ }
+
+ public Path getSplitLogsPath() {
+ return getPath(splitLogsPath);
+ }
+
+ /**
+ * @return path to the directory containing the split logs (raw and proto file).
+ */
+ public Path getSplitLogsDir() {
+ return getPath(splitLogsDir);
+ }
+
+ /**
+ * @return path to the optional zip file of undeclared test outputs.
+ */
+ public Path getUndeclaredOutputsZipPath() {
+ return getPath(undeclaredOutputsZipPath);
+ }
+
+ /**
+ * @return path to the directory to hold undeclared test outputs.
+ */
+ public Path getUndeclaredOutputsDir() {
+ return getPath(undeclaredOutputsDir);
+ }
+
+ /**
+ * @return path to the directory to hold undeclared output annotations parts.
+ */
+ public Path getUndeclaredOutputsAnnotationsDir() {
+ return getPath(undeclaredOutputsAnnotationsDir);
+ }
+
+ /**
+ * @return path to the undeclared output manifest file.
+ */
+ public Path getUndeclaredOutputsManifestPath() {
+ return getPath(undeclaredOutputsManifestPath);
+ }
+
+ /**
+ * @return path to the undeclared output annotations file.
+ */
+ public Path getUndeclaredOutputsAnnotationsPath() {
+ return getPath(undeclaredOutputsAnnotationsPath);
+ }
+
+ @Nullable
+ public Path getTestShard() {
+ return testShard == null ? null : getPath(testShard);
+ }
+
+ public Path getExitSafeFile() {
+ return getPath(testExitSafe);
+ }
+
+ public Path getInfrastructureFailureFile() {
+ return getPath(testInfrastructureFailure);
+ }
+
+ /**
+ * @return path to the optionally created XML output file created by the test.
+ */
+ public Path getXmlOutputPath() {
+ return getPath(xmlOutputPath);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java
new file mode 100644
index 0000000..4905e15
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java
@@ -0,0 +1,388 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Closeables;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.exec.SymlinkTreeHelper;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions;
+import com.google.devtools.build.lib.util.io.FileWatcher;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import com.google.devtools.common.options.Converters.RangeConverter;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A strategy for executing a {@link TestRunnerAction}.
+ */
+public abstract class TestStrategy implements TestActionContext {
+ /**
+ * Converter for the --flaky_test_attempts option.
+ */
+ public static class TestAttemptsConverter extends RangeConverter {
+ public TestAttemptsConverter() {
+ super(1, 10);
+ }
+
+ @Override
+ public Integer convert(String input) throws OptionsParsingException {
+ if ("default".equals(input)) {
+ return -1;
+ } else {
+ return super.convert(input);
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return super.getTypeDescription() + " or the string \"default\"";
+ }
+ }
+
+ public enum TestOutputFormat {
+ SUMMARY, // Provide summary output only.
+ ERRORS, // Print output from failed tests to the stderr after the test failure.
+ ALL, // Print output from all tests to the stderr after the test completion.
+ STREAMED; // Stream output for each test.
+
+ /**
+ * Converts to {@link TestOutputFormat}.
+ */
+ public static class Converter extends EnumConverter<TestOutputFormat> {
+ public Converter() {
+ super(TestOutputFormat.class, "test output");
+ }
+ }
+ }
+
+ public enum TestSummaryFormat {
+ SHORT, // Print information only about tests.
+ TERSE, // Like "SHORT", but even shorter: Do not print PASSED tests.
+ DETAILED, // Print information only about failed test cases.
+ NONE; // Do not print summary.
+
+ /**
+ * Converts to {@link TestSummaryFormat}.
+ */
+ public static class Converter extends EnumConverter<TestSummaryFormat> {
+ public Converter() {
+ super(TestSummaryFormat.class, "test summary");
+ }
+ }
+ }
+
+ public static final PathFragment TEST_TMP_ROOT = new PathFragment("_tmp");
+
+ // Used for selecting subset of testcase / testmethods.
+ private static final String TEST_BRIDGE_TEST_FILTER_ENV = "TESTBRIDGE_TEST_ONLY";
+
+ private final boolean statusServerRunning;
+ protected final ExecutionOptions executionOptions;
+ protected final BinTools binTools;
+
+ public TestStrategy(OptionsClassProvider requestOptionsProvider,
+ OptionsClassProvider startupOptionsProvider, BinTools binTools) {
+ this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class);
+ this.binTools = binTools;
+ BlazeServerStartupOptions startupOptions =
+ startupOptionsProvider.getOptions(BlazeServerStartupOptions.class);
+ statusServerRunning = startupOptions != null && startupOptions.useWebStatusServer > 0;
+ }
+
+ @Override
+ public abstract void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException;
+
+ @Override
+ public abstract String strategyLocality(TestRunnerAction action);
+
+ /**
+ * Callback for determining the strategy locality.
+ *
+ * @param action the test action
+ * @param localRun whether to run it locally
+ */
+ protected String strategyLocality(TestRunnerAction action, boolean localRun) {
+ return strategyLocality(action);
+ }
+
+ /**
+ * Returns mutable map of default testing shell environment. By itself it is incomplete and is
+ * modified further by the specific test strategy implementations (mostly due to the fact that
+ * environments used locally and remotely are different).
+ */
+ protected Map<String, String> getDefaultTestEnvironment(TestRunnerAction action) {
+ Map<String, String> env = new HashMap<>();
+
+ env.putAll(action.getConfiguration().getDefaultShellEnvironment());
+ env.remove("LANG");
+ env.put("TZ", "UTC");
+ env.put("TEST_SIZE", action.getTestProperties().getSize().toString());
+ env.put("TEST_TIMEOUT", Integer.toString(getTimeout(action)));
+
+ if (action.isSharded()) {
+ env.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum()));
+ env.put("TEST_TOTAL_SHARDS",
+ Integer.toString(action.getExecutionSettings().getTotalShards()));
+ }
+
+ // When we run test multiple times, set different TEST_RANDOM_SEED values for each run.
+ if (action.getConfiguration().getRunsPerTestForLabel(action.getOwner().getLabel()) > 1) {
+ env.put("TEST_RANDOM_SEED", Integer.toString(action.getRunNumber() + 1));
+ }
+
+ String testFilter = action.getExecutionSettings().getTestFilter();
+ if (testFilter != null) {
+ env.put(TEST_BRIDGE_TEST_FILTER_ENV, testFilter);
+ }
+
+ return env;
+ }
+
+ /**
+ * Returns the number of attempts specific test action can be retried.
+ *
+ * <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts
+ * option is given and specifies another value.
+ */
+ @VisibleForTesting /* protected */
+ public int getTestAttempts(TestRunnerAction action) {
+ if (executionOptions.testAttempts == -1) {
+ return action.getTestProperties().isFlaky() ? 3 : 1;
+ } else {
+ return executionOptions.testAttempts;
+ }
+ }
+
+ /**
+ * Returns timeout value in seconds that should be used for the given test action. We always use
+ * the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout
+ * but ends up with the same effective value as all other rules in that bucket.
+ */
+ protected final int getTimeout(TestRunnerAction testAction) {
+ return executionOptions.testTimeout.get(testAction.getTestProperties().getTimeout());
+ }
+
+ /**
+ * Returns a subset of the environment from the current shell.
+ *
+ * <p>Warning: Since these variables are not part of the configuration's fingerprint, they
+ * MUST NOT be used by any rule or action in such a way as to affect the semantics of that
+ * build step.
+ */
+ public Map<String, String> getAdmissibleShellEnvironment(BuildConfiguration config,
+ Iterable<String> variables) {
+ return getMapping(variables, config.getClientEnv());
+ }
+
+ /*
+ * Finalize test run: persist the result, and post on the event bus.
+ */
+ protected void postTestResult(Executor executor, TestResult result) throws IOException {
+ result.getTestAction().saveCacheStatus(result.getData());
+ executor.getEventBus().post(result);
+ }
+
+ /**
+ * Parse a test result XML file into a {@link TestCase}.
+ */
+ @Nullable
+ protected TestCase parseTestResult(Path resultFile) {
+ /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
+ a large amount of memory. */
+ if (executionOptions.testSummary != TestSummaryFormat.DETAILED && !statusServerRunning) {
+ return null;
+ }
+
+ try (InputStream fileStream = resultFile.getInputStream()) {
+ return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream);
+ } catch (IOException | TestXmlOutputParserException e) {
+ return null;
+ }
+ }
+
+ /**
+ * For an given environment, returns a subset containing all variables in the given list if they
+ * are defined in the given environment.
+ */
+ @VisibleForTesting
+ public static Map<String, String> getMapping(Iterable<String> variables,
+ Map<String, String> environment) {
+ Map<String, String> result = new HashMap<>();
+ for (String var : variables) {
+ if (environment.containsKey(var)) {
+ result.put(var, environment.get(var));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the runfiles directory associated with the test executable,
+ * creating/updating it if necessary and --build_runfile_links is specified.
+ */
+ protected static Path getLocalRunfilesDirectory(TestRunnerAction testAction,
+ ActionExecutionContext actionExecutionContext, BinTools binTools) throws ExecException,
+ InterruptedException {
+ TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
+
+ // --nobuild_runfile_links disables runfiles generation only for C++ rules.
+ // In that case, getManifest returns the .runfiles_manifest (input) file,
+ // not the MANIFEST output file of the build-runfiles action. So the
+ // extension ".runfiles_manifest" indicates no runfiles tree.
+ if (!execSettings.getManifest().equals(execSettings.getInputManifest())) {
+ return execSettings.getManifest().getPath().getParentDirectory();
+ }
+
+ // We might need to build runfiles tree now, since it was not created yet
+ // local testing is needed.
+ Path program = execSettings.getExecutable().getPath();
+ Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles");
+
+ // Synchronize runfiles tree generation on the runfiles manifest artifact.
+ // This is necessary, because we might end up with multiple test runner actions
+ // trying to generate same runfiles tree in case of --runs_per_test > 1 or
+ // local test sharding.
+ long startTime = Profiler.nanoTimeMaybe();
+ synchronized (execSettings.getManifest()) {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction);
+ updateLocalRunfilesDirectory(testAction, runfilesDir, actionExecutionContext, binTools);
+ }
+
+ return runfilesDir;
+ }
+
+ /**
+ * Ensure the runfiles tree exists and is consistent with the TestAction's manifest
+ * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file
+ * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid
+ * the need for recursion.
+ */
+ private static void updateLocalRunfilesDirectory(TestRunnerAction testAction, Path runfilesDir,
+ ActionExecutionContext actionExecutionContext, BinTools binTools) throws ExecException,
+ InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+
+ TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
+ try {
+ if (Arrays.equals(runfilesDir.getRelative("MANIFEST").getMD5Digest(),
+ execSettings.getManifest().getPath().getMD5Digest())) {
+ return;
+ }
+ } catch (IOException e1) {
+ // Ignore it - we will just try to create runfiles directory.
+ }
+
+ executor.getEventHandler().handle(Event.progress(
+ "Building runfiles directory for '" + execSettings.getExecutable().prettyPrint() + "'."));
+
+ new SymlinkTreeHelper(execSettings.getManifest().getExecPath(),
+ runfilesDir.relativeTo(executor.getExecRoot()), /* filesetTree= */ false)
+ .createSymlinks(testAction, actionExecutionContext, binTools);
+
+ executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage()));
+ }
+
+ /**
+ * In rare cases, we might write something to stderr. Append it to the real test.log.
+ */
+ protected static void appendStderr(Path stdOut, Path stdErr) throws IOException {
+ FileStatus stat = stdErr.statNullable();
+ OutputStream out = null;
+ InputStream in = null;
+ if (stat != null) {
+ try {
+ if (stat.getSize() > 0) {
+ if (stdOut.exists()) {
+ stdOut.setWritable(true);
+ }
+ out = stdOut.getOutputStream(true);
+ in = stdErr.getInputStream();
+ ByteStreams.copy(in, out);
+ }
+ } finally {
+ Closeables.close(out, true);
+ Closeables.close(in, true);
+ stdErr.delete();
+ }
+ }
+ }
+
+ /**
+ * Implements the --test_output=streamed option.
+ */
+ protected static class StreamedTestOutput implements Closeable {
+ private final TestLogHelper.FilterTestHeaderOutputStream headerFilter;
+ private final FileWatcher watcher;
+ private final Path testLogPath;
+ private final OutErr outErr;
+
+ public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException {
+ this.testLogPath = testLogPath;
+ this.outErr = outErr;
+ this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream());
+ this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false);
+ watcher.start();
+ }
+
+ @Override
+ public void close() throws IOException {
+ watcher.stopPumping();
+ try {
+ // The watcher thread might leak if the following call is interrupted.
+ // This is a relatively minor issue since the worst it could do is
+ // write one additional line from the test.log to the console later on
+ // in the build.
+ watcher.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ if (!headerFilter.foundHeader()) {
+ InputStream input = testLogPath.getInputStream();
+ try {
+ ByteStreams.copy(input, outErr.getOutputStream());
+ } finally {
+ input.close();
+ }
+ }
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java
new file mode 100644
index 0000000..ef795aa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java
@@ -0,0 +1,99 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.packages.TestTargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Implementation for the "test_suite" rule.
+ */
+public class TestSuite implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) {
+ checkTestsAndSuites(ruleContext, "tests");
+ checkTestsAndSuites(ruleContext, "suites");
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ //
+ // CAUTION! Keep this logic consistent with lib.query2.TestsExpression!
+ //
+
+ List<String> tagsAttribute = new ArrayList<>(
+ ruleContext.attributes().get("tags", Type.STRING_LIST));
+ tagsAttribute.remove("manual");
+ Pair<Collection<String>, Collection<String>> requiredExcluded =
+ TestTargetUtils.sortTagsBySense(tagsAttribute);
+
+ List<TransitiveInfoCollection> directTestsAndSuitesBuilder = new ArrayList<>();
+
+ // The set of implicit tests is determined in
+ // {@link com.google.devtools.build.lib.packages.Package}.
+ // Manual tests are already filtered out there. That is what $implicit_tests is about.
+ for (TransitiveInfoCollection dep :
+ Iterables.concat(
+ ruleContext.getPrerequisites("tests", Mode.TARGET),
+ ruleContext.getPrerequisites("suites", Mode.TARGET),
+ ruleContext.getPrerequisites("$implicit_tests", Mode.TARGET))) {
+ if (dep.getProvider(TestProvider.class) != null) {
+ List<String> tags = dep.getProvider(TestProvider.class).getTestTags();
+ if (!TestTargetUtils.testMatchesFilters(
+ tags, requiredExcluded.first, requiredExcluded.second, true)) {
+ // This test does not match our filter. Ignore it.
+ continue;
+ }
+ }
+ directTestsAndSuitesBuilder.add(dep);
+ }
+
+ Runfiles runfiles = new Runfiles.Builder()
+ .addTargets(directTestsAndSuitesBuilder, RunfilesProvider.DATA_RUNFILES)
+ .build();
+
+ return new RuleConfiguredTargetBuilder(ruleContext)
+ .add(RunfilesProvider.class,
+ RunfilesProvider.withData(Runfiles.EMPTY, runfiles))
+ .add(TransitiveTestsProvider.class, new TransitiveTestsProvider())
+ .build();
+ }
+
+ private void checkTestsAndSuites(RuleContext ruleContext, String attributeName) {
+ for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attributeName, Mode.TARGET)) {
+ // TODO(bazel-team): Maybe convert the TransitiveTestsProvider into an inner interface.
+ TransitiveTestsProvider provider = dep.getProvider(TransitiveTestsProvider.class);
+ TestProvider testProvider = dep.getProvider(TestProvider.class);
+ if (provider == null && testProvider == null) {
+ ruleContext.attributeError(attributeName,
+ "expecting a test or a test_suite rule but '" + dep.getLabel() + "' is not one");
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java
new file mode 100644
index 0000000..20ad8af
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.packages.TargetUtils;
+
+import java.util.List;
+
+/**
+ * Container for common test execution settings shared by all
+ * all TestRunnerAction instances for the given test target.
+ */
+public final class TestTargetExecutionSettings {
+
+ private final List<String> testArguments;
+ private final String testFilter;
+ private final int totalShards;
+ private final RunUnder runUnder;
+ private final Artifact runUnderExecutable;
+ private final Artifact executable;
+ private final Artifact runfilesManifest;
+ private final Artifact runfilesInputManifest;
+ private final Artifact instrumentedFileManifest;
+
+ TestTargetExecutionSettings(RuleContext ruleContext, RunfilesSupport runfiles,
+ Artifact executable, Artifact instrumentedFileManifest, int shards) {
+ Preconditions.checkArgument(TargetUtils.isTestRule(ruleContext.getRule()));
+ Preconditions.checkArgument(shards >= 0);
+ BuildConfiguration config = ruleContext.getConfiguration();
+
+ List<String> targetArgs = runfiles.getArgs();
+ testArguments = targetArgs.isEmpty()
+ ? config.getTestArguments()
+ : ImmutableList.copyOf(Iterables.concat(targetArgs, config.getTestArguments()));
+
+ totalShards = shards;
+ runUnder = config.getRunUnder();
+ runUnderExecutable = getRunUnderExecutable(ruleContext);
+
+ this.testFilter = config.getTestFilter();
+ this.executable = executable;
+ this.runfilesManifest = runfiles.getRunfilesManifest();
+ this.runfilesInputManifest = runfiles.getRunfilesInputManifest();
+ this.instrumentedFileManifest = instrumentedFileManifest;
+ }
+
+ private static Artifact getRunUnderExecutable(RuleContext ruleContext) {
+ TransitiveInfoCollection runUnderTarget = ruleContext
+ .getPrerequisite(":run_under", Mode.DATA);
+ return runUnderTarget == null
+ ? null
+ : runUnderTarget.getProvider(FilesToRunProvider.class).getExecutable();
+ }
+
+ public List<String> getArgs() {
+ return testArguments;
+ }
+
+ public String getTestFilter() {
+ return testFilter;
+ }
+
+ public int getTotalShards() {
+ return totalShards;
+ }
+
+ public RunUnder getRunUnder() {
+ return runUnder;
+ }
+
+ public Artifact getRunUnderExecutable() {
+ return runUnderExecutable;
+ }
+
+ public Artifact getExecutable() {
+ return executable;
+ }
+
+ /**
+ * Returns the runfiles manifest for this test.
+ *
+ * <p>This returns either the input manifest outside of the runfiles tree,
+ * if blaze is run with --nobuild_runfile_links or the manifest inside the
+ * runfiles tree, if blaze is run with --build_runfile_links.
+ *
+ * @see com.google.devtools.build.lib.analysis.RunfilesSupport#getRunfilesManifest()
+ */
+ public Artifact getManifest() {
+ return runfilesManifest;
+ }
+
+ /**
+ * Returns the input runfiles manifest for this test.
+ *
+ * <p>This always returns the input manifest outside of the runfiles tree.
+ *
+ * @see com.google.devtools.build.lib.analysis.RunfilesSupport#getRunfilesInputManifest()
+ */
+ public Artifact getInputManifest() {
+ return runfilesInputManifest;
+ }
+
+ /**
+ * Returns instrumented file manifest or null if code coverage is not
+ * collected.
+ */
+ public Artifact getInstrumentedFileManifest() {
+ return instrumentedFileManifest;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java
new file mode 100644
index 0000000..8cf26b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java
@@ -0,0 +1,131 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.packages.Type;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Container for test target properties available to the
+ * TestRunnerAction instance.
+ */
+public class TestTargetProperties {
+
+ /**
+ * Resources used by local tests of various sizes.
+ */
+ private static final ResourceSet SMALL_RESOURCES = new ResourceSet(20, 0.9, 0.00);
+ private static final ResourceSet MEDIUM_RESOURCES = new ResourceSet(100, 0.9, 0.1);
+ private static final ResourceSet LARGE_RESOURCES = new ResourceSet(300, 0.8, 0.1);
+ private static final ResourceSet ENORMOUS_RESOURCES = new ResourceSet(800, 0.7, 0.4);
+
+ private static ResourceSet getResourceSetFromSize(TestSize size) {
+ switch (size) {
+ case SMALL: return SMALL_RESOURCES;
+ case MEDIUM: return MEDIUM_RESOURCES;
+ case LARGE: return LARGE_RESOURCES;
+ default: return ENORMOUS_RESOURCES;
+ }
+ }
+
+ private final TestSize size;
+ private final TestTimeout timeout;
+ private final List<String> tags;
+ private final boolean isLocal;
+ private final boolean isFlaky;
+ private final boolean isExternal;
+ private final String language;
+ private final ImmutableMap<String, String> executionInfo;
+
+ /**
+ * Creates test target properties instance. Constructor expects that it
+ * will be called only for test configured targets.
+ */
+ TestTargetProperties(RuleContext ruleContext,
+ ExecutionInfoProvider executionRequirements) {
+ Rule rule = ruleContext.getRule();
+
+ Preconditions.checkState(TargetUtils.isTestRule(rule));
+ size = TestSize.getTestSize(rule);
+ timeout = TestTimeout.getTestTimeout(rule);
+ tags = ruleContext.attributes().get("tags", Type.STRING_LIST);
+ isLocal = TargetUtils.isLocalTestRule(rule) || TargetUtils.isExclusiveTestRule(rule);
+
+ // We need to use method on ruleConfiguredTarget to perform validation.
+ isFlaky = ruleContext.attributes().get("flaky", Type.BOOLEAN);
+ isExternal = TargetUtils.isExternalTestRule(rule);
+
+ Map<String, String> executionInfo = Maps.newLinkedHashMap();
+ executionInfo.putAll(TargetUtils.getExecutionInfo(rule));
+ if (executionRequirements != null) {
+ // This will overwrite whatever TargetUtils put there, which might be confusing.
+ executionInfo.putAll(executionRequirements.getExecutionInfo());
+ }
+ this.executionInfo = ImmutableMap.copyOf(executionInfo);
+
+ language = TargetUtils.getRuleLanguage(rule);
+ }
+
+ public TestSize getSize() {
+ return size;
+ }
+
+ public TestTimeout getTimeout() {
+ return timeout;
+ }
+
+ public List<String> getTags() {
+ return tags;
+ }
+
+ public boolean isLocal() {
+ return isLocal;
+ }
+
+ public boolean isFlaky() {
+ return isFlaky;
+ }
+
+ public boolean isExternal() {
+ return isExternal;
+ }
+
+ public ResourceSet getLocalResourceUsage() {
+ return TestTargetProperties.getResourceSetFromSize(size);
+ }
+
+ /**
+ * Returns a map of execution info. See {@link Spawn#getExecutionInfo}.
+ */
+ public ImmutableMap<String, String> getExecutionInfo() {
+ return executionInfo;
+ }
+
+ public String getLanguage() {
+ return language;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java
new file mode 100644
index 0000000..8d660ec
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java
@@ -0,0 +1,345 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase.Type;
+import com.google.protobuf.UninitializedMessageException;
+
+import java.io.InputStream;
+import java.util.Collection;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+/**
+ * Parses a test.xml generated by jUnit or any testing framework
+ * into a protocol buffer. The schema of the test.xml is a bit hazy, so there is
+ * some guesswork involved.
+ */
+class TestXmlOutputParser {
+ // jUnit can use either "testsuites" or "testsuite".
+ private static final Collection<String> TOPLEVEL_ELEMENT_NAMES =
+ ImmutableSet.of("testsuites", "testsuite");
+
+ public TestCase parseXmlIntoTestResult(InputStream xmlStream)
+ throws TestXmlOutputParserException {
+ return parseXmlToTree(xmlStream);
+ }
+
+ /**
+ * Parses the a test result XML file into the corresponding protocol buffer.
+ * @param xmlStream the XML data stream
+ * @return the protocol buffer with the parsed data, or null if there was
+ * an error while parsing the file.
+ *
+ * @throws TestXmlOutputParserException when the XML file cannot be parsed
+ */
+ private TestCase parseXmlToTree(InputStream xmlStream)
+ throws TestXmlOutputParserException {
+ XMLStreamReader parser = null;
+
+ try {
+ parser = XMLInputFactory.newInstance().createXMLStreamReader(xmlStream);
+
+ while (true) {
+ int event = parser.next();
+ if (event == XMLStreamConstants.END_DOCUMENT) {
+ return null;
+ }
+
+ // First find the topmost node.
+ if (event == XMLStreamConstants.START_ELEMENT) {
+ String elementName = parser.getLocalName();
+ if (TOPLEVEL_ELEMENT_NAMES.contains(elementName)) {
+ TestCase result = parseTestSuite(parser, elementName);
+ return result;
+ }
+ }
+ }
+ } catch (XMLStreamException e) {
+ throw new TestXmlOutputParserException(e);
+ } catch (NumberFormatException e) {
+ // The parser is definitely != null here.
+ throw new TestXmlOutputParserException(
+ "Number could not be parsed at "
+ + parser.getLocation().getLineNumber() + ":"
+ + parser.getLocation().getColumnNumber(),
+ e);
+ } catch (UninitializedMessageException e) {
+ // This happens when the XML does not contain a field that is required
+ // in the protocol buffer
+ throw new TestXmlOutputParserException(e);
+ } catch (RuntimeException e) {
+
+ // Seems like that an XNIException can leak through, even though it is not
+ // specified anywhere.
+ //
+ // It's a bad idea to refer to XNIException directly because the Xerces
+ // documentation says that it may not be available here soon (and it
+ // results in a compile-time warning anyway), so we do it the roundabout
+ // way: check if the class name has something to do with Xerces, and if
+ // so, wrap it in our own exception type, otherwise, let the stack
+ // unwinding continue.
+ String name = e.getClass().getCanonicalName();
+ if (name != null && name.contains("org.apache.xerces")) {
+ throw new TestXmlOutputParserException(e);
+ } else {
+ throw e;
+ }
+ } finally {
+ if (parser != null) {
+ try {
+ parser.close();
+ } catch (XMLStreamException e) {
+
+ // Ignore errors during closure so that we do not interfere with an
+ // already propagating exception.
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates an exception suitable to be thrown when and a bad end tag appears.
+ * The exception could also be thrown from here but that would result in an
+ * extra stack frame, whereas this way, the topmost frame shows the location
+ * where the error occurred.
+ */
+ private TestXmlOutputParserException createBadElementException(
+ String expected, XMLStreamReader parser) {
+ return new TestXmlOutputParserException("Expected end of XML element '"
+ + expected + "' , but got '" + parser.getLocalName() + "' at "
+ + parser.getLocation().getLineNumber() + ":"
+ + parser.getLocation().getColumnNumber());
+ }
+
+ /**
+ * Parses a 'testsuite' element.
+ *
+ * @throws TestXmlOutputParserException if the XML document is malformed
+ * @throws XMLStreamException if there was an error processing the XML
+ * @throws NumberFormatException if one of the numeric fields does not contain
+ * a valid number
+ */
+ private TestCase parseTestSuite(XMLStreamReader parser, String elementName)
+ throws XMLStreamException, TestXmlOutputParserException {
+ TestCase.Builder builder = TestCase.newBuilder();
+ builder.setType(Type.TEST_SUITE);
+ for (int i = 0; i < parser.getAttributeCount(); i++) {
+ String name = parser.getAttributeLocalName(i).intern();
+ String value = parser.getAttributeValue(i);
+
+ if (name.equals("name")) {
+ builder.setName(value);
+ } else if (name.equals("time")) {
+ builder.setRunDurationMillis(parseTime(value));
+ }
+ }
+
+ parseContainedElements(parser, elementName, builder);
+ return builder.build();
+ }
+
+ /**
+ * Parses a time in test.xml format.
+ *
+ * @throws NumberFormatException if the time is malformed (i.e. is neither an
+ * integer nor a decimal fraction with '.' as the fraction separator)
+ */
+ private long parseTime(String string) {
+
+ // This is ugly. For Historical Reasons, we have to check whether the number
+ // contains a decimal point or not. If it does, the number is expressed in
+ // milliseconds, otherwise, in seconds.
+ if (string.contains(".")) {
+ return Math.round(Float.parseFloat(string) * 1000);
+ } else {
+ return Long.parseLong(string);
+ }
+ }
+
+ /**
+ * Parses a 'decorator' element.
+ *
+ * @throws TestXmlOutputParserException if the XML document is malformed
+ * @throws XMLStreamException if there was an error processing the XML
+ * @throws NumberFormatException if one of the numeric fields does not contain
+ * a valid number
+ */
+ private TestCase parseTestDecorator(XMLStreamReader parser)
+ throws XMLStreamException, TestXmlOutputParserException {
+ TestCase.Builder builder = TestCase.newBuilder();
+ builder.setType(Type.TEST_DECORATOR);
+ for (int i = 0; i < parser.getAttributeCount(); i++) {
+ String name = parser.getAttributeLocalName(i);
+ String value = parser.getAttributeValue(i);
+
+ builder.setName(name);
+ if (name.equals("classname")) {
+ builder.setClassName(value);
+ } else if (name.equals("time")) {
+ builder.setRunDurationMillis(parseTime(value));
+ }
+ }
+
+ parseContainedElements(parser, "testdecorator", builder);
+ return builder.build();
+ }
+
+ /**
+ * Parses child elements of the specified tag. Strictly speaking, not every
+ * element can be a child of every other, but the HierarchicalTestResult can
+ * handle that, and (in this case) it does not hurt to be a bit more flexible
+ * than necessary.
+ *
+ * @throws TestXmlOutputParserException if the XML document is malformed
+ * @throws XMLStreamException if there was an error processing the XML
+ * @throws NumberFormatException if one of the numeric fields does not contain
+ * a valid number
+ */
+ private void parseContainedElements(
+ XMLStreamReader parser, String elementName, TestCase.Builder builder)
+ throws XMLStreamException, TestXmlOutputParserException {
+ int failures = 0;
+ int errors = 0;
+
+ while (true) {
+ int event = parser.next();
+ switch (event) {
+ case XMLStreamConstants.START_ELEMENT:
+ String childElementName = parser.getLocalName().intern();
+
+ // We are not parsing four elements here: system-out, system-err,
+ // failure and error. They potentially contain useful information, but
+ // they can be too big to fit in the memory. We add failure and error
+ // elements to the output without a message, so that there is a
+ // difference between passed and failed test cases.
+ if (childElementName.equals("testsuite")) {
+ builder.addChild(parseTestSuite(parser, childElementName));
+ } else if (childElementName.equals("testcase")) {
+ builder.addChild(parseTestCase(parser));
+ } else if (childElementName.equals("failure")) {
+ failures += 1;
+ skipCompleteElement(parser);
+ } else if (childElementName.equals("error")) {
+ errors += 1;
+ skipCompleteElement(parser);
+ } else if (childElementName.equals("testdecorator")) {
+ builder.addChild(parseTestDecorator(parser));
+ } else {
+
+ // Unknown element encountered. Since the schema of the input file
+ // is a bit hazy, just skip it and go merrily on our way. Ignorance
+ // is bliss.
+ skipCompleteElement(parser);
+ }
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ // Propagate errors/failures from children up to the current case
+ for (int i = 0; i < builder.getChildCount(); i += 1) {
+ if (builder.getChild(i).getStatus() == TestCase.Status.ERROR) {
+ errors += 1;
+ }
+ if (builder.getChild(i).getStatus() == TestCase.Status.FAILED) {
+ failures += 1;
+ }
+ }
+
+ if (errors > 0) {
+ builder.setStatus(TestCase.Status.ERROR);
+ } else if (failures > 0) {
+ builder.setStatus(TestCase.Status.FAILED);
+ } else {
+ builder.setStatus(TestCase.Status.PASSED);
+ }
+ // This is the end tag of the element we are supposed to parse.
+ // Hooray, tell our superiors that our mission is complete.
+ if (!parser.getLocalName().equals(elementName)) {
+ throw createBadElementException(elementName, parser);
+ }
+ return;
+ }
+ }
+ }
+
+
+ /**
+ * Parses a 'testcase' element.
+ *
+ * @throws TestXmlOutputParserException if the XML document is malformed
+ * @throws XMLStreamException if there was an error processing the XML
+ * @throws NumberFormatException if the time field does not contain a valid
+ * number
+ */
+ private TestCase parseTestCase(XMLStreamReader parser)
+ throws XMLStreamException, TestXmlOutputParserException {
+ TestCase.Builder builder = TestCase.newBuilder();
+ builder.setType(Type.TEST_CASE);
+ for (int i = 0; i < parser.getAttributeCount(); i++) {
+ String name = parser.getAttributeLocalName(i).intern();
+ String value = parser.getAttributeValue(i);
+
+ if (name.equals("name")) {
+ builder.setName(value);
+ } else if (name.equals("classname")) {
+ builder.setClassName(value);
+ } else if (name.equals("time")) {
+ builder.setRunDurationMillis(parseTime(value));
+ } else if (name.equals("result")) {
+ builder.setResult(value);
+ } else if (name.equals("status")) {
+ if (value.equals("notrun")) {
+ builder.setRun(false);
+ } else if (value.equals("run")) {
+ builder.setRun(true);
+ }
+ }
+ }
+
+ parseContainedElements(parser, "testcase", builder);
+ return builder.build();
+ }
+
+ /**
+ * Skips over a complete XML element on the input.
+ * Precondition: the cursor is at a START_ELEMENT.
+ * Postcondition: the cursor is at an END_ELEMENT.
+ *
+ * @throws XMLStreamException if the XML is malformed
+ */
+ private void skipCompleteElement(XMLStreamReader parser) throws XMLStreamException {
+ int depth = 1;
+ while (true) {
+ int event = parser.next();
+
+ switch (event) {
+ case XMLStreamConstants.START_ELEMENT:
+ depth++;
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ if (--depth == 0) {
+ return;
+ }
+ break;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java
new file mode 100644
index 0000000..c27ca9d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+/**
+ * This exception gets thrown if there was a problem with parsing a test.xml
+ * file.
+ */
+class TestXmlOutputParserException extends Exception {
+ public TestXmlOutputParserException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TestXmlOutputParserException(Throwable cause) {
+ super(cause);
+ }
+
+ public TestXmlOutputParserException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java
new file mode 100644
index 0000000..c46b2a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.test;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+/**
+ * Marker transitive info provider for test_suite rules to recognize one another.
+ */
+@Immutable
+public final class TransitiveTestsProvider implements TransitiveInfoProvider {
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java b/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java
new file mode 100644
index 0000000..49f829a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java
@@ -0,0 +1,125 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.workspace;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FileProvider;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+import com.google.devtools.build.lib.syntax.ClassObject;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
+
+/**
+ * Implementation for the bind rule.
+ */
+public class Bind implements RuleConfiguredTargetFactory {
+
+ /**
+ * This configured target pretends to be whatever type of target "actual" is, returning its
+ * transitive info providers and target, but returning the label for the //external target.
+ */
+ private static class BindConfiguredTarget implements ConfiguredTarget, ClassObject {
+
+ private Label label;
+ private ConfiguredTarget configuredTarget;
+ private BuildConfiguration config;
+
+ BindConfiguredTarget(RuleContext ruleContext) {
+ label = ruleContext.getRule().getLabel();
+ config = ruleContext.getConfiguration();
+ // TODO(bazel-team): we should special case ConfiguredTargetFactory.createConfiguredTarget,
+ // not cast down here.
+ configuredTarget = (ConfiguredTarget) ruleContext.getPrerequisite("actual", Mode.TARGET);
+ }
+
+ @Override
+ public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) {
+ return configuredTarget.getProvider(provider);
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public Object get(String providerKey) {
+ return configuredTarget.get(providerKey);
+ }
+
+ @Override
+ public UnmodifiableIterator<TransitiveInfoProvider> iterator() {
+ return configuredTarget.iterator();
+ }
+
+ @Override
+ public Target getTarget() {
+ return configuredTarget.getTarget();
+ }
+
+ @Override
+ public BuildConfiguration getConfiguration() {
+ return config;
+ }
+
+ /* ClassObject methods */
+
+ @Override
+ public Object getValue(String name) {
+ if (name.equals("label")) {
+ return getLabel();
+ } else if (name.equals("files")) {
+ // A shortcut for files to build in Skylark. FileConfiguredTarget and RunleConfiguredTarget
+ // always has FileProvider and Error- and PackageGroupConfiguredTarget-s shouldn't be
+ // accessible in Skylark.
+ return SkylarkNestedSet.of(
+ Artifact.class, getProvider(FileProvider.class).getFilesToBuild());
+ }
+ return configuredTarget.get(name);
+ }
+
+ @SuppressWarnings("cast")
+ @Override
+ public ImmutableCollection<String> getKeys() {
+ return new ImmutableList.Builder<String>()
+ .add("label", "files")
+ .addAll(configuredTarget.getProvider(RuleConfiguredTarget.SkylarkProviders.class)
+ .getKeys())
+ .build();
+ }
+
+ @Override
+ public String errorMessage(String name) {
+ // Use the default error message.
+ return null;
+ }
+ }
+
+ @Override
+ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException {
+ return new BindConfiguredTarget(ruleContext);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java b/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java
new file mode 100644
index 0000000..c3f2dd2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java
@@ -0,0 +1,126 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.LABEL;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Binds an existing target to a target in the virtual //external package.
+ */
+@BlazeRule(name = "bind",
+ type = RuleClassType.WORKSPACE,
+ ancestors = {BaseRule.class},
+ factoryClass = Bind.class)
+public final class BindRule implements RuleDefinition {
+
+ @Override
+ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) {
+ return builder
+ /* <!-- #BLAZE_RULE(bind).ATTRIBUTE(actual) -->
+ The target to be aliased.
+
+ <p>This target must exist, but can be any type of rule (including bind).</p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("actual", LABEL).allowedFileTypes())
+ .setWorkspaceOnly()
+ .build();
+ }
+}
+/*<!-- #BLAZE_RULE (NAME = bind, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] -->
+
+${ATTRIBUTE_SIGNATURE}
+
+<p>Gives a target an alias in the <code>//external</code> package.</p>
+
+${ATTRIBUTE_DEFINITION}
+
+<p>The <code>//external</code> package is not a "normal" package: there is no external/ directory,
+ so it can be thought of as a "virtual package" that contains all bound targets.</p>
+
+<h4 id="bind_examples">Examples</h4>
+
+<p>To give a target an alias, bind it in the <i>WORKSPACE</i> file. For example, suppose there is
+ a <code>java_library</code> target called <code>//third_party/javacc-v2</code>. This could be
+ aliased by adding the following to the <i>WORKSPACE</i> file:</p>
+
+<pre class="code">
+bind(
+ name = "javacc-latest",
+ actual = "//third_party/javacc-v2",
+)
+</pre>
+
+<p>Now targets can depend on <code>//external:javacc-latest</code> instead of
+ <code>//third_party/javacc-v2</code>. If javacc-v3 is released, the binding can be updated and
+ all of the BUILD files depending on <code>//external:javacc-latest</code> will now depend on
+ javacc-v3 without needing to be edited.</p>
+
+<p>Bind can also be used to refer to external repositories' targets. For example, if there is a
+ remote repository named <code>@my-ssl</code> imported in the WORKSPACE file. If the
+ <code>@my-ssl</code> repository has a cc_library target <code>//src:openssl-lib</code>, you
+ could make this target accessible for your program to depend on by using <code>bind</code>:</p>
+
+<pre class="code">
+bind(
+ name = "openssl",
+ actual = "@my-ssl//src:openssl-lib",
+)
+</pre>
+
+<p>BUILD files cannot use labels that include a repository name
+ ("@repository-name//package-name:target-name"), so the only way to depend on a target from
+ another repository is to <code>bind</code> it in the WORKSPACE file and then refer to it by its
+ aliased name in <code>//external</code> from a BUILD file.</p>
+
+<p>For example, in a BUILD file, the bound target could be used as follows:</p>
+
+<pre class="code">
+cc_library(
+ name = "sign-in",
+ srcs = ["sign_in.cc"],
+ hdrs = ["sign_in.h"],
+ deps = ["//external:openssl"],
+)
+</pre>
+
+<p>Within <code>sign_in.cc</code> and <code>sign_in.h</code>, the header files exposed by
+ <code>//external:openssl</code> can be referred to by their path relative to their repository
+ root. For example, if the rule definition for <code>@my-ssl//src:openssl-lib</code> looks like
+ this:</p>
+
+<pre class="code">
+cc_library(
+ name = "openssl-lib",
+ srcs = ["openssl.cc"],
+ hdrs = ["openssl.h"],
+)
+</pre>
+
+<p>Then <code>sign_in.cc</code>'s first lines might look like this:</p>
+
+<pre class="code">
+#include "sign_in.h"
+#include "src/openssl.h"
+</pre>
+
+<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java
new file mode 100644
index 0000000..9bf7a3f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java
@@ -0,0 +1,120 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+
+import javax.annotation.Nullable;
+
+/**
+ * This class records the critical path for the graph of actions executed.
+ */
+@ThreadCompatible
+public class AbstractCriticalPathComponent<C extends AbstractCriticalPathComponent<C>> {
+
+ /** Wall time start time for the action. In milliseconds. */
+ private final long startTime;
+ /** Wall time finish time for the action. In milliseconds. */
+ private long finishTime = 0;
+ protected volatile boolean isRunning = true;
+
+ /** We keep here the critical path time for the most expensive child. */
+ private long childAggregatedWallTime = 0;
+
+ /** The action for which we are storing the stat. */
+ private final Action action;
+
+ /**
+ * Child with the maximum critical path.
+ */
+ @Nullable
+ private C child;
+
+ public AbstractCriticalPathComponent(Action action, long startTime) {
+ this.action = action;
+ this.startTime = startTime;
+ }
+
+ /** Sets the finish time for the action in milliseconds. */
+ public void setFinishTimeMillis(long finishTime) {
+ Preconditions.checkState(isRunning, "Already stopped! %s.", action);
+ this.finishTime = finishTime;
+ isRunning = false;
+ }
+
+ /** The action for which we are storing the stat. */
+ public Action getAction() {
+ return action;
+ }
+
+ /**
+ * Add statistics for one dependency of this action.
+ */
+ public void addDepInfo(C dep) {
+ Preconditions.checkState(!dep.isRunning,
+ "Cannot add critical path stats when the action is not finished. %s. %s", action,
+ dep.getAction());
+ long childAggregatedWallTime = dep.getAggregatedWallTime();
+ // Replace the child if its critical path had the maximum wall time.
+ if (child == null || childAggregatedWallTime > this.childAggregatedWallTime) {
+ this.childAggregatedWallTime = childAggregatedWallTime;
+ child = dep;
+ }
+ }
+
+ public long getActionWallTime() {
+ Preconditions.checkState(!isRunning, "Still running %s", action);
+ return finishTime - startTime;
+ }
+
+ /**
+ * Returns the current critical path for the action in milliseconds.
+ *
+ * <p>Critical path is defined as : action_execution_time + max(child_critical_path).
+ */
+ public long getAggregatedWallTime() {
+ Preconditions.checkState(!isRunning, "Still running %s", action);
+ return getActionWallTime() + childAggregatedWallTime;
+ }
+
+ /** Time when the action started to execute. Milliseconds since epoch time. */
+ public long getStartTime() {
+ return startTime;
+ }
+
+ /**
+ * Get the child critical path component.
+ *
+ * <p>The component dependency with the maximum total critical path time.
+ */
+ @Nullable
+ public C getChild() {
+ return child;
+ }
+
+ /**
+ * Returns a human readable representation of the critical path stats with all the details.
+ */
+ @Override
+ public String toString() {
+ String currentTime = "still running ";
+ if (!isRunning) {
+ currentTime = String.format("%.2f", getActionWallTime() / 1000.0) + "s ";
+ }
+ return currentTime + action.describe();
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java
new file mode 100644
index 0000000..dd70c35
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Aggregates all the critical path components in one object. This allows us to easily access the
+ * components data and have a proper toString().
+ */
+public class AggregatedCriticalPath<T extends AbstractCriticalPathComponent> {
+
+ private final long totalTime;
+ private final ImmutableList<T> criticalPathComponents;
+
+ protected AggregatedCriticalPath(long totalTime, ImmutableList<T> criticalPathComponents) {
+ this.totalTime = totalTime;
+ this.criticalPathComponents = criticalPathComponents;
+ }
+
+ /** Total wall time in ms spent running the critical path actions. */
+ public long totalTime() {
+ return totalTime;
+ }
+
+ /** Returns a list of all the component stats for the critical path. */
+ public ImmutableList<T> components() {
+ return criticalPathComponents;
+ }
+
+ @Override
+ public String toString() {
+ return toString(false);
+ }
+
+ /**
+ * Returns a summary version of the critical path stats that omits stats that are not useful
+ * to the user.
+ */
+ public String toStringSummary() {
+ return toString(true);
+ }
+
+ private String toString(boolean summary) {
+ StringBuilder sb = new StringBuilder("Critical Path: ");
+ double totalMillis = totalTime;
+ sb.append(String.format("%.2f", totalMillis / 1000.0));
+ sb.append("s");
+ if (summary || criticalPathComponents.isEmpty()) {
+ return sb.toString();
+ }
+ sb.append("\n ");
+ Joiner.on("\n ").appendTo(sb, criticalPathComponents);
+ return sb.toString();
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java
new file mode 100644
index 0000000..cc240c4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java
@@ -0,0 +1,255 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.AllowConcurrentEvents;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisFailureEvent;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.events.ExceptionListener;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * This class aggregates and reports target-wide test statuses in real-time.
+ * It must be public for EventBus invocation.
+ */
+@ThreadSafety.ThreadSafe
+public class AggregatingTestListener {
+ private final ConcurrentMap<Artifact, TestResult> statusMap = new MapMaker().makeMap();
+
+ private final TestResultAnalyzer analyzer;
+ private final EventBus eventBus;
+ private final EventHandlerPreconditions preconditionHelper;
+ private volatile boolean blazeHalted = false;
+
+
+ // summaryLock guards concurrent access to these two collections, which should be kept
+ // synchronized with each other.
+ private final Map<LabelAndConfiguration, TestSummary.Builder> summaries;
+ private final Multimap<LabelAndConfiguration, Artifact> remainingRuns;
+ private final Object summaryLock = new Object();
+
+ public AggregatingTestListener(TestResultAnalyzer analyzer,
+ EventBus eventBus,
+ ExceptionListener listener) {
+ this.analyzer = analyzer;
+ this.eventBus = eventBus;
+ this.preconditionHelper = new EventHandlerPreconditions(listener);
+
+ this.summaries = Maps.newHashMap();
+ this.remainingRuns = HashMultimap.create();
+ }
+
+ /**
+ * @return An unmodifiable copy of the map of test results.
+ */
+ public Map<Artifact, TestResult> getStatusMap() {
+ return ImmutableMap.copyOf(statusMap);
+ }
+
+ /**
+ * Populates the test summary map as soon as test filtering is complete.
+ * This is the earliest at which the final set of targets to test is known.
+ */
+ @Subscribe
+ @AllowConcurrentEvents
+ public void populateTests(TestFilteringCompleteEvent event) {
+ // Add all target runs to the map, assuming 1:1 status artifact <-> result.
+ synchronized (summaryLock) {
+ for (ConfiguredTarget target : event.getTestTargets()) {
+ Iterable<Artifact> statusArtifacts =
+ target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts();
+ preconditionHelper.checkState(remainingRuns.putAll(asKey(target), statusArtifacts));
+
+ // And create an empty summary suitable for incremental analysis.
+ // Also has the nice side effect of mapping labels to RuleConfiguredTargets.
+ TestSummary.Builder summary = TestSummary.newBuilder()
+ .setTarget(target)
+ .setStatus(BlazeTestStatus.NO_STATUS);
+ preconditionHelper.checkState(summaries.put(asKey(target), summary) == null);
+ }
+ }
+ }
+
+ /**
+ * Records a new test run result and incrementally updates the target status.
+ * This event is sent upon completion of executed test runs.
+ */
+ @Subscribe
+ @AllowConcurrentEvents
+ public void testEvent(TestResult result) {
+ Preconditions.checkState(
+ statusMap.put(result.getTestStatusArtifact(), result) == null,
+ "Duplicate result reported for an individual test shard");
+
+ ActionOwner testOwner = result.getTestAction().getOwner();
+ LabelAndConfiguration targetLabel = LabelAndConfiguration.of(
+ testOwner.getLabel(), result.getTestAction().getConfiguration());
+
+ TestSummary finalTestSummary = null;
+ synchronized (summaryLock) {
+ TestSummary.Builder summary = summaries.get(targetLabel);
+ preconditionHelper.checkNotNull(summary);
+ if (!remainingRuns.remove(targetLabel, result.getTestStatusArtifact())) {
+ // This can happen if a buildCompleteEvent() was processed before this event reached us.
+ // This situation is likely to happen if --notest_keep_going is set with multiple targets.
+ return;
+ }
+
+ summary = analyzer.incrementalAnalyze(summary, result);
+
+ // If all runs are processed, the target is finished and ready to report.
+ if (!remainingRuns.containsKey(targetLabel)) {
+ finalTestSummary = summary.build();
+ }
+ }
+
+ // Report finished targets.
+ if (finalTestSummary != null) {
+ eventBus.post(finalTestSummary);
+ }
+ }
+
+ private void targetFailure(LabelAndConfiguration label) {
+ TestSummary finalSummary;
+ synchronized (summaryLock) {
+ if (!remainingRuns.containsKey(label)) {
+ // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
+ // events are in sync. Thus, it is possible that a test event was posted, but the target is
+ // not present in the set of successful targets.
+ return;
+ }
+
+ TestSummary.Builder summary = summaries.get(label);
+ if (summary == null) {
+ // Not a test target; nothing to do.
+ return;
+ }
+ finalSummary = analyzer.markUnbuilt(summary, blazeHalted).build();
+
+ // These are never going to run; removing them marks the target complete.
+ remainingRuns.removeAll(label);
+ }
+ eventBus.post(finalSummary);
+ }
+
+ @VisibleForTesting
+ void buildComplete(
+ Collection<ConfiguredTarget> actualTargets, Collection<ConfiguredTarget> successfulTargets) {
+ if (actualTargets == null || successfulTargets == null) {
+ return;
+ }
+
+ for (ConfiguredTarget target: Sets.difference(
+ ImmutableSet.copyOf(actualTargets), ImmutableSet.copyOf(successfulTargets))) {
+ targetFailure(asKey(target));
+ }
+ }
+
+ @Subscribe
+ public void buildCompleteEvent(BuildCompleteEvent event) {
+ if (event.getResult().wasCatastrophe()) {
+ blazeHalted = true;
+ }
+ buildComplete(event.getResult().getActualTargets(), event.getResult().getSuccessfulTargets());
+ }
+
+ @Subscribe
+ public void analysisFailure(AnalysisFailureEvent event) {
+ targetFailure(event.getFailedTarget());
+ }
+
+ @Subscribe
+ @AllowConcurrentEvents
+ public void buildInterrupted(BuildInterruptedEvent event) {
+ blazeHalted = true;
+ }
+
+ /**
+ * Called when a build action is not executed (e.g. because a dependency failed to build). We want
+ * to catch such events in order to determine when a test target has failed to build.
+ */
+ @Subscribe
+ @AllowConcurrentEvents
+ public void targetComplete(TargetCompleteEvent event) {
+ if (event.failed()) {
+ targetFailure(new LabelAndConfiguration(event.getTarget()));
+ }
+ }
+
+ /**
+ * Returns the known aggregate results for the given target at the current moment.
+ */
+ public TestSummary.Builder getCurrentSummary(ConfiguredTarget target) {
+ synchronized (summaryLock) {
+ return summaries.get(asKey(target));
+ }
+ }
+
+ /**
+ * Returns all test status artifacts associated with a given target
+ * whose runs have yet to finish.
+ */
+ public Collection<Artifact> getIncompleteRuns(ConfiguredTarget target) {
+ synchronized (summaryLock) {
+ return Collections.unmodifiableCollection(remainingRuns.get(asKey(target)));
+ }
+ }
+
+ /**
+ * Returns true iff all runs of the target are accounted for.
+ */
+ public boolean targetReported(ConfiguredTarget target) {
+ synchronized (summaryLock) {
+ return summaries.containsKey(asKey(target)) && !remainingRuns.containsKey(asKey(target));
+ }
+ }
+
+ /**
+ * Returns the {@link TestResultAnalyzer} associated with this listener.
+ */
+ public TestResultAnalyzer getAnalyzer() {
+ return analyzer;
+ }
+
+ private LabelAndConfiguration asKey(ConfiguredTarget target) {
+ return new LabelAndConfiguration(target);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java
new file mode 100644
index 0000000..61f46a8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * Interface implemented by Blaze commands. In addition to implementing this interface, each
+ * command must be annotated with a {@link Command} annotation.
+ */
+public interface BlazeCommand {
+ /**
+ * This method provides the imperative portion of the command. It takes
+ * a {@link OptionsProvider} instance {@code options}, which provides access
+ * to the options instances via {@link OptionsProvider#getOptions(Class)},
+ * and access to the residue (the remainder of the command line) via
+ * {@link OptionsProvider#getResidue()}. The framework parses and makes
+ * available exactly the options that the command class specifies via the
+ * annotation {@link Command#options()}. The command may write to standard
+ * out and standard error via {@code outErr}. It indicates success / failure
+ * via its return value, which becomes the Unix exit status of the Blaze
+ * client process. It may indicate a shutdown request by throwing
+ * {@link BlazeCommandDispatcher.ShutdownBlazeServerException}. In that case,
+ * the Blaze server process (the memory resident portion of Blaze) will
+ * shut down and the exit status will be 0 (in case the shutdown succeeds
+ * without error).
+ *
+ * @param runtime The Blaze runtime requesting the execution of the command
+ * @param options A parsed options instance initialized with the values for
+ * the options specified in {@link Command#options()}.
+ *
+ * @return The Unix exit status for the Blaze client.
+ * @throws BlazeCommandDispatcher.ShutdownBlazeServerException Indicates
+ * that the command wants to shutdown the Blaze server.
+ */
+ ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws BlazeCommandDispatcher.ShutdownBlazeServerException;
+
+ /**
+ * Allows the command to provide command-specific option defaults and/or
+ * requirements. This method is called after all command-line and rc file options have been
+ * parsed.
+ *
+ * @param runtime The Blaze runtime requesting the execution of the command
+ *
+ * @throws AbruptExitException if something went wrong
+ */
+ void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) throws AbruptExitException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
new file mode 100644
index 0000000..cee47ee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
@@ -0,0 +1,692 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.io.Flushables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.AnsiStrippingOutputStream;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.DelegatingOutErr;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.logging.Level;
+
+/**
+ * Dispatches to the Blaze commands; that is, given a command line, this
+ * abstraction looks up the appropriate command object, parses the options
+ * required by the object, and calls its exec method. Also, this object provides
+ * the runtime state (BlazeRuntime) to the commands.
+ */
+public class BlazeCommandDispatcher {
+
+ // Keep in sync with options added in OptionProcessor::AddRcfileArgsAndOptions()
+ private static final Set<String> INTERNAL_COMMAND_OPTIONS = ImmutableSet.of(
+ "rc_source", "default_override", "isatty", "terminal_columns", "ignore_client_env",
+ "client_env", "client_cwd");
+
+ private static final ImmutableList<String> HELP_COMMAND = ImmutableList.of("help");
+
+ private static final Set<String> ALL_HELP_OPTIONS = ImmutableSet.of("--help", "-help", "-h");
+
+ /**
+ * By throwing this exception, a command indicates that it wants to shutdown
+ * the Blaze server process.
+ * See {@link BlazeCommandDispatcher#exec(List, OutErr, long)}.
+ */
+ public static class ShutdownBlazeServerException extends Exception {
+ private final int exitStatus;
+
+ public ShutdownBlazeServerException(int exitStatus, Throwable cause) {
+ super(cause);
+ this.exitStatus = exitStatus;
+ }
+
+ public ShutdownBlazeServerException(int exitStatus) {
+ this.exitStatus = exitStatus;
+ }
+
+ public int getExitStatus() {
+ return exitStatus;
+ }
+ }
+
+ private final BlazeRuntime runtime;
+ private final Map<String, BlazeCommand> commandsByName = new LinkedHashMap<>();
+
+ private OutputStream logOutputStream = null;
+
+ /**
+ * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime}
+ * instance, and no default options, and delegates to {@code commands} as
+ * appropriate.
+ */
+ @VisibleForTesting
+ public BlazeCommandDispatcher(BlazeRuntime runtime, BlazeCommand... commands) {
+ this(runtime, ImmutableList.copyOf(commands));
+ }
+
+ /**
+ * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime}
+ * instance, and delegates to {@code commands} as appropriate.
+ */
+ public BlazeCommandDispatcher(BlazeRuntime runtime, Iterable<BlazeCommand> commands) {
+ this.runtime = runtime;
+ for (BlazeCommand command : commands) {
+ addCommandByName(command);
+ }
+
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ for (BlazeCommand command : module.getCommands()) {
+ addCommandByName(command);
+ }
+ }
+
+ runtime.setCommandMap(commandsByName);
+ }
+
+ /**
+ * Adds the given command under the given name to the map of commands.
+ *
+ * @throws AssertionError if the name is already used by another command.
+ */
+ private void addCommandByName(BlazeCommand command) {
+ String name = command.getClass().getAnnotation(Command.class).name();
+ if (commandsByName.containsKey(name)) {
+ throw new IllegalStateException("Command name or alias " + name + " is already used.");
+ }
+ commandsByName.put(name, command);
+ }
+
+ /**
+ * Only some commands work if cwd != workspaceSuffix in Blaze. In that case, also check if Blaze
+ * was called from the output directory and fail if it was.
+ */
+ private ExitCode checkCwdInWorkspace(Command commandAnnotation, String commandName,
+ OutErr outErr) {
+ if (!commandAnnotation.mustRunInWorkspace()) {
+ return ExitCode.SUCCESS;
+ }
+
+ if (!runtime.inWorkspace()) {
+ outErr.printErrLn("The '" + commandName + "' command is only supported from within a "
+ + "workspace.");
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Path workspace = runtime.getWorkspace();
+ Path doNotBuild = workspace.getParentDirectory().getRelative(
+ BlazeRuntime.DO_NOT_BUILD_FILE_NAME);
+ if (doNotBuild.exists()) {
+ if (!commandAnnotation.canRunInOutputDirectory()) {
+ outErr.printErrLn(getNotInRealWorkspaceError(doNotBuild));
+ return ExitCode.COMMAND_LINE_ERROR;
+ } else {
+ outErr.printErrLn("WARNING: Blaze is run from output directory. This is unsound.");
+ }
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ private CommonCommandOptions checkOptions(OptionsParser optionsParser,
+ Command commandAnnotation, List<String> args, List<String> rcfileNotes, OutErr outErr)
+ throws OptionsParsingException {
+ Function<String, String> commandOptionSourceFunction = new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ if (INTERNAL_COMMAND_OPTIONS.contains(input)) {
+ return "options generated by Blaze launcher";
+ } else {
+ return "command line options";
+ }
+ }
+ };
+
+ // Explicit command-line options:
+ List<String> cmdLineAfterCommand = args.subList(1, args.size());
+ optionsParser.parseWithSourceFunction(OptionPriority.COMMAND_LINE,
+ commandOptionSourceFunction, cmdLineAfterCommand);
+
+ // Command-specific options from .blazerc passed in via --default_override
+ // and --rc_source. A no-op if none are provided.
+ CommonCommandOptions rcFileOptions = optionsParser.getOptions(CommonCommandOptions.class);
+ List<Pair<String, ListMultimap<String, String>>> optionsMap =
+ getOptionsMap(outErr, rcFileOptions.rcSource, rcFileOptions.optionsOverrides,
+ commandsByName.keySet());
+
+ parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap, null);
+
+ // Fix-point iteration until all configs are loaded.
+ List<String> configsLoaded = ImmutableList.of();
+ CommonCommandOptions commonOptions = optionsParser.getOptions(CommonCommandOptions.class);
+ while (!commonOptions.configs.equals(configsLoaded)) {
+ Set<String> missingConfigs = new LinkedHashSet<>(commonOptions.configs);
+ missingConfigs.removeAll(configsLoaded);
+ parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap,
+ missingConfigs);
+ configsLoaded = commonOptions.configs;
+ commonOptions = optionsParser.getOptions(CommonCommandOptions.class);
+ }
+
+ return commonOptions;
+ }
+
+ /**
+ * Sends {@code EventKind.{STDOUT|STDERR}} messages to the given {@link OutErr}.
+ *
+ * <p>This is necessary because we cannot delete the output files from the previous Blaze run
+ * because there can be processes spawned by the previous invocation that are still processing
+ * them, in which case we need to print a warning message about that.
+ *
+ * <p>Thus, messages sent to {@link Reporter#getOutErr} get sent to this event handler, then
+ * to its {@link OutErr}. We need to go deeper!
+ */
+ private static class OutErrEventHandler implements EventHandler {
+ private final OutErr outErr;
+
+ private OutErrEventHandler(OutErr outErr) {
+ this.outErr = outErr;
+ }
+
+ @Override
+ public void handle(Event event) {
+ try {
+ switch (event.getKind()) {
+ case STDOUT:
+ outErr.getOutputStream().write(event.getMessageBytes());
+ break;
+ case STDERR:
+ outErr.getErrorStream().write(event.getMessageBytes());
+ break;
+ }
+ } catch (IOException e) {
+ // We cannot do too much here -- ErrorEventListener#handle does not provide us with ways to
+ // report an error.
+ }
+ }
+ }
+
+ /**
+ * Executes a single command. Returns the Unix exit status for the Blaze
+ * client process, or throws {@link ShutdownBlazeServerException} to
+ * indicate that a command wants to shutdown the Blaze server.
+ */
+ public int exec(List<String> args, OutErr originalOutErr, long firstContactTime)
+ throws ShutdownBlazeServerException {
+ // Record the start time for the profiler and the timestamp granularity monitor. Do not put
+ // anything before this!
+ long execStartTimeNanos = runtime.getClock().nanoTime();
+
+ // Record the command's starting time for use by the commands themselves.
+ runtime.recordCommandStartTime(firstContactTime);
+
+ // Record the command's starting time again, for use by
+ // TimestampGranularityMonitor.waitForTimestampGranularity().
+ // This should be done as close as possible to the start of
+ // the command's execution - that's why we do this separately,
+ // rather than in runtime.beforeCommand().
+ runtime.getTimestampGranularityMonitor().setCommandStartTime();
+ runtime.initEventBus();
+
+ // Give a chance for module.beforeCommand() to report an errors to stdout and stderr.
+ // Once we can close the old streams, this event handler is removed.
+ OutErrEventHandler originalOutErrEventHandler =
+ new OutErrEventHandler(originalOutErr);
+ runtime.getReporter().addHandler(originalOutErrEventHandler);
+ OutErr outErr = originalOutErr;
+ runtime.getReporter().removeHandler(originalOutErrEventHandler);
+
+ if (args.isEmpty()) { // Default to help command if no arguments specified.
+ args = HELP_COMMAND;
+ }
+ String commandName = args.get(0);
+
+ // Be gentle to users who want to find out about Blaze invocation.
+ if (ALL_HELP_OPTIONS.contains(commandName)) {
+ commandName = "help";
+ }
+
+ BlazeCommand command = commandsByName.get(commandName);
+ if (command == null) {
+ outErr.printErrLn("Command '" + commandName + "' not found. " + "Try 'blaze help'.");
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ }
+ Command commandAnnotation = command.getClass().getAnnotation(Command.class);
+
+ AbruptExitException exitCausingException = null;
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ try {
+ module.beforeCommand(runtime, commandAnnotation);
+ } catch (AbruptExitException e) {
+ // Don't let one module's complaints prevent the other modules from doing necessary
+ // setup. We promised to call beforeCommand exactly once per-module before each command
+ // and will be calling afterCommand soon in the future - a module's afterCommand might
+ // rightfully assume its beforeCommand has already been called.
+ outErr.printErrLn(e.getMessage());
+ // It's not ideal but we can only return one exit code, so we just pick the code of the
+ // last exception.
+ exitCausingException = e;
+ }
+ }
+ if (exitCausingException != null) {
+ return exitCausingException.getExitCode().getNumericExitCode();
+ }
+
+ try {
+ Path commandLog = getCommandLogPath(runtime.getOutputBase());
+
+ // Unlink old command log from previous build, if present, so scripts
+ // reading it don't conflate it with the command log we're about to write.
+ commandLog.delete();
+
+ logOutputStream = commandLog.getOutputStream();
+ outErr = tee(originalOutErr, OutErr.create(logOutputStream, logOutputStream));
+ } catch (IOException ioException) {
+ LoggingUtil.logToRemote(
+ Level.WARNING, "Unable to delete or open command.log", ioException);
+ }
+
+ // Create the UUID for this command.
+ runtime.setCommandId(UUID.randomUUID());
+
+ ExitCode result = checkCwdInWorkspace(commandAnnotation, commandName, outErr);
+ if (result != ExitCode.SUCCESS) {
+ return result.getNumericExitCode();
+ }
+
+ OptionsParser optionsParser;
+ CommonCommandOptions commonOptions;
+ // Delay output of notes regarding the parsed rc file, so it's possible to disable this in the
+ // rc file.
+ List<String> rcfileNotes = new ArrayList<>();
+ try {
+ optionsParser = createOptionsParser(command);
+ commonOptions = checkOptions(optionsParser, commandAnnotation, args, rcfileNotes, outErr);
+ } catch (OptionsParsingException e) {
+ for (String note : rcfileNotes) {
+ outErr.printErrLn("INFO: " + note);
+ }
+ outErr.printErrLn(e.getMessage());
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ }
+
+ // Setup log filtering
+ BlazeCommandEventHandler.Options eventHandlerOptions =
+ optionsParser.getOptions(BlazeCommandEventHandler.Options.class);
+ if (!eventHandlerOptions.useColor()) {
+ if (!commandAnnotation.binaryStdOut()) {
+ outErr = ansiStripOut(outErr);
+ }
+
+ if (!commandAnnotation.binaryStdErr()) {
+ outErr = ansiStripErr(outErr);
+ }
+ }
+
+ BlazeRuntime.setupLogging(commonOptions.verbosity);
+
+ // Do this before an actual crash so we don't have to worry about
+ // allocating memory post-crash.
+ String[] crashData = runtime.getCrashData();
+ int numericExitCode = ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode();
+ PrintStream savedOut = System.out;
+ PrintStream savedErr = System.err;
+
+ EventHandler handler = createEventHandler(outErr, eventHandlerOptions);
+ Reporter reporter = runtime.getReporter();
+ reporter.addHandler(handler);
+ try {
+ // While a Blaze command is active, direct all errors to the client's
+ // event handler (and out/err streams).
+ OutErr reporterOutErr = reporter.getOutErr();
+ System.setOut(new PrintStream(reporterOutErr.getOutputStream(), /*autoflush=*/true));
+ System.setErr(new PrintStream(reporterOutErr.getErrorStream(), /*autoflush=*/true));
+
+ if (commonOptions.announceRcOptions) {
+ for (String note : rcfileNotes) {
+ reporter.handle(Event.info(note));
+ }
+ }
+
+ try {
+ // Notify the BlazeRuntime, so it can do some initial setup.
+ runtime.beforeCommand(commandName, optionsParser, commonOptions, execStartTimeNanos);
+ // Allow the command to edit options after parsing:
+ command.editOptions(runtime, optionsParser);
+ } catch (AbruptExitException e) {
+ reporter.handle(Event.error(e.getMessage()));
+ return e.getExitCode().getNumericExitCode();
+ }
+
+ // Print warnings for odd options usage
+ for (String warning : optionsParser.getWarnings()) {
+ reporter.handle(Event.warn(warning));
+ }
+
+ ExitCode outcome = command.exec(runtime, optionsParser);
+ outcome = runtime.precompleteCommand(outcome);
+ numericExitCode = outcome.getNumericExitCode();
+ return numericExitCode;
+ } catch (ShutdownBlazeServerException e) {
+ numericExitCode = e.getExitStatus();
+ throw e;
+ } catch (Throwable e) {
+ BugReport.printBug(outErr, e);
+ BugReport.sendBugReport(e, args, crashData);
+ numericExitCode = e instanceof OutOfMemoryError
+ ? ExitCode.OOM_ERROR.getNumericExitCode()
+ : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode();
+ throw new ShutdownBlazeServerException(numericExitCode, e);
+ } finally {
+ runtime.afterCommand(numericExitCode);
+ // Swallow IOException, as we are already in a finally clause
+ Flushables.flushQuietly(outErr.getOutputStream());
+ Flushables.flushQuietly(outErr.getErrorStream());
+
+ System.setOut(savedOut);
+ System.setErr(savedErr);
+ reporter.removeHandler(handler);
+ releaseHandler(handler);
+ runtime.getTimestampGranularityMonitor().waitForTimestampGranularity(outErr);
+ }
+ }
+
+ /**
+ * For testing ONLY. Same as {@link #exec(List, OutErr, long)}, but automatically uses the current
+ * time.
+ */
+ @VisibleForTesting
+ public int exec(List<String> args, OutErr originalOutErr) throws ShutdownBlazeServerException {
+ return exec(args, originalOutErr, runtime.getClock().currentTimeMillis());
+ }
+
+ /**
+ * Parses the options from .rc files for a command invocation. It works in one of two modes;
+ * either it loads the non-config options, or the config options that are specified in the {@code
+ * configs} parameter.
+ *
+ * <p>This method adds every option pertaining to the specified command to the options parser. To
+ * do that, it needs the command -> option mapping that is generated from the .rc files.
+ *
+ * <p>It is not as trivial as simply taking the list of options for the specified command because
+ * commands can inherit arguments from each other, and we have to respect that (e.g. if an option
+ * is specified for 'build', it needs to take effect for the 'test' command, too).
+ *
+ * <p>Note that the order in which the options are parsed is well-defined: all options from the
+ * same rc file are parsed at the same time, and the rc files are handled in the order in which
+ * they were passed in from the client.
+ *
+ * @param rcfileNotes note message that would be printed during parsing
+ * @param commandAnnotation the command for which options should be parsed.
+ * @param optionsParser parser to receive parsed options.
+ * @param optionsMap .rc files in structured format: a list of pairs, where the first part is the
+ * name of the rc file, and the second part is a multimap of command name (plus config, if
+ * present) to the list of options for that command
+ * @param configs the configs for which to parse options; if {@code null}, non-config options are
+ * parsed
+ * @throws OptionsParsingException
+ */
+ protected static void parseOptionsForCommand(List<String> rcfileNotes, Command commandAnnotation,
+ OptionsParser optionsParser, List<Pair<String, ListMultimap<String, String>>> optionsMap,
+ Iterable<String> configs) throws OptionsParsingException {
+ for (String commandToParse : getCommandNamesToParse(commandAnnotation)) {
+ for (Pair<String, ListMultimap<String, String>> entry : optionsMap) {
+ List<String> allOptions = new ArrayList<>();
+ if (configs == null) {
+ allOptions.addAll(entry.second.get(commandToParse));
+ } else {
+ for (String config : configs) {
+ allOptions.addAll(entry.second.get(commandToParse + ":" + config));
+ }
+ }
+ processOptionList(optionsParser, commandToParse,
+ commandAnnotation.name(), rcfileNotes, entry.first, allOptions);
+ if (allOptions.isEmpty()) {
+ continue;
+ }
+ }
+ }
+ }
+
+ // Processes the option list for an .rc file - command pair.
+ private static void processOptionList(OptionsParser optionsParser, String commandToParse,
+ String originalCommand, List<String> rcfileNotes, String rcfile, List<String> rcfileOptions)
+ throws OptionsParsingException {
+ if (!rcfileOptions.isEmpty()) {
+ String inherited = commandToParse.equals(originalCommand) ? "" : "Inherited ";
+ rcfileNotes.add("Reading options for '" + originalCommand +
+ "' from " + rcfile + ":\n" +
+ " " + inherited + "'" + commandToParse + "' options: "
+ + Joiner.on(' ').join(rcfileOptions));
+ optionsParser.parse(OptionPriority.RC_FILE, rcfile, rcfileOptions);
+ }
+ }
+
+ private static List<String> getCommandNamesToParse(Command commandAnnotation) {
+ List<String> result = new ArrayList<>();
+ getCommandNamesToParseHelper(commandAnnotation, result);
+ result.add("common");
+ // TODO(bazel-team): This statement is a NO-OP: Lists.reverse(result);
+ return result;
+ }
+
+ private static void getCommandNamesToParseHelper(Command commandAnnotation,
+ List<String> accumulator) {
+ for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) {
+ getCommandNamesToParseHelper(base.getAnnotation(Command.class), accumulator);
+ }
+ accumulator.add(commandAnnotation.name());
+ }
+
+ private OutErr ansiStripOut(OutErr outErr) {
+ OutputStream wrappedOut = new AnsiStrippingOutputStream(outErr.getOutputStream());
+ return OutErr.create(wrappedOut, outErr.getErrorStream());
+ }
+
+ private OutErr ansiStripErr(OutErr outErr) {
+ OutputStream wrappedErr = new AnsiStrippingOutputStream(outErr.getErrorStream());
+ return OutErr.create(outErr.getOutputStream(), wrappedErr);
+ }
+
+ private String getNotInRealWorkspaceError(Path doNotBuildFile) {
+ String message = "Blaze should not be called from a Blaze output directory. ";
+ try {
+ String realWorkspace =
+ new String(FileSystemUtils.readContentAsLatin1(doNotBuildFile));
+ message += String.format("The pertinent workspace directory is: '%s'",
+ realWorkspace);
+ } catch (IOException e) {
+ // We are exiting anyway.
+ }
+
+ return message;
+ }
+
+ /**
+ * For a given output_base directory, returns the command log file path.
+ */
+ public static Path getCommandLogPath(Path outputBase) {
+ return outputBase.getRelative("command.log");
+ }
+
+ private OutErr tee(OutErr outErr1, OutErr outErr2) {
+ DelegatingOutErr outErr = new DelegatingOutErr();
+ outErr.addSink(outErr1);
+ outErr.addSink(outErr2);
+ return outErr;
+ }
+
+ private void closeSilently(OutputStream logOutputStream) {
+ if (logOutputStream != null) {
+ try {
+ logOutputStream.close();
+ } catch (IOException e) {
+ LoggingUtil.logToRemote(Level.WARNING, "Unable to close command.log", e);
+ }
+ }
+ }
+
+ /**
+ * Creates an option parser using the common options classes and the
+ * command-specific options classes.
+ *
+ * <p>An overriding method should first call this method and can then
+ * override default values directly or by calling {@link
+ * #parseOptionsForCommand} for command-specific options.
+ *
+ * @throws OptionsParsingException
+ */
+ protected OptionsParser createOptionsParser(BlazeCommand command)
+ throws OptionsParsingException {
+ Command annotation = command.getClass().getAnnotation(Command.class);
+ List<Class<? extends OptionsBase>> allOptions = Lists.newArrayList();
+ allOptions.addAll(BlazeCommandUtils.getOptions(
+ command.getClass(), getRuntime().getBlazeModules(), getRuntime().getRuleClassProvider()));
+ OptionsParser parser = OptionsParser.newOptionsParser(allOptions);
+ parser.setAllowResidue(annotation.allowResidue());
+ return parser;
+ }
+
+ /**
+ * Convert a list of option override specifications to a more easily digestible
+ * form.
+ *
+ * @param overrides list of option override specifications
+ */
+ @VisibleForTesting
+ static List<Pair<String, ListMultimap<String, String>>> getOptionsMap(
+ OutErr outErr,
+ List<String> rcFiles,
+ List<CommonCommandOptions.OptionOverride> overrides,
+ Set<String> validCommands) {
+ List<Pair<String, ListMultimap<String, String>>> result = new ArrayList<>();
+
+ String lastRcFile = null;
+ ListMultimap<String, String> lastMap = null;
+ for (CommonCommandOptions.OptionOverride override : overrides) {
+ if (override.blazeRc < 0 || override.blazeRc >= rcFiles.size()) {
+ outErr.printErrLn("WARNING: inconsistency in generated command line "
+ + "args. Ignoring bogus argument\n");
+ continue;
+ }
+ String rcFile = rcFiles.get(override.blazeRc);
+
+ String command = override.command;
+ int index = command.indexOf(':');
+ if (index > 0) {
+ command = command.substring(0, index);
+ }
+ if (!validCommands.contains(command) && !command.equals("common")) {
+ outErr.printErrLn("WARNING: while reading option defaults file '"
+ + rcFile + "':\n"
+ + " invalid command name '" + override.command + "'.");
+ continue;
+ }
+
+ if (!rcFile.equals(lastRcFile)) {
+ if (lastRcFile != null) {
+ result.add(Pair.of(lastRcFile, lastMap));
+ }
+ lastRcFile = rcFile;
+ lastMap = ArrayListMultimap.create();
+ }
+ lastMap.put(override.command, override.option);
+ }
+ if (lastRcFile != null) {
+ result.add(Pair.of(lastRcFile, lastMap));
+ }
+
+ return result;
+ }
+
+ /**
+ * Returns the event handler to use for this Blaze command.
+ */
+ private EventHandler createEventHandler(OutErr outErr,
+ BlazeCommandEventHandler.Options eventOptions) {
+ EventHandler eventHandler;
+ if ((eventOptions.useColor() || eventOptions.useCursorControl())) {
+ eventHandler = new FancyTerminalEventHandler(outErr, eventOptions);
+ } else {
+ eventHandler = new BlazeCommandEventHandler(outErr, eventOptions);
+ }
+
+ return RateLimitingEventHandler.create(eventHandler, eventOptions.showProgressRateLimit);
+ }
+
+ /**
+ * Unsets the event handler.
+ */
+ private void releaseHandler(EventHandler eventHandler) {
+ if (eventHandler instanceof FancyTerminalEventHandler) {
+ // Make sure that the terminal state of the old event handler is clear
+ // before creating a new one.
+ ((FancyTerminalEventHandler)eventHandler).resetTerminal();
+ }
+ }
+
+ /**
+ * Returns the runtime instance shared by the commands that this dispatcher
+ * dispatches to.
+ */
+ public BlazeRuntime getRuntime() {
+ return runtime;
+ }
+
+ /**
+ * The map from command names to commands that this dispatcher dispatches to.
+ */
+ Map<String, BlazeCommand> getCommandsByName() {
+ return Collections.unmodifiableMap(commandsByName);
+ }
+
+ /**
+ * Shuts down all the registered commands to give them a chance to cleanup or
+ * close resources. Should be called by the owner of this command dispatcher
+ * in all termination cases.
+ */
+ public void shutdown() {
+ closeSilently(logOutputStream);
+ logOutputStream = null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
new file mode 100644
index 0000000..603b0be
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
@@ -0,0 +1,246 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * BlazeCommandEventHandler: an event handler established for the duration of a
+ * single Blaze command.
+ */
+public class BlazeCommandEventHandler implements EventHandler {
+
+ public enum UseColor { YES, NO, AUTO }
+ public enum UseCurses { YES, NO, AUTO }
+
+ public static class UseColorConverter extends EnumConverter<UseColor> {
+ public UseColorConverter() {
+ super(UseColor.class, "--color setting");
+ }
+ }
+
+ public static class UseCursesConverter extends EnumConverter<UseCurses> {
+ public UseCursesConverter() {
+ super(UseCurses.class, "--curses setting");
+ }
+ }
+
+ public static class Options extends OptionsBase {
+
+ @Option(name = "show_progress",
+ defaultValue = "true",
+ category = "verbosity",
+ help = "Display progress messages during a build.")
+ public boolean showProgress;
+
+ @Option(name = "show_task_finish",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Display progress messages when tasks complete, not just when they start.")
+ public boolean showTaskFinish;
+
+ @Option(name = "show_progress_rate_limit",
+ defaultValue = "0.03", // A nice middle ground; snappy but not too spammy in logs.
+ category = "verbosity",
+ help = "Minimum number of seconds between progress messages in the output.")
+ public double showProgressRateLimit;
+
+ @Option(name = "color",
+ defaultValue = "auto",
+ converter = UseColorConverter.class,
+ category = "verbosity",
+ help = "Use terminal controls to colorize output.")
+ public UseColor useColorEnum;
+
+ @Option(name = "curses",
+ defaultValue = "auto",
+ converter = UseCursesConverter.class,
+ category = "verbosity",
+ help = "Use terminal cursor controls to minimize scrolling output")
+ public UseCurses useCursesEnum;
+
+ @Option(name = "terminal_columns",
+ defaultValue = "80",
+ category = "hidden",
+ help = "A system-generated parameter which specifies the terminal "
+ + " width in columns.")
+ public int terminalColumns;
+
+ @Option(name = "isatty",
+ defaultValue = "false",
+ category = "hidden",
+ help = "A system-generated parameter which is used to notify the "
+ + "server whether this client is running in a terminal. "
+ + "If this is set to false, then '--color=auto' will be treated as '--color=no'. "
+ + "If this is set to true, then '--color=auto' will be treated as '--color=yes'.")
+ public boolean isATty;
+
+ // This lives here (as opposed to the more logical BuildRequest.Options)
+ // because the client passes it to the server *always*. We don't want the
+ // client to have to figure out when it should or shouldn't to send it.
+ @Option(name = "emacs",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "A system-generated parameter which is true iff EMACS=t in the environment of "
+ + "the client. This option controls certain display features.")
+ public boolean runningInEmacs;
+
+ @Option(name = "show_timestamps",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Include timestamps in messages")
+ public boolean showTimestamp;
+
+ @Option(name = "progress_in_terminal_title",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Show the command progress in the terminal title. "
+ + "Useful to see what blaze is doing when having multiple terminal tabs.")
+ public boolean progressInTermTitle;
+
+
+ public boolean useColor() {
+ return useColorEnum == UseColor.YES || (useColorEnum == UseColor.AUTO && isATty);
+ }
+
+ public boolean useCursorControl() {
+ return useCursesEnum == UseCurses.YES || (useCursesEnum == UseCurses.AUTO && isATty);
+ }
+ }
+
+ private static final DateTimeFormatter TIMESTAMP_FORMAT =
+ DateTimeFormat.forPattern("(MM-dd HH:mm:ss.SSS) ");
+
+ protected final OutErr outErr;
+
+ private final PrintStream errPrintStream;
+
+ protected final Set<EventKind> eventMask =
+ EnumSet.copyOf(EventKind.ERRORS_WARNINGS_AND_INFO_AND_OUTPUT);
+
+ protected final boolean showTimestamp;
+
+ public BlazeCommandEventHandler(OutErr outErr, Options eventOptions) {
+ this.outErr = outErr;
+ this.errPrintStream = new PrintStream(outErr.getErrorStream(), true);
+ if (eventOptions.showProgress) {
+ eventMask.add(EventKind.PROGRESS);
+ eventMask.add(EventKind.START);
+ } else {
+ // Skip PASS events if --noshow_progress is requested.
+ eventMask.remove(EventKind.PASS);
+ }
+ if (eventOptions.showTaskFinish) {
+ eventMask.add(EventKind.FINISH);
+ }
+ eventMask.add(EventKind.SUBCOMMAND);
+ this.showTimestamp = eventOptions.showTimestamp;
+ }
+
+ /** See EventHandler.handle. */
+ @Override
+ public void handle(Event event) {
+ if (!eventMask.contains(event.getKind())) {
+ return;
+ }
+ String prefix;
+ switch (event.getKind()) {
+ case STDOUT:
+ putOutput(outErr.getOutputStream(), event);
+ return;
+ case STDERR:
+ putOutput(outErr.getErrorStream(), event);
+ return;
+ case PASS:
+ case FAIL:
+ case TIMEOUT:
+ case ERROR:
+ case WARNING:
+ case DEPCHECKER:
+ prefix = event.getKind() + ": ";
+ break;
+ case SUBCOMMAND:
+ prefix = ">>>>>>>>> ";
+ break;
+ case INFO:
+ case PROGRESS:
+ case START:
+ case FINISH:
+ prefix = "____";
+ break;
+ default:
+ throw new IllegalStateException("" + event.getKind());
+ }
+ StringBuilder buf = new StringBuilder();
+ buf.append(prefix);
+
+ if (showTimestamp) {
+ buf.append(timestamp());
+ }
+
+ Location location = event.getLocation();
+ if (location != null) {
+ buf.append(location.print()).append(": ");
+ }
+
+ buf.append(event.getMessage());
+ if (event.getKind() == EventKind.FINISH) {
+ buf.append(" DONE");
+ }
+
+ // Add a trailing period for ERROR and WARNING messages, which are
+ // typically English sentences composed from exception messages.
+ if (event.getKind() == EventKind.WARNING ||
+ event.getKind() == EventKind.ERROR) {
+ buf.append('.');
+ }
+
+ // Event messages go to stderr; results (e.g. 'blaze query') go to stdout.
+ errPrintStream.println(buf);
+ }
+
+ private void putOutput(OutputStream out, Event event) {
+ try {
+ out.write(event.getMessageBytes());
+ out.flush();
+ } catch (IOException e) {
+ // This can happen in server mode if the blaze client has exited,
+ // or if output is redirected to a file and the disk is full, etc.
+ // Ignore.
+ }
+ }
+
+ /**
+ * @return a string representing the current time, eg "04-26 13:47:32.124".
+ */
+ protected String timestamp() {
+ return TIMESTAMP_FORMAT.print(System.currentTimeMillis());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java
new file mode 100644
index 0000000..ff738db
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java
@@ -0,0 +1,166 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.util.ResourceFileLoader;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Utility class for functionality related to Blaze commands.
+ */
+public class BlazeCommandUtils {
+ /**
+ * Options classes used as startup options in Blaze core.
+ */
+ private static final List<Class<? extends OptionsBase>> DEFAULT_STARTUP_OPTIONS =
+ ImmutableList.<Class<? extends OptionsBase>>of(
+ BlazeServerStartupOptions.class,
+ HostJvmStartupOptions.class);
+
+ /**
+ * The set of option-classes that are common to all Blaze commands.
+ */
+ private static final Collection<Class<? extends OptionsBase>> COMMON_COMMAND_OPTIONS =
+ ImmutableList.of(CommonCommandOptions.class, BlazeCommandEventHandler.Options.class);
+
+
+ private BlazeCommandUtils() {}
+
+ public static ImmutableList<Class<? extends OptionsBase>> getStartupOptions(
+ Iterable<BlazeModule> modules) {
+ Set<Class<? extends OptionsBase>> options = new HashSet<>();
+ options.addAll(DEFAULT_STARTUP_OPTIONS);
+ for (BlazeModule blazeModule : modules) {
+ Iterables.addAll(options, blazeModule.getStartupOptions());
+ }
+
+ return ImmutableList.copyOf(options);
+ }
+
+ /**
+ * Returns the set of all options (including those inherited directly and
+ * transitively) for this AbstractCommand's @Command annotation.
+ *
+ * <p>Why does metaprogramming always seem like such a bright idea in the
+ * beginning?
+ */
+ public static ImmutableList<Class<? extends OptionsBase>> getOptions(
+ Class<? extends BlazeCommand> clazz,
+ Iterable<BlazeModule> modules,
+ ConfiguredRuleClassProvider ruleClassProvider) {
+ Command commandAnnotation = clazz.getAnnotation(Command.class);
+ if (commandAnnotation == null) {
+ throw new IllegalStateException("@Command missing for " + clazz.getName());
+ }
+
+ Set<Class<? extends OptionsBase>> options = new HashSet<>();
+ options.addAll(COMMON_COMMAND_OPTIONS);
+ Collections.addAll(options, commandAnnotation.options());
+
+ if (commandAnnotation.usesConfigurationOptions()) {
+ options.addAll(ruleClassProvider.getConfigurationOptions());
+ }
+
+ for (BlazeModule blazeModule : modules) {
+ Iterables.addAll(options, blazeModule.getCommandOptions(commandAnnotation));
+ }
+
+ for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) {
+ options.addAll(getOptions(base, modules, ruleClassProvider));
+ }
+ return ImmutableList.copyOf(options);
+ }
+
+ /**
+ * Returns the expansion of the specified help topic.
+ *
+ * @param topic the name of the help topic; used in %{command} expansion.
+ * @param help the text template of the help message. Certain %{x} variables
+ * will be expanded. A prefix of "resource:" means use the .jar
+ * resource of that name.
+ * @param categoryDescriptions a mapping from option category names to
+ * descriptions, passed to {@link OptionsParser#describeOptions}.
+ * @param helpVerbosity a tri-state verbosity option selecting between just
+ * names, names and syntax, and full description.
+ */
+ public static final String expandHelpTopic(String topic, String help,
+ Class<? extends BlazeCommand> commandClass,
+ Collection<Class<? extends OptionsBase>> options,
+ Map<String, String> categoryDescriptions,
+ OptionsParser.HelpVerbosity helpVerbosity) {
+ OptionsParser parser = OptionsParser.newOptionsParser(options);
+
+ String template;
+ if (help.startsWith("resource:")) {
+ String resourceName = help.substring("resource:".length());
+ try {
+ template = ResourceFileLoader.loadResource(commandClass, resourceName);
+ } catch (IOException e) {
+ throw new IllegalStateException("failed to load help resource '" + resourceName
+ + "' due to I/O error: " + e.getMessage(), e);
+ }
+ } else {
+ template = help;
+ }
+
+ if (!template.contains("%{options}")) {
+ throw new IllegalStateException("Help template for '" + topic + "' omits %{options}!");
+ }
+
+ return template.
+ replace("%{command}", topic).
+ replace("%{options}", parser.describeOptions(categoryDescriptions, helpVerbosity)).
+ trim()
+ + "\n\n"
+ + (helpVerbosity == OptionsParser.HelpVerbosity.MEDIUM
+ ? "(Use 'help --long' for full details or --short to just enumerate options.)\n"
+ : "");
+ }
+
+ /**
+ * The help page for this command.
+ *
+ * @param categoryDescriptions a mapping from option category names to
+ * descriptions, passed to {@link OptionsParser#describeOptions}.
+ * @param verbosity a tri-state verbosity option selecting between just names,
+ * names and syntax, and full description.
+ */
+ public static String getUsage(
+ Class<? extends BlazeCommand> commandClass,
+ Map<String, String> categoryDescriptions,
+ OptionsParser.HelpVerbosity verbosity,
+ Iterable<BlazeModule> blazeModules,
+ ConfiguredRuleClassProvider ruleClassProvider) {
+ Command commandAnnotation = commandClass.getAnnotation(Command.class);
+ return BlazeCommandUtils.expandHelpTopic(
+ commandAnnotation.name(),
+ commandAnnotation.help(),
+ commandClass,
+ BlazeCommandUtils.getOptions(commandClass, blazeModules, ruleClassProvider),
+ categoryDescriptions,
+ verbosity);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java
new file mode 100644
index 0000000..6855cbd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java
@@ -0,0 +1,420 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.ActionContextConsumer;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.exec.OutputService;
+import com.google.devtools.build.lib.packages.MakeEnvironment;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.PackageArgument;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+import com.google.devtools.build.lib.query2.output.OutputFormatter;
+import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.skyframe.DiffAwareness;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * A module Blaze can load at the beginning of its execution. Modules are supplied with extension
+ * points to augment the functionality at specific, well-defined places.
+ *
+ * <p>The constructors of individual Blaze modules should be empty. All work should be done in the
+ * methods (e.g. {@link #blazeStartup}).
+ */
+public abstract class BlazeModule {
+
+ /**
+ * Returns the extra startup options this module contributes.
+ *
+ * <p>This method will be called at the beginning of Blaze startup (before #blazeStartup).
+ */
+ public Iterable<Class<? extends OptionsBase>> getStartupOptions() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Called before {@link #getFileSystem} and {@link #blazeStartup}.
+ *
+ * <p>This method will be called at the beginning of Blaze startup.
+ */
+ @SuppressWarnings("unused")
+ public void globalInit(OptionsProvider startupOptions) throws AbruptExitException {
+ }
+
+ /**
+ * Returns the file system implementation used by Blaze. It is an error if more than one module
+ * returns a file system. If all return null, the default unix file system is used.
+ *
+ * <p>This method will be called at the beginning of Blaze startup (in-between #globalInit and
+ * #blazeStartup).
+ */
+ @SuppressWarnings("unused")
+ public FileSystem getFileSystem(OptionsProvider startupOptions, PathFragment outputPath) {
+ return null;
+ }
+
+ /**
+ * Called when Blaze starts up.
+ */
+ @SuppressWarnings("unused")
+ public void blazeStartup(OptionsProvider startupOptions,
+ BlazeVersionInfo versionInfo, UUID instanceId, BlazeDirectories directories,
+ Clock clock) throws AbruptExitException {
+ }
+
+ /**
+ * Returns the set of directories under which blaze may assume all files are immutable.
+ */
+ public Set<Path> getImmutableDirectories() {
+ return ImmutableSet.<Path>of();
+ }
+
+ /**
+ * May yield a supplier that provides factories for the Preprocessor to apply. Only one of the
+ * configured modules may return non-null.
+ *
+ * The factory yielded by the supplier will be checked with
+ * {@link Preprocessor.Factory#isStillValid} at the beginning of each incremental build. This
+ * allows modules to have preprocessors customizable by flags.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Preprocessor.Factory.Supplier getPreprocessorFactorySupplier() {
+ return null;
+ }
+
+ /**
+ * Adds the rule classes supported by this module.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ @SuppressWarnings("unused")
+ public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
+ }
+
+ /**
+ * Returns the list of commands this module contributes to Blaze.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Iterable<? extends BlazeCommand> getCommands() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the list of query output formatters this module provides.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Iterable<OutputFormatter> getQueryOutputFormatters() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the {@link DiffAwareness} strategies this module contributes. These will be used to
+ * determine which files, if any, changed between Blaze commands.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ @SuppressWarnings("unused")
+ public Iterable<? extends DiffAwareness.Factory> getDiffAwarenessFactories(boolean watchFS) {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the workspace status action factory contributed by this module.
+ *
+ * <p>There should always be exactly one of these in a Blaze instance.
+ */
+ public WorkspaceStatusAction.Factory getWorkspaceStatusActionFactory() {
+ return null;
+ }
+
+ /**
+ * PlatformSet is a group of platforms characterized by a regular expression. For example, the
+ * entry "oldlinux": "i[34]86-libc[345]-linux" might define a set of platforms representing
+ * certain older linux releases.
+ *
+ * <p>Platform-set names are used in BUILD files in the third argument to <tt>vardef</tt>, to
+ * define per-platform tweaks to variables such as CFLAGS.
+ *
+ * <p>vardef is a legacy mechanism: it needs explicit support in the rule implementations,
+ * and cannot express conditional dependencies, only conditional attribute values. This
+ * mechanism will be supplanted by configuration dependent attributes, and its effect can
+ * usually also be achieved with abi_deps.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Map<String, String> getPlatformSetRegexps() {
+ return ImmutableMap.<String, String>of();
+ }
+
+ /**
+ * Services provided for Blaze modules via BlazeRuntime.
+ */
+ public interface ModuleEnvironment {
+ /**
+ * Gets a file from the depot based on its label and returns the {@link Path} where it can
+ * be found.
+ */
+ Path getFileFromDepot(Label label)
+ throws NoSuchThingException, InterruptedException, IOException;
+
+ /**
+ * Exits Blaze as early as possible. This is currently a hack and should only be called in
+ * event handlers for {@code BuildStartingEvent}, {@code GotOptionsEvent} and
+ * {@code LoadingPhaseCompleteEvent}.
+ */
+ void exit(AbruptExitException exception);
+ }
+
+ /**
+ * Called before each command.
+ */
+ @SuppressWarnings("unused")
+ public void beforeCommand(BlazeRuntime blazeRuntime, Command command)
+ throws AbruptExitException {
+ }
+
+ /**
+ * Returns the output service to be used. It is an error if more than one module returns an
+ * output service.
+ *
+ * <p>This method will be called at the beginning of each command (after #beforeCommand).
+ */
+ @SuppressWarnings("unused")
+ public OutputService getOutputService() throws AbruptExitException {
+ return null;
+ }
+
+ /**
+ * Returns the extra options this module contributes to a specific command.
+ *
+ * <p>This method will be called at the beginning of each command (after #beforeCommand).
+ */
+ @SuppressWarnings("unused")
+ public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns a map of option categories to descriptive strings. This is used by {@code HelpCommand}
+ * to show a more readable list of flags.
+ */
+ public Map<String, String> getOptionCategories() {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * A item that is returned by "blaze info".
+ */
+ public interface InfoItem {
+ /**
+ * The name of the info key.
+ */
+ String getName();
+
+ /**
+ * The help description of the info key.
+ */
+ String getDescription();
+
+ /**
+ * Whether the key is printed when "blaze info" is invoked without arguments.
+ *
+ * <p>This is usually true for info keys that take multiple lines, thus, cannot really be
+ * included in the output of argumentless "blaze info".
+ */
+ boolean isHidden();
+
+ /**
+ * Returns the value of the info key. The return value is directly printed to stdout.
+ */
+ byte[] get(Supplier<BuildConfiguration> configurationSupplier) throws AbruptExitException;
+ }
+
+ /**
+ * Returns the additional information this module provides to "blaze info".
+ *
+ * <p>This method will be called at the beginning of each "blaze info" command (after
+ * #beforeCommand).
+ */
+ public Iterable<InfoItem> getInfoItems() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the list of query functions this module provides to "blaze query".
+ *
+ * <p>This method will be called at the beginning of each "blaze query" command (after
+ * #beforeCommand).
+ */
+ public Iterable<QueryFunction> getQueryFunctions() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the action context provider the module contributes to Blaze, if any.
+ *
+ * <p>This method will be called at the beginning of the execution phase, e.g. of the
+ * "blaze build" command.
+ */
+ public ActionContextProvider getActionContextProvider() {
+ return null;
+ }
+
+ /**
+ * Returns the action context consumer that pulls in action contexts required by this module,
+ * if any.
+ *
+ * <p>This method will be called at the beginning of the execution phase, e.g. of the
+ * "blaze build" command.
+ */
+ public ActionContextConsumer getActionContextConsumer() {
+ return null;
+ }
+
+ /**
+ * Called after each command.
+ */
+ public void afterCommand() {
+ }
+
+ /**
+ * Called when Blaze shuts down.
+ */
+ public void blazeShutdown() {
+ }
+
+ /**
+ * Action inputs are allowed to be missing for all inputs where this predicate returns true.
+ */
+ public Predicate<PathFragment> getAllowedMissingInputs() {
+ return null;
+ }
+
+ /**
+ * Optionally specializes the cache that ensures source files are looked at just once during
+ * a build. Only one module may do so.
+ */
+ public ActionInputFileCache createActionInputCache(String cwd, FileSystem fs) {
+ return null;
+ }
+
+ /**
+ * Returns the extensions this module contributes to the global namespace of the BUILD language.
+ */
+ public PackageFactory.EnvironmentExtension getPackageEnvironmentExtension() {
+ return new PackageFactory.EnvironmentExtension() {
+ @Override
+ public void update(
+ Environment environment, MakeEnvironment.Builder pkgMakeEnv, Label buildFileLabel) {
+ }
+
+ @Override
+ public Iterable<PackageArgument<?>> getPackageArguments() {
+ return ImmutableList.of();
+ }
+ };
+ }
+
+ /**
+ * Returns a factory for creating {@link SkyframeExecutor} objects. If the module does not
+ * provide any SkyframeExecutorFactory, it returns null. Note that only one factory per
+ * Bazel/Blaze runtime is allowed.
+ */
+ public SkyframeExecutorFactory getSkyframeExecutorFactory() {
+ return null;
+ }
+
+ /** Returns a map of "extra" SkyFunctions for SkyValues that this module may want to build. */
+ public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * Returns the extra precomputed values that the module makes available in Skyframe.
+ *
+ * <p>This method is called once per Blaze instance at the very beginning of its life.
+ * If it creates the injected values by using a {@code com.google.common.base.Supplier},
+ * that supplier is asked for the value it contains just before the loading phase begins. This
+ * functionality can be used to implement precomputed values that are not constant during the
+ * lifetime of a Blaze instance (naturally, they must be constant over the course of a build)
+ *
+ * <p>The following things must be done in order to define a new precomputed values:
+ * <ul>
+ * <li> Create a public static final variable of type
+ * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed}
+ * <li> Set its value by adding an {@link Injected} in this method (it can be created using the
+ * aforementioned variable and the value or a supplier of the value)
+ * <li> Reference the value in Skyframe functions by calling get {@code get} method on the
+ * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed} variable. This
+ * will never return null, because its value will have been injected before most of the Skyframe
+ * values are computed.
+ * </ul>
+ */
+ public Iterable<Injected> getPrecomputedSkyframeValues() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Optionally returns a provider for project files that can be used to bundle targets and
+ * command-line options.
+ */
+ @Nullable
+ public ProjectFile.Provider createProjectFileProvider() {
+ return null;
+ }
+
+ /**
+ * Optionally returns a factory to create coverage report actions.
+ */
+ @Nullable
+ public CoverageReportActionFactory getCoverageReportFactory() {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
new file mode 100644
index 0000000..0251e83
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -0,0 +1,1795 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.SubscriberExceptionContext;
+import com.google.common.eventbus.SubscriberExceptionHandler;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache;
+import com.google.devtools.build.lib.actions.cache.NullActionCache;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.buildtool.BuildTool;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.OutputFilter;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.exec.OutputService;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.profiler.MemoryProfiler;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.Profiler.ProfiledTaskKinds;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.query2.output.OutputFormatter;
+import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.runtime.commands.BuildCommand;
+import com.google.devtools.build.lib.runtime.commands.CanonicalizeCommand;
+import com.google.devtools.build.lib.runtime.commands.CleanCommand;
+import com.google.devtools.build.lib.runtime.commands.HelpCommand;
+import com.google.devtools.build.lib.runtime.commands.InfoCommand;
+import com.google.devtools.build.lib.runtime.commands.ProfileCommand;
+import com.google.devtools.build.lib.runtime.commands.QueryCommand;
+import com.google.devtools.build.lib.runtime.commands.RunCommand;
+import com.google.devtools.build.lib.runtime.commands.ShutdownCommand;
+import com.google.devtools.build.lib.runtime.commands.SkylarkCommand;
+import com.google.devtools.build.lib.runtime.commands.TestCommand;
+import com.google.devtools.build.lib.runtime.commands.VersionCommand;
+import com.google.devtools.build.lib.server.RPCServer;
+import com.google.devtools.build.lib.server.ServerCommand;
+import com.google.devtools.build.lib.server.signal.InterruptSignalHandler;
+import com.google.devtools.build.lib.skyframe.DiffAwareness;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutorFactory;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.util.ThreadUtils;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixFileSystem;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+import com.google.devtools.common.options.TriState;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * The BlazeRuntime class encapsulates the runtime settings and services that
+ * are available to most parts of any Blaze application for the duration of the
+ * batch run or server lifetime. A single instance of this runtime will exist
+ * and will be passed around as needed.
+ */
+public final class BlazeRuntime {
+ /**
+ * The threshold for memory reserved by a 32-bit JVM before trouble may be expected.
+ *
+ * <p>After the JVM starts, it reserves memory for heap (controlled by -Xmx) and non-heap
+ * (code, PermGen, etc.). Furthermore, as Blaze spawns threads, each thread reserves memory
+ * for the stack (controlled by -Xss). Thus even if Blaze starts fine, with high memory settings
+ * it will die from a stack allocation failure in the middle of a build. We prefer failing
+ * upfront by setting a safe threshold.
+ *
+ * <p>This does not apply to 64-bit VMs.
+ */
+ private static final long MAX_BLAZE32_RESERVED_MEMORY = 3400 * 1048576L;
+
+ // Less than this indicates tampering with -Xmx settings.
+ private static final long MIN_BLAZE32_HEAP_SIZE = 3000 * 1000000L;
+
+ public static final String DO_NOT_BUILD_FILE_NAME = "DO_NOT_BUILD_HERE";
+
+ private static final Pattern suppressFromLog = Pattern.compile(".*(auth|pass|cookie).*",
+ Pattern.CASE_INSENSITIVE);
+
+ private static final Logger LOG = Logger.getLogger(BlazeRuntime.class.getName());
+
+ private final BlazeDirectories directories;
+ private Path workingDirectory;
+ private long commandStartTime;
+
+ // Application-specified constants
+ private final PathFragment runfilesPrefix;
+
+ private final SkyframeExecutor skyframeExecutor;
+
+ private final Reporter reporter;
+ private EventBus eventBus;
+ private final LoadingPhaseRunner loadingPhaseRunner;
+ private final PackageFactory packageFactory;
+ private final ConfigurationFactory configurationFactory;
+ private final ConfiguredRuleClassProvider ruleClassProvider;
+ private final BuildView view;
+ private ActionCache actionCache;
+ private final TimestampGranularityMonitor timestampGranularityMonitor;
+ private final Clock clock;
+ private final BuildTool buildTool;
+
+ private OutputService outputService;
+
+ private final Iterable<BlazeModule> blazeModules;
+ private final BlazeModule.ModuleEnvironment blazeModuleEnvironment;
+
+ private UUID commandId; // Unique identifier for the command being run
+
+ private final AtomicInteger storedExitCode = new AtomicInteger();
+
+ private final Map<String, String> clientEnv;
+
+ // We pass this through here to make it available to the MasterLogWriter.
+ private final OptionsProvider startupOptionsProvider;
+
+ private String outputFileSystem;
+ private Map<String, BlazeCommand> commandMap;
+
+ private AbruptExitException pendingException;
+
+ private final SubscriberExceptionHandler eventBusExceptionHandler;
+
+ private final BinTools binTools;
+
+ private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
+
+ private final ProjectFile.Provider projectFileProvider;
+
+ private class BlazeModuleEnvironment implements BlazeModule.ModuleEnvironment {
+ @Override
+ public Path getFileFromDepot(Label label)
+ throws NoSuchThingException, InterruptedException, IOException {
+ Target target = getPackageManager().getTarget(reporter, label);
+ return (outputService != null)
+ ? outputService.stageTool(target)
+ : target.getPackage().getPackageDirectory().getRelative(target.getName());
+ }
+
+ @Override
+ public void exit(AbruptExitException exception) {
+ Preconditions.checkState(pendingException == null);
+ pendingException = exception;
+ }
+ }
+
+ private BlazeRuntime(BlazeDirectories directories, Reporter reporter,
+ WorkspaceStatusAction.Factory workspaceStatusActionFactory,
+ final SkyframeExecutor skyframeExecutor,
+ PackageFactory pkgFactory, ConfiguredRuleClassProvider ruleClassProvider,
+ ConfigurationFactory configurationFactory, PathFragment runfilesPrefix, Clock clock,
+ OptionsProvider startupOptionsProvider, Iterable<BlazeModule> blazeModules,
+ Map<String, String> clientEnv,
+ TimestampGranularityMonitor timestampGranularityMonitor,
+ SubscriberExceptionHandler eventBusExceptionHandler,
+ BinTools binTools, ProjectFile.Provider projectFileProvider) {
+ this.workspaceStatusActionFactory = workspaceStatusActionFactory;
+ this.directories = directories;
+ this.workingDirectory = directories.getWorkspace();
+ this.reporter = reporter;
+ this.runfilesPrefix = runfilesPrefix;
+ this.packageFactory = pkgFactory;
+ this.binTools = binTools;
+ this.projectFileProvider = projectFileProvider;
+
+ this.skyframeExecutor = skyframeExecutor;
+ this.loadingPhaseRunner = new LoadingPhaseRunner(
+ skyframeExecutor.getPackageManager(),
+ pkgFactory.getRuleClassNames());
+
+ this.clientEnv = clientEnv;
+
+ this.blazeModules = blazeModules;
+ this.ruleClassProvider = ruleClassProvider;
+ this.configurationFactory = configurationFactory;
+ this.view = new BuildView(directories, getPackageManager(), ruleClassProvider,
+ skyframeExecutor, binTools, getCoverageReportActionFactory(blazeModules));
+ this.clock = clock;
+ this.timestampGranularityMonitor = Preconditions.checkNotNull(timestampGranularityMonitor);
+ this.startupOptionsProvider = startupOptionsProvider;
+
+ this.eventBusExceptionHandler = eventBusExceptionHandler;
+ this.blazeModuleEnvironment = new BlazeModuleEnvironment();
+ this.buildTool = new BuildTool(this);
+ initEventBus();
+
+ if (inWorkspace()) {
+ writeOutputBaseReadmeFile();
+ writeOutputBaseDoNotBuildHereFile();
+ }
+ setupExecRoot();
+ }
+
+ @Nullable private CoverageReportActionFactory getCoverageReportActionFactory(
+ Iterable<BlazeModule> blazeModules) {
+ CoverageReportActionFactory firstFactory = null;
+ for (BlazeModule module : blazeModules) {
+ CoverageReportActionFactory factory = module.getCoverageReportFactory();
+ if (factory != null) {
+ Preconditions.checkState(firstFactory == null,
+ "only one Blaze Module can have a Coverage Report Factory");
+ firstFactory = factory;
+ }
+ }
+ return firstFactory;
+ }
+
+ /**
+ * Figures out what file system we are writing output to. Here we use
+ * outputBase instead of outputPath because we need a file system to create the latter.
+ */
+ private String determineOutputFileSystem() {
+ if (getOutputService() != null) {
+ return getOutputService().getFilesSystemName();
+ }
+ long startTime = Profiler.nanoTimeMaybe();
+ String fileSystem = FileSystemUtils.getFileSystem(getOutputBase());
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Finding output file system");
+ return fileSystem;
+ }
+
+ public String getOutputFileSystem() {
+ return outputFileSystem;
+ }
+
+ @VisibleForTesting
+ public void initEventBus() {
+ setEventBus(new EventBus(eventBusExceptionHandler));
+ }
+
+ private void clearEventBus() {
+ // EventBus does not have an unregister() method, so this is how we release memory associated
+ // with handlers.
+ setEventBus(null);
+ }
+
+ private void setEventBus(EventBus eventBus) {
+ this.eventBus = eventBus;
+ skyframeExecutor.setEventBus(eventBus);
+ }
+
+ /**
+ * Conditionally enable profiling.
+ */
+ private final boolean initProfiler(CommonCommandOptions options,
+ UUID buildID, long execStartTimeNanos) {
+ OutputStream out = null;
+ boolean recordFullProfilerData = false;
+ ProfiledTaskKinds profiledTasks = ProfiledTaskKinds.NONE;
+
+ try {
+ if (options.profilePath != null) {
+ Path profilePath = getWorkspace().getRelative(options.profilePath);
+
+ recordFullProfilerData = options.recordFullProfilerData;
+ out = new BufferedOutputStream(profilePath.getOutputStream(), 1024 * 1024);
+ getReporter().handle(Event.info("Writing profile data to '" + profilePath + "'"));
+ profiledTasks = ProfiledTaskKinds.ALL;
+ } else if (options.alwaysProfileSlowOperations) {
+ recordFullProfilerData = false;
+ out = null;
+ profiledTasks = ProfiledTaskKinds.SLOWEST;
+ }
+ if (profiledTasks != ProfiledTaskKinds.NONE) {
+ Profiler.instance().start(profiledTasks, out,
+ "Blaze profile for " + getOutputBase() + " at " + new Date()
+ + ", build ID: " + buildID,
+ recordFullProfilerData, clock, execStartTimeNanos);
+ return true;
+ }
+ } catch (IOException e) {
+ getReporter().handle(Event.error("Error while creating profile file: " + e.getMessage()));
+ }
+ return false;
+ }
+
+ /**
+ * Generates a README file in the output base directory. This README file
+ * contains the name of the workspace directory, so that users can figure out
+ * which output base directory corresponds to which workspace.
+ */
+ private void writeOutputBaseReadmeFile() {
+ Preconditions.checkNotNull(getWorkspace());
+ Path outputBaseReadmeFile = getOutputBase().getRelative("README");
+ try {
+ FileSystemUtils.writeIsoLatin1(outputBaseReadmeFile, "WORKSPACE: " + getWorkspace(), "",
+ "The first line of this file is intentionally easy to parse for various",
+ "interactive scripting and debugging purposes. But please DO NOT write programs",
+ "that exploit it, as they will be broken by design: it is not possible to",
+ "reverse engineer the set of source trees or the --package_path from the output",
+ "tree, and if you attempt it, you will fail, creating subtle and",
+ "hard-to-diagnose bugs, that will no doubt get blamed on changes made by the",
+ "Blaze team.", "", "This directory was generated by Blaze.",
+ "Do not attempt to modify or delete any files in this directory.",
+ "Among other issues, Blaze's file system caching assumes that",
+ "only Blaze will modify this directory and the files in it,",
+ "so if you change anything here you may mess up Blaze's cache.");
+ } catch (IOException e) {
+ LOG.warning("Couldn't write to '" + outputBaseReadmeFile + "': " + e.getMessage());
+ }
+ }
+
+ private void writeOutputBaseDoNotBuildHereFile() {
+ Preconditions.checkNotNull(getWorkspace());
+ Path filePath = getOutputBase().getRelative(DO_NOT_BUILD_FILE_NAME);
+ try {
+ FileSystemUtils.writeContent(filePath, ISO_8859_1, getWorkspace().toString());
+ } catch (IOException e) {
+ LOG.warning("Couldn't write to '" + filePath + "': " + e.getMessage());
+ }
+ }
+
+ /**
+ * Creates the execRoot dir under outputBase.
+ */
+ private void setupExecRoot() {
+ try {
+ FileSystemUtils.createDirectoryAndParents(directories.getExecRoot());
+ } catch (IOException e) {
+ LOG.warning("failed to create execution root '" + directories.getExecRoot() + "': "
+ + e.getMessage());
+ }
+ }
+
+ public void recordCommandStartTime(long commandStartTime) {
+ this.commandStartTime = commandStartTime;
+ }
+
+ public long getCommandStartTime() {
+ return commandStartTime;
+ }
+
+ public String getWorkspaceName() {
+ Path workspace = directories.getWorkspace();
+ if (workspace == null) {
+ return "";
+ }
+ return workspace.getBaseName();
+ }
+
+ /**
+ * Returns any prefix to be inserted between relative source paths and the runfiles directory.
+ */
+ public PathFragment getRunfilesPrefix() {
+ return runfilesPrefix;
+ }
+
+ /**
+ * Returns the Blaze directories object for this runtime.
+ */
+ public BlazeDirectories getDirectories() {
+ return directories;
+ }
+
+ /**
+ * Returns the working directory of the server.
+ *
+ * <p>This is often the first entry on the {@code --package_path}, but not always.
+ * Callers should certainly not make this assumption. The Path returned may be null.
+ *
+ * @see #getWorkingDirectory()
+ */
+ public Path getWorkspace() {
+ return directories.getWorkspace();
+ }
+
+ /**
+ * Returns the working directory of the {@code blaze} client process.
+ *
+ * <p>This may be equal to {@code getWorkspace()}, or beneath it.
+ *
+ * @see #getWorkspace()
+ */
+ public Path getWorkingDirectory() {
+ return workingDirectory;
+ }
+
+ /**
+ * Returns if the client passed a valid workspace to be used for the build.
+ */
+ public boolean inWorkspace() {
+ return directories.inWorkspace();
+ }
+
+ /**
+ * Returns the output base directory associated with this Blaze server
+ * process. This is the base directory for shared Blaze state as well as tool
+ * and strategy specific subdirectories.
+ */
+ public Path getOutputBase() {
+ return directories.getOutputBase();
+ }
+
+ /**
+ * Returns the output path associated with this Blaze server process..
+ */
+ public Path getOutputPath() {
+ return directories.getOutputPath();
+ }
+
+ /**
+ * The directory in which blaze stores the server state - that is, the socket
+ * file and a log.
+ */
+ public Path getServerDirectory() {
+ return getOutputBase().getChild("server");
+ }
+
+ /**
+ * Returns the execution root directory associated with this Blaze server
+ * process. This is where all input and output files visible to the actual
+ * build reside.
+ */
+ public Path getExecRoot() {
+ return directories.getExecRoot();
+ }
+
+ /**
+ * Returns the reporter for events.
+ */
+ public Reporter getReporter() {
+ return reporter;
+ }
+
+ /**
+ * Returns the current event bus. Only valid within the scope of a single Blaze command.
+ */
+ public EventBus getEventBus() {
+ return eventBus;
+ }
+
+ public BinTools getBinTools() {
+ return binTools;
+ }
+
+ /**
+ * Returns the skyframe executor.
+ */
+ public SkyframeExecutor getSkyframeExecutor() {
+ return skyframeExecutor;
+ }
+
+ /**
+ * Returns the package factory.
+ */
+ public PackageFactory getPackageFactory() {
+ return packageFactory;
+ }
+
+ /**
+ * Returns the build tool.
+ */
+ public BuildTool getBuildTool() {
+ return buildTool;
+ }
+
+ public ImmutableList<OutputFormatter> getQueryOutputFormatters() {
+ ImmutableList.Builder<OutputFormatter> result = ImmutableList.builder();
+ result.addAll(OutputFormatter.getDefaultFormatters());
+ for (BlazeModule module : blazeModules) {
+ result.addAll(module.getQueryOutputFormatters());
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Returns the package manager.
+ */
+ public PackageManager getPackageManager() {
+ return skyframeExecutor.getPackageManager();
+ }
+
+ public WorkspaceStatusAction.Factory getworkspaceStatusActionFactory() {
+ return workspaceStatusActionFactory;
+ }
+
+ public BlazeModule.ModuleEnvironment getBlazeModuleEnvironment() {
+ return blazeModuleEnvironment;
+ }
+
+ /**
+ * Returns the rule class provider.
+ */
+ public ConfiguredRuleClassProvider getRuleClassProvider() {
+ return ruleClassProvider;
+ }
+
+ public LoadingPhaseRunner getLoadingPhaseRunner() {
+ return loadingPhaseRunner;
+ }
+
+ /**
+ * Returns the build view.
+ */
+ public BuildView getView() {
+ return view;
+ }
+
+ public Iterable<BlazeModule> getBlazeModules() {
+ return blazeModules;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends BlazeModule> T getBlazeModule(Class<T> moduleClass) {
+ for (BlazeModule module : blazeModules) {
+ if (module.getClass() == moduleClass) {
+ return (T) module;
+ }
+ }
+
+ return null;
+ }
+
+ public ConfigurationFactory getConfigurationFactory() {
+ return configurationFactory;
+ }
+
+ /**
+ * Returns the target pattern parser.
+ */
+ public TargetPatternEvaluator getTargetPatternEvaluator() {
+ return loadingPhaseRunner.getTargetPatternEvaluator();
+ }
+
+ /**
+ * Returns reference to the lazily instantiated persistent action cache
+ * instance. Note, that method may recreate instance between different build
+ * requests, so return value should not be cached.
+ */
+ public ActionCache getPersistentActionCache() throws IOException {
+ if (actionCache == null) {
+ if (OS.getCurrent() == OS.WINDOWS) {
+ // TODO(bazel-team): Add support for a persistent action cache on Windows.
+ actionCache = new NullActionCache();
+ return actionCache;
+ }
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock);
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Failed to load action cache: " + e.getMessage(), e);
+ LoggingUtil.logToRemote(Level.WARNING, "Failed to load action cache: "
+ + e.getMessage(), e);
+ getReporter().handle(
+ Event.error("Error during action cache initialization: " + e.getMessage()
+ + ". Corrupted files were renamed to '" + getCacheDirectory() + "/*.bad'. "
+ + "Blaze will now reset action cache data, causing a full rebuild"));
+ actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock);
+ } finally {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Loading action cache");
+ }
+ }
+ return actionCache;
+ }
+
+ /**
+ * Removes in-memory caches.
+ */
+ public void clearCaches() throws IOException {
+ clearSkyframeRelevantCaches();
+ actionCache = null;
+ FileSystemUtils.deleteTree(getCacheDirectory());
+ }
+
+ /** Removes skyframe cache and other caches that must be kept synchronized with skyframe. */
+ private void clearSkyframeRelevantCaches() {
+ skyframeExecutor.resetEvaluator();
+ view.clear();
+ }
+
+ /**
+ * Returns the TimestampGranularityMonitor. The same monitor object is used
+ * across multiple Blaze commands, but it doesn't hold any persistent state
+ * across different commands.
+ */
+ public TimestampGranularityMonitor getTimestampGranularityMonitor() {
+ return timestampGranularityMonitor;
+ }
+
+ /**
+ * Returns path to the cache directory. Path must be inside output base to
+ * ensure that users can run concurrent instances of blaze in different
+ * clients without attempting to concurrently write to the same action cache
+ * on disk, which might not be safe.
+ */
+ private Path getCacheDirectory() {
+ return getOutputBase().getChild("action_cache");
+ }
+
+ /**
+ * Returns a provider for project file objects. Can be null if no such provider was set by any of
+ * the modules.
+ */
+ @Nullable
+ public ProjectFile.Provider getProjectFileProvider() {
+ return projectFileProvider;
+ }
+
+ /**
+ * Hook method called by the BlazeCommandDispatcher prior to the dispatch of
+ * each command.
+ *
+ * @param options The CommonCommandOptions used by every command.
+ * @throws AbruptExitException if this command is unsuitable to be run as specified
+ */
+ void beforeCommand(String commandName, OptionsParser optionsParser,
+ CommonCommandOptions options, long execStartTimeNanos)
+ throws AbruptExitException {
+ commandStartTime -= options.startupTime;
+
+ eventBus.post(new GotOptionsEvent(startupOptionsProvider,
+ optionsParser));
+ throwPendingException();
+
+ outputService = null;
+ BlazeModule outputModule = null;
+ for (BlazeModule module : blazeModules) {
+ OutputService moduleService = module.getOutputService();
+ if (moduleService != null) {
+ if (outputService != null) {
+ throw new IllegalStateException(String.format(
+ "More than one module (%s and %s) returns an output service",
+ module.getClass(), outputModule.getClass()));
+ }
+ outputService = moduleService;
+ outputModule = module;
+ }
+ }
+
+ skyframeExecutor.setBatchStatter(outputService == null
+ ? null
+ : outputService.getBatchStatter());
+
+ outputFileSystem = determineOutputFileSystem();
+
+ // Ensure that the working directory will be under the workspace directory.
+ Path workspace = getWorkspace();
+ if (inWorkspace()) {
+ workingDirectory = workspace.getRelative(options.clientCwd);
+ } else {
+ workspace = FileSystemUtils.getWorkingDirectory(directories.getFileSystem());
+ workingDirectory = workspace;
+ }
+ updateClientEnv(options.clientEnv, options.ignoreClientEnv);
+ loadingPhaseRunner.updatePatternEvaluator(workingDirectory.relativeTo(workspace));
+
+ // Fail fast in the case where a Blaze command forgets to install the package path correctly.
+ skyframeExecutor.setActive(false);
+ // Let skyframe figure out if it needs to store graph edges for this build.
+ skyframeExecutor.decideKeepIncrementalState(
+ startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).batch,
+ optionsParser.getOptions(BuildView.Options.class));
+
+ // Conditionally enable profiling
+ // We need to compensate for launchTimeNanos (measurements taken outside of the jvm).
+ long startupTimeNanos = options.startupTime * 1000000L;
+ if (initProfiler(options, this.getCommandId(), execStartTimeNanos - startupTimeNanos)) {
+ Profiler profiler = Profiler.instance();
+
+ // Instead of logEvent() we're calling the low level function to pass the timings we took in
+ // the launcher. We're setting the INIT phase marker so that it follows immediately the LAUNCH
+ // phase.
+ profiler.logSimpleTaskDuration(execStartTimeNanos - startupTimeNanos, 0, ProfilerTask.PHASE,
+ ProfilePhase.LAUNCH.description);
+ profiler.logSimpleTaskDuration(execStartTimeNanos, 0, ProfilerTask.PHASE,
+ ProfilePhase.INIT.description);
+ }
+
+ if (options.memoryProfilePath != null) {
+ Path memoryProfilePath = getWorkingDirectory().getRelative(options.memoryProfilePath);
+ try {
+ MemoryProfiler.instance().start(memoryProfilePath.getOutputStream());
+ } catch (IOException e) {
+ getReporter().handle(
+ Event.error("Error while creating memory profile file: " + e.getMessage()));
+ }
+ }
+
+ eventBus.post(new CommandStartEvent(commandName, commandId, clientEnv, workingDirectory));
+ // Initialize exit code to dummy value for afterCommand.
+ storedExitCode.set(ExitCode.RESERVED.getNumericExitCode());
+ }
+
+ /**
+ * Hook method called by the BlazeCommandDispatcher right before the dispatch
+ * of each command ends (while its outcome can still be modified).
+ */
+ ExitCode precompleteCommand(ExitCode originalExit) {
+ eventBus.post(new CommandPrecompleteEvent(originalExit));
+ // If Blaze did not suffer an infrastructure failure, check for errors in modules.
+ ExitCode exitCode = originalExit;
+ if (!originalExit.isInfrastructureFailure()) {
+ if (pendingException != null) {
+ exitCode = pendingException.getExitCode();
+ }
+ }
+ pendingException = null;
+ return exitCode;
+ }
+
+ /**
+ * Posts the {@link CommandCompleteEvent}, so that listeners can tidy up. Called by {@link
+ * #afterCommand}, and by BugReport when crashing from an exception in an async thread.
+ */
+ public void notifyCommandComplete(int exitCode) {
+ if (!storedExitCode.compareAndSet(ExitCode.RESERVED.getNumericExitCode(), exitCode)) {
+ // This command has already been called, presumably because there is a race between the main
+ // thread and a worker thread that crashed. Don't try to arbitrate the dispute. If the main
+ // thread won the race (unlikely, but possible), this may be incorrectly logged as a success.
+ return;
+ }
+ eventBus.post(new CommandCompleteEvent(exitCode));
+ }
+
+ /**
+ * Hook method called by the BlazeCommandDispatcher after the dispatch of each
+ * command.
+ */
+ @VisibleForTesting
+ public void afterCommand(int exitCode) {
+ // Remove any filters that the command might have added to the reporter.
+ getReporter().setOutputFilter(OutputFilter.OUTPUT_EVERYTHING);
+
+ notifyCommandComplete(exitCode);
+
+ for (BlazeModule module : blazeModules) {
+ module.afterCommand();
+ }
+
+ clearEventBus();
+
+ try {
+ Profiler.instance().stop();
+ MemoryProfiler.instance().stop();
+ } catch (IOException e) {
+ getReporter().handle(Event.error("Error while writing profile file: " + e.getMessage()));
+ }
+ }
+
+ // Make sure we keep a strong reference to this logger, so that the
+ // configuration isn't lost when the gc kicks in.
+ private static Logger templateLogger = Logger.getLogger("com.google.devtools.build");
+
+ /**
+ * Configures "com.google.devtools.build.*" loggers to the given
+ * {@code level}. Note: This code relies on static state.
+ */
+ public static void setupLogging(Level level) {
+ templateLogger.setLevel(level);
+ templateLogger.info("Log level: " + templateLogger.getLevel());
+ }
+
+ /**
+ * Return an unmodifiable view of the blaze client's environment when it
+ * invoked the most recent command. Updates from future requests will be
+ * accessible from this view.
+ */
+ public Map<String, String> getClientEnv() {
+ return Collections.unmodifiableMap(clientEnv);
+ }
+
+ @VisibleForTesting
+ void updateClientEnv(List<Map.Entry<String, String>> clientEnvList, boolean ignoreClientEnv) {
+ clientEnv.clear();
+
+ Collection<Map.Entry<String, String>> env =
+ ignoreClientEnv ? System.getenv().entrySet() : clientEnvList;
+ for (Map.Entry<String, String> entry : env) {
+ clientEnv.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Returns the Clock-instance used for the entire build. Before,
+ * individual classes (such as Profiler) used to specify the type
+ * of clock (e.g. EpochClock) they wanted to use. This made it
+ * difficult to get Blaze working on Windows as some of the clocks
+ * available for Linux aren't (directly) available on Windows.
+ * Setting the Blaze-wide clock upon construction of BlazeRuntime
+ * allows injecting whatever Clock instance should be used from
+ * BlazeMain.
+ *
+ * @return The Blaze-wide clock
+ */
+ public Clock getClock() {
+ return clock;
+ }
+
+ public OptionsProvider getStartupOptionsProvider() {
+ return startupOptionsProvider;
+ }
+
+ /**
+ * An array of String values useful if Blaze crashes.
+ * For now, just returns the size of the action cache and the build id.
+ */
+ public String[] getCrashData() {
+ return new String[]{
+ getFileSizeString(CompactPersistentActionCache.cacheFile(getCacheDirectory()),
+ "action cache"),
+ commandIdString(),
+ };
+ }
+
+ private String commandIdString() {
+ UUID uuid = getCommandId();
+ return (uuid == null)
+ ? "no build id"
+ : uuid + " (build id)";
+ }
+
+ /**
+ * @return the OutputService in use, or null if none.
+ */
+ public OutputService getOutputService() {
+ return outputService;
+ }
+
+ private String getFileSizeString(Path path, String type) {
+ try {
+ return String.format("%d bytes (%s)", path.getFileSize(), type);
+ } catch (IOException e) {
+ return String.format("unknown file size (%s)", type);
+ }
+ }
+
+ /**
+ * Returns the UUID that Blaze uses to identify everything
+ * logged from the current build command.
+ */
+ public UUID getCommandId() {
+ return commandId;
+ }
+
+ void setCommandMap(Map<String, BlazeCommand> commandMap) {
+ this.commandMap = ImmutableMap.copyOf(commandMap);
+ }
+
+ public Map<String, BlazeCommand> getCommandMap() {
+ return commandMap;
+ }
+
+ /**
+ * Sets the UUID that Blaze uses to identify everything
+ * logged from the current build command.
+ */
+ @VisibleForTesting
+ public void setCommandId(UUID runId) {
+ commandId = runId;
+ }
+
+ /**
+ * Constructs a build configuration key for the given options.
+ */
+ public BuildConfigurationKey getBuildConfigurationKey(BuildOptions buildOptions,
+ ImmutableSortedSet<String> multiCpu) {
+ return new BuildConfigurationKey(buildOptions, directories, clientEnv, multiCpu);
+ }
+
+ /**
+ * This method only exists for the benefit of InfoCommand, which needs to construct a {@link
+ * BuildConfigurationCollection} without running a full loading phase. Don't add any more clients;
+ * instead, we should change info so that it doesn't need the configuration.
+ */
+ public BuildConfigurationCollection getConfigurations(OptionsProvider optionsProvider)
+ throws InvalidConfigurationException, InterruptedException {
+ BuildConfigurationKey configurationKey = getBuildConfigurationKey(
+ createBuildOptions(optionsProvider), ImmutableSortedSet.<String>of());
+ boolean keepGoing = optionsProvider.getOptions(BuildView.Options.class).keepGoing;
+ LoadedPackageProvider loadedPackageProvider =
+ loadingPhaseRunner.loadForConfigurations(reporter,
+ ImmutableSet.copyOf(configurationKey.getLabelsToLoadUnconditionally().values()),
+ keepGoing);
+ if (loadedPackageProvider == null) {
+ throw new InvalidConfigurationException("Configuration creation failed");
+ }
+ return skyframeExecutor.createConfigurations(keepGoing, configurationFactory,
+ configurationKey);
+ }
+
+ /**
+ * Initializes the package cache using the given options, and syncs the package cache. Also
+ * injects a defaults package using the options for the {@link BuildConfiguration}.
+ *
+ * @see DefaultsPackage
+ */
+ public void setupPackageCache(PackageCacheOptions packageCacheOptions,
+ String defaultsPackageContents) throws InterruptedException, AbruptExitException {
+ if (!skyframeExecutor.hasIncrementalState()) {
+ clearSkyframeRelevantCaches();
+ }
+ skyframeExecutor.sync(packageCacheOptions, getWorkingDirectory(),
+ defaultsPackageContents, getCommandId());
+ }
+
+ public void shutdown() {
+ for (BlazeModule module : blazeModules) {
+ module.blazeShutdown();
+ }
+ }
+
+ /**
+ * Throws the exception currently queued by a Blaze module.
+ *
+ * <p>This should be called as often as is practical so that errors are reported as soon as
+ * possible. Ideally, we'd not need this, but the event bus swallows exceptions so we raise
+ * the exception this way.
+ */
+ public void throwPendingException() throws AbruptExitException {
+ if (pendingException != null) {
+ AbruptExitException exception = pendingException;
+ pendingException = null;
+ throw exception;
+ }
+ }
+
+ /**
+ * Returns the defaults package for the default settings. Should only be called by commands that
+ * do <i>not</i> process {@link BuildOptions}, since build options can alter the contents of the
+ * defaults package, which will not be reflected here.
+ */
+ public String getDefaultsPackageContent() {
+ return ruleClassProvider.getDefaultsPackageContent();
+ }
+
+ /**
+ * Returns the defaults package for the given options taken from an optionsProvider.
+ */
+ public String getDefaultsPackageContent(OptionsClassProvider optionsProvider) {
+ return ruleClassProvider.getDefaultsPackageContent(optionsProvider);
+ }
+
+ /**
+ * Creates a BuildOptions class for the given options taken from an optionsProvider.
+ */
+ public BuildOptions createBuildOptions(OptionsClassProvider optionsProvider) {
+ return ruleClassProvider.createBuildOptions(optionsProvider);
+ }
+
+ /**
+ * An EventBus exception handler that will report the exception to a remote server, if a
+ * handler is registered.
+ */
+ public static final class RemoteExceptionHandler implements SubscriberExceptionHandler {
+ @Override
+ public void handleException(Throwable exception, SubscriberExceptionContext context) {
+ LoggingUtil.logToRemote(Level.SEVERE, "Failure in EventBus subscriber.", exception);
+ }
+ }
+
+ /**
+ * An EventBus exception handler that will call BugReport.handleCrash exiting
+ * the current thread.
+ */
+ public static final class BugReportingExceptionHandler implements SubscriberExceptionHandler {
+ @Override
+ public void handleException(Throwable exception, SubscriberExceptionContext context) {
+ BugReport.handleCrash(exception);
+ }
+ }
+
+ /**
+ * Main method for the Blaze server startup. Note: This method logs
+ * exceptions to remote servers. Do not add this to a unittest.
+ */
+ public static void main(Iterable<Class<? extends BlazeModule>> moduleClasses, String[] args) {
+ setupUncaughtHandler(args);
+ List<BlazeModule> modules = createModules(moduleClasses);
+ if (args.length >= 1 && args[0].equals("--batch")) {
+ // Run Blaze in batch mode.
+ System.exit(batchMain(modules, args));
+ }
+ LOG.info("Starting Blaze server with args " + Arrays.toString(args));
+ try {
+ // Run Blaze in server mode.
+ System.exit(serverMain(modules, OutErr.SYSTEM_OUT_ERR, args));
+ } catch (RuntimeException | Error e) { // A definite bug...
+ BugReport.printBug(OutErr.SYSTEM_OUT_ERR, e);
+ BugReport.sendBugReport(e, Arrays.asList(args));
+ System.exit(ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode());
+ throw e; // Shouldn't get here.
+ }
+ }
+
+ @VisibleForTesting
+ public static List<BlazeModule> createModules(
+ Iterable<Class<? extends BlazeModule>> moduleClasses) {
+ ImmutableList.Builder<BlazeModule> result = ImmutableList.builder();
+ for (Class<? extends BlazeModule> moduleClass : moduleClasses) {
+ try {
+ BlazeModule module = moduleClass.newInstance();
+ result.add(module);
+ } catch (Throwable e) {
+ throw new IllegalStateException("Cannot instantiate module " + moduleClass.getName(), e);
+ }
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Generates a string form of a request to be written to the logs,
+ * filtering the user environment to remove anything that looks private.
+ * The current filter criteria removes any variable whose name includes
+ * "auth", "pass", or "cookie".
+ *
+ * @param requestStrings
+ * @return the filtered request to write to the log.
+ */
+ @VisibleForTesting
+ public static String getRequestLogString(List<String> requestStrings) {
+ StringBuilder buf = new StringBuilder();
+ buf.append('[');
+ String sep = "";
+ for (String s : requestStrings) {
+ buf.append(sep);
+ if (s.startsWith("--client_env")) {
+ int varStart = "--client_env=".length();
+ int varEnd = s.indexOf('=', varStart);
+ String varName = s.substring(varStart, varEnd);
+ if (suppressFromLog.matcher(varName).matches()) {
+ buf.append("--client_env=");
+ buf.append(varName);
+ buf.append("=__private_value_removed__");
+ } else {
+ buf.append(s);
+ }
+ } else {
+ buf.append(s);
+ }
+ sep = ", ";
+ }
+ buf.append(']');
+ return buf.toString();
+ }
+
+ /**
+ * Command line options split in to two parts: startup options and everything else.
+ */
+ @VisibleForTesting
+ static class CommandLineOptions {
+ private final List<String> startupArgs;
+ private final List<String> otherArgs;
+
+ CommandLineOptions(List<String> startupArgs, List<String> otherArgs) {
+ this.startupArgs = ImmutableList.copyOf(startupArgs);
+ this.otherArgs = ImmutableList.copyOf(otherArgs);
+ }
+
+ public List<String> getStartupArgs() {
+ return startupArgs;
+ }
+
+ public List<String> getOtherArgs() {
+ return otherArgs;
+ }
+ }
+
+ /**
+ * Splits given arguments into two lists - arguments matching options defined in this class
+ * and everything else, while preserving order in each list.
+ */
+ static CommandLineOptions splitStartupOptions(
+ Iterable<BlazeModule> modules, String... args) {
+ List<String> prefixes = new ArrayList<>();
+ List<Field> startupFields = Lists.newArrayList();
+ for (Class<? extends OptionsBase> defaultOptions
+ : BlazeCommandUtils.getStartupOptions(modules)) {
+ startupFields.addAll(ImmutableList.copyOf(defaultOptions.getFields()));
+ }
+
+ for (Field field : startupFields) {
+ if (field.isAnnotationPresent(Option.class)) {
+ prefixes.add("--" + field.getAnnotation(Option.class).name());
+ if (field.getType() == boolean.class || field.getType() == TriState.class) {
+ prefixes.add("--no" + field.getAnnotation(Option.class).name());
+ }
+ }
+ }
+
+ List<String> startupArgs = new ArrayList<>();
+ List<String> otherArgs = Lists.newArrayList(args);
+
+ for (Iterator<String> argi = otherArgs.iterator(); argi.hasNext(); ) {
+ String arg = argi.next();
+ if (!arg.startsWith("--")) {
+ break; // stop at command - all startup options would be specified before it.
+ }
+ for (String prefix : prefixes) {
+ if (arg.startsWith(prefix)) {
+ startupArgs.add(arg);
+ argi.remove();
+ break;
+ }
+ }
+ }
+ return new CommandLineOptions(startupArgs, otherArgs);
+ }
+
+ private static void captureSigint() {
+ final Thread mainThread = Thread.currentThread();
+ final AtomicInteger numInterrupts = new AtomicInteger();
+
+ final Runnable interruptWatcher = new Runnable() {
+ @Override
+ public void run() {
+ int count = 0;
+ // Not an actual infinite loop because it's run in a daemon thread.
+ while (true) {
+ count++;
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
+ LOG.warning("Slow interrupt number " + count + " in batch mode");
+ ThreadUtils.warnAboutSlowInterrupt();
+ }
+ }
+ };
+
+ new InterruptSignalHandler() {
+ @Override
+ public void run() {
+ LOG.info("User interrupt");
+ OutErr.SYSTEM_OUT_ERR.printErrLn("Blaze received an interrupt");
+ mainThread.interrupt();
+
+ int curNumInterrupts = numInterrupts.incrementAndGet();
+ if (curNumInterrupts == 1) {
+ Thread interruptWatcherThread = new Thread(interruptWatcher, "interrupt-watcher");
+ interruptWatcherThread.setDaemon(true);
+ interruptWatcherThread.start();
+ } else if (curNumInterrupts == 2) {
+ LOG.warning("Second --batch interrupt: Reverting to JVM SIGINT handler");
+ uninstall();
+ }
+ }
+ };
+ }
+
+ /**
+ * A main method that runs blaze commands in batch mode. The return value indicates the desired
+ * exit status of the program.
+ */
+ private static int batchMain(Iterable<BlazeModule> modules, String[] args) {
+ captureSigint();
+ CommandLineOptions commandLineOptions = splitStartupOptions(modules, args);
+ LOG.info("Running Blaze in batch mode with startup args "
+ + commandLineOptions.getStartupArgs());
+
+ String memoryWarning = validateJvmMemorySettings();
+ if (memoryWarning != null) {
+ OutErr.SYSTEM_OUT_ERR.printErrLn(memoryWarning);
+ }
+
+ BlazeRuntime runtime;
+ try {
+ runtime = newRuntime(modules, parseOptions(modules, commandLineOptions.getStartupArgs()));
+ } catch (OptionsParsingException e) {
+ OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage());
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ } catch (AbruptExitException e) {
+ OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage());
+ return e.getExitCode().getNumericExitCode();
+ }
+
+ BlazeCommandDispatcher dispatcher =
+ new BlazeCommandDispatcher(runtime, getBuiltinCommandList());
+
+ try {
+ LOG.info(getRequestLogString(commandLineOptions.getOtherArgs()));
+ return dispatcher.exec(commandLineOptions.getOtherArgs(), OutErr.SYSTEM_OUT_ERR,
+ runtime.getClock().currentTimeMillis());
+ } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) {
+ return e.getExitStatus();
+ } finally {
+ runtime.shutdown();
+ dispatcher.shutdown();
+ }
+ }
+
+ /**
+ * A main method that does not send email. The return value indicates the desired exit status of
+ * the program.
+ */
+ private static int serverMain(Iterable<BlazeModule> modules, OutErr outErr, String[] args) {
+ try {
+ createBlazeRPCServer(modules, Arrays.asList(args)).serve();
+ return ExitCode.SUCCESS.getNumericExitCode();
+ } catch (OptionsParsingException e) {
+ outErr.printErr(e.getMessage());
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ } catch (IOException e) {
+ outErr.printErr("I/O Error: " + e.getMessage());
+ return ExitCode.BUILD_FAILURE.getNumericExitCode();
+ } catch (AbruptExitException e) {
+ outErr.printErr(e.getMessage());
+ return e.getExitCode().getNumericExitCode();
+ }
+ }
+
+ private static FileSystem fileSystemImplementation() {
+ // The JNI-based UnixFileSystem is faster, but on Windows it is not available.
+ return OS.getCurrent() == OS.WINDOWS ? new JavaIoFileSystem() : new UnixFileSystem();
+ }
+
+ /**
+ * Creates and returns a new Blaze RPCServer. Call {@link RPCServer#serve()} to start the server.
+ */
+ private static RPCServer createBlazeRPCServer(Iterable<BlazeModule> modules, List<String> args)
+ throws IOException, OptionsParsingException, AbruptExitException {
+ OptionsProvider options = parseOptions(modules, args);
+ BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class);
+
+ final BlazeRuntime runtime = newRuntime(modules, options);
+ final BlazeCommandDispatcher dispatcher =
+ new BlazeCommandDispatcher(runtime, getBuiltinCommandList());
+ final String memoryWarning = validateJvmMemorySettings();
+
+ final ServerCommand blazeCommand;
+
+ // Adaptor from RPC mechanism to BlazeCommandDispatcher:
+ blazeCommand = new ServerCommand() {
+ private boolean shutdown = false;
+
+ @Override
+ public int exec(List<String> args, OutErr outErr, long firstContactTime) {
+ LOG.info(getRequestLogString(args));
+ if (memoryWarning != null) {
+ outErr.printErrLn(memoryWarning);
+ }
+
+ try {
+ return dispatcher.exec(args, outErr, firstContactTime);
+ } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) {
+ if (e.getCause() != null) {
+ StringWriter message = new StringWriter();
+ message.write("Shutting down due to exception:\n");
+ PrintWriter writer = new PrintWriter(message, true);
+ e.printStackTrace(writer);
+ writer.flush();
+ LOG.severe(message.toString());
+ }
+ shutdown = true;
+ runtime.shutdown();
+ dispatcher.shutdown();
+ return e.getExitStatus();
+ }
+ }
+
+ @Override
+ public boolean shutdown() {
+ return shutdown;
+ }
+ };
+
+ RPCServer server = RPCServer.newServerWith(runtime.getClock(), blazeCommand,
+ runtime.getServerDirectory(), runtime.getWorkspace(), startupOptions.maxIdleSeconds);
+ return server;
+ }
+
+ private static Function<String, String> sourceFunctionForMap(final Map<String, String> map) {
+ return new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ if (!map.containsKey(input)) {
+ return "default";
+ }
+
+ if (map.get(input).isEmpty()) {
+ return "command line";
+ }
+
+ return map.get(input);
+ }
+ };
+ }
+
+ /**
+ * Parses the command line arguments into a {@link OptionsParser} object.
+ *
+ * <p>This function needs to parse the --option_sources option manually so that the real option
+ * parser can set the source for every option correctly. If that cannot be parsed or is missing,
+ * we just report an unknown source for every startup option.
+ */
+ private static OptionsProvider parseOptions(
+ Iterable<BlazeModule> modules, List<String> args) throws OptionsParsingException {
+ Set<Class<? extends OptionsBase>> optionClasses = Sets.newHashSet();
+ optionClasses.addAll(BlazeCommandUtils.getStartupOptions(modules));
+ // First parse the command line so that we get the option_sources argument
+ OptionsParser parser = OptionsParser.newOptionsParser(optionClasses);
+ parser.setAllowResidue(false);
+ parser.parse(OptionPriority.COMMAND_LINE, null, args);
+ Function<? super String, String> sourceFunction =
+ sourceFunctionForMap(parser.getOptions(BlazeServerStartupOptions.class).optionSources);
+
+ // Then parse the command line again, this time with the correct option sources
+ parser = OptionsParser.newOptionsParser(optionClasses);
+ parser.setAllowResidue(false);
+ parser.parseWithSourceFunction(OptionPriority.COMMAND_LINE, sourceFunction, args);
+ return parser;
+ }
+
+ /**
+ * Creates a new blaze runtime, given the install and output base directories.
+ *
+ * <p>Note: This method can and should only be called once per startup, as it also creates the
+ * filesystem object that will be used for the runtime. So it should only ever be called from the
+ * main method of the Blaze program.
+ *
+ * @param options Blaze startup options.
+ *
+ * @return a new BlazeRuntime instance initialized with the given filesystem and directories, and
+ * an error string that, if not null, describes a fatal initialization failure that makes
+ * this runtime unsuitable for real commands
+ */
+ private static BlazeRuntime newRuntime(
+ Iterable<BlazeModule> blazeModules, OptionsProvider options) throws AbruptExitException {
+ for (BlazeModule module : blazeModules) {
+ module.globalInit(options);
+ }
+
+ BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class);
+ PathFragment workspaceDirectory = startupOptions.workspaceDirectory;
+ PathFragment installBase = startupOptions.installBase;
+ PathFragment outputBase = startupOptions.outputBase;
+
+ OsUtils.maybeForceJNI(installBase); // Must be before first use of JNI.
+
+ // From the point of view of the Java program --install_base and --output_base
+ // are mandatory options, despite the comment in their declarations.
+ if (installBase == null || !installBase.isAbsolute()) { // (includes "" default case)
+ throw new IllegalArgumentException(
+ "Bad --install_base option specified: '" + installBase + "'");
+ }
+ if (outputBase != null && !outputBase.isAbsolute()) { // (includes "" default case)
+ throw new IllegalArgumentException(
+ "Bad --output_base option specified: '" + outputBase + "'");
+ }
+
+ PathFragment outputPathFragment = BlazeDirectories.outputPathFromOutputBase(
+ outputBase, workspaceDirectory);
+ FileSystem fs = null;
+ for (BlazeModule module : blazeModules) {
+ FileSystem moduleFs = module.getFileSystem(options, outputPathFragment);
+ if (moduleFs != null) {
+ Preconditions.checkState(fs == null, "more than one module returns a file system");
+ fs = moduleFs;
+ }
+ }
+
+ if (fs == null) {
+ fs = fileSystemImplementation();
+ }
+ Path.setFileSystemForSerialization(fs);
+
+ Path installBasePath = fs.getPath(installBase);
+ Path outputBasePath = fs.getPath(outputBase);
+ Path workspaceDirectoryPath = null;
+ if (!workspaceDirectory.equals(PathFragment.EMPTY_FRAGMENT)) {
+ workspaceDirectoryPath = fs.getPath(workspaceDirectory);
+ }
+
+ BlazeDirectories directories =
+ new BlazeDirectories(installBasePath, outputBasePath, workspaceDirectoryPath);
+
+ Clock clock = BlazeClock.instance();
+
+ BinTools binTools;
+ try {
+ binTools = BinTools.forProduction(directories);
+ } catch (IOException e) {
+ throw new AbruptExitException(
+ "Cannot enumerate embedded binaries: " + e.getMessage(),
+ ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
+ }
+
+ BlazeRuntime.Builder runtimeBuilder = new BlazeRuntime.Builder().setDirectories(directories)
+ .setStartupOptionsProvider(options)
+ .setBinTools(binTools)
+ .setClock(clock)
+ // TODO(bazel-team): Make BugReportingExceptionHandler the default.
+ // See bug "Make exceptions in EventBus subscribers fatal"
+ .setEventBusExceptionHandler(
+ startupOptions.fatalEventBusExceptions || !BlazeVersionInfo.instance().isReleasedBlaze()
+ ? new BlazeRuntime.BugReportingExceptionHandler()
+ : new BlazeRuntime.RemoteExceptionHandler());
+
+ runtimeBuilder.setRunfilesPrefix(new PathFragment(Constants.RUNFILES_PREFIX));
+ for (BlazeModule blazeModule : blazeModules) {
+ runtimeBuilder.addBlazeModule(blazeModule);
+ }
+
+ BlazeRuntime runtime = runtimeBuilder.build();
+ BugReport.setRuntime(runtime);
+ return runtime;
+ }
+
+ /**
+ * Returns null if JVM memory settings are considered safe, and an error string otherwise.
+ */
+ private static String validateJvmMemorySettings() {
+ boolean is64BitVM = "64".equals(System.getProperty("sun.arch.data.model"));
+ if (is64BitVM) {
+ return null;
+ }
+ MemoryMXBean mem = ManagementFactory.getMemoryMXBean();
+ long heapSize = mem.getHeapMemoryUsage().getMax();
+ long nonHeapSize = mem.getNonHeapMemoryUsage().getMax();
+ if (heapSize == -1 || nonHeapSize == -1) {
+ return null;
+ }
+
+ if (heapSize + nonHeapSize > MAX_BLAZE32_RESERVED_MEMORY) {
+ return String.format(
+ "WARNING: JVM reserved %d MB of virtual memory (above threshold of %d MB). "
+ + "This may result in OOMs at runtime. Use lower values of MaxPermSize "
+ + "or switch to blaze64.",
+ (heapSize + nonHeapSize) >> 20, MAX_BLAZE32_RESERVED_MEMORY >> 20);
+ } else if (heapSize < MIN_BLAZE32_HEAP_SIZE) {
+ return String.format(
+ "WARNING: JVM heap size is %d MB. You probably have a custom -Xmx setting in your "
+ + "local Blaze configuration. This may result in OOMs. Removing overrides of -Xmx "
+ + "settings is advised.",
+ heapSize >> 20);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Make sure async threads cannot be orphaned. This method makes sure bugs are reported to
+ * telemetry and the proper exit code is reported.
+ */
+ private static void setupUncaughtHandler(final String[] args) {
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread thread, Throwable throwable) {
+ BugReport.handleCrash(throwable, args);
+ }
+ });
+ }
+
+
+ /**
+ * Returns an immutable list containing new instances of each Blaze command.
+ */
+ @VisibleForTesting
+ public static List<BlazeCommand> getBuiltinCommandList() {
+ return ImmutableList.of(
+ new BuildCommand(),
+ new CanonicalizeCommand(),
+ new CleanCommand(),
+ new HelpCommand(),
+ new SkylarkCommand(),
+ new InfoCommand(),
+ new ProfileCommand(),
+ new QueryCommand(),
+ new RunCommand(),
+ new ShutdownCommand(),
+ new TestCommand(),
+ new VersionCommand());
+ }
+
+ /**
+ * A builder for {@link BlazeRuntime} objects. The only required fields are the {@link
+ * BlazeDirectories}, and the {@link RuleClassProvider} (except for testing). All other fields
+ * have safe default values.
+ *
+ * <p>If a {@link ConfigurationFactory} is set, then the builder ignores the host system flag.
+ * <p>The default behavior of the BlazeRuntime's EventBus is to exit when a subscriber throws
+ * an exception. Please plan appropriately.
+ */
+ public static class Builder {
+
+ private PathFragment runfilesPrefix = PathFragment.EMPTY_FRAGMENT;
+ private BlazeDirectories directories;
+ private Reporter reporter;
+ private ConfigurationFactory configurationFactory;
+ private Clock clock;
+ private OptionsProvider startupOptionsProvider;
+ private final List<BlazeModule> blazeModules = Lists.newArrayList();
+ private SubscriberExceptionHandler eventBusExceptionHandler =
+ new RemoteExceptionHandler();
+ private BinTools binTools;
+ private UUID instanceId;
+
+ public BlazeRuntime build() throws AbruptExitException {
+ Preconditions.checkNotNull(directories);
+ Preconditions.checkNotNull(startupOptionsProvider);
+ Reporter reporter = (this.reporter == null) ? new Reporter() : this.reporter;
+
+ Clock clock = (this.clock == null) ? BlazeClock.instance() : this.clock;
+ UUID instanceId = (this.instanceId == null) ? UUID.randomUUID() : this.instanceId;
+
+ Preconditions.checkNotNull(clock);
+ Map<String, String> clientEnv = new HashMap<>();
+ TimestampGranularityMonitor timestampMonitor = new TimestampGranularityMonitor(clock);
+
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier = null;
+ SkyframeExecutorFactory skyframeExecutorFactory = null;
+ for (BlazeModule module : blazeModules) {
+ module.blazeStartup(startupOptionsProvider,
+ BlazeVersionInfo.instance(), instanceId, directories, clock);
+ Preprocessor.Factory.Supplier modulePreprocessorFactorySupplier =
+ module.getPreprocessorFactorySupplier();
+ if (modulePreprocessorFactorySupplier != null) {
+ Preconditions.checkState(preprocessorFactorySupplier == null,
+ "more than one module defines a preprocessor factory supplier");
+ preprocessorFactorySupplier = modulePreprocessorFactorySupplier;
+ }
+ SkyframeExecutorFactory skyFactory = module.getSkyframeExecutorFactory();
+ if (skyFactory != null) {
+ Preconditions.checkState(skyframeExecutorFactory == null,
+ "At most one skyframe factory supported. But found two: %s and %s", skyFactory,
+ skyframeExecutorFactory);
+ skyframeExecutorFactory = skyFactory;
+ }
+ }
+ if (skyframeExecutorFactory == null) {
+ skyframeExecutorFactory = new SequencedSkyframeExecutorFactory();
+ }
+ if (preprocessorFactorySupplier == null) {
+ preprocessorFactorySupplier = Preprocessor.Factory.Supplier.NullSupplier.INSTANCE;
+ }
+
+ ConfiguredRuleClassProvider.Builder ruleClassBuilder =
+ new ConfiguredRuleClassProvider.Builder();
+ for (BlazeModule module : blazeModules) {
+ module.initializeRuleClasses(ruleClassBuilder);
+ }
+
+ Map<String, String> platformRegexps = null;
+ {
+ ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
+ for (BlazeModule module : blazeModules) {
+ builder.putAll(module.getPlatformSetRegexps());
+ }
+ platformRegexps = builder.build();
+ if (platformRegexps.isEmpty()) {
+ platformRegexps = null; // Use the default.
+ }
+ }
+
+ Set<Path> immutableDirectories = null;
+ {
+ ImmutableSet.Builder<Path> builder = new ImmutableSet.Builder<>();
+ for (BlazeModule module : blazeModules) {
+ builder.addAll(module.getImmutableDirectories());
+ }
+ immutableDirectories = builder.build();
+ }
+
+ Iterable<DiffAwareness.Factory> diffAwarenessFactories = null;
+ {
+ ImmutableList.Builder<DiffAwareness.Factory> builder = new ImmutableList.Builder<>();
+ boolean watchFS = startupOptionsProvider != null
+ && startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).watchFS;
+ for (BlazeModule module : blazeModules) {
+ builder.addAll(module.getDiffAwarenessFactories(watchFS));
+ }
+ diffAwarenessFactories = builder.build();
+ }
+
+ // Merge filters from Blaze modules that allow some action inputs to be missing.
+ Predicate<PathFragment> allowedMissingInputs = null;
+ for (BlazeModule module : blazeModules) {
+ Predicate<PathFragment> modulePredicate = module.getAllowedMissingInputs();
+ if (modulePredicate != null) {
+ Preconditions.checkArgument(allowedMissingInputs == null,
+ "More than one Blaze module allows missing inputs.");
+ allowedMissingInputs = modulePredicate;
+ }
+ }
+ if (allowedMissingInputs == null) {
+ allowedMissingInputs = Predicates.alwaysFalse();
+ }
+
+ ConfiguredRuleClassProvider ruleClassProvider = ruleClassBuilder.build();
+ WorkspaceStatusAction.Factory workspaceStatusActionFactory = null;
+ for (BlazeModule module : blazeModules) {
+ WorkspaceStatusAction.Factory candidate = module.getWorkspaceStatusActionFactory();
+ if (candidate != null) {
+ Preconditions.checkState(workspaceStatusActionFactory == null,
+ "more than one module defines a workspace status action factory");
+ workspaceStatusActionFactory = candidate;
+ }
+ }
+
+ List<PackageFactory.EnvironmentExtension> extensions = new ArrayList<>();
+ for (BlazeModule module : blazeModules) {
+ extensions.add(module.getPackageEnvironmentExtension());
+ }
+
+ // We use an immutable map builder for the nice side effect that it throws if a duplicate key
+ // is inserted.
+ ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
+ for (BlazeModule module : blazeModules) {
+ skyFunctions.putAll(module.getSkyFunctions(directories));
+ }
+
+ ImmutableList.Builder<PrecomputedValue.Injected> precomputedValues = ImmutableList.builder();
+ for (BlazeModule module : blazeModules) {
+ precomputedValues.addAll(module.getPrecomputedSkyframeValues());
+ }
+
+ final PackageFactory pkgFactory =
+ new PackageFactory(ruleClassProvider, platformRegexps, extensions);
+ SkyframeExecutor skyframeExecutor = skyframeExecutorFactory.create(reporter, pkgFactory,
+ timestampMonitor, directories, workspaceStatusActionFactory,
+ ruleClassProvider.getBuildInfoFactories(), immutableDirectories, diffAwarenessFactories,
+ allowedMissingInputs, preprocessorFactorySupplier, skyFunctions.build(),
+ precomputedValues.build());
+
+ if (configurationFactory == null) {
+ configurationFactory = new ConfigurationFactory(
+ ruleClassProvider.getConfigurationCollectionFactory(),
+ ruleClassProvider.getConfigurationFragments());
+ }
+
+ ProjectFile.Provider projectFileProvider = null;
+ for (BlazeModule module : blazeModules) {
+ ProjectFile.Provider candidate = module.createProjectFileProvider();
+ if (candidate != null) {
+ Preconditions.checkState(projectFileProvider == null,
+ "more than one module defines a project file provider");
+ projectFileProvider = candidate;
+ }
+ }
+
+ return new BlazeRuntime(directories, reporter, workspaceStatusActionFactory, skyframeExecutor,
+ pkgFactory, ruleClassProvider, configurationFactory,
+ runfilesPrefix == null ? PathFragment.EMPTY_FRAGMENT : runfilesPrefix,
+ clock, startupOptionsProvider, ImmutableList.copyOf(blazeModules),
+ clientEnv, timestampMonitor,
+ eventBusExceptionHandler, binTools, projectFileProvider);
+ }
+
+ public Builder setRunfilesPrefix(PathFragment prefix) {
+ this.runfilesPrefix = prefix;
+ return this;
+ }
+
+ public Builder setBinTools(BinTools binTools) {
+ this.binTools = binTools;
+ return this;
+ }
+
+ public Builder setDirectories(BlazeDirectories directories) {
+ this.directories = directories;
+ return this;
+ }
+
+ /**
+ * Creates and sets a new {@link BlazeDirectories} instance with the given
+ * parameters.
+ */
+ public Builder setDirectories(Path installBase, Path outputBase,
+ Path workspace) {
+ this.directories = new BlazeDirectories(installBase, outputBase, workspace);
+ return this;
+ }
+
+ public Builder setReporter(Reporter reporter) {
+ this.reporter = reporter;
+ return this;
+ }
+
+ public Builder setConfigurationFactory(ConfigurationFactory configurationFactory) {
+ this.configurationFactory = configurationFactory;
+ return this;
+ }
+
+ public Builder setClock(Clock clock) {
+ this.clock = clock;
+ return this;
+ }
+
+ public Builder setStartupOptionsProvider(OptionsProvider startupOptionsProvider) {
+ this.startupOptionsProvider = startupOptionsProvider;
+ return this;
+ }
+
+ public Builder addBlazeModule(BlazeModule blazeModule) {
+ blazeModules.add(blazeModule);
+ return this;
+ }
+
+ public Builder setInstanceId(UUID id) {
+ instanceId = id;
+ return this;
+ }
+
+ @VisibleForTesting
+ public Builder setEventBusExceptionHandler(
+ SubscriberExceptionHandler eventBusExceptionHandler) {
+ this.eventBusExceptionHandler = eventBusExceptionHandler;
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
new file mode 100644
index 0000000..1f9bcea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
@@ -0,0 +1,225 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.Map;
+
+/**
+ * Options that will be evaluated by the blaze client startup code and passed
+ * to the blaze server upon startup.
+ *
+ * <h4>IMPORTANT</h4> These options and their defaults must be kept in sync with those in the
+ * source of the launcher. The latter define the actual default values; this class exists only to
+ * provide the help message, which displays the default values.
+ *
+ * The same relationship holds between {@link HostJvmStartupOptions} and the launcher.
+ */
+public class BlazeServerStartupOptions extends OptionsBase {
+ /**
+ * Converter for the <code>option_sources</code> option. Takes a string in the form of
+ * "option_name1:source1:option_name2:source2:.." and converts it into an option name to
+ * source map.
+ */
+ public static class OptionSourcesConverter implements Converter<Map<String, String>> {
+ private String unescape(String input) {
+ return input.replace("_C", ":").replace("_U", "_");
+ }
+
+ @Override
+ public Map<String, String> convert(String input) {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ if (input.isEmpty()) {
+ return builder.build();
+ }
+
+ String[] elements = input.split(":");
+ for (int i = 0; i < (elements.length + 1) / 2; i++) {
+ String name = elements[i * 2];
+ String value = "";
+ if (elements.length > i * 2 + 1) {
+ value = elements[i * 2 + 1];
+ }
+ builder.put(unescape(name), unescape(value));
+ }
+ return builder.build();
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a list of option-source pairs";
+ }
+ }
+
+ /* Passed from the client to the server, specifies the installation
+ * location. The location should be of the form:
+ * $OUTPUT_BASE/_blaze_${USER}/install/${MD5_OF_INSTALL_MANIFEST}.
+ * The server code will only accept a non-empty path; it's the
+ * responsibility of the client to compute a proper default if
+ * necessary.
+ */
+ @Option(name = "install_base",
+ defaultValue = "", // NOTE: purely decorative! See class docstring.
+ category = "hidden",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "This launcher option is intended for use only by tests.")
+ public PathFragment installBase;
+
+ /* Note: The help string in this option applies to the client code; not
+ * the server code. The server code will only accept a non-empty path; it's
+ * the responsibility of the client to compute a proper default if
+ * necessary.
+ */
+ @Option(name = "output_base",
+ defaultValue = "null", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, specifies the output location to which all build output will be written. "
+ + "Otherwise, the location will be "
+ + "${OUTPUT_ROOT}/_blaze_${USER}/${MD5_OF_WORKSPACE_ROOT}. Note: If you specify a "
+ + "different option from one to the next Blaze invocation for this value, you'll likely "
+ + "start up a new, additional Blaze server. Blaze starts exactly one server per "
+ + "specified output base. Typically there is one output base per workspace--however, "
+ + "with this option you may have multiple output bases per workspace and thereby run "
+ + "multiple builds for the same client on the same machine concurrently. See "
+ + "'blaze help shutdown' on how to shutdown a Blaze server.")
+ public PathFragment outputBase;
+
+ /* Note: This option is only used by the C++ client, never by the Java server.
+ * It is included here to make sure that the option is documented in the help
+ * output, which is auto-generated by Java code.
+ */
+ @Option(name = "output_user_root",
+ defaultValue = "null", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "The user-specific directory beneath which all build outputs are written; "
+ + "by default, this is a function of $USER, but by specifying a constant, build outputs "
+ + "can be shared between collaborating users.")
+ public PathFragment outputUserRoot;
+
+ @Option(name = "workspace_directory",
+ defaultValue = "",
+ category = "hidden",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "The root of the workspace, that is, the directory that Blaze uses as the root of the "
+ + "build. This flag is only to be set by the blaze client.")
+ public PathFragment workspaceDirectory;
+
+ @Option(name = "max_idle_secs",
+ defaultValue = "" + (3 * 3600), // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ help = "The number of seconds the build server will wait idling " +
+ "before shutting down. Note: Blaze will ignore this option " +
+ "unless you are starting a new instance. See also 'blaze help " +
+ "shutdown'.")
+ public int maxIdleSeconds;
+
+ @Option(name = "batch",
+ defaultValue = "false", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ help = "If set, Blaze will be run in batch mode, instead of " +
+ "the standard client/server. Doing so may provide " +
+ "more predictable semantics with respect to signal handling and job control, " +
+ "Batch mode retains proper queueing semantics within the same output_base. " +
+ "That is, simultaneous invocations will be processed in order, without overlap. " +
+ "If a batch mode Blaze is run on a client with a running server, it first kills " +
+ "the server before processing the command." +
+ "Blaze will run slower in batch mode, compared to client/server mode. " +
+ "Among other things, the build file cache is memory-resident, so it is not " +
+ "preserved between sequential batch invocations. Therefore, using batch mode " +
+ "often makes more sense in cases where performance is less critical, " +
+ "such as continuous builds.")
+ public boolean batch;
+
+ @Option(name = "block_for_lock",
+ defaultValue = "true", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ help = "If set, Blaze will exit immediately instead of waiting for other " +
+ "Blaze commands holding the server lock to complete.")
+ public boolean noblock_for_lock;
+
+ @Option(name = "io_nice_level",
+ defaultValue = "-1", // NOTE: purely decorative!
+ category = "server startup",
+ help = "Set a level from 0-7 for best-effort IO scheduling. 0 is highest priority, " +
+ "7 is lowest. The anticipatory scheduler may only honor up to priority 4. " +
+ "Negative values are ignored.")
+ public int ioNiceLevel;
+
+ @Option(name = "batch_cpu_scheduling",
+ defaultValue = "false", // NOTE: purely decorative!
+ category = "server startup",
+ help = "Use 'batch' CPU scheduling for Blaze. This policy is useful for workloads that " +
+ "are non-interactive, but do not want to lower their nice value. " +
+ "See 'man 2 sched_setscheduler'.")
+ public boolean batchCpuScheduling;
+
+ @Option(name = "blazerc",
+ // NOTE: purely decorative!
+ defaultValue = "In the current directory, then in the user's home directory, the file named "
+ + ".$(basename $0)rc (i.e. .bazelrc for Bazel or .blazerc for Blaze)",
+ category = "misc",
+ help = "The location of the .bazelrc/.blazerc file containing default values of "
+ + "Blaze command options. Use /dev/null to disable the search for a "
+ + "blazerc file, e.g. in release builds.")
+ public String blazerc;
+
+ @Option(name = "master_blazerc",
+ defaultValue = "true", // NOTE: purely decorative!
+ category = "misc",
+ help = "If this option is false, the master blazerc/bazelrc next to the binary "
+ + "is not read.")
+ public boolean masterBlazerc;
+
+ @Option(name = "skyframe",
+ defaultValue = "full",
+ category = "undocumented",
+ help = "Unused.")
+ public String unusedSkyframe;
+
+ @Option(name = "fatal_event_bus_exceptions",
+ defaultValue = "false", // NOTE: purely decorative!
+ category = "undocumented",
+ help = "Whether or not to allow EventBus exceptions to be fatal. Experimental.")
+ public boolean fatalEventBusExceptions;
+
+ @Option(name = "option_sources",
+ converter = OptionSourcesConverter.class,
+ defaultValue = "",
+ category = "hidden",
+ help = "")
+ public Map<String, String> optionSources;
+
+ // TODO(bazel-team): In order to make it easier to have local watchers in open source Bazel,
+ // turn this into a non-startup option.
+ @Option(name = "watchfs",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "If true, Blaze tries to use the operating system's file watch service for local "
+ + "changes instead of scanning every file for a change.")
+ public boolean watchFS;
+
+ @Option(name = "use_webstatusserver",
+ defaultValue = "0",
+ category = "server startup",
+ help = "Specifies port to run web status server on (0 to disable, which is default).")
+ public int useWebStatusServer;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
new file mode 100644
index 0000000..ee1e429
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Utility methods for sending bug reports.
+ *
+ * <p> Note, code in this class must be extremely robust. There's nothing
+ * worse than a crash-handler that itself crashes!
+ */
+public abstract class BugReport {
+
+ private BugReport() {}
+
+ private static Logger LOG = Logger.getLogger(BugReport.class.getName());
+
+ private static BlazeVersionInfo versionInfo = BlazeVersionInfo.instance();
+
+ private static BlazeRuntime runtime = null;
+
+ public static void setRuntime(BlazeRuntime newRuntime) {
+ Preconditions.checkNotNull(newRuntime);
+ Preconditions.checkState(runtime == null, "runtime already set: %s, %s", runtime, newRuntime);
+ runtime = newRuntime;
+ }
+
+ /**
+ * Logs the unhandled exception with a special prefix signifying that this was a crash.
+ *
+ * @param exception the unhandled exception to display.
+ * @param args additional values to record in the message.
+ * @param values Additional string values to clarify the exception.
+ */
+ public static void sendBugReport(Throwable exception, List<String> args, String... values) {
+ if (!versionInfo.isReleasedBlaze()) {
+ LOG.info("(Not a released binary; not logged.)");
+ return;
+ }
+
+ logException(exception, filterClientEnv(args), values);
+ }
+
+ /**
+ * Print and send a bug report, and exit with the proper Blaze code.
+ */
+ public static void handleCrash(Throwable throwable, String... args) {
+ BugReport.sendBugReport(throwable, Arrays.asList(args));
+ BugReport.printBug(OutErr.SYSTEM_OUT_ERR, throwable);
+ System.err.println("Blaze crash in async thread:");
+ throwable.printStackTrace();
+ int exitCode =
+ (throwable instanceof OutOfMemoryError) ? ExitCode.OOM_ERROR.getNumericExitCode()
+ : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode();
+ if (runtime != null) {
+ runtime.notifyCommandComplete(exitCode);
+ // We don't call runtime#shutDown() here because all it does is shut down the modules, and who
+ // knows if they can be trusted.
+ }
+ System.exit(exitCode);
+ }
+
+ private static void printThrowableTo(OutErr outErr, Throwable e) {
+ PrintStream err = new PrintStream(outErr.getErrorStream());
+ e.printStackTrace(err);
+ err.flush();
+ LOG.log(Level.SEVERE, "Blaze crashed", e);
+ }
+
+ /**
+ * Print user-helpful information about the bug/crash to the output.
+ *
+ * @param outErr where to write the output
+ * @param e the exception thrown
+ */
+ public static void printBug(OutErr outErr, Throwable e) {
+ if (e instanceof OutOfMemoryError) {
+ outErr.printErr(e.getMessage() + "\n\n" +
+ "Blaze ran out of memory and crashed.\n");
+ } else {
+ printThrowableTo(outErr, e);
+ }
+ }
+
+ /**
+ * Filters {@code args} by removing any item that starts with "--client_env",
+ * then returns this as an immutable list.
+ *
+ * <p>The client's environment variables may contain sensitive data, so we filter it out.
+ */
+ private static List<String> filterClientEnv(Iterable<String> args) {
+ if (args == null) {
+ return null;
+ }
+
+ ImmutableList.Builder<String> filteredArgs = ImmutableList.builder();
+ for (String arg : args) {
+ if (arg != null && !arg.startsWith("--client_env")) {
+ filteredArgs.add(arg);
+ }
+ }
+ return filteredArgs.build();
+ }
+
+ // Log the exception. Because this method is only called in a blaze release,
+ // this will result in a report being sent to a remote logging service.
+ private static void logException(Throwable exception, List<String> args, String... values) {
+ // The preamble is used in the crash watcher, so don't change it
+ // unless you know what you're doing.
+ String preamble = exception instanceof OutOfMemoryError
+ ? "Blaze OOMError: "
+ : "Blaze crashed with args: ";
+
+ LoggingUtil.logToRemote(Level.SEVERE, preamble + Joiner.on(' ').join(args), exception,
+ values);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java
new file mode 100644
index 0000000..5175a15
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+/**
+ * Represents how far into the build a given target has gone.
+ * Used primarily for master log status reporting and representation.
+ */
+public enum BuildPhase {
+ PARSING("parsing-failed", false),
+ LOADING("loading-failed", false),
+ ANALYSIS("analysis-failed", false),
+ TEST_FILTERING("test-filtered", true),
+ TARGET_FILTERING("target-filtered", true),
+ NOT_BUILT("not-built", false),
+ NOT_ANALYZED("not-analyzed", false),
+ EXECUTION("build-failed", false),
+ BLAZE_HALTED("blaze-halted", false),
+ COMPLETE("built", true);
+
+ private final String msg;
+ private final boolean success;
+
+ BuildPhase(String msg, boolean success) {
+ this.msg = msg;
+ this.success = success;
+ }
+
+ public String getMessage() {
+ return msg;
+ }
+
+ public boolean getSuccess() {
+ return success;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java
new file mode 100644
index 0000000..8b072c7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.BlazeClock;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * Blaze module for the build summary message that reports various stats to the user.
+ */
+public class BuildSummaryStatsModule extends BlazeModule {
+
+ private static final Logger LOG = Logger.getLogger(BuildSummaryStatsModule.class.getName());
+
+ private SimpleCriticalPathComputer criticalPathComputer;
+ private EventBus eventBus;
+ private Reporter reporter;
+
+ @Override
+ public void beforeCommand(BlazeRuntime runtime, Command command) {
+ this.reporter = runtime.getReporter();
+ this.eventBus = runtime.getEventBus();
+ eventBus.register(this);
+ }
+
+ @Subscribe
+ public void executionPhaseStarting(ExecutionStartingEvent event) {
+ criticalPathComputer = new SimpleCriticalPathComputer(BlazeClock.instance());
+ eventBus.register(criticalPathComputer);
+ }
+
+ @Subscribe
+ public void buildComplete(BuildCompleteEvent event) {
+ try {
+ // We might want to make this conditional on a flag; it can sometimes be a bit of a nuisance.
+ List<String> items = new ArrayList<>();
+ items.add(String.format("Elapsed time: %.3fs", event.getResult().getElapsedSeconds()));
+
+ if (criticalPathComputer != null) {
+ Profiler.instance().startTask(ProfilerTask.CRITICAL_PATH, "Critical path");
+ AggregatedCriticalPath<SimpleCriticalPathComponent> criticalPath =
+ criticalPathComputer.aggregate();
+ items.add(criticalPath.toStringSummary());
+ LOG.info(criticalPath.toString());
+ LOG.info("Slowest actions:\n " + Joiner.on("\n ")
+ .join(criticalPathComputer.getSlowestComponents()));
+ // We reverse the critical path because the profiler expect events ordered by the time
+ // when the actions were executed while critical path computation is stored in the reverse
+ // way.
+ for (SimpleCriticalPathComponent stat : criticalPath.components().reverse()) {
+ Profiler.instance().logSimpleTaskDuration(
+ TimeUnit.MILLISECONDS.toNanos(stat.getStartTime()),
+ TimeUnit.MILLISECONDS.toNanos(stat.getActionWallTime()),
+ ProfilerTask.CRITICAL_PATH_COMPONENT, stat.getAction());
+ }
+ Profiler.instance().completeTask(ProfilerTask.CRITICAL_PATH);
+ }
+
+ reporter.handle(Event.info(Joiner.on(", ").join(items)));
+ } finally {
+ criticalPathComputer = null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/Command.java b/src/main/java/com/google/devtools/build/lib/runtime/Command.java
new file mode 100644
index 0000000..1797cd3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/Command.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation that lets blaze commands specify their options and their help.
+ * The annotations are processed by {@link BlazeCommand}.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Command {
+ /**
+ * The name of the command, as the user would type it.
+ */
+ String name();
+
+ /**
+ * Options processed by the command, indicated by options interfaces.
+ * These interfaces must contain methods annotated with {@link Option}.
+ */
+ Class<? extends OptionsBase>[] options() default {};
+
+ /**
+ * The set of other Blaze commands that this annotation's command "inherits"
+ * options from. These classes must be annotated with {@link Command}.
+ */
+ Class<? extends BlazeCommand>[] inherits() default {};
+
+ /**
+ * A short description, which appears in 'blaze help'.
+ */
+ String shortDescription();
+
+ /**
+ * True if the configuration-specific options should be available for this command.
+ */
+ boolean usesConfigurationOptions() default false;
+
+ /**
+ * True if the command runs a build.
+ */
+ boolean builds() default false;
+
+ /**
+ * True if the command should not be shown in the output of 'blaze help'.
+ */
+ boolean hidden() default false;
+
+ /**
+ * Specifies whether this command allows a residue after the parsed options.
+ * For example, a command might expect a list of targets to build in the
+ * residue.
+ */
+ boolean allowResidue() default false;
+
+ /**
+ * Returns true if this command wants to write binary data to stdout.
+ * Enabling this flag will disable ANSI escape stripping for this command.
+ */
+ boolean binaryStdOut() default false;
+
+ /**
+ * Returns true if this command wants to write binary data to stderr.
+ * Enabling this flag will disable ANSI escape stripping for this command.
+ */
+ boolean binaryStdErr() default false;
+
+ /**
+ * The help message for this command. If the value starts with "resource:",
+ * the remainder is interpreted as the name of a text file resource (in the
+ * .jar file that provides the Command implementation class).
+ */
+ String help();
+
+ /**
+ * Returns true iff this command may only be run from within a Blaze workspace. Broadly, this
+ * should be true for any command that interprets the package-path, since it's potentially
+ * confusing otherwise.
+ */
+ boolean mustRunInWorkspace() default true;
+
+ /**
+ * Returns true iff this command is allowed to run in the output directory,
+ * i.e. $OUTPUT_BASE/_blaze_$USER/$MD5/... . No command should be allowed to run here,
+ * but there are some legacy uses of 'blaze query'.
+ */
+ boolean canRunInOutputDirectory() default false;
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java
new file mode 100644
index 0000000..fb92781
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+/**
+ * This event is fired when the Blaze command is complete
+ * (clean, build, test, etc.).
+ */
+public class CommandCompleteEvent extends CommandEvent {
+
+ private final int exitCode;
+
+ /**
+ * @param exitCode the exit code of the blaze command
+ */
+ public CommandCompleteEvent(int exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * @return the exit code of the blaze command
+ */
+ public int getExitCode() {
+ return exitCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java
new file mode 100644
index 0000000..3e59dce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.BlazeClock;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.Date;
+
+/**
+ * Base class for Command events that includes some resource fields.
+ */
+public abstract class CommandEvent {
+
+ private final long eventTimeInNanos;
+ private final long eventTimeInEpochTime;
+ private final long gcTimeInMillis;
+
+ protected CommandEvent() {
+ eventTimeInNanos = BlazeClock.nanoTime();
+ eventTimeInEpochTime = new Date().getTime();
+ gcTimeInMillis = collectGcTimeInMillis();
+ }
+
+ /**
+ * Returns time spent in garbage collection since the start of the JVM process.
+ */
+ private static long collectGcTimeInMillis() {
+ long gcTime = 0;
+ for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
+ gcTime += gcBean.getCollectionTime();
+ }
+ return gcTime;
+ }
+
+ /**
+ * Get the time-stamp in ns for the event.
+ */
+ public long getEventTimeInNanos() {
+ return eventTimeInNanos;
+ }
+
+ /**
+ * Get the time-stamp as epoch-time for the event.
+ */
+ public long getEventTimeInEpochTime() {
+ return eventTimeInEpochTime;
+ }
+
+ /**
+ * Get the cumulative GC time for the event.
+ */
+ public long getGCTimeInMillis() {
+ return gcTimeInMillis;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java
new file mode 100644
index 0000000..9a44086
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.ExitCode;
+
+/**
+ * This message is fired right before the Blaze command completes,
+ * and can be used to modify the command's exit code.
+ */
+public class CommandPrecompleteEvent {
+ private final ExitCode exitCode;
+
+ /**
+ * @param exitCode the exit code of the blaze command
+ */
+ public CommandPrecompleteEvent(ExitCode exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * @return the exit code of the blaze command
+ */
+ public ExitCode getExitCode() {
+ return exitCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java
new file mode 100644
index 0000000..32834a2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * This event is fired when the Blaze command is started (clean, build, test,
+ * etc.).
+ */
+public class CommandStartEvent extends CommandEvent {
+ private final String commandName;
+ private final UUID commandId;
+ private final Map<String, String> clientEnv;
+ private final Path workingDirectory;
+
+ /**
+ * @param commandName the name of the command
+ */
+ public CommandStartEvent(String commandName, UUID commandId, Map<String, String> clientEnv,
+ Path workingDirectory) {
+ this.commandName = commandName;
+ this.commandId = commandId;
+ this.clientEnv = clientEnv;
+ this.workingDirectory = workingDirectory;
+ }
+
+ public String getCommandName() {
+ return commandName;
+ }
+
+ public UUID getCommandId() {
+ return commandId;
+ }
+
+ public Map<String, String> getClientEnv() {
+ return clientEnv;
+ }
+
+ public Path getWorkingDirectory() {
+ return workingDirectory;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
new file mode 100644
index 0000000..7054975
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
@@ -0,0 +1,250 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+
+/**
+ * Options common to all commands.
+ */
+public class CommonCommandOptions extends OptionsBase {
+ /**
+ * A class representing a blazerc option. blazeRc is serial number of the rc
+ * file this option came from, option is the name of the option and value is
+ * its value (or null if not specified).
+ */
+ public static class OptionOverride {
+ final int blazeRc;
+ final String command;
+ final String option;
+
+ public OptionOverride(int blazeRc, String command, String option) {
+ this.blazeRc = blazeRc;
+ this.command = command;
+ this.option = option;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%d:%s=%s", blazeRc, command, option);
+ }
+ }
+
+ /**
+ * Converter for --default_override. The format is:
+ * --default_override=blazerc:command=option.
+ */
+ public static class OptionOverrideConverter implements Converter<OptionOverride> {
+ static final String ERROR_MESSAGE = "option overrides must be in form "
+ + " rcfile:command=option, where rcfile is a nonzero integer";
+
+ public OptionOverrideConverter() {}
+
+ @Override
+ public OptionOverride convert(String input) throws OptionsParsingException {
+ int colonPos = input.indexOf(':');
+ int assignmentPos = input.indexOf('=');
+
+ if (colonPos < 0) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ if (assignmentPos <= colonPos + 1) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ int blazeRc;
+ try {
+ blazeRc = Integer.valueOf(input.substring(0, colonPos));
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ if (blazeRc < 0) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ String command = input.substring(colonPos + 1, assignmentPos);
+ String option = input.substring(assignmentPos + 1);
+
+ return new OptionOverride(blazeRc, command, option);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "blazerc option override";
+ }
+ }
+
+
+ @Option(name = "config",
+ defaultValue = "",
+ category = "misc",
+ allowMultiple = true,
+ help = "Selects additional config sections from the rc files; for every <command>, it "
+ + "also pulls in the options from <command>:<config> if such a section exists. "
+ + "Note that it is currently only possible to provide these options on the "
+ + "command line, not in the rc files. The config sections and flag combinations "
+ + "they are equivalent to are located in the tools/*.blazerc config files.")
+ public List<String> configs;
+
+ @Option(name = "logging",
+ defaultValue = "3", // Level.INFO
+ category = "verbosity",
+ converter = Converters.LogLevelConverter.class,
+ help = "The logging level.")
+ public Level verbosity;
+
+ @Option(name = "client_env",
+ defaultValue = "",
+ category = "hidden",
+ converter = Converters.AssignmentConverter.class,
+ allowMultiple = true,
+ help = "A system-generated parameter which specifies the client's environment")
+ public List<Map.Entry<String, String>> clientEnv;
+
+ @Option(name = "ignore_client_env",
+ defaultValue = "false",
+ category = "hidden",
+ help = "If true, ignore the '--client_env' flag, and use the JVM environment instead")
+ public boolean ignoreClientEnv;
+
+ @Option(name = "client_cwd",
+ defaultValue = "",
+ category = "hidden",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "A system-generated parameter which specifies the client's working directory")
+ public PathFragment clientCwd;
+
+ @Option(name = "announce_rc",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Whether to announce rc options.")
+ public boolean announceRcOptions;
+
+ /**
+ * These are the actual default overrides.
+ * Each value is a pair of (command name, value).
+ *
+ * For example: "--default_override=build=--cpu=piii"
+ */
+ @Option(name = "default_override",
+ defaultValue = "",
+ allowMultiple = true,
+ category = "hidden",
+ converter = OptionOverrideConverter.class,
+ help = "")
+ public List<OptionOverride> optionsOverrides;
+
+ /**
+ * This is the filename that the Blaze client parsed.
+ */
+ @Option(name = "rc_source",
+ defaultValue = "",
+ allowMultiple = true,
+ category = "hidden",
+ help = "")
+ public List<String> rcSource;
+
+ @Option(name = "always_profile_slow_operations",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Whether profiling slow operations is always turned on")
+ public boolean alwaysProfileSlowOperations;
+
+ @Option(name = "profile",
+ defaultValue = "null",
+ category = "misc",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, profile Blaze and write data to the specified "
+ + "file. Use blaze analyze-profile to analyze the profile.")
+ public PathFragment profilePath;
+
+ @Option(name = "record_full_profiler_data",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "By default, Blaze profiler will record only aggregated data for fast but numerous "
+ + "events (such as statting the file). If this option is enabled, profiler will record "
+ + "each event - resulting in more precise profiling data but LARGE performance "
+ + "hit. Option only has effect if --profile used as well.")
+ public boolean recordFullProfilerData;
+
+ @Option(name = "memory_profile",
+ defaultValue = "null",
+ category = "undocumented",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, write memory usage data to the specified "
+ + "file at phase ends.")
+ public PathFragment memoryProfilePath;
+
+ @Option(name = "gc_watchdog",
+ defaultValue = "false",
+ category = "undocumented",
+ deprecationWarning = "Ignoring: this option is no longer supported",
+ help = "Deprecated.")
+ public boolean gcWatchdog;
+
+ @Option(name = "startup_time",
+ defaultValue = "0",
+ category = "hidden",
+ help = "The time in ms the launcher spends before sending the request to the blaze server.")
+ public long startupTime;
+
+ @Option(name = "extract_data_time",
+ defaultValue = "0",
+ category = "hidden",
+ help = "The time spend on extracting the new blaze version.")
+ public long extractDataTime;
+
+ @Option(name = "command_wait_time",
+ defaultValue = "0",
+ category = "hidden",
+ help = "The time in ms a command had to wait on a busy Blaze server process.")
+ public long waitTime;
+
+ @Option(name = "tool_tag",
+ defaultValue = "",
+ allowMultiple = true,
+ category = "misc",
+ help = "A tool name to attribute this Blaze invocation to.")
+ public List<String> toolTag;
+
+ @Option(name = "restart_reason",
+ defaultValue = "no_restart",
+ category = "hidden",
+ help = "The reason for the server restart.")
+ public String restartReason;
+
+ @Option(name = "binary_path",
+ defaultValue = "",
+ category = "hidden",
+ help = "The absolute path of the blaze binary.")
+ public String binaryPath;
+
+ @Option(name = "experimental_allow_project_files",
+ defaultValue = "false",
+ category = "hidden",
+ help = "Enable processing of +<file> parameters.")
+ public boolean allowProjectFiles;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java
new file mode 100644
index 0000000..2546492
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java
@@ -0,0 +1,231 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCompletionEvent;
+import com.google.devtools.build.lib.actions.ActionMetadata;
+import com.google.devtools.build.lib.actions.ActionMiddlemanEvent;
+import com.google.devtools.build.lib.actions.ActionStartedEvent;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.CachedActionEvent;
+import com.google.devtools.build.lib.util.Clock;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * Computes the critical path in the action graph based on events published to the event bus.
+ *
+ * <p>After instantiation, this object needs to be registered on the event bus to work.
+ */
+@ThreadSafe
+public abstract class CriticalPathComputer<C extends AbstractCriticalPathComponent<C>,
+ A extends AggregatedCriticalPath<C>> {
+
+ /** Number of top actions to record. */
+ static final int SLOWEST_COMPONENTS_SIZE = 30;
+ // outputArtifactToComponent is accessed from multiple event handlers.
+ protected final ConcurrentMap<Artifact, C> outputArtifactToComponent = Maps.newConcurrentMap();
+
+ /** Maximum critical path found. */
+ private C maxCriticalPath;
+ private final Clock clock;
+
+ /**
+ * The list of slowest individual components, ignoring the time to build dependencies.
+ *
+ * <p>This data is a useful metric when running non highly incremental builds, where multiple
+ * tasks could run un parallel and critical path would only record the longest path.
+ */
+ private final PriorityQueue<C> slowestComponents = new PriorityQueue<>(SLOWEST_COMPONENTS_SIZE,
+ new Comparator<C>() {
+ @Override
+ public int compare(C o1, C o2) {
+ return Long.compare(o1.getActionWallTime(), o2.getActionWallTime());
+ }
+ }
+ );
+
+ private final Object lock = new Object();
+
+ protected CriticalPathComputer(Clock clock) {
+ this.clock = clock;
+ maxCriticalPath = null;
+ }
+
+ /**
+ * Creates a critical path component for an action.
+ * @param action the action for the critical path component
+ * @param startTimeMillis time when the action started to run
+ */
+ protected abstract C createComponent(Action action, long startTimeMillis);
+
+ /**
+ * Return the critical path stats for the current command execution.
+ *
+ * <p>This method allows us to calculate lazily the aggregate statistics of the critical path,
+ * avoiding the memory and cpu penalty for doing it for all the actions executed.
+ */
+ public abstract A aggregate();
+
+ /**
+ * Record an action that has started to run.
+ *
+ * @param event information about the started action
+ */
+ @Subscribe
+ public void actionStarted(ActionStartedEvent event) {
+ Action action = event.getAction();
+ C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart()));
+ for (Artifact output : action.getOutputs()) {
+ C old = outputArtifactToComponent.put(output, component);
+ Preconditions.checkState(old == null, "Duplicate output artifact found. This could happen"
+ + " if a previous event registered the action %s. Artifact: %s", action, output);
+ }
+ }
+
+ /**
+ * Record a middleman action execution. Even if middleman are almost instant, we record them
+ * because they depend on other actions and we need them for constructing the critical path.
+ *
+ * <p>For some rules with incorrect configuration transitions we might get notified several times
+ * for the same middleman. This should only happen if the actions are shared.
+ */
+ @Subscribe
+ public void middlemanAction(ActionMiddlemanEvent event) {
+ Action action = event.getAction();
+ C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart()));
+ boolean duplicate = false;
+ for (Artifact output : action.getOutputs()) {
+ C old = outputArtifactToComponent.putIfAbsent(output, component);
+ if (old != null) {
+ if (!Actions.canBeShared(action, old.getAction())) {
+ throw new IllegalStateException("Duplicate output artifact found for middleman."
+ + "This could happen if a previous event registered the action.\n"
+ + "Old action: " + old.getAction() + "\n\n"
+ + "New action: " + action + "\n\n"
+ + "Artifact: " + output + "\n");
+ }
+ duplicate = true;
+ }
+ }
+ if (!duplicate) {
+ finalizeActionStat(action, component);
+ }
+ }
+
+ /**
+ * Record an action that was not executed because it was in the (disk) cache. This is needed so
+ * that we can calculate correctly the dependencies tree if we have some cached actions in the
+ * middle of the critical path.
+ */
+ @Subscribe
+ public void actionCached(CachedActionEvent event) {
+ Action action = event.getAction();
+ C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart()));
+ for (Artifact output : action.getOutputs()) {
+ outputArtifactToComponent.put(output, component);
+ }
+ finalizeActionStat(action, component);
+ }
+
+ /**
+ * Records the elapsed time stats for the action. For each input artifact, it finds the real
+ * dependent artifacts and records the critical path stats.
+ */
+ @Subscribe
+ public void actionComplete(ActionCompletionEvent event) {
+ ActionMetadata action = event.getActionMetadata();
+ C component = Preconditions.checkNotNull(
+ outputArtifactToComponent.get(action.getPrimaryOutput()));
+ finalizeActionStat(action, component);
+ }
+
+ /** Maximum critical path component found during the build. */
+ protected C getMaxCriticalPath() {
+ synchronized (lock) {
+ return maxCriticalPath;
+ }
+ }
+
+ /**
+ * The list of slowest individual components, ignoring the time to build dependencies.
+ */
+ public ImmutableList<C> getSlowestComponents() {
+ ArrayList<C> list;
+ synchronized (lock) {
+ list = new ArrayList<>(slowestComponents);
+ Collections.sort(list, slowestComponents.comparator());
+ }
+ return ImmutableList.copyOf(list).reverse();
+ }
+
+ private void finalizeActionStat(ActionMetadata action, C component) {
+ component.setFinishTimeMillis(getTime());
+ for (Artifact input : action.getInputs()) {
+ addArtifactDependency(component, input);
+ }
+
+ synchronized (lock) {
+ if (isBiggestCriticalPath(component)) {
+ maxCriticalPath = component;
+ }
+
+ if (slowestComponents.size() == SLOWEST_COMPONENTS_SIZE) {
+ // The new component is faster than any of the slow components, avoid insertion.
+ if (slowestComponents.peek().getActionWallTime() >= component.getActionWallTime()) {
+ return;
+ }
+ // Remove the head element to make space (The fastest component in the queue).
+ slowestComponents.remove();
+ }
+ slowestComponents.add(component);
+ }
+ }
+
+ private long getTime() {
+ return TimeUnit.NANOSECONDS.toMillis(clock.nanoTime());
+ }
+
+ private boolean isBiggestCriticalPath(C newCriticalPath) {
+ synchronized (lock) {
+ return maxCriticalPath == null
+ || maxCriticalPath.getAggregatedWallTime() < newCriticalPath.getAggregatedWallTime();
+ }
+ }
+
+ /**
+ * If "input" is a generated artifact, link its critical path to the one we're building.
+ */
+ private void addArtifactDependency(C actionStats, Artifact input) {
+ C depComponent = outputArtifactToComponent.get(input);
+ if (depComponent != null) {
+ actionStats.addDepInfo(depComponent);
+ }
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java
new file mode 100644
index 0000000..f4ef8e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.events.ExceptionListener;
+import com.google.devtools.build.lib.util.LoggingUtil;
+
+import java.util.logging.Level;
+
+/**
+ * Reports precondition failures from within an event handler.
+ * Necessary because the EventBus silently ignores exceptions thrown from within a handler.
+ * This class logs the exceptions and creates some noise when a precondition check fails.
+ */
+public class EventHandlerPreconditions {
+
+ private final ExceptionListener listener;
+
+ /**
+ * Creates a new precondition helper which outputs errors to the given reporter.
+ */
+ public EventHandlerPreconditions(ExceptionListener listener) {
+ this.listener = listener;
+ }
+
+ /**
+ * Verifies that the given condition (a check on an argument) is true,
+ * throwing an IllegalArgumentException if not.
+ *
+ * @param condition a condition to check for truth.
+ * @throws IllegalArgumentException if the condition is false.
+ */
+ @SuppressWarnings("unused")
+ public void checkArgument(boolean condition) {
+ checkArgument(condition, null);
+ }
+
+ /**
+ * Verifies that the given condition (a check on an argument) is true,
+ * throwing an IllegalArgumentException with the given message if not.
+ *
+ * @param condition a condition to check for truth.
+ * @param message extra information to output if the condition is false.
+ * @throws IllegalArgumentException if the condition is false.
+ */
+ public void checkArgument(boolean condition, String message) {
+ try {
+ Preconditions.checkArgument(condition, message);
+ } catch (IllegalArgumentException iae) {
+ String error = "Event handler argument check failed";
+ LoggingUtil.logToRemote(Level.SEVERE, error, iae);
+ listener.error(null, error, iae);
+ throw iae; // Still terminate the handler.
+ }
+ }
+
+ /**
+ * Verifies that the given condition (a check against the program's current state) is true,
+ * throwing an IllegalStateException if not.
+ *
+ * @param condition a condition to check for truth.
+ * @throws IllegalStateException if the condition is false.
+ */
+ public void checkState(boolean condition) {
+ checkState(condition, null);
+ }
+
+ /**
+ * Verifies that the given condition (a check against the program's current state) is true,
+ * throwing an IllegalStateException with the given message if not.
+ *
+ * @param condition a condition to check for truth.
+ * @param message extra information to output if the condition is false.
+ * @throws IllegalStateException if the condition is false.
+ */
+ public void checkState(boolean condition, String message) {
+ try {
+ Preconditions.checkState(condition, message);
+ } catch (IllegalStateException ise) {
+ String error = "Event handler state check failed";
+ LoggingUtil.logToRemote(Level.SEVERE, error, ise);
+ listener.error(null, error, ise);
+ throw ise; // Still terminate the handler.
+ }
+ }
+
+ /**
+ * Fails with an IllegalStateException when invoked.
+ */
+ public void fail(String message) {
+ String error = "Event handler failed: " + message;
+ IllegalStateException ise = new IllegalStateException(message);
+ LoggingUtil.logToRemote(Level.SEVERE, error, ise);
+ listener.error(null, error, ise);
+ throw ise;
+ }
+
+ /**
+ * Verifies that the given argument is not null, throwing a NullPointerException if it is null.
+ * Returns the original argument or throws.
+ *
+ * @param object an object to test for null.
+ * @return the reference which was checked.
+ * @throws NullPointerException if the object is null.
+ */
+ public <T> T checkNotNull(T object) {
+ return checkNotNull(object, null);
+ }
+
+ /**
+ * Verifies that the given argument is not null, throwing a
+ * NullPointerException with the given message if it is null.
+ * Returns the original argument or throws.
+ *
+ * @param object an object to test for null.
+ * @param message extra information to output if the object is null.
+ * @return the reference which was checked.
+ * @throws NullPointerException if the object is null.
+ */
+ public <T> T checkNotNull(T object, String message) {
+ try {
+ return Preconditions.checkNotNull(object, message);
+ } catch (NullPointerException npe) {
+ String error = "Event handler not-null check failed";
+ LoggingUtil.logToRemote(Level.SEVERE, error, npe);
+ listener.error(null, error, npe);
+ throw npe;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java
new file mode 100644
index 0000000..e55ad2f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java
@@ -0,0 +1,355 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Splitter;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.AnsiTerminal;
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * An event handler for ANSI terminals which uses control characters to
+ * provide eye-candy, reduce scrolling, and generally improve usability
+ * for users running directly from the shell.
+ *
+ * <p/>
+ * This event handler differs from a normal terminal because it only adds
+ * control characters to stderr, not stdout. All blaze status feedback
+ * is sent to stderr, so adding control characters just to that stream gives
+ * the benefits described above without modifying the normal output stream.
+ * For commands like build that don't generate stdout output this doesn't
+ * matter, but for commands like query and ide_build_info, inserting these
+ * control characters in stdout invalidated their output.
+ *
+ * <p/>
+ * The underlying streams may be either line-bufferred or unbuffered.
+ * Normally each event will write out a sequence of output to a single
+ * stream, and will end with a newline, which ensures a flush.
+ * But care is required when outputting incomplete lines, or when mixing
+ * output between the two different streams (stdout and stderr):
+ * it may be necessary to explicitly flush the output in those cases.
+ * However, we also don't want to flush too often; that can lead to
+ * a choppy UI experience.
+ */
+public class FancyTerminalEventHandler extends BlazeCommandEventHandler {
+ private static Logger LOG = Logger.getLogger(FancyTerminalEventHandler.class.getName());
+ private static final Pattern progressPattern = Pattern.compile(
+ // Match strings that look like they start with progress info:
+ // [42%] Compiling base/base.cc
+ // [1,442 / 23,476] Compiling base/base.cc
+ "^\\[(?:(?:\\d\\d?\\d?%)|(?:[\\d+,]+ / [\\d,]+))\\] ");
+ private static final Splitter LINEBREAK_SPLITTER = Splitter.on('\n');
+
+ private final AnsiTerminal terminal;
+
+ private final boolean useColor;
+ private final boolean useCursorControls;
+ private final boolean progressInTermTitle;
+ public final int terminalWidth;
+
+ private boolean terminalClosed = false;
+ private boolean previousLineErasable = false;
+ private int numLinesPreviousErasable = 0;
+
+ public FancyTerminalEventHandler(OutErr outErr, BlazeCommandEventHandler.Options options) {
+ super(outErr, options);
+ this.terminal = new AnsiTerminal(outErr.getErrorStream());
+ this.terminalWidth = (options.terminalColumns > 0 ? options.terminalColumns : 80);
+ useColor = options.useColor();
+ useCursorControls = options.useCursorControl();
+ progressInTermTitle = options.progressInTermTitle;
+ }
+
+ @Override
+ public void handle(Event event) {
+ if (terminalClosed) {
+ return;
+ }
+ if (!eventMask.contains(event.getKind())) {
+ return;
+ }
+
+ try {
+ boolean previousLineErased = false;
+ if (previousLineErasable) {
+ previousLineErased = maybeOverwritePreviousMessage();
+ }
+ switch (event.getKind()) {
+ case PROGRESS:
+ case START:
+ {
+ String message = event.getMessage();
+ Pair<String,String> progressPair = matchProgress(message);
+ if (progressPair != null) {
+ progress(progressPair.getFirst(), progressPair.getSecond());
+ } else {
+ progress("INFO: ", message);
+ }
+ break;
+ }
+ case FINISH:
+ {
+ String message = event.getMessage();
+ Pair<String,String> progressPair = matchProgress(message);
+ if (progressPair != null) {
+ String percentage = progressPair.getFirst();
+ String rest = progressPair.getSecond();
+ progress(percentage, rest + " DONE");
+ } else {
+ progress("INFO: ", message + " DONE");
+ }
+ break;
+ }
+ case PASS:
+ progress("PASS: ", event.getMessage());
+ break;
+ case INFO:
+ info(event);
+ break;
+ case ERROR:
+ case FAIL:
+ case TIMEOUT:
+ // For errors, scroll the message, so it appears above the status
+ // line, and highlight the word "ERROR" or "FAIL" in boldface red.
+ errorOrFail(event);
+ break;
+ case WARNING:
+ // For warnings, highlight the word "Warning" in boldface magenta,
+ // and scroll it.
+ warning(event);
+ break;
+ case SUBCOMMAND:
+ subcmd(event);
+ break;
+ case STDOUT:
+ if (previousLineErased) {
+ terminal.flush();
+ }
+ previousLineErasable = false;
+ super.handle(event);
+ // We don't need to flush stdout here, because
+ // super.handle(event) will take care of that.
+ break;
+ case STDERR:
+ putOutput(event);
+ break;
+ default:
+ // Ignore all other event types.
+ break;
+ }
+ } catch (IOException e) {
+ // The terminal shouldn't have IO errors, unless the shell is killed, which
+ // should also kill the blaze client. So this isn't something that should
+ // occur here; it will show up in the client/server interface as a broken
+ // pipe.
+ LOG.warning("Terminal was closed during build: " + e);
+ terminalClosed = true;
+ }
+ }
+
+ /**
+ * Displays a progress message that may be erased by subsequent messages.
+ *
+ * @param prefix a short string such as "[99%] " or "INFO: ", which will be highlighted
+ * @param rest the remainder of the message; may be multiple lines
+ */
+ private void progress(String prefix, String rest) throws IOException {
+ previousLineErasable = true;
+
+ if (progressInTermTitle) {
+ int newlinePos = rest.indexOf('\n');
+ if (newlinePos == -1) {
+ terminal.setTitle(prefix + rest);
+ } else {
+ terminal.setTitle(prefix + rest.substring(0, newlinePos));
+ }
+ }
+
+ if (useColor) {
+ terminal.textGreen();
+ }
+ int prefixWidth = prefix.length();
+ terminal.writeString(prefix);
+ terminal.resetTerminal();
+ if (showTimestamp) {
+ String timestamp = timestamp();
+ prefixWidth += timestamp.length();
+ terminal.writeString(timestamp);
+ }
+ int numLines = 0;
+ Iterator<String> lines = LINEBREAK_SPLITTER.split(rest).iterator();
+ String firstLine = lines.next();
+ terminal.writeString(firstLine);
+ // Subtract one, because when the line length is the same as the terminal
+ // width, the terminal doesn't line-advance, so we don't want to erase
+ // two lines.
+ numLines += (prefixWidth + firstLine.length() - 1) / terminalWidth + 1;
+ crlf();
+ while (lines.hasNext()) {
+ String line = lines.next();
+ terminal.writeString(line);
+ crlf();
+ numLines += (line.length() - 1) / terminalWidth + 1;
+ }
+ numLinesPreviousErasable = numLines;
+ }
+
+ /**
+ * Try to match a message against the "progress message" pattern. If it
+ * matches, return the progress percentage, and the rest of the message.
+ * @param message the message to match
+ * @return a pair containing the progress percentage, and the rest of the
+ * progress message, or null if the message isn't a progress message.
+ */
+ private Pair<String,String> matchProgress(String message) {
+ Matcher m = progressPattern.matcher(message);
+ if (m.find()) {
+ return Pair.of(message.substring(0, m.end()), message.substring(m.end()));
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Send the terminal controls that will put the cursor on the beginning
+ * of the same line if cursor control is on, or the next line if not.
+ * @returns True if it did any output; if so, caller is responsible for
+ * flushing the terminal if needed.
+ */
+ private boolean maybeOverwritePreviousMessage() throws IOException {
+ if (useCursorControls && numLinesPreviousErasable != 0) {
+ for (int i = 0; i < numLinesPreviousErasable; i++) {
+ terminal.cr();
+ terminal.cursorUp(1);
+ terminal.clearLine();
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void errorOrFail(Event event) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textRed();
+ terminal.textBold();
+ }
+ terminal.writeString(event.getKind().toString() + ": ");
+ if (useColor) {
+ terminal.resetTerminal();
+ }
+ writeTimestampAndLocation(event);
+ terminal.writeString(event.getMessage());
+ terminal.writeString(".");
+ crlf();
+ }
+
+ private void warning(Event warning) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textMagenta();
+ }
+ terminal.writeString("WARNING: ");
+ terminal.resetTerminal();
+ writeTimestampAndLocation(warning);
+ terminal.writeString(warning.getMessage());
+ terminal.writeString(".");
+ crlf();
+ }
+
+ private void info(Event event) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textGreen();
+ }
+ terminal.writeString(event.getKind().toString() + ": ");
+ terminal.resetTerminal();
+ writeTimestampAndLocation(event);
+ terminal.writeString(event.getMessage());
+ // No period; info messages often end in '...'.
+ crlf();
+ }
+
+ private void subcmd(Event subcmd) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textBlue();
+ }
+ terminal.writeString(">>>>> ");
+ terminal.resetTerminal();
+ writeTimestampAndLocation(subcmd);
+ terminal.writeString(subcmd.getMessage());
+ crlf();
+ }
+
+ /* Handle STDERR events. */
+ private void putOutput(Event event) throws IOException {
+ previousLineErasable = false;
+ terminal.writeBytes(event.getMessageBytes());
+/*
+ * The following code doesn't work because buildtool.TerminalTestNotifier
+ * writes ANSI-formatted text via this mechanism, one character at a time,
+ * and if we try to insert additional ANSI sequences in between the characters
+ * of another ANSI escape sequence, we screw things up. (?)
+ * TODO(bazel-team): (2009) fix this. TerminalTestNotifier should go via the Reporter
+ * rather than via an AnsiTerminalWriter.
+ */
+// terminal.resetTerminal();
+// writeTimestampAndLocation(event);
+// if (useColor) {
+// terminal.textNormal();
+// }
+// terminal.writeBytes(event.getMessageBytes());
+// terminal.resetTerminal();
+ }
+
+ /**
+ * Add a carriage return, shifting to the next line on the terminal, while
+ * guaranteeing that the terminal control codes don't cause any strange
+ * effects. Without the CR before the "\n", the "\n" can cause a line-break
+ * moving text to the next line, where the new message will be generated.
+ * Emitting a "CR" before means that the actual terminal controls generated
+ * here are CR+CR+LF; the double-CR resets the terminal line state, which
+ * prevents the potentially ugly formatting issue.
+ */
+ private void crlf() throws IOException {
+ terminal.cr();
+ terminal.writeString("\n");
+ }
+
+ private void writeTimestampAndLocation(Event event) throws IOException {
+ if (showTimestamp) {
+ terminal.writeString(timestamp());
+ }
+ if (event.getLocation() != null) {
+ terminal.writeString(event.getLocation() + ": ");
+ }
+ }
+
+ public void resetTerminal() {
+ try {
+ terminal.resetTerminal();
+ } catch (IOException e) {
+ LOG.warning("IO Error writing to user terminal: " + e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java
new file mode 100644
index 0000000..48e366d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Record GC stats for a build.
+ */
+public class GCStatsRecorder {
+
+ private final Iterable<GarbageCollectorMXBean> mxBeans;
+ private final ImmutableMap<String, GCStat> initialData;
+
+ public GCStatsRecorder(Iterable<GarbageCollectorMXBean> mxBeans) {
+ this.mxBeans = mxBeans;
+ ImmutableMap.Builder<String, GCStat> initialData = ImmutableMap.builder();
+ for (GarbageCollectorMXBean mxBean : mxBeans) {
+ String name = mxBean.getName();
+ initialData.put(name, new GCStat(name, mxBean.getCollectionCount(),
+ mxBean.getCollectionTime()));
+ }
+ this.initialData = initialData.build();
+ }
+
+ public Iterable<GCStat> getCurrentGcStats() {
+ List<GCStat> stats = new ArrayList<>();
+ for (GarbageCollectorMXBean mxBean : mxBeans) {
+ String name = mxBean.getName();
+ GCStat initStat = Preconditions.checkNotNull(initialData.get(name));
+ stats.add(new GCStat(name,
+ mxBean.getCollectionCount() - initStat.getNumCollections(),
+ mxBean.getCollectionTime() - initStat.getTotalTimeInMs()));
+ }
+ return stats;
+ }
+
+ /** Represents the garbage collections statistics for one collector (For example CMS). */
+ public static class GCStat {
+
+ private final String name;
+ private final long numCollections;
+ private final long totalTimeInMs;
+
+ public GCStat(String name, long numCollections, long totalTimeInMs) {
+ this.name = name;
+ this.numCollections = numCollections;
+ this.totalTimeInMs = totalTimeInMs;
+ }
+
+ /** Name of the Collector. For example CMS. */
+ public String getName() { return name; }
+
+ /** Number of invocations for a build. */
+ public long getNumCollections() { return numCollections; }
+
+ /**
+ * Total time spend in GC for the collector. Note that the time does need to be exclusive (aka a
+ * stop-the-world GC).
+ */
+ public long getTotalTimeInMs() { return totalTimeInMs; }
+
+ @Override
+ public String toString() {
+ return "GC time for '" + name + "' collector: " + numCollections
+ + " collections using " + totalTimeInMs + "ms";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java
new file mode 100644
index 0000000..622d112
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * An event in which the command line options
+ * are discovered.
+ */
+public class GotOptionsEvent {
+
+ private final OptionsProvider startupOptions;
+ private final OptionsProvider options;
+
+ /**
+ * Construct the options event.
+ *
+ * @param startupOptions the parsed startup options
+ * @param options the parsed options
+ */
+ public GotOptionsEvent(OptionsProvider startupOptions, OptionsProvider options) {
+ this.startupOptions = startupOptions;
+ this.options = options;
+ }
+
+ /**
+ * @return the parsed startup options
+ */
+ public OptionsProvider getStartupOptions() {
+ return startupOptions;
+ }
+
+ /**
+ * @return the parsed options.
+ */
+ public OptionsProvider getOptions() {
+ return options;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java
new file mode 100644
index 0000000..305c048
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+/**
+ * Options that will be evaluated by the blaze client startup code only.
+ *
+ * The only reason we have this interface is that we'd like to print a nice
+ * help page for the client startup options. These options do not affect the
+ * server's behavior in any way.
+ */
+public class HostJvmStartupOptions extends OptionsBase {
+
+ @Option(name = "host_jvm_args",
+ defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions.
+ category = "host jvm startup",
+ help = "Flags to pass to the JVM executing Blaze. Note: Blaze " +
+ "will ignore this option unless you are starting a new " +
+ "instance. See also 'blaze help shutdown'.")
+ public String hostJvmArgs;
+
+ @Option(name = "host_jvm_profile",
+ defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions.
+ category = "host jvm startup",
+ help = "Run the JVM executing Blaze in the given profiler. " +
+ "Blaze will search for hardcoded paths based on the " +
+ "profiler. Note: Blaze will ignore this option unless you " +
+ "are starting a new instance. See also 'blaze help shutdown'.")
+ public String hostJvmProfile;
+
+ @Option(name = "host_jvm_debug",
+ defaultValue = "false", // NOTE: purely decorative! See BlazeServerStartupOptions.
+ category = "host jvm startup",
+ help = "Run the JVM executing Blaze so that it listens for a " +
+ "connection from a JDWP-compliant debugger. Note: Blaze " +
+ "will ignore this option unless you are starting a new " +
+ "instance. See also 'blaze help shutdown'.")
+ public boolean hostJvmDebug;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java
new file mode 100644
index 0000000..56747d8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * A file that describes a project - for large source trees that are worked on by multiple
+ * independent teams, it is useful to have a larger unit than a package which combines a set of
+ * target patterns and a set of corresponding options.
+ */
+public interface ProjectFile {
+
+ /**
+ * A provider for a project file - we generally expect the provider to cache parsed files
+ * internally and return a cached version if it can ascertain that that is still correct.
+ *
+ * <p>Note in particular that packages may be moved between different package path entries, which
+ * should lead to cache invalidation.
+ */
+ public interface Provider {
+ /**
+ * Returns an (optionally cached) project file instance. If there is no such file, or if the
+ * file cannot be parsed, then it throws an exception.
+ */
+ ProjectFile getProjectFile(List<Path> packagePath, PathFragment path)
+ throws AbruptExitException;
+ }
+
+ /**
+ * A string name of the project file that is reported to the user. It should be in such a format
+ * that passing it back in on the command line works.
+ */
+ String getName();
+
+ /**
+ * A list of strings that are parsed into the options for the command.
+ *
+ * @param command An action from the command line, e.g. "build" or "test".
+ * @throws UnsupportedOperationException if an unknown command is passed.
+ */
+ List<String> getCommandLineFor(String command);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java
new file mode 100644
index 0000000..5e90f2e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+
+/**
+ * An event handler that rate limits events.
+ */
+public class RateLimitingEventHandler implements EventHandler {
+
+ private final EventHandler outputHandler;
+ private final double intervalMillis;
+ private final Clock clock;
+ private long lastEventMillis = -1;
+
+ /**
+ * Creates a new Event handler that rate limits the events of type PROGRESS
+ * to one per event "rateLimitation" seconds. Events that arrive too quickly are dropped;
+ * all others are are forwarded to the handler "delegateTo".
+ *
+ * @param delegateTo The event handler that ultimately handles the events
+ * @param rateLimitation The minimum number of seconds between events that will be forwarded
+ * to the delegateTo-handler.
+ * If less than zero (or NaN), all events will be forwarded.
+ */
+ public static EventHandler create(EventHandler delegateTo, double rateLimitation) {
+ if (rateLimitation < 0.0 || Double.isNaN(rateLimitation)) {
+ return delegateTo;
+ }
+ return new RateLimitingEventHandler(delegateTo, rateLimitation);
+ }
+
+ private RateLimitingEventHandler(EventHandler delegateTo, double rateLimitation) {
+ clock = BlazeClock.instance();
+ outputHandler = delegateTo;
+ this.intervalMillis = rateLimitation * 1000;
+ }
+
+ @Override
+ public void handle(Event event) {
+ switch (event.getKind()) {
+ case PROGRESS:
+ case START:
+ case FINISH:
+ long currentTime = clock.currentTimeMillis();
+ if (lastEventMillis + intervalMillis <= currentTime) {
+ lastEventMillis = currentTime;
+ outputHandler.handle(event);
+ }
+ break;
+ default:
+ outputHandler.handle(event);
+ break;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java
new file mode 100644
index 0000000..b8d5d45
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.actions.Action;
+
+/**
+ * This class records the critical path for the graph of actions executed.
+ */
+public class SimpleCriticalPathComponent
+ extends AbstractCriticalPathComponent<SimpleCriticalPathComponent> {
+
+ public SimpleCriticalPathComponent(Action action, long startTime) { super(action, startTime); }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java
new file mode 100644
index 0000000..65a9c95
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.util.Clock;
+
+/**
+ * Computes the critical path during a build.
+ */
+public class SimpleCriticalPathComputer
+ extends CriticalPathComputer<SimpleCriticalPathComponent,
+ AggregatedCriticalPath<SimpleCriticalPathComponent>> {
+
+ public SimpleCriticalPathComputer(Clock clock) {
+ super(clock);
+ }
+
+ @Override
+ public SimpleCriticalPathComponent createComponent(Action action, long startTimeMillis) {
+ return new SimpleCriticalPathComponent(action, startTimeMillis);
+ }
+
+ /**
+ * Return the critical path stats for the current command execution.
+ *
+ * <p>This method allow us to calculate lazily the aggregate statistics of the critical path,
+ * avoiding the memory and cpu penalty for doing it for all the actions executed.
+ */
+ @Override
+ public AggregatedCriticalPath<SimpleCriticalPathComponent> aggregate() {
+ ImmutableList.Builder<SimpleCriticalPathComponent> components = ImmutableList.builder();
+ SimpleCriticalPathComponent maxCriticalPath = getMaxCriticalPath();
+ if (maxCriticalPath == null) {
+ return new AggregatedCriticalPath<>(0, components.build());
+ }
+ SimpleCriticalPathComponent child = maxCriticalPath;
+ while (child != null) {
+ components.add(child);
+ child = child.getChild();
+ }
+ return new AggregatedCriticalPath<>(maxCriticalPath.getAggregatedWallTime(),
+ components.build());
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
new file mode 100644
index 0000000..0134f55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
@@ -0,0 +1,220 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.rules.test.TestLogHelper;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat;
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Prints the test results to a terminal.
+ */
+public class TerminalTestResultNotifier implements TestResultNotifier {
+ private static class TestResultStats {
+ int numberOfTargets;
+ int passCount;
+ int failedToBuildCount;
+ int failedCount;
+ int failedRemotelyCount;
+ int failedLocallyCount;
+ int noStatusCount;
+ int numberOfExecutedTargets;
+ boolean wasUnreportedWrongSize;
+ }
+
+ /**
+ * Flags specific to test summary reporting.
+ */
+ public static class TestSummaryOptions extends OptionsBase {
+ @Option(name = "verbose_test_summary",
+ defaultValue = "true",
+ category = "verbosity",
+ help = "If true, print additional information (timing, number of failed runs, etc) in the"
+ + " test summary.")
+ public boolean verboseSummary;
+
+ @Option(name = "test_verbose_timeout_warnings",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "If true, print additional warnings when the actual test execution time does not " +
+ "match the timeout defined by the test (whether implied or explicit).")
+ public boolean testVerboseTimeoutWarnings;
+ }
+
+ private final AnsiTerminalPrinter printer;
+ private final OptionsProvider options;
+ private final TestSummaryOptions summaryOptions;
+
+ /**
+ * @param printer The terminal to print to
+ */
+ public TerminalTestResultNotifier(AnsiTerminalPrinter printer, OptionsProvider options) {
+ this.printer = printer;
+ this.options = options;
+ this.summaryOptions = options.getOptions(TestSummaryOptions.class);
+ }
+
+ /**
+ * Prints a test result summary that contains only failed tests.
+ */
+ private void printDetailedTestResultSummary(Set<TestSummary> summaries) {
+ for (TestSummary entry : summaries) {
+ if (entry.getStatus() != BlazeTestStatus.PASSED) {
+ TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, true);
+ }
+ }
+ }
+
+ /**
+ * Prints a full test result summary.
+ */
+ private void printShortSummary(Set<TestSummary> summaries, boolean showPassingTests) {
+ for (TestSummary entry : summaries) {
+ if (entry.getStatus() != BlazeTestStatus.PASSED || showPassingTests) {
+ TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, false);
+ }
+ }
+ }
+
+ /**
+ * Returns true iff the --check_tests_up_to_date option is enabled.
+ */
+ private boolean optionCheckTestsUpToDate() {
+ return options.getOptions(ExecutionOptions.class).testCheckUpToDate;
+ }
+
+
+ /**
+ * Prints a test summary information for all tests to the terminal.
+ *
+ * @param summaries Summary of all targets that were ran
+ * @param numberOfExecutedTargets the number of targets that were actually ran
+ */
+ @Override
+ public void notify(Set<TestSummary> summaries, int numberOfExecutedTargets) {
+ TestResultStats stats = new TestResultStats();
+ stats.numberOfTargets = summaries.size();
+ stats.numberOfExecutedTargets = numberOfExecutedTargets;
+
+ TestOutputFormat testOutput = options.getOptions(ExecutionOptions.class).testOutput;
+
+ for (TestSummary summary : summaries) {
+ if (summary.isLocalActionCached()
+ && TestLogHelper.shouldOutputTestLog(testOutput,
+ TestResult.isBlazeTestStatusPassed(summary.getStatus()))) {
+ TestSummaryPrinter.printCachedOutput(summary, testOutput, printer);
+ }
+ }
+
+ for (TestSummary summary : summaries) {
+ if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
+ stats.passCount++;
+ } else if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) {
+ stats.failedToBuildCount++;
+ } else if (summary.ranRemotely()) {
+ stats.failedRemotelyCount++;
+ } else {
+ stats.failedLocallyCount++;
+ }
+
+ if (summary.getStatus() == BlazeTestStatus.NO_STATUS) {
+ stats.noStatusCount++;
+ }
+
+ if (summary.wasUnreportedWrongSize()) {
+ stats.wasUnreportedWrongSize = true;
+ }
+ }
+
+ stats.failedCount = summaries.size() - stats.passCount;
+
+ TestSummaryFormat testSummaryFormat = options.getOptions(ExecutionOptions.class).testSummary;
+ switch (testSummaryFormat) {
+ case DETAILED:
+ printDetailedTestResultSummary(summaries);
+ break;
+
+ case SHORT:
+ printShortSummary(summaries, /*printSuccess=*/true);
+ break;
+
+ case TERSE:
+ printShortSummary(summaries, /*printSuccess=*/false);
+ break;
+
+ case NONE:
+ break;
+ }
+
+ printStats(stats);
+ }
+
+ private void addToErrorList(List<String> list, String failureDescription, int count) {
+ if (count > 0) {
+ list.add(String.format("%s%d %s %s%s",
+ AnsiTerminalPrinter.Mode.ERROR,
+ count,
+ count == 1 ? "fails" : "fail",
+ failureDescription,
+ AnsiTerminalPrinter.Mode.DEFAULT));
+ }
+ }
+
+ private void printStats(TestResultStats stats) {
+ if (!optionCheckTestsUpToDate()) {
+ List<String> results = new ArrayList<>();
+ if (stats.passCount == 1) {
+ results.add(stats.passCount + " test passes");
+ } else if (stats.passCount > 0) {
+ results.add(stats.passCount + " tests pass");
+ }
+ addToErrorList(results, "to build", stats.failedToBuildCount);
+ addToErrorList(results, "locally", stats.failedLocallyCount);
+ addToErrorList(results, "remotely", stats.failedRemotelyCount);
+ printer.print(String.format("\nExecuted %d out of %d tests: %s.\n",
+ stats.numberOfExecutedTargets,
+ stats.numberOfTargets,
+ StringUtil.joinEnglishList(results, "and")));
+ } else {
+ int failingUpToDateCount = stats.failedCount - stats.noStatusCount;
+ printer.print(String.format(
+ "\nFinished with %d passing and %s%d failing%s tests up to date, %s%d out of date.%s\n",
+ stats.passCount,
+ failingUpToDateCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "",
+ failingUpToDateCount,
+ AnsiTerminalPrinter.Mode.DEFAULT,
+ stats.noStatusCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "",
+ stats.noStatusCount,
+ AnsiTerminalPrinter.Mode.DEFAULT));
+ }
+
+ if (stats.wasUnreportedWrongSize) {
+ printer.print("There were tests whose specified size is too big. Use the "
+ + "--test_verbose_timeout_warnings command line option to see which "
+ + "ones these are.\n");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
new file mode 100644
index 0000000..ed9120b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
@@ -0,0 +1,349 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Prints results to the terminal, showing the results of each test target.
+ */
+@ThreadCompatible
+public class TestResultAnalyzer {
+ private final Path execRoot;
+ private final TestSummaryOptions summaryOptions;
+ private final ExecutionOptions executionOptions;
+ private final EventBus eventBus;
+
+ /**
+ * @param summaryOptions Parsed test summarization options.
+ * @param executionOptions Parsed build/test execution options.
+ * @param eventBus For reporting failed to build and cached tests.
+ */
+ public TestResultAnalyzer(Path execRoot,
+ TestSummaryOptions summaryOptions,
+ ExecutionOptions executionOptions,
+ EventBus eventBus) {
+ this.execRoot = execRoot;
+ this.summaryOptions = summaryOptions;
+ this.executionOptions = executionOptions;
+ this.eventBus = eventBus;
+ }
+
+ /**
+ * Prints out the results of the given tests, and returns true if they all passed.
+ * Posts any targets which weren't already completed by the listener to the EventBus.
+ * Reports all targets on the console via the given notifier.
+ * Run at the end of the build, run only once.
+ *
+ * @param testTargets The list of targets being run
+ * @param listener An aggregating listener with intermediate results
+ * @param notifier A console notifier to echo results to.
+ * @return true if all the tests passed, else false
+ */
+ public boolean differentialAnalyzeAndReport(
+ Collection<ConfiguredTarget> testTargets,
+ AggregatingTestListener listener,
+ TestResultNotifier notifier) {
+
+ Preconditions.checkNotNull(testTargets);
+ Preconditions.checkNotNull(listener);
+ Preconditions.checkNotNull(notifier);
+
+ // The natural ordering of the summaries defines their output order.
+ Set<TestSummary> summaries = Sets.newTreeSet();
+
+ int totalRun = 0; // Number of targets running at least one non-cached test.
+ int passCount = 0;
+
+ for (ConfiguredTarget testTarget : testTargets) {
+ TestSummary summary = aggregateAndReportSummary(testTarget, listener).build();
+ summaries.add(summary);
+
+ // Finished aggregating; build the final console output.
+ if (summary.actionRan()) {
+ totalRun++;
+ }
+
+ if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
+ passCount++;
+ }
+ }
+
+ Preconditions.checkState(summaries.size() == testTargets.size());
+
+ notifier.notify(summaries, totalRun);
+ return passCount == testTargets.size();
+ }
+
+ private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
+ return status.ordinal() > other.ordinal() ? status : other;
+ }
+
+ /**
+ * Helper for differential analysis which aggregates the TestSummary
+ * for an individual target, reporting runs on the EventBus if necessary.
+ */
+ private TestSummary.Builder aggregateAndReportSummary(
+ ConfiguredTarget testTarget,
+ AggregatingTestListener listener) {
+
+ // If already reported by the listener, no work remains for this target.
+ TestSummary.Builder summary = listener.getCurrentSummary(testTarget);
+ Label testLabel = testTarget.getLabel();
+ Preconditions.checkNotNull(summary,
+ "%s did not complete test filtering, but has a test result", testLabel);
+ if (listener.targetReported(testTarget)) {
+ return summary;
+ }
+
+ Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget);
+ Map<Artifact, TestResult> statusMap = listener.getStatusMap();
+
+ // We will get back multiple TestResult instances if test had to be retried several
+ // times before passing. Sharding and multiple runs of the same test without retries
+ // will be represented by separate artifacts and will produce exactly one TestResult.
+ for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
+ // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
+ // be null for an unrelated test because we were not able to even try to execute the test.
+ // In that case, for tests that were previously passing we return null ( == NO STATUS),
+ // because checking if the cached test target is up-to-date would require running the
+ // dependency checker transitively.
+ TestResult runResult = statusMap.get(testStatus);
+ boolean isIncompleteRun = incompleteRuns.contains(testStatus);
+ if (runResult == null) {
+ summary = markIncomplete(summary);
+ } else if (isIncompleteRun) {
+ // Only process results which were not recorded by the listener.
+
+ boolean newlyFetched = !statusMap.containsKey(testStatus);
+ summary = incrementalAnalyze(summary, runResult);
+ if (newlyFetched) {
+ eventBus.post(runResult);
+ }
+ Preconditions.checkState(
+ listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun,
+ "TestListener changed in differential analysis. Ensure it isn't still registered.");
+ }
+ }
+
+ // The target was not posted by the listener and must be posted now.
+ eventBus.post(summary.build());
+ return summary;
+ }
+
+ /**
+ * Incrementally updates a TestSummary given an existing summary
+ * and a new TestResult. Only call on built targets.
+ *
+ * @param summaryBuilder Existing unbuilt test summary associated with a target.
+ * @param result New test result to aggregate into the summary.
+ * @return The updated TestSummary.
+ */
+ public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder,
+ TestResult result) {
+ // Cache retrieval should have been performed already.
+ Preconditions.checkNotNull(result);
+ Preconditions.checkNotNull(summaryBuilder);
+ TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek());
+
+ TransitiveInfoCollection target = existingSummary.getTarget();
+ Preconditions.checkNotNull(
+ target, "The existing TestSummary must be associated with a target");
+
+ BlazeTestStatus status = existingSummary.getStatus();
+ int numCached = existingSummary.numCached();
+ int numLocalActionCached = existingSummary.numLocalActionCached();
+
+ if (!existingSummary.actionRan() && !result.isCached()) {
+ // At least one run of the test actually ran uncached.
+ summaryBuilder.setActionRan(true);
+
+ // Coverage data artifact will be identical for all test results - it is provided by the
+ // TestRunnerAction and all results in this collection associate with the same action.
+ PathFragment coverageData = result.getCoverageData();
+ if (coverageData != null) {
+ summaryBuilder.addCoverageFiles(
+ Collections.singletonList(execRoot.getRelative(coverageData)));
+ }
+ }
+
+ if (result.isCached() || result.getData().getRemotelyCached()) {
+ numCached++;
+ }
+ if (result.isCached()) {
+ numLocalActionCached++;
+ }
+
+ if (!executionOptions.runsPerTestDetectsFlakes) {
+ status = aggregateStatus(status, result.getData().getStatus());
+ } else {
+ int shardNumber = result.getShardNum();
+ int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns();
+ List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus(
+ shardNumber, result.getData().getStatus());
+ if (singleShardStatuses.size() == runsPerTestForLabel) {
+ BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
+ int passes = 0;
+ for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
+ shardStatus = aggregateStatus(shardStatus, runStatusForShard);
+ if (TestResult.isBlazeTestStatusPassed(shardStatus)) {
+ passes++;
+ }
+ }
+ // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass.
+ // If all results pass or fail, aggregate the passing/failing shardStatus.
+ if (passes == 0 || passes == runsPerTestForLabel) {
+ status = aggregateStatus(status, shardStatus);
+ } else {
+ status = aggregateStatus(status, BlazeTestStatus.FLAKY);
+ }
+ }
+ }
+
+ List<String> filtered = new ArrayList<>();
+ warningLoop: for (String warning : result.getData().getWarningList()) {
+ for (String ignoredPrefix : Constants.IGNORED_TEST_WARNING_PREFIXES) {
+ if (warning.startsWith(ignoredPrefix)) {
+ continue warningLoop;
+ }
+ }
+
+ filtered.add(warning);
+ }
+
+ List<Path> passed = new ArrayList<>();
+ if (result.getData().hasPassedLog()) {
+ passed.add(result.getTestAction().getTestLog().getPath().getRelative(
+ result.getData().getPassedLog()));
+ }
+
+ List<Path> failed = new ArrayList<>();
+ for (String path : result.getData().getFailedLogsList()) {
+ failed.add(result.getTestAction().getTestLog().getPath().getRelative(path));
+ }
+
+ summaryBuilder
+ .addTestTimes(result.getData().getTestTimesList())
+ .addPassedLogs(passed)
+ .addFailedLogs(failed)
+ .addWarnings(filtered)
+ .collectFailedTests(result.getData().getTestCase())
+ .setRanRemotely(result.getData().getIsRemoteStrategy());
+
+ List<String> warnings = new ArrayList<>();
+ if (status == BlazeTestStatus.PASSED) {
+ if (shouldEmitTestSizeWarningInSummary(
+ summaryOptions.testVerboseTimeoutWarnings,
+ warnings, result.getData().getTestProcessTimesList(), target)) {
+ summaryBuilder.setWasUnreportedWrongSize(true);
+ }
+ }
+
+ return summaryBuilder
+ .setStatus(status)
+ .setNumCached(numCached)
+ .setNumLocalActionCached(numLocalActionCached)
+ .addWarnings(warnings);
+ }
+
+ private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) {
+ // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
+ // tests with no status and post it here.
+ TestSummary summary = summaryBuilder.peek();
+ BlazeTestStatus status = summary.getStatus();
+ if (status != BlazeTestStatus.NO_STATUS) {
+ status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
+ }
+
+ return summaryBuilder.setStatus(status);
+ }
+
+ TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) {
+ BlazeTestStatus runStatus = blazeHalted ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
+ : (executionOptions.testCheckUpToDate
+ ? BlazeTestStatus.NO_STATUS
+ : BlazeTestStatus.FAILED_TO_BUILD);
+
+ return summary.setStatus(runStatus);
+ }
+
+ /**
+ * Checks whether the specified test timeout could have been smaller and adds
+ * a warning message if verbose is true.
+ *
+ * <p>Returns true if there was a test with the wrong timeout, but if was not
+ * reported.
+ */
+ private static boolean shouldEmitTestSizeWarningInSummary(boolean verbose,
+ List<String> warnings, List<Long> testTimes, TransitiveInfoCollection target) {
+
+ TestTimeout specifiedTimeout =
+ target.getProvider(TestProvider.class).getTestParams().getTimeout();
+ long maxTimeOfShard = 0;
+
+ for (Long shardTime : testTimes) {
+ if (shardTime != null) {
+ maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
+ }
+ }
+
+ int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);
+
+ if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
+ TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
+ TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
+ if (verbose) {
+ StringBuilder builder = new StringBuilder(String.format(
+ "Test execution time (%.1fs excluding execution overhead) outside of "
+ + "range for %s tests. Consider setting timeout=\"%s\"",
+ maxTimeOfShard / 1000.0,
+ specifiedTimeout.prettyPrint(),
+ expectedTimeout));
+ if (expectedSize != null) {
+ builder.append(" or size=\"").append(expectedSize).append("\"");
+ }
+ builder.append(". You need not modify the size if you think it is correct.");
+ warnings.add(builder.toString());
+ return false;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java
new file mode 100644
index 0000000..d7dbebb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import java.util.Set;
+
+/**
+ * Used to notify interested parties of test results.
+ */
+public interface TestResultNotifier {
+
+ /**
+ * @param summaries Summary of all targets that were supposed to be tested
+ * (regardless whether they actually were executed).
+ * @param numberOfExecutedTargets the number of targets that were actually run.
+ * Must not exceed summaries.size().
+ */
+ void notify(Set<TestSummary> summaries, int numberOfExecutedTargets);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
new file mode 100644
index 0000000..171f150
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
@@ -0,0 +1,428 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Test summary entry. Stores summary information for a single test rule.
+ * Also used to sort summary output by status.
+ *
+ * <p>Invariant:
+ * All TestSummary mutations should be performed through the Builder.
+ * No direct TestSummary methods (except the constructor) may mutate the object.
+ */
+@VisibleForTesting // Ideally package-scoped.
+public class TestSummary implements Comparable<TestSummary> {
+ /**
+ * Builder class responsible for creating and altering TestSummary objects.
+ */
+ public static class Builder {
+ private TestSummary summary;
+ private boolean built;
+
+ private Builder() {
+ summary = new TestSummary();
+ built = false;
+ }
+
+ private void mergeFrom(TestSummary existingSummary) {
+ // Yuck, manually fill in fields.
+ summary.shardRunStatuses = ArrayListMultimap.create(existingSummary.shardRunStatuses);
+ setTarget(existingSummary.target);
+ setStatus(existingSummary.status);
+ addCoverageFiles(existingSummary.coverageFiles);
+ addPassedLogs(existingSummary.passedLogs);
+ addFailedLogs(existingSummary.failedLogs);
+
+ if (existingSummary.failedTestCasesStatus != null) {
+ addFailedTestCases(existingSummary.getFailedTestCases(),
+ existingSummary.getFailedTestCasesStatus());
+ }
+
+ addTestTimes(existingSummary.testTimes);
+ addWarnings(existingSummary.warnings);
+ setActionRan(existingSummary.actionRan);
+ setNumCached(existingSummary.numCached);
+ setRanRemotely(existingSummary.ranRemotely);
+ setWasUnreportedWrongSize(existingSummary.wasUnreportedWrongSize);
+ }
+
+ // Implements copy on write logic, allowing reuse of the same builder.
+ private void checkMutation() {
+ // If mutating the builder after an object was built, create another copy.
+ if (built) {
+ built = false;
+ TestSummary lastSummary = summary;
+ summary = new TestSummary();
+ mergeFrom(lastSummary);
+ }
+ }
+
+ // This used to return a reference to the value on success.
+ // However, since it can alter the summary member, inlining it in an
+ // assignment to a property of summary was unsafe.
+ private void checkMutation(Object value) {
+ Preconditions.checkNotNull(value);
+ checkMutation();
+ }
+
+ public Builder setTarget(ConfiguredTarget target) {
+ checkMutation(target);
+ summary.target = target;
+ return this;
+ }
+
+ public Builder setStatus(BlazeTestStatus status) {
+ checkMutation(status);
+ summary.status = status;
+ return this;
+ }
+
+ public Builder addCoverageFiles(List<Path> coverageFiles) {
+ checkMutation(coverageFiles);
+ summary.coverageFiles.addAll(coverageFiles);
+ return this;
+ }
+
+ public Builder addPassedLogs(List<Path> passedLogs) {
+ checkMutation(passedLogs);
+ summary.passedLogs.addAll(passedLogs);
+ return this;
+ }
+
+ public Builder addFailedLogs(List<Path> failedLogs) {
+ checkMutation(failedLogs);
+ summary.failedLogs.addAll(failedLogs);
+ return this;
+ }
+
+ public Builder collectFailedTests(TestCase testCase) {
+ if (testCase == null) {
+ summary.failedTestCasesStatus = FailedTestCasesStatus.NOT_AVAILABLE;
+ return this;
+ }
+ summary.failedTestCasesStatus = FailedTestCasesStatus.FULL;
+ return collectFailedTestCases(testCase);
+ }
+
+ private Builder collectFailedTestCases(TestCase testCase) {
+ if (testCase.getChildCount() > 0) {
+ // This is a non-leaf result. Traverse its children, but do not add its
+ // name to the output list. It should not contain any 'failure' or
+ // 'error' tags, but we want to be lax here, because the syntax of the
+ // test.xml file is also lax.
+ for (TestCase child : testCase.getChildList()) {
+ collectFailedTestCases(child);
+ }
+ } else {
+ // This is a leaf result. If it passed, don't add it.
+ if (testCase.getStatus() == TestCase.Status.PASSED) {
+ return this;
+ }
+
+ String name = testCase.getName();
+ String className = testCase.getClassName();
+ if (name == null || className == null) {
+ // A test case detail is not really interesting if we cannot tell which
+ // one it is.
+ this.summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL;
+ return this;
+ }
+
+ this.summary.failedTestCases.add(testCase);
+ }
+ return this;
+ }
+
+ public Builder addFailedTestCases(List<TestCase> testCases, FailedTestCasesStatus status) {
+ checkMutation(status);
+ checkMutation(testCases);
+
+ if (summary.failedTestCasesStatus == null) {
+ summary.failedTestCasesStatus = status;
+ } else if (summary.failedTestCasesStatus != status) {
+ summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL;
+ }
+
+ if (testCases.isEmpty()) {
+ return this;
+ }
+
+ // union of summary.failedTestCases, testCases
+ Map<String, TestCase> allCases = new TreeMap<>();
+ if (summary.failedTestCases != null) {
+ for (TestCase detail : summary.failedTestCases) {
+ allCases.put(detail.getClassName() + "." + detail.getName(), detail);
+ }
+ }
+ for (TestCase detail : testCases) {
+ allCases.put(detail.getClassName() + "." + detail.getName(), detail);
+ }
+
+ summary.failedTestCases = new ArrayList<TestCase>(allCases.values());
+ return this;
+ }
+
+ public Builder addTestTimes(List<Long> testTimes) {
+ checkMutation(testTimes);
+ summary.testTimes.addAll(testTimes);
+ return this;
+ }
+
+ public Builder addWarnings(List<String> warnings) {
+ checkMutation(warnings);
+ summary.warnings.addAll(warnings);
+ return this;
+ }
+
+ public Builder setActionRan(boolean actionRan) {
+ checkMutation();
+ summary.actionRan = actionRan;
+ return this;
+ }
+
+ public Builder setNumCached(int numCached) {
+ checkMutation();
+ summary.numCached = numCached;
+ return this;
+ }
+
+ public Builder setNumLocalActionCached(int numLocalActionCached) {
+ checkMutation();
+ summary.numLocalActionCached = numLocalActionCached;
+ return this;
+ }
+
+ public Builder setRanRemotely(boolean ranRemotely) {
+ checkMutation();
+ summary.ranRemotely = ranRemotely;
+ return this;
+ }
+
+ public Builder setWasUnreportedWrongSize(boolean wasUnreportedWrongSize) {
+ checkMutation();
+ summary.wasUnreportedWrongSize = wasUnreportedWrongSize;
+ return this;
+ }
+
+ /**
+ * Records a new result for the given shard of the test.
+ *
+ * @return an immutable view of the statuses associated with the shard, with the new element.
+ */
+ public List<BlazeTestStatus> addShardStatus(int shardNumber, BlazeTestStatus status) {
+ Preconditions.checkState(summary.shardRunStatuses.put(shardNumber, status),
+ "shardRunStatuses must allow duplicate statuses");
+ return ImmutableList.copyOf(summary.shardRunStatuses.get(shardNumber));
+ }
+
+ /**
+ * Returns the created TestSummary object.
+ * Any actions following a build() will create another copy of the same values.
+ * Since no mutators are provided directly by TestSummary, a copy will not
+ * be produced if two builds are invoked in a row without calling a setter.
+ */
+ public TestSummary build() {
+ peek();
+ if (!built) {
+ makeSummaryImmutable();
+ // else: it is already immutable.
+ }
+ Preconditions.checkState(built, "Built flag was not set");
+ return summary;
+ }
+
+ /**
+ * Within-package, it is possible to read directly from an
+ * incompletely-built TestSummary. Used to pass Builders around directly.
+ */
+ TestSummary peek() {
+ Preconditions.checkNotNull(summary.target, "Target cannot be null");
+ Preconditions.checkNotNull(summary.status, "Status cannot be null");
+ return summary;
+ }
+
+ private void makeSummaryImmutable() {
+ // Once finalized, the list types are immutable.
+ summary.passedLogs = Collections.unmodifiableList(summary.passedLogs);
+ summary.failedLogs = Collections.unmodifiableList(summary.failedLogs);
+ summary.warnings = Collections.unmodifiableList(summary.warnings);
+ summary.coverageFiles = Collections.unmodifiableList(summary.coverageFiles);
+ summary.testTimes = Collections.unmodifiableList(summary.testTimes);
+
+ built = true;
+ }
+ }
+
+ private ConfiguredTarget target;
+ private BlazeTestStatus status;
+ // Currently only populated if --runs_per_test_detects_flakes is enabled.
+ private Multimap<Integer, BlazeTestStatus> shardRunStatuses = ArrayListMultimap.create();
+ private int numCached;
+ private int numLocalActionCached;
+ private boolean actionRan;
+ private boolean ranRemotely;
+ private boolean wasUnreportedWrongSize;
+ private List<TestCase> failedTestCases = new ArrayList<>();
+ private List<Path> passedLogs = new ArrayList<>();
+ private List<Path> failedLogs = new ArrayList<>();
+ private List<String> warnings = new ArrayList<>();
+ private List<Path> coverageFiles = new ArrayList<>();
+ private List<Long> testTimes = new ArrayList<>();
+ private FailedTestCasesStatus failedTestCasesStatus = null;
+
+ // Don't allow public instantiation; go through the Builder.
+ private TestSummary() {
+ }
+
+ /**
+ * Creates a new Builder allowing construction of a new TestSummary object.
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /**
+ * Creates a new Builder initialized with a copy of the existing object's values.
+ */
+ public static Builder newBuilderFromExisting(TestSummary existing) {
+ Builder builder = new Builder();
+ builder.mergeFrom(existing);
+ return builder;
+ }
+
+ public ConfiguredTarget getTarget() {
+ return target;
+ }
+
+ public BlazeTestStatus getStatus() {
+ return status;
+ }
+
+ public boolean isCached() {
+ return numCached > 0;
+ }
+
+ public boolean isLocalActionCached() {
+ return numLocalActionCached > 0;
+ }
+
+ public int numLocalActionCached() {
+ return numLocalActionCached;
+ }
+
+ public int numCached() {
+ return numCached;
+ }
+
+ private int numUncached() {
+ return totalRuns() - numCached;
+ }
+
+ public boolean actionRan() {
+ return actionRan;
+ }
+
+ public boolean ranRemotely() {
+ return ranRemotely;
+ }
+
+ public boolean wasUnreportedWrongSize() {
+ return wasUnreportedWrongSize;
+ }
+
+ public List<TestCase> getFailedTestCases() {
+ return failedTestCases;
+ }
+
+ public List<Path> getCoverageFiles() {
+ return coverageFiles;
+ }
+
+ public List<Path> getPassedLogs() {
+ return passedLogs;
+ }
+
+ public List<Path> getFailedLogs() {
+ return failedLogs;
+ }
+
+ public FailedTestCasesStatus getFailedTestCasesStatus() {
+ return failedTestCasesStatus;
+ }
+
+ /**
+ * Returns an immutable view of the warnings associated with this test.
+ */
+ public List<String> getWarnings() {
+ return Collections.unmodifiableList(warnings);
+ }
+
+ private static int getSortKey(BlazeTestStatus status) {
+ return status == BlazeTestStatus.PASSED ? -1 : status.ordinal();
+ }
+
+ @Override
+ public int compareTo(TestSummary that) {
+ if (this.isCached() != that.isCached()) {
+ return this.isCached() ? -1 : 1;
+ } else if ((this.isCached() && that.isCached()) && (this.numUncached() != that.numUncached())) {
+ return this.numUncached() - that.numUncached();
+ } else if (this.status != that.status) {
+ return getSortKey(this.status) - getSortKey(that.status);
+ } else {
+ Artifact thisExecutable = this.target.getProvider(FilesToRunProvider.class).getExecutable();
+ Artifact thatExecutable = that.target.getProvider(FilesToRunProvider.class).getExecutable();
+ return thisExecutable.getPath().compareTo(thatExecutable.getPath());
+ }
+ }
+
+ public List<Long> getTestTimes() {
+ // The return result is unmodifiable (UnmodifiableList instance)
+ return testTimes;
+ }
+
+ public int getNumCached() {
+ return numCached;
+ }
+
+ public int totalRuns() {
+ return testTimes.size();
+ }
+
+ static Mode getStatusMode(BlazeTestStatus status) {
+ return status == BlazeTestStatus.PASSED
+ ? Mode.INFO
+ : (status == BlazeTestStatus.FLAKY ? Mode.WARNING : Mode.ERROR);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java
new file mode 100644
index 0000000..91c1488
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java
@@ -0,0 +1,255 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.devtools.build.lib.rules.test.TestLogHelper;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+
+/**
+ * Print test statistics in human readable form.
+ */
+public class TestSummaryPrinter {
+
+ /**
+ * Print the cached test log to the given printer.
+ */
+ public static void printCachedOutput(TestSummary summary,
+ TestOutputFormat testOutput,
+ AnsiTerminalPrinter printer) {
+
+ String testName = summary.getTarget().getLabel().toString();
+ List<String> allLogs = new ArrayList<>();
+ for (Path path : summary.getFailedLogs()) {
+ allLogs.add(path.getPathString());
+ }
+ for (Path path : summary.getPassedLogs()) {
+ allLogs.add(path.getPathString());
+ }
+ printer.printLn("" + TestSummary.getStatusMode(summary.getStatus()) + summary.getStatus() + ": "
+ + Mode.DEFAULT + testName + " (see " + Joiner.on(' ').join(allLogs) + ")");
+ printer.printLn(Mode.INFO + "INFO: " + Mode.DEFAULT + "From Testing " + testName);
+
+ // Whether to output the target at all was checked by the caller.
+ // Now check whether to output failing shards.
+ if (TestLogHelper.shouldOutputTestLog(testOutput, false)) {
+ for (Path path : summary.getFailedLogs()) {
+ try {
+ TestLogHelper.writeTestLog(path, testName, printer.getOutputStream());
+ } catch (IOException e) {
+ printer.printLn("==================== Could not read test output for " + testName);
+ LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e);
+ }
+ }
+ }
+
+ // And passing shards, independently.
+ if (TestLogHelper.shouldOutputTestLog(testOutput, true)) {
+ for (Path path : summary.getPassedLogs()) {
+ try {
+ TestLogHelper.writeTestLog(path, testName, printer.getOutputStream());
+ } catch (Exception e) {
+ printer.printLn("==================== Could not read test output for " + testName);
+ LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e);
+ }
+ }
+ }
+ }
+
+ private static String statusString(BlazeTestStatus status) {
+ return status.toString().replace('_', ' ');
+ }
+
+ /**
+ * Prints summary status for a single test.
+ * @param terminalPrinter The printer to print to
+ */
+ public static void print(
+ TestSummary summary,
+ AnsiTerminalPrinter terminalPrinter,
+ boolean verboseSummary, boolean printFailedTestCases) {
+ // Skip output for tests that failed to build.
+ if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) {
+ return;
+ }
+ String message = getCacheMessage(summary) + statusString(summary.getStatus());
+ terminalPrinter.print(
+ Strings.padEnd(summary.getTarget().getLabel().toString(), 78 - message.length(), ' ')
+ + " " + TestSummary.getStatusMode(summary.getStatus()) + message + Mode.DEFAULT
+ + (verboseSummary ? getAttemptSummary(summary) + getTimeSummary(summary) : "") + "\n");
+
+ if (printFailedTestCases && summary.getStatus() == BlazeTestStatus.FAILED) {
+ if (summary.getFailedTestCasesStatus() == FailedTestCasesStatus.NOT_AVAILABLE) {
+ terminalPrinter.print(
+ Mode.WARNING + " (individual test case information not available) "
+ + Mode.DEFAULT + "\n");
+ } else {
+ for (TestCase testCase : summary.getFailedTestCases()) {
+ if (testCase.getStatus() != TestCase.Status.PASSED) {
+ TestSummaryPrinter.printTestCase(terminalPrinter, testCase);
+ }
+ }
+
+ if (summary.getFailedTestCasesStatus() != FailedTestCasesStatus.FULL) {
+ terminalPrinter.print(
+ Mode.WARNING
+ + " (some shards did not report details, list of failed test"
+ + " cases incomplete)\n"
+ + Mode.DEFAULT);
+ }
+ }
+ }
+
+ if (printFailedTestCases) {
+ // In this mode, test output and coverage files would just clutter up
+ // the output.
+ return;
+ }
+
+ for (String warning : summary.getWarnings()) {
+ terminalPrinter.print(" " + AnsiTerminalPrinter.Mode.WARNING + "WARNING: "
+ + AnsiTerminalPrinter.Mode.DEFAULT + warning + "\n");
+ }
+
+ for (Path path : summary.getFailedLogs()) {
+ if (path.exists()) {
+ // Don't use getPrettyPath() here - we want to print the absolute path,
+ // so that it cut and paste into a different terminal, and we don't
+ // want to use the blaze-bin etc. symbolic links because they could be changed
+ // by a subsequent build with different options.
+ terminalPrinter.print(" " + path.getPathString() + "\n");
+ }
+ }
+ for (Path path : summary.getCoverageFiles()) {
+ // Print only non-trivial coverage files.
+ try {
+ if (path.exists() && path.getFileSize() > 0) {
+ terminalPrinter.print(" " + path.getPathString() + "\n");
+ }
+ } catch (IOException e) {
+ LoggingUtil.logToRemote(Level.WARNING, "Error while reading coverage data file size",
+ e);
+ }
+ }
+ }
+
+ /**
+ * Prints the result of an individual test case. It is assumed not to have
+ * passed, since passed test cases are not reported.
+ */
+ static void printTestCase(
+ AnsiTerminalPrinter terminalPrinter, TestCase testCase) {
+ String timeSummary;
+ if (testCase.hasRunDurationMillis()) {
+ timeSummary = " ("
+ + timeInSec(testCase.getRunDurationMillis(), TimeUnit.MILLISECONDS)
+ + ")";
+ } else {
+ timeSummary = "";
+ }
+
+ terminalPrinter.print(
+ " "
+ + Mode.ERROR
+ + Strings.padEnd(testCase.getStatus().toString(), 8, ' ')
+ + Mode.DEFAULT
+ + testCase.getClassName()
+ + "."
+ + testCase.getName()
+ + timeSummary
+ + "\n");
+ }
+
+ /**
+ * Return the given time in seconds, to 1 decimal place,
+ * i.e. "32.1s".
+ */
+ static String timeInSec(long time, TimeUnit unit) {
+ double ms = TimeUnit.MILLISECONDS.convert(time, unit);
+ return String.format("%.1fs", ms / 1000.0);
+ }
+
+ static String getAttemptSummary(TestSummary summary) {
+ int attempts = summary.getPassedLogs().size() + summary.getFailedLogs().size();
+ if (attempts > 1) {
+ // Print number of failed runs for failed tests if testing was completed.
+ if (summary.getStatus() == BlazeTestStatus.FLAKY) {
+ return ", failed in " + summary.getFailedLogs().size() + " out of " + attempts;
+ }
+ if (summary.getStatus() == BlazeTestStatus.TIMEOUT
+ || summary.getStatus() == BlazeTestStatus.FAILED) {
+ return " in " + summary.getFailedLogs().size() + " out of " + attempts;
+ }
+ }
+ return "";
+ }
+
+ static String getCacheMessage(TestSummary summary) {
+ if (summary.getNumCached() == 0 || summary.getStatus() == BlazeTestStatus.INCOMPLETE) {
+ return "";
+ } else if (summary.getNumCached() == summary.totalRuns()) {
+ return "(cached) ";
+ } else {
+ return String.format("(%d/%d cached) ", summary.getNumCached(), summary.totalRuns());
+ }
+ }
+
+ static String getTimeSummary(TestSummary summary) {
+ if (summary.getTestTimes().isEmpty()) {
+ return "";
+ } else if (summary.getTestTimes().size() == 1) {
+ return " in " + timeInSec(summary.getTestTimes().get(0), TimeUnit.MILLISECONDS);
+ } else {
+ // We previously used com.google.math for this, which added about 1 MB of deps to the total
+ // size. If we re-introduce a dependency on that package, we could revert this change.
+ long min = summary.getTestTimes().get(0).longValue(), max = min, sum = 0;
+ double sumOfSquares = 0.0;
+ for (Long l : summary.getTestTimes()) {
+ long value = l.longValue();
+ min = value < min ? value : min;
+ max = value > max ? value : max;
+ sum += value;
+ sumOfSquares += ((double) value) * (double) value;
+ }
+ double mean = ((double) sum) / summary.getTestTimes().size();
+ double stddev = Math.sqrt((sumOfSquares - sum * mean) / summary.getTestTimes().size());
+ // For sharded tests, we print the max time on the same line as
+ // the test, and then print more detailed info about the
+ // distribution of times on the next line.
+ String maxTime = timeInSec(max, TimeUnit.MILLISECONDS);
+ return String.format(
+ " in %s\n Stats over %d runs: max = %s, min = %s, avg = %s, dev = %s",
+ maxTime,
+ summary.getTestTimes().size(),
+ maxTime,
+ timeInSec(min, TimeUnit.MILLISECONDS),
+ timeInSec((long) mean, TimeUnit.MILLISECONDS),
+ timeInSec((long) stddev, TimeUnit.MILLISECONDS));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
new file mode 100644
index 0000000..d6f61eb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.List;
+
+/**
+ * Handles the 'build' command on the Blaze command line, including targets
+ * named by arguments passed to Blaze.
+ */
+@Command(name = "build",
+ builds = true,
+ options = { BuildRequestOptions.class,
+ ExecutionOptions.class,
+ PackageCacheOptions.class,
+ BuildView.Options.class,
+ LoadingPhaseRunner.Options.class,
+ BuildConfiguration.Options.class,
+ },
+ usesConfigurationOptions = true,
+ shortDescription = "Builds the specified targets.",
+ allowResidue = true,
+ help = "resource:build.txt")
+public final class BuildCommand implements BlazeCommand {
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser)
+ throws AbruptExitException {
+ ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "build");
+ }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ List<String> targets = ProjectFileSupport.getTargets(runtime, options);
+
+ BuildRequest request = BuildRequest.create(
+ getClass().getAnnotation(Command.class).name(), options,
+ runtime.getStartupOptionsProvider(),
+ targets,
+ runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime());
+ return runtime.getBuildTool().processRequest(request, null).getExitCondition();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java
new file mode 100644
index 0000000..0bb5a0e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java
@@ -0,0 +1,95 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandUtils;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * The 'blaze canonicalize-flags' command.
+ */
+@Command(name = "canonicalize-flags",
+ options = { CanonicalizeCommand.Options.class },
+ allowResidue = true,
+ mustRunInWorkspace = false,
+ shortDescription = "Canonicalizes a list of Blaze options.",
+ help = "This command canonicalizes a list of Blaze options. Don't forget to prepend '--' "
+ + "to end option parsing before the flags to canonicalize.\n"
+ + "%{options}")
+public final class CanonicalizeCommand implements BlazeCommand {
+
+ public static class CommandConverter implements Converter<String> {
+
+ @Override
+ public String convert(String input) throws OptionsParsingException {
+ if (input.equals("build")) {
+ return input;
+ } else if (input.equals("test")) {
+ return input;
+ }
+ throw new OptionsParsingException("Not a valid command: '" + input + "' (should be "
+ + getTypeDescription() + ")");
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "build or test";
+ }
+ }
+
+ public static class Options extends OptionsBase {
+
+ @Option(name = "for_command",
+ defaultValue = "build",
+ category = "misc",
+ converter = CommandConverter.class,
+ help = "The command for which the options should be canonicalized.")
+ public String forCommand;
+ }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ BlazeCommand command = runtime.getCommandMap().get(
+ options.getOptions(Options.class).forCommand);
+ Collection<Class<? extends OptionsBase>> optionsClasses =
+ BlazeCommandUtils.getOptions(
+ command.getClass(), runtime.getBlazeModules(), runtime.getRuleClassProvider());
+ try {
+ List<String> result = OptionsParser.canonicalize(optionsClasses, options.getResidue());
+ for (String piece : result) {
+ runtime.getReporter().getOutErr().printOutLn(piece);
+ }
+ } catch (OptionsParsingException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
new file mode 100644
index 0000000..3fd300e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
@@ -0,0 +1,185 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.util.CommandBuilder;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.ProcessUtils;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+/**
+ * Implements 'blaze clean'.
+ */
+@Command(name = "clean",
+ builds = true, // Does not, but people expect build options to be there
+ options = { CleanCommand.Options.class },
+ help = "resource:clean.txt",
+ shortDescription = "Removes output files and optionally stops the server.",
+ // TODO(bazel-team): Remove this - we inherit a huge number of unused options.
+ inherits = { BuildCommand.class })
+public final class CleanCommand implements BlazeCommand {
+
+ /**
+ * An interface for special options for the clean command.
+ */
+ public static class Options extends OptionsBase {
+ @Option(name = "clean_style",
+ defaultValue = "",
+ category = "clean",
+ help = "Can be either 'expunge' or 'expunge_async'.")
+ public String cleanStyle;
+
+ @Option(name = "expunge",
+ defaultValue = "false",
+ category = "clean",
+ expansion = "--clean_style=expunge",
+ help = "If specified, clean will remove the entire working tree for this Blaze " +
+ "instance, which includes all Blaze-created temporary and build output " +
+ "files, and it will stop the Blaze server if it is running.")
+ public boolean expunge;
+
+ @Option(name = "expunge_async",
+ defaultValue = "false",
+ category = "clean",
+ expansion = "--clean_style=expunge_async",
+ help = "If specified, clean will asynchronously remove the entire working tree for " +
+ "this Blaze instance, which includes all Blaze-created temporary and build " +
+ "output files, and it will stop the Blaze server if it is running. When this " +
+ "command completes, it will be safe to execute new commands in the same client, " +
+ "even though the deletion may continue in the background.")
+ public boolean expunge_async;
+ }
+
+ private static Logger LOG = Logger.getLogger(CleanCommand.class.getName());
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws ShutdownBlazeServerException {
+ Options cleanOptions = options.getOptions(Options.class);
+ cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async");
+ cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge");
+
+ if (cleanOptions.expunge == false && cleanOptions.expunge_async == false &&
+ !cleanOptions.cleanStyle.isEmpty()) {
+ runtime.getReporter().handle(Event.error(
+ null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ String cleanBanner = cleanOptions.expunge_async ?
+ "Starting clean." :
+ "Starting clean (this may take a while). " +
+ "Consider using --expunge_async if the clean takes more than several minutes.";
+
+ runtime.getReporter().handle(Event.info(null/*location*/, cleanBanner));
+ try {
+ String symlinkPrefix =
+ options.getOptions(BuildRequest.BuildRequestOptions.class).symlinkPrefix;
+ actuallyClean(runtime, runtime.getOutputBase(), cleanOptions, symlinkPrefix);
+ return ExitCode.SUCCESS;
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ } catch (CommandException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.RUN_FAILURE;
+ } catch (ExecException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.RUN_FAILURE;
+ } catch (InterruptedException e) {
+ runtime.getReporter().handle(Event.error("clean interrupted"));
+ return ExitCode.INTERRUPTED;
+ }
+ }
+
+ private void actuallyClean(BlazeRuntime runtime,
+ Path outputBase, Options cleanOptions, String symlinkPrefix) throws IOException,
+ ShutdownBlazeServerException, CommandException, ExecException, InterruptedException {
+ if (runtime.getOutputService() != null) {
+ runtime.getOutputService().clean();
+ }
+ if (cleanOptions.expunge) {
+ LOG.info("Expunging...");
+ // Delete the big subdirectories with the important content first--this
+ // will take the most time. Then quickly delete the little locks, logs
+ // and links right before we exit. Once the lock file is gone there will
+ // be a small possibility of a server race if a client is waiting, but
+ // all significant files will be gone by then.
+ FileSystemUtils.deleteTreesBelow(outputBase);
+ FileSystemUtils.deleteTree(outputBase);
+ } else if (cleanOptions.expunge_async) {
+ LOG.info("Expunging asynchronously...");
+ String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();
+
+ // Keeping tempOutputBase in the same directory ensures it remains in the
+ // same file system, and therefore the mv will be atomic and fast.
+ Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
+ outputBase.renameTo(tempOutputBase);
+ runtime.getReporter().handle(Event.info(
+ null, "Output base moved to " + tempOutputBase + " for deletion"));
+
+ // Daemonize the shell and use the double-fork idiom to ensure that the shell
+ // exits even while the "rm -rf" command continues.
+ String command = String.format("exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
+ ShellEscaper.escapeString(tempOutputBase.getPathString()));
+
+ LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));
+
+ // Doesn't throw iff command exited and was successful.
+ new CommandBuilder().addArg(command).useShell(true)
+ .setWorkingDir(tempOutputBase.getParentDirectory())
+ .build().execute();
+ } else {
+ LOG.info("Output cleaning...");
+ runtime.clearCaches();
+ for (String directory : new String[] {
+ BlazeDirectories.RELATIVE_OUTPUT_PATH, runtime.getWorkspaceName() }) {
+ Path child = outputBase.getChild(directory);
+ if (child.exists()) {
+ LOG.finest("Cleaning " + child);
+ FileSystemUtils.deleteTreesBelow(child);
+ }
+ }
+ }
+ // remove convenience links
+ OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
+ runtime.getWorkspaceName(), runtime.getWorkspace(), runtime.getReporter(), symlinkPrefix);
+ // shutdown on expunge cleans
+ if (cleanOptions.expunge || cleanOptions.expunge_async) {
+ throw new ShutdownBlazeServerException(0);
+ }
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
new file mode 100644
index 0000000..5267e71
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
@@ -0,0 +1,248 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.docgen.BlazeRuleHelpPrinter;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandUtils;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The 'blaze help' command, which prints all available commands as well as
+ * specific help pages.
+ */
+@Command(name = "help",
+ options = { HelpCommand.Options.class },
+ allowResidue = true,
+ mustRunInWorkspace = false,
+ shortDescription = "Prints help for commands, or the index.",
+ help = "resource:help.txt")
+public final class HelpCommand implements BlazeCommand {
+ public static class Options extends OptionsBase {
+
+ @Option(name = "help_verbosity",
+ category = "help",
+ defaultValue = "medium",
+ converter = Converters.HelpVerbosityConverter.class,
+ help = "Select the verbosity of the help command.")
+ public OptionsParser.HelpVerbosity helpVerbosity;
+
+ @Option(name = "long",
+ abbrev = 'l',
+ defaultValue = "null",
+ category = "help",
+ expansion = {"--help_verbosity", "long"},
+ help = "Show full description of each option, instead of just its name.")
+ public Void showLongFormOptions;
+
+ @Option(name = "short",
+ defaultValue = "null",
+ category = "help",
+ expansion = {"--help_verbosity", "short"},
+ help = "Show only the names of the options, not their types or meanings.")
+ public Void showShortFormOptions;
+ }
+
+ /**
+ * Returns a map that maps option categories to descriptive help strings for categories that
+ * are not part of the Bazel core.
+ */
+ private ImmutableMap<String, String> getOptionCategories(BlazeRuntime runtime) {
+ ImmutableMap.Builder<String, String> optionCategoriesBuilder = ImmutableMap.builder();
+ optionCategoriesBuilder
+ .put("checking",
+ "Checking options, which control Blaze's error checking and/or warnings")
+ .put("coverage",
+ "Options that affect how Blaze generates code coverage information")
+ .put("experimental",
+ "Experimental options, which control experimental (and potentially risky) features")
+ .put("flags",
+ "Flags options, for passing options to other tools")
+ .put("help",
+ "Help options")
+ .put("host jvm startup",
+ "Options that affect the startup of the Blaze server's JVM")
+ .put("misc",
+ "Miscellaneous options")
+ .put("package loading",
+ "Options that specify how to locate packages")
+ .put("query",
+ "Options affecting the 'blaze query' dependency query command")
+ .put("run",
+ "Options specific to 'blaze run'")
+ .put("semantics",
+ "Semantics options, which affect the build commands and/or output file contents")
+ .put("server startup",
+ "Startup options, which affect the startup of the Blaze server")
+ .put("strategy",
+ "Strategy options, which affect how Blaze will execute the build")
+ .put("testing",
+ "Options that affect how Blaze runs tests")
+ .put("verbosity",
+ "Verbosity options, which control what Blaze prints")
+ .put("version",
+ "Version options, for selecting which version of other tools will be used")
+ .put("what",
+ "Output selection options, for determining what to build/test");
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ optionCategoriesBuilder.putAll(module.getOptionCategories());
+ }
+ return optionCategoriesBuilder.build();
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ OutErr outErr = runtime.getReporter().getOutErr();
+ Options helpOptions = options.getOptions(Options.class);
+ if (options.getResidue().isEmpty()) {
+ emitBlazeVersionInfo(outErr);
+ emitGenericHelp(runtime, outErr);
+ return ExitCode.SUCCESS;
+ }
+ if (options.getResidue().size() != 1) {
+ runtime.getReporter().handle(Event.error("You must specify exactly one command"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ String helpSubject = options.getResidue().get(0);
+ if (helpSubject.equals("startup_options")) {
+ emitBlazeVersionInfo(outErr);
+ emitStartupOptions(outErr, helpOptions.helpVerbosity, runtime, getOptionCategories(runtime));
+ return ExitCode.SUCCESS;
+ } else if (helpSubject.equals("target-syntax")) {
+ emitBlazeVersionInfo(outErr);
+ emitTargetSyntaxHelp(outErr, getOptionCategories(runtime));
+ return ExitCode.SUCCESS;
+ } else if (helpSubject.equals("info-keys")) {
+ emitInfoKeysHelp(runtime, outErr);
+ return ExitCode.SUCCESS;
+ }
+
+ BlazeCommand command = runtime.getCommandMap().get(helpSubject);
+ if (command == null) {
+ ConfiguredRuleClassProvider provider = runtime.getRuleClassProvider();
+ RuleClass ruleClass = provider.getRuleClassMap().get(helpSubject);
+ if (ruleClass != null && ruleClass.isDocumented()) {
+ // There is a rule with a corresponding name
+ outErr.printOut(BlazeRuleHelpPrinter.getRuleDoc(helpSubject, provider));
+ return ExitCode.SUCCESS;
+ } else {
+ runtime.getReporter().handle(Event.error(
+ null, "'" + helpSubject + "' is neither a command nor a build rule"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ }
+ emitBlazeVersionInfo(outErr);
+ outErr.printOut(BlazeCommandUtils.getUsage(
+ command.getClass(),
+ getOptionCategories(runtime),
+ helpOptions.helpVerbosity,
+ runtime.getBlazeModules(),
+ runtime.getRuleClassProvider()));
+ return ExitCode.SUCCESS;
+ }
+
+ private void emitBlazeVersionInfo(OutErr outErr) {
+ String releaseInfo = BlazeVersionInfo.instance().getReleaseName();
+ String line = "[Blaze " + releaseInfo + "]";
+ outErr.printOut(String.format("%80s\n", line));
+ }
+
+ @SuppressWarnings("unchecked") // varargs generic array creation
+ private void emitStartupOptions(OutErr outErr, OptionsParser.HelpVerbosity helpVerbosity,
+ BlazeRuntime runtime, ImmutableMap<String, String> optionCategories) {
+ outErr.printOut(
+ BlazeCommandUtils.expandHelpTopic("startup_options",
+ "resource:startup_options.txt",
+ getClass(),
+ BlazeCommandUtils.getStartupOptions(runtime.getBlazeModules()),
+ optionCategories,
+ helpVerbosity));
+ }
+
+ private void emitTargetSyntaxHelp(OutErr outErr, ImmutableMap<String, String> optionCategories) {
+ outErr.printOut(BlazeCommandUtils.expandHelpTopic("target-syntax",
+ "resource:target-syntax.txt",
+ getClass(),
+ ImmutableList.<Class<? extends OptionsBase>>of(),
+ optionCategories,
+ OptionsParser.HelpVerbosity.MEDIUM));
+ }
+
+ private void emitInfoKeysHelp(BlazeRuntime runtime, OutErr outErr) {
+ for (InfoKey key : InfoKey.values()) {
+ outErr.printOut(String.format("%-23s %s\n", key.getName(), key.getDescription()));
+ }
+
+ for (BlazeModule.InfoItem item : InfoCommand.getInfoItemMap(runtime,
+ OptionsParser.newOptionsParser(
+ ImmutableList.<Class<? extends OptionsBase>>of())).values()) {
+ outErr.printOut(String.format("%-23s %s\n", item.getName(), item.getDescription()));
+ }
+ }
+
+ private void emitGenericHelp(BlazeRuntime runtime, OutErr outErr) {
+ outErr.printOut("Usage: blaze <command> <options> ...\n\n");
+
+ outErr.printOut("Available commands:\n");
+
+ Map<String, BlazeCommand> commandsByName = runtime.getCommandMap();
+ List<String> namesInOrder = new ArrayList<>(commandsByName.keySet());
+ Collections.sort(namesInOrder);
+
+ for (String name : namesInOrder) {
+ BlazeCommand command = commandsByName.get(name);
+ Command annotation = command.getClass().getAnnotation(Command.class);
+ if (annotation.hidden()) {
+ continue;
+ }
+
+ String shortDescription = annotation.shortDescription();
+ outErr.printOut(String.format(" %-19s %s\n", name, shortDescription));
+ }
+
+ outErr.printOut("\n");
+ outErr.printOut("Getting more help:\n");
+ outErr.printOut(" blaze help <command>\n");
+ outErr.printOut(" Prints help and options for <command>.\n");
+ outErr.printOut(" blaze help startup_options\n");
+ outErr.printOut(" Options for the JVM hosting Blaze.\n");
+ outErr.printOut(" blaze help target-syntax\n");
+ outErr.printOut(" Explains the syntax for specifying targets.\n");
+ outErr.printOut(" blaze help info-keys\n");
+ outErr.printOut(" Displays a list of keys used by the info command.\n");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java
new file mode 100644
index 0000000..31aaeb1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java
@@ -0,0 +1,448 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.ProtoUtils;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.AllowedRuleClassInfo;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.AttributeDefinition;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.BuildLanguage;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.RuleDefinition;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Implementation of 'blaze info'.
+ */
+@Command(name = "info",
+ // TODO(bazel-team): this is not really a build command, but needs access to the
+ // configuration options to do its job
+ builds = true,
+ allowResidue = true,
+ binaryStdOut = true,
+ help = "resource:info.txt",
+ shortDescription = "Displays runtime info about the blaze server.",
+ options = { InfoCommand.Options.class },
+ // We have InfoCommand inherit from {@link BuildCommand} because we want all
+ // configuration defaults specified in ~/.blazerc for {@code build} to apply to
+ // {@code info} too, even though it doesn't actually do a build.
+ //
+ // (Ideally there would be a way to make {@code info} inherit just the bare
+ // minimum of relevant options from {@code build}, i.e. those that affect the
+ // values it prints. But there's no such mechanism.)
+ inherits = { BuildCommand.class })
+public class InfoCommand implements BlazeCommand {
+
+ public static class Options extends OptionsBase {
+ @Option(name = "show_make_env",
+ defaultValue = "false",
+ category = "misc",
+ help = "Include the \"Make\" environment in the output.")
+ public boolean showMakeEnvironment;
+ }
+
+ /**
+ * Unchecked variant of ExitCausingException. Below, we need to throw from the Supplier interface,
+ * which does not allow checked exceptions.
+ */
+ public static class ExitCausingRuntimeException extends RuntimeException {
+
+ private final ExitCode exitCode;
+
+ public ExitCausingRuntimeException(String message, ExitCode exitCode) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ public ExitCausingRuntimeException(ExitCode exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ public ExitCode getExitCode() {
+ return exitCode;
+ }
+ }
+
+ private static class HardwiredInfoItem implements BlazeModule.InfoItem {
+ private final InfoKey key;
+ private final BlazeRuntime runtime;
+ private final OptionsProvider commandOptions;
+
+ private HardwiredInfoItem(InfoKey key, BlazeRuntime runtime, OptionsProvider commandOptions) {
+ this.key = key;
+ this.runtime = runtime;
+ this.commandOptions = commandOptions;
+ }
+
+ @Override
+ public String getName() {
+ return key.getName();
+ }
+
+ @Override
+ public String getDescription() {
+ return key.getDescription();
+ }
+
+ @Override
+ public boolean isHidden() {
+ return key.isHidden();
+ }
+
+ @Override
+ public byte[] get(Supplier<BuildConfiguration> configurationSupplier) {
+ return print(getInfoItem(runtime, key, configurationSupplier, commandOptions));
+ }
+ }
+
+ private static class MakeInfoItem implements BlazeModule.InfoItem {
+ private final String name;
+ private final String value;
+
+ private MakeInfoItem(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Make environment variable '" + name + "'";
+ }
+
+ @Override
+ public boolean isHidden() {
+ return false;
+ }
+
+ @Override
+ public byte[] get(Supplier<BuildConfiguration> configurationSupplier) {
+ return print(value);
+ }
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { }
+
+ @Override
+ public ExitCode exec(final BlazeRuntime runtime, final OptionsProvider optionsProvider) {
+ Options infoOptions = optionsProvider.getOptions(Options.class);
+
+ OutErr outErr = runtime.getReporter().getOutErr();
+ // Creating a BuildConfiguration is expensive and often unnecessary. Delay the creation until
+ // it is needed.
+ Supplier<BuildConfiguration> configurationSupplier = new Supplier<BuildConfiguration>() {
+ private BuildConfiguration configuration;
+ @Override
+ public BuildConfiguration get() {
+ if (configuration != null) {
+ return configuration;
+ }
+ try {
+ // In order to be able to answer configuration-specific queries, we need to setup the
+ // package path. Since info inherits all the build options, all the necessary information
+ // is available here.
+ runtime.setupPackageCache(
+ optionsProvider.getOptions(PackageCacheOptions.class),
+ runtime.getDefaultsPackageContent(optionsProvider));
+ // TODO(bazel-team): What if there are multiple configurations? [multi-config]
+ configuration = runtime
+ .getConfigurations(optionsProvider)
+ .getTargetConfigurations().get(0);
+ return configuration;
+ } catch (InvalidConfigurationException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ throw new ExitCausingRuntimeException(ExitCode.COMMAND_LINE_ERROR);
+ } catch (AbruptExitException e) {
+ throw new ExitCausingRuntimeException("unknown error: " + e.getMessage(),
+ e.getExitCode());
+ } catch (InterruptedException e) {
+ runtime.getReporter().handle(Event.error("interrupted"));
+ throw new ExitCausingRuntimeException(ExitCode.INTERRUPTED);
+ }
+ }
+ };
+
+ Map<String, BlazeModule.InfoItem> items = getInfoItemMap(runtime, optionsProvider);
+
+ try {
+ if (infoOptions.showMakeEnvironment) {
+ Map<String, String> makeEnv = configurationSupplier.get().getMakeEnvironment();
+ for (Map.Entry<String, String> entry : makeEnv.entrySet()) {
+ BlazeModule.InfoItem item = new MakeInfoItem(entry.getKey(), entry.getValue());
+ items.put(item.getName(), item);
+ }
+ }
+
+ List<String> residue = optionsProvider.getResidue();
+ if (residue.size() > 1) {
+ runtime.getReporter().handle(Event.error("at most one key may be specified"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ String key = residue.size() == 1 ? residue.get(0) : null;
+ if (key != null) { // print just the value for the specified key:
+ byte[] value;
+ if (items.containsKey(key)) {
+ value = items.get(key).get(configurationSupplier);
+ } else {
+ runtime.getReporter().handle(Event.error("unknown key: '" + key + "'"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ try {
+ outErr.getOutputStream().write(value);
+ outErr.getOutputStream().flush();
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error("Cannot write info block: " + e.getMessage()));
+ return ExitCode.ANALYSIS_FAILURE;
+ }
+ } else { // print them all
+ configurationSupplier.get(); // We'll need this later anyway
+ for (BlazeModule.InfoItem infoItem : items.values()) {
+ if (infoItem.isHidden()) {
+ continue;
+ }
+ outErr.getOutputStream().write(
+ (infoItem.getName() + ": ").getBytes(StandardCharsets.UTF_8));
+ outErr.getOutputStream().write(infoItem.get(configurationSupplier));
+ }
+ }
+ } catch (AbruptExitException e) {
+ return e.getExitCode();
+ } catch (ExitCausingRuntimeException e) {
+ return e.getExitCode();
+ } catch (IOException e) {
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ /**
+ * Compute and return the info for the given key. Only keys that are not hidden are supported
+ * here.
+ */
+ private static Object getInfoItem(BlazeRuntime runtime, InfoKey key,
+ Supplier<BuildConfiguration> configurationSupplier, OptionsProvider options) {
+ switch (key) {
+ // directories
+ case WORKSPACE : return runtime.getWorkspace();
+ case INSTALL_BASE : return runtime.getDirectories().getInstallBase();
+ case OUTPUT_BASE : return runtime.getOutputBase();
+ case EXECUTION_ROOT : return runtime.getExecRoot();
+ case OUTPUT_PATH : return runtime.getDirectories().getOutputPath();
+ // These are the only (non-hidden) info items that require a configuration, because the
+ // corresponding paths contain the short name. Maybe we should recommend using the symlinks
+ // or make them hidden by default?
+ case BLAZE_BIN : return configurationSupplier.get().getBinDirectory().getPath();
+ case BLAZE_GENFILES : return configurationSupplier.get().getGenfilesDirectory().getPath();
+ case BLAZE_TESTLOGS : return configurationSupplier.get().getTestLogsDirectory().getPath();
+
+ // logs
+ case COMMAND_LOG : return BlazeCommandDispatcher.getCommandLogPath(runtime.getOutputBase());
+ case MESSAGE_LOG :
+ // NB: Duplicated in EventLogModule
+ return runtime.getOutputBase().getRelative("message.log");
+
+ // misc
+ case RELEASE : return BlazeVersionInfo.instance().getReleaseName();
+ case SERVER_PID : return OsUtils.getpid();
+ case PACKAGE_PATH : return getPackagePath(options);
+
+ // memory statistics
+ case GC_COUNT :
+ case GC_TIME :
+ // The documentation is not very clear on what it means to have more than
+ // one GC MXBean, so we just sum them up.
+ int gcCount = 0;
+ int gcTime = 0;
+ for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
+ gcCount += gcBean.getCollectionCount();
+ gcTime += gcBean.getCollectionTime();
+ }
+ if (key == InfoKey.GC_COUNT) {
+ return gcCount + "";
+ } else {
+ return gcTime + "ms";
+ }
+
+ case MAX_HEAP_SIZE :
+ return StringUtilities.prettyPrintBytes(getMemoryUsage().getMax());
+ case USED_HEAP_SIZE :
+ case COMMITTED_HEAP_SIZE :
+ return StringUtilities.prettyPrintBytes(key == InfoKey.USED_HEAP_SIZE ?
+ getMemoryUsage().getUsed() : getMemoryUsage().getCommitted());
+
+ case USED_HEAP_SIZE_AFTER_GC :
+ // Note that this info value is not printed by default, but only when explicitly requested.
+ System.gc();
+ return StringUtilities.prettyPrintBytes(getMemoryUsage().getUsed());
+
+ case DEFAULTS_PACKAGE:
+ return runtime.getDefaultsPackageContent();
+
+ case BUILD_LANGUAGE:
+ return getBuildLanguageDefinition(runtime.getRuleClassProvider());
+
+ case DEFAULT_PACKAGE_PATH:
+ return Joiner.on(":").join(Constants.DEFAULT_PACKAGE_PATH);
+
+ default:
+ throw new IllegalArgumentException("missing implementation for " + key);
+ }
+ }
+
+ private static MemoryUsage getMemoryUsage() {
+ MemoryMXBean memBean = ManagementFactory.getMemoryMXBean();
+ return memBean.getHeapMemoryUsage();
+ }
+
+ /**
+ * Get the package_path variable for the given set of options.
+ */
+ private static String getPackagePath(OptionsProvider options) {
+ PackageCacheOptions packageCacheOptions =
+ options.getOptions(PackageCacheOptions.class);
+ return Joiner.on(":").join(packageCacheOptions.packagePath);
+ }
+
+ private static AllowedRuleClassInfo getAllowedRuleClasses(
+ Collection<RuleClass> ruleClasses, Attribute attr) {
+ AllowedRuleClassInfo.Builder info = AllowedRuleClassInfo.newBuilder();
+ info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.ANY);
+
+ if (attr.isStrictLabelCheckingEnabled()) {
+ if (attr.getAllowedRuleClassesPredicate() != Predicates.<RuleClass>alwaysTrue()) {
+ info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.SPECIFIED);
+ Predicate<RuleClass> filter = attr.getAllowedRuleClassesPredicate();
+ for (RuleClass otherClass : Iterables.filter(
+ ruleClasses, filter)) {
+ if (otherClass.isDocumented()) {
+ info.addAllowedRuleClass(otherClass.getName());
+ }
+ }
+ }
+ }
+
+ return info.build();
+ }
+
+ /**
+ * Returns a byte array containing a proto-buffer describing the build language.
+ */
+ private static byte[] getBuildLanguageDefinition(RuleClassProvider provider) {
+ BuildLanguage.Builder resultPb = BuildLanguage.newBuilder();
+ Collection<RuleClass> ruleClasses = provider.getRuleClassMap().values();
+ for (RuleClass ruleClass : ruleClasses) {
+ if (!ruleClass.isDocumented()) {
+ continue;
+ }
+
+ RuleDefinition.Builder rulePb = RuleDefinition.newBuilder();
+ rulePb.setName(ruleClass.getName());
+ for (Attribute attr : ruleClass.getAttributes()) {
+ if (!attr.isDocumented()) {
+ continue;
+ }
+
+ AttributeDefinition.Builder attrPb = AttributeDefinition.newBuilder();
+ attrPb.setName(attr.getName());
+ // The protocol compiler, in its infinite wisdom, generates the field as one of the
+ // integer type and the getTypeEnum() method is missing. WTF?
+ attrPb.setType(ProtoUtils.getDiscriminatorFromType(attr.getType()));
+ attrPb.setMandatory(attr.isMandatory());
+
+ if (Type.isLabelType(attr.getType())) {
+ attrPb.setAllowedRuleClasses(getAllowedRuleClasses(ruleClasses, attr));
+ }
+
+ rulePb.addAttribute(attrPb);
+ }
+
+ resultPb.addRule(rulePb);
+ }
+
+ return resultPb.build().toByteArray();
+ }
+
+ private static byte[] print(Object value) {
+ if (value instanceof byte[]) {
+ return (byte[]) value;
+ }
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ PrintWriter writer = new PrintWriter(outputStream);
+ writer.print(value.toString() + "\n");
+ writer.flush();
+ return outputStream.toByteArray();
+ }
+
+ static Map<String, BlazeModule.InfoItem> getInfoItemMap(
+ BlazeRuntime runtime, OptionsProvider commandOptions) {
+ Map<String, BlazeModule.InfoItem> result = new TreeMap<>(); // order by key
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ for (BlazeModule.InfoItem item : module.getInfoItems()) {
+ result.put(item.getName(), item);
+ }
+ }
+
+ for (InfoKey key : InfoKey.values()) {
+ BlazeModule.InfoItem item = new HardwiredInfoItem(key, runtime, commandOptions);
+ result.put(item.getName(), item);
+ }
+
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java
new file mode 100644
index 0000000..d2e7bc0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+
+/**
+ * An enumeration of all the valid info keys, excepting the make environment
+ * variables.
+ */
+public enum InfoKey {
+ // directories
+ WORKSPACE("workspace", "The working directory of the server."),
+ INSTALL_BASE("install_base", "The installation base directory."),
+ OUTPUT_BASE("output_base",
+ "A directory for shared Blaze state as well as tool and strategy specific subdirectories."),
+ EXECUTION_ROOT("execution_root",
+ "A directory that makes all input and output files visible to the build."),
+ OUTPUT_PATH("output_path", "Output directory"),
+ BLAZE_BIN("blaze-bin", "Configuration dependent directory for binaries."),
+ BLAZE_GENFILES("blaze-genfiles", "Configuration dependent directory for generated files."),
+ BLAZE_TESTLOGS("blaze-testlogs", "Configuration dependent directory for logs from a test run."),
+
+ // logs
+ COMMAND_LOG("command_log", "Location of the log containg the output from the build commands."),
+ MESSAGE_LOG("message_log" ,
+ "Location of a log containing machine readable message in LogMessage protobuf format."),
+
+ // misc
+ RELEASE("release", "Blaze release identifier"),
+ SERVER_PID("server_pid", "Blaze process id"),
+ PACKAGE_PATH("package_path", "The search path for resolving package labels."),
+
+ // memory statistics
+ USED_HEAP_SIZE("used-heap-size", "The amount of used memory in bytes. Note that this is not a "
+ + "good indicator of the actual memory use, as it includes any remaining inaccessible "
+ + "memory."),
+ USED_HEAP_SIZE_AFTER_GC("used-heap-size-after-gc",
+ "The amount of used memory in bytes after a call to System.gc().", true),
+ COMMITTED_HEAP_SIZE("committed-heap-size",
+ "The amount of memory in bytes that is committed for the Java virtual machine to use"),
+ MAX_HEAP_SIZE("max-heap-size",
+ "The maximum amount of memory in bytes that can be used for memory management."),
+ GC_COUNT("gc-count", "Number of garbage collection runs."),
+ GC_TIME("gc-time", "The approximate accumulated time spend on garbage collection."),
+
+ // These are deprecated, they still work, when explicitly requested, but are not shown by default
+
+ // These keys print multi-line messages and thus don't play well with grep. We don't print them
+ // unless explicitly requested
+ DEFAULTS_PACKAGE("defaults-package", "Default packages used as implicit dependencies", true),
+ BUILD_LANGUAGE("build-language", "A protobuffer with the build language structure", true),
+ DEFAULT_PACKAGE_PATH("default-package-path", "The default package path", true);
+
+ private final String name;
+ private final String description;
+ private final boolean hidden;
+
+ private InfoKey(String name, String description) {
+ this(name, description, false);
+ }
+
+ private InfoKey(String name, String description, boolean hidden) {
+ this.name = name;
+ this.description = description;
+ this.hidden = hidden;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public boolean isHidden() {
+ return hidden;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java
new file mode 100644
index 0000000..7b91dc7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java
@@ -0,0 +1,771 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.TreeMultimap;
+import com.google.devtools.build.lib.actions.MiddlemanAction;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.profiler.ProfileInfo;
+import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry;
+import com.google.devtools.build.lib.profiler.ProfileInfo.InfoListener;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.profiler.chart.AggregatingChartCreator;
+import com.google.devtools.build.lib.profiler.chart.Chart;
+import com.google.devtools.build.lib.profiler.chart.ChartCreator;
+import com.google.devtools.build.lib.profiler.chart.DetailedChartCreator;
+import com.google.devtools.build.lib.profiler.chart.HtmlChartVisitor;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.build.lib.util.TimeUtilities;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Command line wrapper for analyzing Blaze build profiles.
+ */
+@Command(name = "analyze-profile",
+ options = { ProfileCommand.ProfileOptions.class },
+ shortDescription = "Analyzes build profile data.",
+ help = "resource:analyze-profile.txt",
+ allowResidue = true,
+ mustRunInWorkspace = false)
+public final class ProfileCommand implements BlazeCommand {
+
+ private final String TWO_COLUMN_FORMAT = "%-37s %10s\n";
+ private final String THREE_COLUMN_FORMAT = "%-28s %10s %8s\n";
+
+ public static class DumpConverter extends Converters.StringSetConverter {
+ public DumpConverter() {
+ super("text", "raw", "text-unsorted", "raw-unsorted");
+ }
+ }
+
+ public static class ProfileOptions extends OptionsBase {
+ @Option(name = "dump",
+ abbrev='d',
+ converter = DumpConverter.class,
+ defaultValue = "null",
+ help = "output full profile data dump either in human-readable 'text' format or"
+ + " script-friendly 'raw' format, either sorted or unsorted.")
+ public String dumpMode;
+
+ @Option(name = "html",
+ defaultValue = "false",
+ help = "If present, an HTML file visualizing the tasks of the profiled build is created. "
+ + "The name of the html file is the name of the profile file plus '.html'.")
+ public boolean html;
+
+ @Option(name = "html_pixels_per_second",
+ defaultValue = "50",
+ help = "Defines the scale of the time axis of the task diagram. The unit is "
+ + "pixels per second. Default is 50 pixels per second. ")
+ public int htmlPixelsPerSecond;
+
+ @Option(name = "html_details",
+ defaultValue = "false",
+ help = "If --html_details is present, the task diagram contains all tasks of the profile. "
+ + "If --nohtml_details is present, an aggregated diagram is generated. The default is "
+ + "to generate an aggregated diagram.")
+ public boolean htmlDetails;
+
+ @Option(name = "vfs_stats",
+ defaultValue = "false",
+ help = "If present, include VFS path statistics.")
+ public boolean vfsStats;
+
+ @Option(name = "vfs_stats_limit",
+ defaultValue = "-1",
+ help = "Maximum number of VFS path statistics to print.")
+ public int vfsStatsLimit;
+ }
+
+ private Function<String, String> currentPathMapping = Functions.<String>identity();
+
+ private InfoListener getInfoListener(final BlazeRuntime runtime) {
+ return new InfoListener() {
+ private final EventHandler reporter = runtime.getReporter();
+
+ @Override
+ public void info(String text) {
+ reporter.handle(Event.info(text));
+ }
+
+ @Override
+ public void warn(String text) {
+ reporter.handle(Event.warn(text));
+ }
+ };
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(final BlazeRuntime runtime, OptionsProvider options) {
+ ProfileOptions opts =
+ options.getOptions(ProfileOptions.class);
+
+ if (!opts.vfsStats) {
+ opts.vfsStatsLimit = 0;
+ }
+
+ currentPathMapping = new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ if (runtime.getWorkspaceName().isEmpty()) {
+ return input;
+ } else {
+ return input.substring(input.lastIndexOf("/" + runtime.getWorkspaceName()) + 1);
+ }
+ }
+ };
+
+ PrintStream out = new PrintStream(runtime.getReporter().getOutErr().getOutputStream());
+ try {
+ runtime.getReporter().handle(Event.warn(
+ null, "This information is intended for consumption by Blaze developers"
+ + " only, and may change at any time. Script against it at your own risk"));
+
+ for (String name : options.getResidue()) {
+ Path profileFile = runtime.getWorkingDirectory().getRelative(name);
+ try {
+ ProfileInfo info = ProfileInfo.loadProfileVerbosely(
+ profileFile, getInfoListener(runtime));
+ if (opts.dumpMode != null) {
+ dumpProfile(runtime, info, out, opts.dumpMode);
+ } else if (opts.html) {
+ createHtml(runtime, info, profileFile, opts);
+ } else {
+ createText(runtime, info, out, opts);
+ }
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error(
+ null, "Failed to process file " + name + ": " + e.getMessage()));
+ }
+ }
+ } finally {
+ out.flush();
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ private void createText(BlazeRuntime runtime, ProfileInfo info, PrintStream out,
+ ProfileOptions opts) {
+ List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts);
+
+ for (ProfilePhaseStatistics stat : statistics) {
+ String title = stat.getTitle();
+
+ if (!title.equals("")) {
+ out.println("\n=== " + title.toUpperCase() + " ===\n");
+ }
+ out.print(stat.getStatistics());
+ }
+ }
+
+ private void createHtml(BlazeRuntime runtime, ProfileInfo info, Path profileFile,
+ ProfileOptions opts)
+ throws IOException {
+ Path htmlFile =
+ profileFile.getParentDirectory().getChild(profileFile.getBaseName() + ".html");
+ List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts);
+
+ runtime.getReporter().handle(Event.info("Creating HTML output in " + htmlFile));
+
+ ChartCreator chartCreator =
+ opts.htmlDetails ? new DetailedChartCreator(info, statistics)
+ : new AggregatingChartCreator(info, statistics);
+ Chart chart = chartCreator.create();
+ OutputStream out = new BufferedOutputStream(htmlFile.getOutputStream());
+ try {
+ chart.accept(new HtmlChartVisitor(new PrintStream(out), opts.htmlPixelsPerSecond));
+ } finally {
+ try {
+ out.close();
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+ }
+
+ private List<ProfilePhaseStatistics> getStatistics(
+ BlazeRuntime runtime, ProfileInfo info, ProfileOptions opts) {
+ try {
+ ProfileInfo.aggregateProfile(info, getInfoListener(runtime));
+ runtime.getReporter().handle(Event.info("Analyzing relationships"));
+
+ info.analyzeRelationships();
+
+ List<ProfilePhaseStatistics> statistics = new ArrayList<>();
+
+ // Print phase durations and total execution time
+ ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(byteOutput, false, "UTF-8");
+ long duration = 0;
+ for (ProfilePhase phase : ProfilePhase.values()) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask != null) {
+ duration += info.getPhaseDuration(phaseTask);
+ }
+ }
+ for (ProfilePhase phase : ProfilePhase.values()) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask != null) {
+ long phaseDuration = info.getPhaseDuration(phaseTask);
+ out.printf(THREE_COLUMN_FORMAT, "Total " + phase.nick + " phase time",
+ TimeUtilities.prettyTime(phaseDuration), prettyPercentage(phaseDuration, duration));
+ }
+ }
+ out.printf(THREE_COLUMN_FORMAT, "Total run time", TimeUtilities.prettyTime(duration),
+ "100.00%");
+ statistics.add(new ProfilePhaseStatistics("Phase Summary Information",
+ new String(byteOutput.toByteArray(), "UTF-8")));
+
+ // Print details of major phases
+ if (duration > 0) {
+ statistics.add(formatInitPhaseStatistics(info, opts));
+ statistics.add(formatLoadingPhaseStatistics(info, opts));
+ statistics.add(formatAnalysisPhaseStatistics(info, opts));
+ ProfilePhaseStatistics stat = formatExecutionPhaseStatistics(info, opts);
+ if (stat != null) {
+ statistics.add(stat);
+ }
+ }
+
+ return statistics;
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError("Should not happen since, UTF8 is available on all JVMs");
+ }
+ }
+
+ private void dumpProfile(
+ BlazeRuntime runtime, ProfileInfo info, PrintStream out, String dumpMode) {
+ if (!dumpMode.contains("unsorted")) {
+ ProfileInfo.aggregateProfile(info, getInfoListener(runtime));
+ }
+ if (dumpMode.contains("raw")) {
+ for (ProfileInfo.Task task : info.allTasksById) {
+ dumpRaw(task, out);
+ }
+ } else if (dumpMode.contains("unsorted")) {
+ for (ProfileInfo.Task task : info.allTasksById) {
+ dumpTask(task, out, 0);
+ }
+ } else {
+ for (ProfileInfo.Task task : info.rootTasksById) {
+ dumpTask(task, out, 0);
+ }
+ }
+ }
+
+ private void dumpTask(ProfileInfo.Task task, PrintStream out, int indent) {
+ StringBuilder builder = new StringBuilder(String.format(
+ "\n%s %s\nThread: %-6d Id: %-6d Parent: %d\nStart time: %-12s Duration: %s",
+ task.type, task.getDescription(), task.threadId, task.id, task.parentId,
+ TimeUtilities.prettyTime(task.startTime), TimeUtilities.prettyTime(task.duration)));
+ if (task.hasStats()) {
+ builder.append("\n");
+ ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray();
+ for (ProfilerTask type : ProfilerTask.values()) {
+ ProfileInfo.AggregateAttr attr = stats[type.ordinal()];
+ if (attr != null) {
+ builder.append(type.toString().toLowerCase()).append("=(").
+ append(attr.count).append(", ").
+ append(TimeUtilities.prettyTime(attr.totalTime)).append(") ");
+ }
+ }
+ }
+ out.println(StringUtil.indent(builder.toString(), indent));
+ for (ProfileInfo.Task subtask : task.subtasks) {
+ dumpTask(subtask, out, indent + 1);
+ }
+ }
+
+ private void dumpRaw(ProfileInfo.Task task, PrintStream out) {
+ StringBuilder aggregateString = new StringBuilder();
+ ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray();
+ for (ProfilerTask type : ProfilerTask.values()) {
+ ProfileInfo.AggregateAttr attr = stats[type.ordinal()];
+ if (attr != null) {
+ aggregateString.append(type.toString().toLowerCase()).append(",").
+ append(attr.count).append(",").append(attr.totalTime).append(" ");
+ }
+ }
+ out.println(
+ task.threadId + "|" + task.id + "|" + task.parentId + "|"
+ + task.startTime + "|" + task.duration + "|"
+ + aggregateString.toString().trim() + "|"
+ + task.type + "|" + task.getDescription());
+ }
+
+ /**
+ * Converts relative duration to the percentage string
+ * @return formatted percentage string or "N/A" if result is undefined.
+ */
+ private static String prettyPercentage(long duration, long total) {
+ if (total == 0) {
+ // Return "not available" string if total is 0 and result is undefined.
+ return "N/A";
+ }
+ return String.format("%5.2f%%", duration*100.0/total);
+ }
+
+ private void printCriticalPath(String title, PrintStream out, CriticalPathEntry path) {
+ out.println(String.format("\n%s (%s):", title,
+ TimeUtilities.prettyTime(path.cumulativeDuration)));
+
+ boolean lightCriticalPath = isLightCriticalPath(path);
+ out.println(lightCriticalPath ?
+ String.format("%6s %11s %8s %s", "Id", "Time", "Percentage", "Description")
+ : String.format("%6s %11s %8s %8s %s", "Id", "Time", "Share", "Critical", "Description"));
+
+ long totalPathTime = path.cumulativeDuration;
+ int middlemanCount = 0;
+ long middlemanDuration = 0L;
+ long middlemanCritTime = 0L;
+
+ for (; path != null ; path = path.next) {
+ if (path.task.id < 0) {
+ // Ignore fake actions.
+ continue;
+ } else if (path.task.getDescription().startsWith(MiddlemanAction.MIDDLEMAN_MNEMONIC + " ")
+ || path.task.getDescription().startsWith("TargetCompletionMiddleman")) {
+ // Aggregate middleman actions.
+ middlemanCount++;
+ middlemanDuration += path.duration;
+ middlemanCritTime += path.getCriticalTime();
+ } else {
+ String desc = path.task.getDescription().replace(':', ' ');
+ if (lightCriticalPath) {
+ out.println(String.format("%6d %11s %8s %s", path.task.id,
+ TimeUtilities.prettyTime(path.duration),
+ prettyPercentage(path.duration, totalPathTime),
+ desc));
+ } else {
+ out.println(String.format("%6d %11s %8s %8s %s", path.task.id,
+ TimeUtilities.prettyTime(path.duration),
+ prettyPercentage(path.duration, totalPathTime),
+ prettyPercentage(path.getCriticalTime(), totalPathTime), desc));
+ }
+ }
+ }
+ if (middlemanCount > 0) {
+ if (lightCriticalPath) {
+ out.println(String.format(" %11s %8s [%d middleman actions]",
+ TimeUtilities.prettyTime(middlemanDuration),
+ prettyPercentage(middlemanDuration, totalPathTime),
+ middlemanCount));
+ } else {
+ out.println(String.format(" %11s %8s %8s [%d middleman actions]",
+ TimeUtilities.prettyTime(middlemanDuration),
+ prettyPercentage(middlemanDuration, totalPathTime),
+ prettyPercentage(middlemanCritTime, totalPathTime), middlemanCount));
+ }
+ }
+ }
+
+ private boolean isLightCriticalPath(CriticalPathEntry path) {
+ return path.task.type == ProfilerTask.CRITICAL_PATH_COMPONENT;
+ }
+
+ private void printShortPhaseAnalysis(ProfileInfo info, PrintStream out, ProfilePhase phase) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask != null) {
+ long phaseDuration = info.getPhaseDuration(phaseTask);
+ out.printf(TWO_COLUMN_FORMAT, "Total " + phase.nick + " phase time",
+ TimeUtilities.prettyTime(phaseDuration));
+ printTimeDistributionByType(info, out, phaseTask);
+ }
+ }
+
+ private void printTimeDistributionByType(ProfileInfo info, PrintStream out,
+ ProfileInfo.Task phaseTask) {
+ List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask);
+ long phaseDuration = info.getPhaseDuration(phaseTask);
+ long totalDuration = phaseDuration;
+ for (ProfileInfo.Task task : taskList) {
+ // Tasks on the phaseTask thread already accounted for in the phaseDuration.
+ if (task.threadId != phaseTask.threadId) {
+ totalDuration += task.duration;
+ }
+ }
+ boolean headerNeeded = true;
+ for (ProfilerTask type : ProfilerTask.values()) {
+ ProfileInfo.AggregateAttr stats = info.getStatsForType(type, taskList);
+ if (stats.count > 0 && stats.totalTime > 0) {
+ if (headerNeeded) {
+ out.println("\nTotal time (across all threads) spent on:");
+ out.println(String.format("%18s %8s %8s %11s", "Type", "Total", "Count", "Average"));
+ headerNeeded = false;
+ }
+ out.println(String.format("%18s %8s %8d %11s", type.toString(),
+ prettyPercentage(stats.totalTime, totalDuration), stats.count,
+ TimeUtilities.prettyTime(stats.totalTime / stats.count)));
+ }
+ }
+ }
+
+ static class Stat implements Comparable<Stat> {
+ public long duration;
+ public long frequency;
+
+ @Override
+ public int compareTo(Stat o) {
+ return this.duration == o.duration ? Long.compare(this.frequency, o.frequency)
+ : Long.compare(this.duration, o.duration);
+ }
+ }
+
+ /**
+ * Print the time spent on VFS operations on each path. Output is grouped by operation and sorted
+ * by descending duration. If multiple of the same VFS operation were logged for the same path,
+ * print the total duration.
+ *
+ * @param info profiling data.
+ * @param out output stream.
+ * @param phase build phase.
+ * @param limit maximum number of statistics to print, or -1 for no limit.
+ */
+ private void printVfsStatistics(ProfileInfo info, PrintStream out,
+ ProfilePhase phase, int limit) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask == null) {
+ return;
+ }
+
+ if (limit == 0) {
+ return;
+ }
+
+ // Group into VFS operations and build maps from path to duration.
+
+ List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask);
+ EnumMap<ProfilerTask, Map<String, Stat>> stats = Maps.newEnumMap(ProfilerTask.class);
+
+ collectVfsEntries(stats, taskList);
+
+ if (!stats.isEmpty()) {
+ out.printf("\nVFS path statistics:\n");
+ out.printf("%15s %10s %10s %s\n", "Type", "Frequency", "Duration", "Path");
+ }
+
+ // Reverse the maps to get maps from duration to path. We use a TreeMultimap to sort by duration
+ // and because durations are not unique.
+
+ for (ProfilerTask type : stats.keySet()) {
+ Map<String, Stat> statsForType = stats.get(type);
+ TreeMultimap<Stat, String> sortedStats =
+ TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural());
+
+ for (Map.Entry<String, Stat> stat : statsForType.entrySet()) {
+ sortedStats.put(stat.getValue(), stat.getKey());
+ }
+
+ int numPrinted = 0;
+ for (Map.Entry<Stat, String> stat : sortedStats.entries()) {
+ if (limit != -1 && numPrinted++ == limit) {
+ out.printf("... %d more ...\n", sortedStats.size() - limit);
+ break;
+ }
+ out.printf("%15s %10d %10s %s\n",
+ type.name(), stat.getKey().frequency, TimeUtilities.prettyTime(stat.getKey().duration),
+ stat.getValue());
+ }
+ }
+ }
+
+ private void collectVfsEntries(EnumMap<ProfilerTask, Map<String, Stat>> stats,
+ List<ProfileInfo.Task> taskList) {
+ for (ProfileInfo.Task task : taskList) {
+ collectVfsEntries(stats, Arrays.asList(task.subtasks));
+ if (!task.type.name().startsWith("VFS_")) {
+ continue;
+ }
+
+ Map<String, Stat> statsForType = stats.get(task.type);
+ if (statsForType == null) {
+ statsForType = Maps.newHashMap();
+ stats.put(task.type, statsForType);
+ }
+
+ String path = currentPathMapping.apply(task.getDescription());
+
+ Stat stat = statsForType.get(path);
+ if (stat == null) {
+ stat = new Stat();
+ }
+
+ stat.duration += task.duration;
+ stat.frequency++;
+ statsForType.put(path, stat);
+ }
+ }
+
+ /**
+ * Returns set of profiler tasks to be filtered from critical path.
+ * Also always filters out ACTION_LOCK and WAIT tasks to simulate
+ * unlimited resource critical path (see comments inside formatExecutionPhaseStatistics()
+ * method).
+ */
+ private EnumSet<ProfilerTask> getTypeFilter(ProfilerTask... tasks) {
+ EnumSet<ProfilerTask> filter = EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT);
+ for (ProfilerTask task : tasks) {
+ filter.add(task);
+ }
+ return filter;
+ }
+
+ private ProfilePhaseStatistics formatInitPhaseStatistics(ProfileInfo info, ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ return formatSimplePhaseStatistics(info, opts, "Init", ProfilePhase.INIT);
+ }
+
+ private ProfilePhaseStatistics formatLoadingPhaseStatistics(ProfileInfo info, ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ return formatSimplePhaseStatistics(info, opts, "Loading", ProfilePhase.LOAD);
+ }
+
+ private ProfilePhaseStatistics formatAnalysisPhaseStatistics(ProfileInfo info,
+ ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ return formatSimplePhaseStatistics(info, opts, "Analysis", ProfilePhase.ANALYZE);
+ }
+
+ private ProfilePhaseStatistics formatSimplePhaseStatistics(ProfileInfo info,
+ ProfileOptions opts,
+ String name,
+ ProfilePhase phase)
+ throws UnsupportedEncodingException {
+ ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(byteOutput, false, "UTF-8");
+
+ printShortPhaseAnalysis(info, out, phase);
+ printVfsStatistics(info, out, phase, opts.vfsStatsLimit);
+ return new ProfilePhaseStatistics(name + " Phase Information",
+ new String(byteOutput.toByteArray(), "UTF-8"));
+ }
+
+ private ProfilePhaseStatistics formatExecutionPhaseStatistics(ProfileInfo info,
+ ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(byteOutput, false, "UTF-8");
+
+ ProfileInfo.Task prepPhase = info.getPhaseTask(ProfilePhase.PREPARE);
+ ProfileInfo.Task execPhase = info.getPhaseTask(ProfilePhase.EXECUTE);
+ ProfileInfo.Task finishPhase = info.getPhaseTask(ProfilePhase.FINISH);
+ if (execPhase == null) {
+ return null;
+ }
+
+ List<ProfileInfo.Task> execTasks = info.getTasksForPhase(execPhase);
+ long graphTime = info.getStatsForType(ProfilerTask.ACTION_GRAPH, execTasks).totalTime;
+ long execTime = info.getPhaseDuration(execPhase) - graphTime;
+
+ if (prepPhase != null) {
+ out.printf(TWO_COLUMN_FORMAT, "Total preparation time",
+ TimeUtilities.prettyTime(info.getPhaseDuration(prepPhase)));
+ }
+ out.printf(TWO_COLUMN_FORMAT, "Total execution phase time",
+ TimeUtilities.prettyTime(info.getPhaseDuration(execPhase)));
+ if (finishPhase != null) {
+ out.printf(TWO_COLUMN_FORMAT, "Total time finalizing build",
+ TimeUtilities.prettyTime(info.getPhaseDuration(finishPhase)));
+ }
+ out.println("");
+ out.printf(TWO_COLUMN_FORMAT, "Action dependency map creation",
+ TimeUtilities.prettyTime(graphTime));
+ out.printf(TWO_COLUMN_FORMAT, "Actual execution time",
+ TimeUtilities.prettyTime(execTime));
+
+ EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class);
+ CriticalPathEntry totalPath = info.getCriticalPath(typeFilter);
+ info.analyzeCriticalPath(typeFilter, totalPath);
+
+ typeFilter = getTypeFilter();
+ CriticalPathEntry optimalPath = info.getCriticalPath(typeFilter);
+ info.analyzeCriticalPath(typeFilter, optimalPath);
+
+ if (totalPath != null) {
+ printCriticalPathTimingBreakdown(info, totalPath, optimalPath, execTime, out);
+ } else {
+ out.println("\nCritical path not available because no action graph was generated.");
+ }
+
+ printTimeDistributionByType(info, out, execPhase);
+
+ if (totalPath != null) {
+ printCriticalPath("Critical path", out, totalPath);
+ // In light critical path we do not record scheduling delay data so it does not make sense
+ // to differentiate it.
+ if (!isLightCriticalPath(totalPath)) {
+ printCriticalPath("Critical path excluding scheduling delays", out, optimalPath);
+ }
+ }
+
+ if (info.getMissingActionsCount() > 0) {
+ out.println("\n" + info.getMissingActionsCount() + " action(s) are present in the"
+ + " action graph but missing instrumentation data. Most likely profile file"
+ + " has been created for the failed or aborted build.");
+ }
+
+ printVfsStatistics(info, out, ProfilePhase.EXECUTE, opts.vfsStatsLimit);
+
+ return new ProfilePhaseStatistics("Execution Phase Information",
+ new String(byteOutput.toByteArray(), "UTF-8"));
+ }
+
+ void printCriticalPathTimingBreakdown(ProfileInfo info, CriticalPathEntry totalPath,
+ CriticalPathEntry optimalPath, long execTime, PrintStream out) {
+ Preconditions.checkNotNull(totalPath);
+ Preconditions.checkNotNull(optimalPath);
+ // TODO(bazel-team): Print remote vs build stats recorded by CriticalPathStats
+ if (isLightCriticalPath(totalPath)) {
+ return;
+ }
+ out.println(totalPath.task.type);
+ // Worker thread pool scheduling delays for the actual critical path.
+ long workerWaitTime = 0;
+ long mainThreadWaitTime = 0;
+ for (ProfileInfo.CriticalPathEntry entry = totalPath; entry != null; entry = entry.next) {
+ workerWaitTime += info.getActionWaitTime(entry.task);
+ mainThreadWaitTime += info.getActionQueueTime(entry.task);
+ }
+ out.printf(TWO_COLUMN_FORMAT, "Worker thread scheduling delays",
+ TimeUtilities.prettyTime(workerWaitTime));
+ out.printf(TWO_COLUMN_FORMAT, "Main thread scheduling delays",
+ TimeUtilities.prettyTime(mainThreadWaitTime));
+
+ out.println("\nCritical path time:");
+ // Actual critical path.
+ long totalTime = totalPath.cumulativeDuration;
+ out.printf("%-37s %10s (%s of execution time)\n", "Actual time",
+ TimeUtilities.prettyTime(totalTime),
+ prettyPercentage(totalTime, execTime));
+ // Unlimited resource critical path. Essentially, we assume that if we
+ // remove all scheduling delays caused by resource semaphore contention,
+ // each action execution time would not change (even though load now would
+ // be substantially higher - so this assumption might be incorrect but it is
+ // still useful for modeling). Given those assumptions we calculate critical
+ // path excluding scheduling delays.
+ long optimalTime = optimalPath.cumulativeDuration;
+ out.printf("%-37s %10s (%s of execution time)\n", "Time excluding scheduling delays",
+ TimeUtilities.prettyTime(optimalTime),
+ prettyPercentage(optimalTime, execTime));
+
+ // Artificial critical path if we ignore all the time spent in all tasks,
+ // except time directly attributed to the ACTION tasks.
+ out.println("\nTime related to:");
+
+ EnumSet<ProfilerTask> typeFilter = EnumSet.allOf(ProfilerTask.class);
+ ProfileInfo.CriticalPathEntry path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the builder overhead",
+ prettyPercentage(path.cumulativeDuration, totalTime));
+
+ typeFilter = getTypeFilter();
+ for (ProfilerTask task : ProfilerTask.values()) {
+ if (task.name().startsWith("VFS_")) {
+ typeFilter.add(task);
+ }
+ }
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the VFS calls",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.ACTION_CHECK);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the dependency checking",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.ACTION_EXECUTE);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the execution setup",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.SPAWN, ProfilerTask.LOCAL_EXECUTION);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "local execution",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.SCANNER);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the include scanner",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION, ProfilerTask.PROCESS_TIME,
+ ProfilerTask.LOCAL_PARSE, ProfilerTask.UPLOAD_TIME,
+ ProfilerTask.REMOTE_QUEUE, ProfilerTask.REMOTE_SETUP, ProfilerTask.FETCH);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "Remote execution (cumulative)",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter( ProfilerTask.UPLOAD_TIME, ProfilerTask.REMOTE_SETUP);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " file uploads",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.FETCH);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " file fetching",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.PROCESS_TIME);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " process time",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.REMOTE_QUEUE);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " remote queueing",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.LOCAL_PARSE);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " remote execution parse",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " other remote activities",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java
new file mode 100644
index 0000000..2e5faf6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.CommonCommandOptions;
+import com.google.devtools.build.lib.runtime.ProjectFile;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.List;
+
+/**
+ * Provides support for implementations for {@link BlazeCommand} to work with {@link ProjectFile}.
+ */
+public final class ProjectFileSupport {
+ static final String PROJECT_FILE_PREFIX = "+";
+
+ private ProjectFileSupport() {}
+
+ /**
+ * Reads any project files specified on the command line and updates the options parser
+ * accordingly. If project files cannot be read or if they contain unparsable options, or if they
+ * are not enabled, then it throws an exception instead.
+ */
+ public static void handleProjectFiles(BlazeRuntime runtime, OptionsParser optionsParser,
+ String command) throws AbruptExitException {
+ List<String> targets = optionsParser.getResidue();
+ ProjectFile.Provider projectFileProvider = runtime.getProjectFileProvider();
+ if (projectFileProvider != null && targets.size() > 0
+ && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) {
+ if (targets.size() > 1) {
+ throw new AbruptExitException("Cannot handle more than one +<file> argument yet",
+ ExitCode.COMMAND_LINE_ERROR);
+ }
+ if (!optionsParser.getOptions(CommonCommandOptions.class).allowProjectFiles) {
+ throw new AbruptExitException("project file support is not enabled",
+ ExitCode.COMMAND_LINE_ERROR);
+ }
+ // TODO(bazel-team): This is currently treated as a path relative to the workspace - if the
+ // cwd is a subdirectory of the workspace, that will be surprising, and we should interpret it
+ // relative to the cwd instead.
+ PathFragment projectFilePath = new PathFragment(targets.get(0).substring(1));
+ List<Path> packagePath = PathPackageLocator.create(
+ optionsParser.getOptions(PackageCacheOptions.class).packagePath, runtime.getReporter(),
+ runtime.getWorkspace(), runtime.getWorkingDirectory()).getPathEntries();
+ ProjectFile projectFile = projectFileProvider.getProjectFile(packagePath, projectFilePath);
+ runtime.getReporter().handle(Event.info("Using " + projectFile.getName()));
+
+ try {
+ optionsParser.parse(
+ OptionPriority.RC_FILE, projectFile.getName(), projectFile.getCommandLineFor(command));
+ } catch (OptionsParsingException e) {
+ throw new AbruptExitException(e.getMessage(), ExitCode.COMMAND_LINE_ERROR);
+ }
+ }
+ }
+
+ /**
+ * Returns a list of targets from the options residue. If a project file is supplied as the first
+ * argument, it will be ignored, on the assumption that handleProjectFiles() has been called to
+ * process it.
+ */
+ public static List<String> getTargets(BlazeRuntime runtime, OptionsProvider options) {
+ List<String> targets = options.getResidue();
+ if (runtime.getProjectFileProvider() != null && targets.size() > 0
+ && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) {
+ return targets.subList(1, targets.size());
+ }
+ return targets;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
new file mode 100644
index 0000000..c5120cb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
@@ -0,0 +1,173 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.query2.BlazeQueryEnvironment;
+import com.google.devtools.build.lib.query2.SkyframeQueryEnvironment;
+import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting;
+import com.google.devtools.build.lib.query2.engine.QueryException;
+import com.google.devtools.build.lib.query2.engine.QueryExpression;
+import com.google.devtools.build.lib.query2.output.OutputFormatter;
+import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter;
+import com.google.devtools.build.lib.query2.output.QueryOptions;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.channels.ClosedByInterruptException;
+import java.util.Set;
+
+/**
+ * Command line wrapper for executing a query with blaze.
+ */
+@Command(name = "query",
+ options = { PackageCacheOptions.class,
+ QueryOptions.class },
+ help = "resource:query.txt",
+ shortDescription = "Executes a dependency graph query.",
+ allowResidue = true,
+ binaryStdOut = true,
+ canRunInOutputDirectory = true)
+public final class QueryCommand implements BlazeCommand {
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { }
+
+ /**
+ * Exit codes:
+ * 0 on successful evaluation.
+ * 1 if query evaluation did not complete.
+ * 2 if query parsing failed.
+ * 3 if errors were reported but evaluation produced a partial result
+ * (only when --keep_going is in effect.)
+ */
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ QueryOptions queryOptions = options.getOptions(QueryOptions.class);
+
+ try {
+ runtime.setupPackageCache(
+ options.getOptions(PackageCacheOptions.class),
+ runtime.getDefaultsPackageContent());
+ } catch (InterruptedException e) {
+ runtime.getReporter().handle(Event.error("query interrupted"));
+ return ExitCode.INTERRUPTED;
+ } catch (AbruptExitException e) {
+ runtime.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage()));
+ return e.getExitCode();
+ }
+
+ if (options.getResidue().isEmpty()) {
+ runtime.getReporter().handle(Event.error(
+ "missing query expression. Type 'blaze help query' for syntax and help"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Iterable<OutputFormatter> formatters = runtime.getQueryOutputFormatters();
+ OutputFormatter formatter =
+ OutputFormatter.getFormatter(formatters, queryOptions.outputFormat);
+ if (formatter == null) {
+ runtime.getReporter().handle(Event.error(
+ String.format("Invalid output format '%s'. Valid values are: %s",
+ queryOptions.outputFormat, OutputFormatter.formatterNames(formatters))));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ String query = Joiner.on(' ').join(options.getResidue());
+
+ Set<Setting> settings = queryOptions.toSettings();
+ BlazeQueryEnvironment env = newQueryEnvironment(
+ runtime,
+ queryOptions.keepGoing,
+ queryOptions.loadingPhaseThreads,
+ settings);
+
+ // 1. Parse query:
+ QueryExpression expr;
+ try {
+ expr = QueryExpression.parse(query, env);
+ } catch (QueryException e) {
+ runtime.getReporter().handle(Event.error(
+ null, "Error while parsing '" + query + "': " + e.getMessage()));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ // 2. Evaluate expression:
+ BlazeQueryEvalResult<Target> result;
+ try {
+ result = env.evaluateQuery(expr);
+ } catch (QueryException e) {
+ // Keep consistent with reportBuildFileError()
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.ANALYSIS_FAILURE;
+ }
+
+ // 3. Output results:
+ OutputFormatter.UnorderedFormatter unorderedFormatter = null;
+ if (!queryOptions.orderResults && formatter instanceof UnorderedFormatter) {
+ unorderedFormatter = (UnorderedFormatter) formatter;
+ }
+
+ PrintStream output = new PrintStream(runtime.getReporter().getOutErr().getOutputStream());
+ try {
+ if (unorderedFormatter != null) {
+ unorderedFormatter.outputUnordered(queryOptions, result.getResultSet(), output);
+ } else {
+ formatter.output(queryOptions, result.getResultGraph(), output);
+ }
+ } catch (ClosedByInterruptException e) {
+ runtime.getReporter().handle(Event.error("query interrupted"));
+ return ExitCode.INTERRUPTED;
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error("I/O error: " + e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ } finally {
+ output.flush();
+ }
+ if (result.getResultSet().isEmpty()) {
+ runtime.getReporter().handle(Event.info("Empty results"));
+ }
+
+ return result.getSuccess() ? ExitCode.SUCCESS : ExitCode.PARTIAL_ANALYSIS_FAILURE;
+ }
+
+ @VisibleForTesting // for com.google.devtools.deps.gquery.test.QueryResultTestUtil
+ public static BlazeQueryEnvironment newQueryEnvironment(BlazeRuntime runtime,
+ boolean keepGoing, int loadingPhaseThreads, Set<Setting> settings) {
+ ImmutableList.Builder<QueryFunction> functions = ImmutableList.builder();
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ functions.addAll(module.getQueryFunctions());
+ }
+ return new SkyframeQueryEnvironment(
+ runtime.getPackageManager().newTransitiveLoader(),
+ runtime.getPackageManager(),
+ runtime.getTargetPatternEvaluator(),
+ keepGoing, loadingPhaseThreads, runtime.getReporter(), settings, functions.build());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
new file mode 100644
index 0000000..b128d37
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
@@ -0,0 +1,519 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.buildtool.BuildResult;
+import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils;
+import com.google.devtools.build.lib.buildtool.TargetValidator;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.exec.SymlinkTreeHelper;
+import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.shell.AbnormalTerminationException;
+import com.google.devtools.build.lib.shell.BadExitStatusException;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.util.CommandBuilder;
+import com.google.devtools.build.lib.util.CommandDescriptionForm;
+import com.google.devtools.build.lib.util.CommandFailureUtils;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Builds and run a target with the given command line arguments.
+ */
+@Command(name = "run",
+ builds = true,
+ options = { RunCommand.RunOptions.class },
+ inherits = { BuildCommand.class },
+ shortDescription = "Runs the specified target.",
+ help = "resource:run.txt",
+ allowResidue = true,
+ binaryStdOut = true,
+ binaryStdErr = true)
+public class RunCommand implements BlazeCommand {
+
+ public static class RunOptions extends OptionsBase {
+ @Option(name = "script_path",
+ category = "run",
+ defaultValue = "null",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, write a shell script to the given file which invokes the "
+ + "target. If this option is set, the target is not run from Blaze. "
+ + "Use 'blaze run --script_path=foo //foo && foo' to invoke target '//foo' "
+ + "This differs from 'blaze run //foo' in that the Blaze lock is released "
+ + "and the executable is connected to the terminal's stdin.")
+ public PathFragment scriptPath;
+ }
+
+ @VisibleForTesting
+ public static final String SINGLE_TARGET_MESSAGE = "Blaze can only run a single target. "
+ + "Do not use wildcards that match more than one target";
+ @VisibleForTesting
+ public static final String NO_TARGET_MESSAGE = "No targets found to run";
+
+ private static final String PROCESS_WRAPPER = "process-wrapper";
+
+ // Value of --run_under as of the most recent command invocation.
+ private RunUnder currentRunUnder;
+
+ private static final FileType RUNFILES_MANIFEST = FileType.of(".runfiles_manifest");
+
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ protected BuildResult processRequest(final BlazeRuntime runtime, BuildRequest request) {
+ return runtime.getBuildTool().processRequest(request, new TargetValidator() {
+ @Override
+ public void validateTargets(Collection<Target> targets, boolean keepGoing)
+ throws LoadingFailedException {
+ RunCommand.this.validateTargets(runtime.getReporter(), targets, keepGoing);
+ }
+ });
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ RunOptions runOptions = options.getOptions(RunOptions.class);
+ // This list should look like: ["//executable:target", "arg1", "arg2"]
+ List<String> targetAndArgs = options.getResidue();
+
+ // The user must at the least specify an executable target.
+ if (targetAndArgs.isEmpty()) {
+ runtime.getReporter().handle(Event.error("Must specify a target to run"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ String targetString = targetAndArgs.get(0);
+ List<String> runTargetArgs = targetAndArgs.subList(1, targetAndArgs.size());
+ RunUnder runUnder = options.getOptions(BuildConfiguration.Options.class).runUnder;
+
+ OutErr outErr = runtime.getReporter().getOutErr();
+ List<String> targets = (runUnder != null) && (runUnder.getLabel() != null)
+ ? ImmutableList.of(targetString, runUnder.getLabel().toString())
+ : ImmutableList.of(targetString);
+ BuildRequest request = BuildRequest.create(
+ this.getClass().getAnnotation(Command.class).name(), options,
+ runtime.getStartupOptionsProvider(), targets, outErr,
+ runtime.getCommandId(), runtime.getCommandStartTime());
+ if (request.getBuildOptions().compileOnly) {
+ String message = "The '" + getClass().getAnnotation(Command.class).name() +
+ "' command is incompatible with the --compile_only option";
+ runtime.getReporter().handle(Event.error(message));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ currentRunUnder = runUnder;
+ BuildResult result;
+ try {
+ result = processRequest(runtime, request);
+ } finally {
+ currentRunUnder = null;
+ }
+
+ if (!result.getSuccess()) {
+ runtime.getReporter().handle(Event.error("Build failed. Not running target"));
+ return result.getExitCondition();
+ }
+
+ // Make sure that we have exactly 1 built target (excluding --run_under),
+ // and that it is executable.
+ // These checks should only fail if keepGoing is true, because we already did
+ // validation before the build began. See {@link #validateTargets()}.
+ Collection<ConfiguredTarget> targetsBuilt = result.getSuccessfulTargets();
+ ConfiguredTarget targetToRun = null;
+ ConfiguredTarget runUnderTarget = null;
+
+ if (targetsBuilt != null) {
+ int maxTargets = runUnder != null && runUnder.getLabel() != null ? 2 : 1;
+ if (targetsBuilt.size() > maxTargets) {
+ runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ for (ConfiguredTarget target : targetsBuilt) {
+ ExitCode targetValidation = fullyValidateTarget(runtime, target);
+ if (targetValidation != ExitCode.SUCCESS) {
+ return targetValidation;
+ }
+ if (runUnder != null && target.getLabel().equals(runUnder.getLabel())) {
+ if (runUnderTarget != null) {
+ runtime.getReporter().handle(Event.error(
+ null, "Can't identify the run_under target from multiple options?"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ runUnderTarget = target;
+ } else if (targetToRun == null) {
+ targetToRun = target;
+ } else {
+ runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ }
+ }
+ // Handle target & run_under referring to the same target.
+ if ((targetToRun == null) && (runUnderTarget != null)) {
+ targetToRun = runUnderTarget;
+ }
+ if (targetToRun == null) {
+ runtime.getReporter().handle(Event.error(NO_TARGET_MESSAGE));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Path executablePath = Preconditions.checkNotNull(
+ targetToRun.getProvider(FilesToRunProvider.class).getExecutable().getPath());
+ BuildConfiguration configuration = targetToRun.getConfiguration();
+ if (configuration == null) {
+ // The target may be an input file, which doesn't have a configuration. In that case, we
+ // choose any target configuration.
+ configuration = runtime.getBuildTool().getView().getConfigurationCollection()
+ .getTargetConfigurations().get(0);
+ }
+ Path workingDir;
+ try {
+ workingDir = ensureRunfilesBuilt(runtime, targetToRun);
+ } catch (CommandException e) {
+ runtime.getReporter().handle(Event.error("Error creating runfiles: " + e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ }
+
+ List<String> args = runTargetArgs;
+
+ FilesToRunProvider provider = targetToRun.getProvider(FilesToRunProvider.class);
+ RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport();
+ if (runfilesSupport != null && runfilesSupport.getArgs() != null) {
+ List<String> targetArgs = runfilesSupport.getArgs();
+ if (!targetArgs.isEmpty()) {
+ args = Lists.newArrayListWithCapacity(targetArgs.size() + runTargetArgs.size());
+ args.addAll(targetArgs);
+ args.addAll(runTargetArgs);
+ }
+ }
+
+ //
+ // We now have a unique executable ready to be run.
+ //
+ // We build up two different versions of the command to run: one with an absolute path, which
+ // we'll actually run, and a prettier one with the long absolute path to the executable
+ // replaced with a shorter relative path that uses the symlinks in the workspace.
+ PathFragment prettyExecutablePath =
+ OutputDirectoryLinksUtils.getPrettyPath(executablePath,
+ runtime.getWorkspaceName(), runtime.getWorkspace(),
+ options.getOptions(BuildRequestOptions.class).symlinkPrefix);
+ List<String> cmdLine = new ArrayList<>();
+ if (runOptions.scriptPath == null) {
+ cmdLine.add(runtime.getDirectories().getExecRoot()
+ .getRelative(runtime.getBinTools().getExecPath(PROCESS_WRAPPER)).getPathString());
+ cmdLine.add("-1");
+ cmdLine.add("15");
+ cmdLine.add("-");
+ cmdLine.add("-");
+ }
+ List<String> prettyCmdLine = new ArrayList<>();
+ // Insert the command prefix specified by the "--run_under=<command-prefix>" option
+ // at the start of the command line.
+ if (runUnder != null) {
+ String runUnderValue = runUnder.getValue();
+ if (runUnderTarget != null) {
+ // --run_under specifies a target. Get the corresponding executable.
+ // This must be an absolute path, because the run_under target is only
+ // in the runfiles of test targets.
+ runUnderValue = runUnderTarget
+ .getProvider(FilesToRunProvider.class).getExecutable().getPath().getPathString();
+ // If the run_under command contains any options, make sure to add them
+ // to the command line as well.
+ List<String> opts = runUnder.getOptions();
+ if (!opts.isEmpty()) {
+ runUnderValue += " " + ShellEscaper.escapeJoinAll(opts);
+ }
+ }
+ cmdLine.add(configuration.getShExecutable().getPathString());
+ cmdLine.add("-c");
+ cmdLine.add(runUnderValue + " " + executablePath.getPathString() + " " +
+ ShellEscaper.escapeJoinAll(args));
+ prettyCmdLine.add(configuration.getShExecutable().getPathString());
+ prettyCmdLine.add("-c");
+ prettyCmdLine.add(runUnderValue + " " + prettyExecutablePath.getPathString() + " " +
+ ShellEscaper.escapeJoinAll(args));
+ } else {
+ cmdLine.add(executablePath.getPathString());
+ cmdLine.addAll(args);
+ prettyCmdLine.add(prettyExecutablePath.getPathString());
+ prettyCmdLine.addAll(args);
+ }
+
+ // Add a newline between the blaze output and the binary's output.
+ outErr.printErrLn("");
+
+ if (runOptions.scriptPath != null) {
+ String unisolatedCommand = CommandFailureUtils.describeCommand(
+ CommandDescriptionForm.COMPLETE_UNISOLATED,
+ cmdLine, null, workingDir.getPathString());
+ if (writeScript(runtime, runOptions.scriptPath, unisolatedCommand)) {
+ return ExitCode.SUCCESS;
+ } else {
+ return ExitCode.RUN_FAILURE;
+ }
+ }
+
+ runtime.getReporter().handle(Event.info(
+ null, "Running command line: " + ShellEscaper.escapeJoinAll(prettyCmdLine)));
+
+ com.google.devtools.build.lib.shell.Command command = new CommandBuilder()
+ .addArgs(cmdLine).setEnv(runtime.getClientEnv()).setWorkingDir(workingDir).build();
+
+ try {
+ // The command API is a little strange in that the following statement
+ // will return normally only if the program exits with exit code 0.
+ // If it ends with any other code, we have to catch BadExitStatusException.
+ command.execute(com.google.devtools.build.lib.shell.Command.NO_INPUT,
+ com.google.devtools.build.lib.shell.Command.NO_OBSERVER,
+ outErr.getOutputStream(),
+ outErr.getErrorStream(),
+ true /* interruptible */).getTerminationStatus().getExitCode();
+ return ExitCode.SUCCESS;
+ } catch (BadExitStatusException e) {
+ String message = "Non-zero return code '"
+ + e.getResult().getTerminationStatus().getExitCode()
+ + "' from command: " + e.getMessage();
+ runtime.getReporter().handle(Event.error(message));
+ return ExitCode.RUN_FAILURE;
+ } catch (AbnormalTerminationException e) {
+ // The process was likely terminated by a signal in this case.
+ return ExitCode.INTERRUPTED;
+ } catch (CommandException e) {
+ runtime.getReporter().handle(Event.error("Error running program: " + e.getMessage()));
+ return ExitCode.RUN_FAILURE;
+ }
+ }
+
+ /**
+ * Ensures that runfiles are built for the specified target. If they already
+ * are, does nothing, otherwise builds them.
+ *
+ * @param target the target to build runfiles for.
+ * @return the path of the runfiles directory.
+ * @throws CommandException
+ */
+ private Path ensureRunfilesBuilt(BlazeRuntime runtime, ConfiguredTarget target)
+ throws CommandException {
+ FilesToRunProvider provider = target.getProvider(FilesToRunProvider.class);
+ RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport();
+ if (runfilesSupport == null) {
+ return runtime.getWorkingDirectory();
+ }
+
+ Artifact manifest = runfilesSupport.getRunfilesManifest();
+ PathFragment runfilesDir = runfilesSupport.getRunfilesDirectoryExecPath();
+ Path workingDir = runtime.getExecRoot()
+ .getRelative(runfilesDir)
+ .getRelative(runtime.getRunfilesPrefix());
+
+ // When runfiles are not generated, getManifest() returns the
+ // .runfiles_manifest file, otherwise it returns the MANIFEST file. This is
+ // a handy way to check whether runfiles were built or not.
+ if (!RUNFILES_MANIFEST.matches(manifest.getFilename())) {
+ // Runfiles already built, nothing to do.
+ return workingDir;
+ }
+
+ SymlinkTreeHelper helper = new SymlinkTreeHelper(
+ manifest.getExecPath(),
+ runfilesDir,
+ false);
+ helper.createSymlinksUsingCommand(runtime.getExecRoot(), target.getConfiguration(),
+ runtime.getBinTools());
+ return workingDir;
+ }
+
+ private boolean writeScript(BlazeRuntime runtime, PathFragment scriptPathFrag, String cmd) {
+ final String SH_SHEBANG = "#!/bin/sh";
+ Path scriptPath = runtime.getWorkingDirectory().getRelative(scriptPathFrag);
+ try {
+ FileSystemUtils.writeContent(scriptPath, StandardCharsets.ISO_8859_1,
+ SH_SHEBANG + "\n" + cmd + " \"$@\"");
+ scriptPath.setExecutable(true);
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error("Error writing run script:" + e.getMessage()));
+ return false;
+ }
+ return true;
+ }
+
+ // Make sure we are building exactly 1 binary target.
+ // If keepGoing, we'll build all the targets even if they are non-binary.
+ private void validateTargets(Reporter reporter, Collection<Target> targets, boolean keepGoing)
+ throws LoadingFailedException {
+ Target targetToRun = null;
+ Target runUnderTarget = null;
+
+ boolean singleTargetWarningWasOutput = false;
+ int maxTargets = currentRunUnder != null && currentRunUnder.getLabel() != null ? 2 : 1;
+ if (targets.size() > maxTargets) {
+ warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing);
+ singleTargetWarningWasOutput = true;
+ }
+ for (Target target : targets) {
+ String targetError = validateTarget(target);
+ if (targetError != null) {
+ warningOrException(reporter, targetError, keepGoing);
+ }
+
+ if (currentRunUnder != null && target.getLabel().equals(currentRunUnder.getLabel())) {
+ // It's impossible to have two targets with the same label.
+ Preconditions.checkState(runUnderTarget == null);
+ runUnderTarget = target;
+ } else if (targetToRun == null) {
+ targetToRun = target;
+ } else {
+ if (!singleTargetWarningWasOutput) {
+ warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing);
+ }
+ return;
+ }
+ }
+ // Handle target & run_under referring to the same target.
+ if ((targetToRun == null) && (runUnderTarget != null)) {
+ targetToRun = runUnderTarget;
+ }
+ if (targetToRun == null) {
+ warningOrException(reporter, NO_TARGET_MESSAGE, keepGoing);
+ }
+ }
+
+ // If keepGoing, print a warning and return the given collection.
+ // Otherwise, throw InvalidTargetException.
+ private void warningOrException(Reporter reporter, String message,
+ boolean keepGoing) throws LoadingFailedException {
+ if (keepGoing) {
+ reporter.handle(Event.warn(message + ". Will continue anyway"));
+ } else {
+ throw new LoadingFailedException(message);
+ }
+ }
+
+ private static String notExecutableError(Target target) {
+ return "Cannot run target " + target.getLabel() + ": Not executable";
+ }
+
+ /** Returns null if the target is a runnable rule, or an appropriate error message otherwise. */
+ private static String validateTarget(Target target) {
+ return isExecutable(target)
+ ? null
+ : notExecutableError(target);
+ }
+
+ /**
+ * Performs all available validation checks on an individual target.
+ *
+ * @param target ConfiguredTarget to validate
+ * @return ExitCode.SUCCESS if all checks succeeded, otherwise a different error code.
+ */
+ private ExitCode fullyValidateTarget(BlazeRuntime runtime, ConfiguredTarget target) {
+ String targetError = validateTarget(target.getTarget());
+
+ if (targetError != null) {
+ runtime.getReporter().handle(Event.error(targetError));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Artifact executable = target.getProvider(FilesToRunProvider.class).getExecutable();
+ if (executable == null) {
+ runtime.getReporter().handle(Event.error(notExecutableError(target.getTarget())));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ // Shouldn't happen: We just validated the target.
+ Preconditions.checkState(executable != null,
+ "Could not find executable for target %s", target);
+ Path executablePath = executable.getPath();
+ try {
+ if (!executablePath.exists() || !executablePath.isExecutable()) {
+ runtime.getReporter().handle(Event.error(
+ null, "Non-existent or non-executable " + executablePath));
+ return ExitCode.BLAZE_INTERNAL_ERROR;
+ }
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error(
+ "Error checking " + executablePath.getPathString() + ": " + e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ }
+
+ return ExitCode.SUCCESS;
+ }
+
+ /**
+ * Return true iff {@code target} is a rule that has an executable file. This includes
+ * *_test rules, *_binary rules, and generated outputs.
+ */
+ private static boolean isExecutable(Target target) {
+ return isOutputFile(target) || isExecutableNonTestRule(target)
+ || TargetUtils.isTestRule(target);
+ }
+
+ /**
+ * Return true iff {@code target} is a rule that generates an executable file and is user-executed
+ * code.
+ */
+ private static boolean isExecutableNonTestRule(Target target) {
+ if (!(target instanceof Rule)) {
+ return false;
+ }
+ Rule rule = ((Rule) target);
+ if (rule.getRuleClassObject().hasAttr("$is_executable", Type.BOOLEAN)) {
+ return NonconfigurableAttributeMapper.of(rule).get("$is_executable", Type.BOOLEAN);
+ }
+ return false;
+ }
+
+ private static boolean isOutputFile(Target target) {
+ return (target instanceof OutputFile);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java
new file mode 100644
index 0000000..fb9ba39
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * The 'blaze shutdown' command.
+ */
+@Command(name = "shutdown",
+ options = { ShutdownCommand.Options.class },
+ allowResidue = false,
+ mustRunInWorkspace = false,
+ shortDescription = "Stops the Blaze server.",
+ help = "This command shuts down the memory resident Blaze server process.\n%{options}")
+public final class ShutdownCommand implements BlazeCommand {
+
+ public static class Options extends OptionsBase {
+
+ @Option(name="iff_heap_size_greater_than",
+ defaultValue = "0",
+ category = "misc",
+ help="Iff non-zero, then shutdown will only shut down the " +
+ "server if the total memory (in MB) consumed by the JVM " +
+ "exceeds this value.")
+ public int heapSizeLimit;
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws ShutdownBlazeServerException {
+
+ int limit = options.getOptions(Options.class).heapSizeLimit;
+
+ // Iff limit is non-zero, shut down the server if total memory exceeds the
+ // limit. totalMemory is the actual heap size that the VM currently uses
+ // *from the OS perspective*. That is, it's not the size occupied by all
+ // objects (which is totalMemory() - freeMemory()), and not the -Xmx
+ // (which is maxMemory()). It's really how much memory this process
+ // currently consumes, in addition to the JVM code and C heap.
+
+ if (limit == 0 ||
+ Runtime.getRuntime().totalMemory() > limit * 1000L * 1000) {
+ throw new ShutdownBlazeServerException(0);
+ }
+ return ExitCode.SUCCESS;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java
new file mode 100644
index 0000000..70082ef
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.docgen.SkylarkDocumentationProcessor;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Map;
+
+/**
+ * The 'doc_ext' command, which prints the extension API doc.
+ */
+@Command(name = "doc_ext",
+allowResidue = true,
+mustRunInWorkspace = false,
+shortDescription = "Prints help for commands, or the index.",
+help = "resource:skylark.txt")
+public final class SkylarkCommand implements BlazeCommand {
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws ShutdownBlazeServerException {
+ OutErr outErr = runtime.getReporter().getOutErr();
+ if (options.getResidue().isEmpty()) {
+ printTopLevelAPIDoc(outErr);
+ return ExitCode.SUCCESS;
+ }
+ if (options.getResidue().size() != 1) {
+ runtime.getReporter().handle(Event.error("Cannot specify more than one parameters"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ return printAPIDoc(options.getResidue().get(0), outErr, runtime.getReporter());
+ }
+
+ private ExitCode printAPIDoc(String param, OutErr outErr, Reporter reporter) {
+ String params[] = param.split("\\.");
+ if (params.length > 2) {
+ reporter.handle(Event.error("Identifier not found: " + param));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor();
+ String doc = processor.getCommandLineAPIDoc(params);
+ if (doc == null) {
+ reporter.handle(Event.error("Identifier not found: " + param));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ outErr.printOut(doc);
+ return ExitCode.SUCCESS;
+ }
+
+ private void printTopLevelAPIDoc(OutErr outErr) {
+ SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor();
+ outErr.printOut("Top level language modules, methods and objects:\n\n");
+ for (Map.Entry<String, String> entry : processor.collectTopLevelModules().entrySet()) {
+ outErr.printOut(entry.getKey() + ": " + entry.getValue());
+ }
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java
new file mode 100644
index 0000000..561c54a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java
@@ -0,0 +1,161 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildResult;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.rules.test.TestStrategy;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.runtime.AggregatingTestListener;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
+import com.google.devtools.build.lib.runtime.TestResultAnalyzer;
+import com.google.devtools.build.lib.runtime.TestResultNotifier;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Handles the 'test' command on the Blaze command line.
+ */
+@Command(name = "test",
+ builds = true,
+ inherits = { BuildCommand.class },
+ options = { TestSummaryOptions.class },
+ shortDescription = "Builds and runs the specified test targets.",
+ help = "resource:test.txt",
+ allowResidue = true)
+public class TestCommand implements BlazeCommand {
+ private AnsiTerminalPrinter printer;
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser)
+ throws AbruptExitException {
+ ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "test");
+
+ TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput;
+
+ if (testOutput == TestStrategy.TestOutputFormat.STREAMED) {
+ runtime.getReporter().handle(Event.warn(
+ "Streamed test output requested so all tests will be run locally, without sharding, " +
+ "one at a time"));
+ try {
+ optionsParser.parse(OptionPriority.SOFTWARE_REQUIREMENT,
+ "streamed output requires locally run tests, without sharding",
+ ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive"));
+ } catch (OptionsParsingException e) {
+ throw new IllegalStateException("Known options failed to parse", e);
+ }
+ }
+ }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ TestResultAnalyzer resultAnalyzer = new TestResultAnalyzer(
+ runtime.getExecRoot(),
+ options.getOptions(TestSummaryOptions.class),
+ options.getOptions(ExecutionOptions.class),
+ runtime.getEventBus());
+
+ printer = new AnsiTerminalPrinter(runtime.getReporter().getOutErr().getOutputStream(),
+ options.getOptions(BlazeCommandEventHandler.Options.class).useColor());
+
+ // Initialize test handler.
+ AggregatingTestListener testListener = new AggregatingTestListener(
+ resultAnalyzer, runtime.getEventBus(), runtime.getReporter());
+
+ runtime.getEventBus().register(testListener);
+ return doTest(runtime, options, testListener);
+ }
+
+ private ExitCode doTest(BlazeRuntime runtime,
+ OptionsProvider options,
+ AggregatingTestListener testListener) {
+ // Run simultaneous build and test.
+ List<String> targets = ProjectFileSupport.getTargets(runtime, options);
+ BuildRequest request = BuildRequest.create(
+ getClass().getAnnotation(Command.class).name(), options,
+ runtime.getStartupOptionsProvider(), targets,
+ runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime());
+ if (request.getBuildOptions().compileOnly) {
+ String message = "The '" + getClass().getAnnotation(Command.class).name() +
+ "' command is incompatible with the --compile_only option";
+ runtime.getReporter().handle(Event.error(message));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ request.setRunTests();
+
+ BuildResult buildResult = runtime.getBuildTool().processRequest(request, null);
+
+ Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets();
+ // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests
+ if (buildResult.getSuccessfulTargets() == null) {
+ // This can happen if there were errors in the target parsing or loading phase
+ // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given
+ // (original exitcode=SUCCESS).
+ runtime.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests"));
+ return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition();
+ }
+ // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks
+ // more tests
+ if (testTargets.isEmpty()) {
+ runtime.getReporter().handle(Event.error(
+ null, "No test targets were found, yet testing was requested"));
+ return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition();
+ }
+
+ boolean buildSuccess = buildResult.getSuccess();
+ boolean testSuccess = analyzeTestResults(testTargets, testListener, options);
+
+ if (testSuccess && !buildSuccess) {
+ // If all tests run successfully, test summary should include warning if
+ // there were build errors not associated with the test targets.
+ printer.printLn(AnsiTerminalPrinter.Mode.ERROR
+ + "One or more non-test targets failed to build.\n"
+ + AnsiTerminalPrinter.Mode.DEFAULT);
+ }
+
+ return buildSuccess ?
+ (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED)
+ : buildResult.getExitCondition();
+ }
+
+ /**
+ * Analyzes test results and prints summary information.
+ * Returns true if and only if all tests were successful.
+ */
+ private boolean analyzeTestResults(Collection<ConfiguredTarget> testTargets,
+ AggregatingTestListener listener,
+ OptionsProvider options) {
+ TestResultNotifier notifier = new TerminalTestResultNotifier(printer, options);
+ return listener.getAnalyzer().differentialAnalyzeAndReport(
+ testTargets, listener, notifier);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java
new file mode 100644
index 0000000..0804cf6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * The 'blaze version' command, which informs users about the blaze version
+ * information.
+ */
+@Command(name = "version",
+ options = {},
+ allowResidue = false,
+ mustRunInWorkspace = false,
+ help = "resource:version.txt",
+ shortDescription = "Prints version information for Blaze.")
+public final class VersionCommand implements BlazeCommand {
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ BlazeVersionInfo info = BlazeVersionInfo.instance();
+ if (info.getSummary() == null) {
+ runtime.getReporter().handle(Event.error("Version information not available"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ runtime.getReporter().getOutErr().printOutLn(info.getSummary());
+ return ExitCode.SUCCESS;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt
new file mode 100644
index 0000000..0ef55a8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt
@@ -0,0 +1,14 @@
+
+Usage: blaze %{command} <options> <profile-files> [<profile-file> ...]
+
+Analyzes build profile data for the given profile data files.
+
+Analyzes each specified profile data file and prints the results. The
+input files must have been produced by the 'blaze build
+--profile=file' command.
+
+By default, a summary of the analysis is printed. For post-processing
+with scripts, the --dump=raw option is recommended, causing this
+command to dump profile data in easily-parsed format.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt
new file mode 100644
index 0000000..5e8d88a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt
@@ -0,0 +1,10 @@
+
+Usage: blaze %{command} <options> <targets>
+
+Builds the specified targets, using the options.
+
+See 'blaze help target-syntax' for details and examples on how to
+specify targets to build.
+
+%{options}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt
new file mode 100644
index 0000000..11541ff
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt
@@ -0,0 +1,8 @@
+
+Usage: blaze canonicalize-flags <options> -- <options-to-canonicalize>
+
+Canonicalizes Blaze flags for the test and build commands. This command is
+intended to be used for tools that wish to check if two lists of options have
+the same effect at runtime.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt
new file mode 100644
index 0000000..7633888
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt
@@ -0,0 +1,10 @@
+
+Usage: blaze %{command} [<option> ...]
+
+Removes Blaze-created output, including all object files, and Blaze
+metadata.
+
+If '--expunge' is specified, the entire working tree will be removed
+and the server stopped.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt
new file mode 100644
index 0000000..a2040c8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt
@@ -0,0 +1,7 @@
+
+Usage: blaze help [<command>]
+
+Prints a help page for the given command, or, if no command is
+specified, prints the index of available commands.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt
new file mode 100644
index 0000000..9c8b552
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt
@@ -0,0 +1,23 @@
+
+Usage: blaze info <options> [key]
+
+Displays information about the state of the blaze process in the
+form of several "key: value" pairs. This includes the locations of
+several output directories. Because some of the
+values are affected by the options passed to 'blaze build', the
+info command accepts the same set of options.
+
+A single non-option argument may be specified (e.g. "blaze-bin"), in
+which case only the value for that key will be printed.
+
+If --show_make_env is specified, the output includes the set of key/value
+pairs in the "Make" environment, accessible within BUILD files.
+
+The full list of keys and the meaning of their values is documented in
+the Blaze User Manual, and can be programmatically obtained with
+'blaze help info-keys'.
+
+See also 'blaze version' for more detailed blaze version
+information.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt
new file mode 100644
index 0000000..ce10211
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt
@@ -0,0 +1,19 @@
+
+Usage: blaze %{command} <options> <query-expression>
+
+Executes a query language expression over a specified subgraph of the
+build dependency graph.
+
+For example, to show all C++ test rules in the strings package, use:
+
+ % blaze query 'kind("cc_.*test", strings:*)'
+
+or to find all dependencies of chubby lockserver, use:
+
+ % blaze query 'deps(//path/to/package:target)'
+
+or to find a dependency path between //path/to/package:target and //dependency:
+
+ % blaze query 'somepath(//path/to/package:target, //dependency)'
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt
new file mode 100644
index 0000000..57283d5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt
@@ -0,0 +1,12 @@
+
+Usage: blaze %{command} <options> -- <binary target> <flags to binary>
+
+Build the specified target and run it with the given arguments.
+
+'run' accepts any 'build' options, and will inherit any defaults
+provided by .blazerc.
+
+If your script needs stdin or execution not constrained by the Blaze lock,
+use 'blaze run --script_path' to write a script and then execute it.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt
new file mode 100644
index 0000000..5414707
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt
@@ -0,0 +1,14 @@
+
+Startup options
+===============
+
+These options affect how Blaze starts up, or more specifically, how
+the virtual machine hosting Blaze starts up, and how the Blaze server
+starts up. These options must be specified to the left of the Blaze
+command (e.g. 'build'), and they must not contain any space between
+option name and value.
+
+Example:
+ % blaze --host_jvm_args=-Xmx1400m --output_base=/tmp/foo build //base
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt
new file mode 100644
index 0000000..1fac498
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt
@@ -0,0 +1,64 @@
+
+Target pattern syntax
+=====================
+
+The BUILD file label syntax is used to specify a single target. Target
+patterns generalize this syntax to sets of targets, and also support
+working-directory-relative forms, recursion, subtraction and filtering.
+Examples:
+
+Specifying a single target:
+
+ //foo/bar:wiz The single target '//foo/bar:wiz'.
+ foo/bar/wiz Equivalent to the first existing one of these:
+ //foo/bar:wiz
+ //foo:bar/wiz
+ //foo/bar Equivalent to '//foo/bar:bar'.
+
+Specifying all rules in a package:
+
+ //foo/bar:all Matches all rules in package 'foo/bar'.
+
+Specifying all rules recursively beneath a package:
+
+ //foo/...:all Matches all rules in all packages beneath directory 'foo'.
+ //foo/... (ditto)
+
+Working-directory relative forms: (assume cwd = 'workspace/foo')
+
+ Target patterns which do not begin with '//' are taken relative to
+ the working directory. Patterns which begin with '//' are always
+ absolute.
+
+ ...:all Equivalent to '//foo/...:all'.
+ ... (ditto)
+
+ bar/...:all Equivalent to '//foo/bar/...:all'.
+ bar/... (ditto)
+
+ bar:wiz Equivalent to '//foo/bar:wiz'.
+ :foo Equivalent to '//foo:foo'.
+
+ bar:all Equivalent to '//foo/bar:all'.
+ :all Equivalent to '//foo:all'.
+
+Summary of target wildcards:
+
+ :all, Match all rules in the specified packages.
+ :*, :all-targets Match all targets (rules and files) in the specified
+ packages, including .par and _deploy.jar files.
+
+Subtractive patterns:
+
+ Target patterns may be preceded by '-', meaning they should be
+ subtracted from the set of targets accumulated by preceding
+ patterns. For example:
+
+ % blaze build -- foo/... -foo/contrib/...
+
+ builds everything in 'foo', except 'contrib'. In case a target not
+ under 'contrib' depends on something under 'contrib' though, in order to
+ build the former blaze has to build the latter too. As usual, the '--' is
+ required to prevent '-b' from being interpreted as an option.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt
new file mode 100644
index 0000000..a1f0523
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt
@@ -0,0 +1,15 @@
+
+Usage: blaze %{command} <options> <test-targets>
+
+Builds the specified targets and runs all test targets among them (test targets
+might also need to satisfy provided tag, size or language filters) using
+the specified options.
+
+This command accepts all valid options to 'build', and inherits
+defaults for 'build' from your .blazerc. If you don't use .blazerc,
+don't forget to pass all your 'build' options to '%{command}' too.
+
+See 'blaze help target-syntax' for details and examples on how to
+specify targets.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt
new file mode 100644
index 0000000..10e1df7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt
@@ -0,0 +1,3 @@
+Prints the version information that was embedded when blaze was built.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java b/src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java
new file mode 100644
index 0000000..ad3e475
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java
@@ -0,0 +1,158 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.server;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.ProcMeminfoParser;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import java.io.IOException;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * Run cleanup-related tasks during idle periods in the server.
+ * idle() and busy() must be called in that order, and only once.
+ */
+class IdleServerTasks {
+
+ private final Path workspaceDir;
+ private final ScheduledThreadPoolExecutor executor;
+ private static final Logger LOG = Logger.getLogger(IdleServerTasks.class.getName());
+
+ private static final long FIVE_MIN_MILLIS = 1000 * 60 * 5;
+
+ /**
+ * Must be called from the main thread.
+ */
+ public IdleServerTasks(@Nullable Path workspaceDir) {
+ this.executor = new ScheduledThreadPoolExecutor(1);
+ this.workspaceDir = workspaceDir;
+ }
+
+ /**
+ * Called when the server becomes idle. Should not block, but may invoke
+ * new threads.
+ */
+ public void idle() {
+ Preconditions.checkState(!executor.isShutdown());
+
+ // Do a GC cycle while the server is idle.
+ executor.schedule(new Runnable() {
+ @Override public void run() {
+ long before = System.currentTimeMillis();
+ System.gc();
+ LOG.info("Idle GC: " + (System.currentTimeMillis() - before) + "ms");
+ }
+ }, 10, TimeUnit.SECONDS);
+ }
+
+ /**
+ * Called by the main thread when the server gets to work.
+ * Should return quickly.
+ */
+ public void busy() {
+ Preconditions.checkState(!executor.isShutdown());
+
+ // Make sure tasks are finished after shutdown(), so they do not intefere
+ // with subsequent server invocations.
+ executor.shutdown();
+ executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
+ executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+
+ boolean interrupted = false;
+ while (true) {
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS);
+ break;
+ } catch (InterruptedException e) {
+ // It's unsafe to leak threads - just reset the interrupt bit later.
+ interrupted = true;
+ }
+ }
+
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Return true iff the server should continue processing requests.
+ * Called from the main thread, so it should return quickly.
+ */
+ public boolean continueProcessing(long idleMillis) {
+ if (!memoryHeuristic(idleMillis)) {
+ return false;
+ }
+ if (workspaceDir == null) {
+ return false;
+ }
+
+ FileStatus stat;
+ try {
+ stat = workspaceDir.statIfFound(Symlinks.FOLLOW);
+ } catch (IOException e) {
+ // Do not terminate the server if the workspace is temporarily inaccessible, for example,
+ // if it is on a network filesystem and the connection is down.
+ return true;
+ }
+ return stat != null && stat.isDirectory();
+ }
+
+ private boolean memoryHeuristic(long idleMillis) {
+ if (idleMillis < FIVE_MIN_MILLIS) {
+ // Don't check memory health until after five minutes.
+ return true;
+ }
+
+ ProcMeminfoParser memInfo = null;
+ try {
+ memInfo = new ProcMeminfoParser();
+ } catch (IOException e) {
+ LOG.info("Could not process /proc/meminfo: " + e);
+ return true;
+ }
+
+ long totalPhysical, totalFree;
+ try {
+ totalPhysical = memInfo.getTotalKb();
+ totalFree = memInfo.getFreeRamKb(); // See method javadoc.
+ } catch (IllegalArgumentException e) {
+ // Ugly capture of unchecked exception, similar to that in
+ // LocalHostCapacity.
+ LoggingUtil.logToRemote(Level.WARNING,
+ "Could not read memInfo during idle query", e);
+ return true;
+ }
+ double fractionFree = (double) totalFree / totalPhysical;
+
+ // If the system as a whole is low on memory, let this server die.
+ if (fractionFree < .1) {
+ LOG.info("Terminating due to memory constraints");
+ LOG.info(String.format("Total physical:%d\nTotal free: %d\n",
+ totalPhysical, totalFree));
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/server/RPCServer.java b/src/main/java/com/google/devtools/build/lib/server/RPCServer.java
new file mode 100644
index 0000000..a1e9982
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/server/RPCServer.java
@@ -0,0 +1,562 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.server;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.server.RPCService.UnknownCommandException;
+import com.google.devtools.build.lib.server.signal.InterruptSignalHandler;
+import com.google.devtools.build.lib.unix.FilesystemUtils;
+import com.google.devtools.build.lib.unix.LocalClientSocket;
+import com.google.devtools.build.lib.unix.LocalServerSocket;
+import com.google.devtools.build.lib.unix.LocalSocketAddress;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.ThreadUtils;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.util.io.StreamMultiplexer;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+/**
+ * An RPCServer server is a Java object that sits and waits for RPC requests
+ * (the sit-and-wait is implemented in {@link #serve()}). These requests
+ * arrive via UNIX file sockets. The RPCServer then calls the application
+ * (which implements ServerCommand) to handle the request. (Since the Blaze
+ * server may need to stat hundreds of directories during initialization, this
+ * is a significant speedup.) The server thread will terminate after idling
+ * for a user-specified time.
+ *
+ * Note: If you are contemplating to call into the RPCServer from
+ * within Java, consider using the {@link RPCService} class instead.
+ */
+// TODO(bazel-team): Signal handling.
+// TODO(bazel-team): Gives clients status information when the server is busy. One
+// way to do this is to put the server status in a file (pid, the current
+// target, etc) in the server directory. Alternatively, we can have a separate
+// thread taking care of the server socket and put the information into socket
+// handshakes.
+// TODO(bazel-team): Use Reporter for server-side messages.
+public final class RPCServer {
+
+ private final Clock clock;
+ private final RPCService rpcService;
+ private final LocalServerSocket serverSocket;
+ private final long maxIdleMillis;
+ private final long statusCheckMillis;
+ private final Path serverDirectory;
+ private final Path workspaceDir;
+ private static final Logger LOG = Logger.getLogger(RPCServer.class.getName());
+ private volatile boolean lameDuck;
+
+ private static final long STATUS_CHECK_PERIOD_MILLIS = 1000 * 60; // 1 minute.
+ private static final Splitter NULLTERMINATOR_SPLITTER = Splitter.on('\0');
+
+ /**
+ * Create a new server instance. After creating the server, you can start it
+ * by calling the {@link #serve()} method.
+ *
+ * @param clock The clock to take time measurements
+ * @param rpcService The underlying service object, which takes
+ * care of dispatching to the {@link ServerCommand}
+ * instances, as requests arrive.
+ * @param maxIdleMillis The maximum time the server will wait idly.
+ * @param statusCheckPeriodMillis How long to wait between system status checks.
+ * @param serverDirectory Directory to put file socket and pid files, etc.
+ * @param workspaceDir The workspace. Used solely to ensure it persists.
+ * @throws IOException
+ */
+ public RPCServer(Clock clock, RPCService rpcService,
+ long maxIdleMillis, long statusCheckPeriodMillis,
+ Path serverDirectory, Path workspaceDir)
+ throws IOException {
+ this.clock = clock;
+ this.rpcService = rpcService;
+ this.maxIdleMillis = maxIdleMillis;
+ this.statusCheckMillis = statusCheckPeriodMillis;
+ this.serverDirectory = serverDirectory;
+ this.workspaceDir = workspaceDir;
+
+ this.serverSocket = openServerSocket();
+ serverSocket.setSoTimeout(Math.min(maxIdleMillis, statusCheckMillis));
+ lameDuck = false;
+ }
+
+ /**
+ * Create a new server instance. After creating the server, you can start it
+ * by calling the {@link #serve()} method.
+ *
+ * @param clock The clock to take time measurements
+ * @param rpcService The underlying service object, which takes
+ * care of dispatching to the {@link ServerCommand}
+ * instances, as requests arrive.
+ * @param maxIdleMillis The maximum time the server will wait idly.
+ * @param serverDirectory Directory to put file socket and pid files, etc.
+ * @param workspaceDir The workspace. Used solely to ensure it persists.
+ * @throws IOException
+ */
+ public RPCServer(Clock clock, RPCService rpcService,
+ long maxIdleMillis, Path serverDirectory, Path workspaceDir)
+ throws IOException {
+ this(clock, rpcService, maxIdleMillis, STATUS_CHECK_PERIOD_MILLIS,
+ serverDirectory, workspaceDir);
+ }
+
+ private static void printStack(IOException e) {
+ /*
+ * Hopefully this never happens. It's not very nice to just write this
+ * to the user's console, but I'm not sure what better choice we have.
+ */
+ StringWriter err = new StringWriter();
+ PrintWriter printErr = new PrintWriter(err);
+ printErr.println("=======[BLAZE SERVER: ENCOUNTERED IO EXCEPTION]=======");
+ e.printStackTrace(printErr);
+ printErr.println("=====================================================");
+ LOG.severe(err.toString());
+ }
+
+ /**
+ * Wait on a socket for business (answer requests). Note that this
+ * method won't return until the server shuts down.
+ */
+ public void serve() {
+ // Register the signal handler.
+ final AtomicBoolean inAction = new AtomicBoolean(false);
+ final AtomicBoolean allowingInterrupt = new AtomicBoolean(true);
+ final AtomicLong cmdNum = new AtomicLong();
+ final Thread mainThread = Thread.currentThread();
+ final Object interruptLock = new Object();
+
+ InterruptSignalHandler sigintHandler = new InterruptSignalHandler() {
+ @Override
+ public void run() {
+ LOG.severe("User interrupt");
+
+ // Only interrupt during actions - otherwise we may end up setting the interrupt bit
+ // at the end of a build and responding to it at the beginning of the subsequent build.
+ synchronized (interruptLock) {
+ if (allowingInterrupt.get()) {
+ mainThread.interrupt();
+ }
+ }
+
+ Runnable interruptWatcher = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ long originalCmd = cmdNum.get();
+ Thread.sleep(10 * 1000);
+ if (inAction.get() && cmdNum.get() == originalCmd) {
+ // We're still operating on the same command.
+ // Interrupt took too long.
+ ThreadUtils.warnAboutSlowInterrupt();
+ }
+ } catch (InterruptedException e) {
+ // Ignore.
+ }
+ }
+ };
+
+ if (inAction.get()) {
+ Thread interruptWatcherThread =
+ new Thread(interruptWatcher, "interrupt-watcher-" + cmdNum);
+ interruptWatcherThread.setDaemon(true);
+ interruptWatcherThread.start();
+ }
+ }
+ };
+
+ try {
+ while (!lameDuck) {
+ try {
+ IdleServerTasks idleChecker = new IdleServerTasks(workspaceDir);
+ idleChecker.idle();
+ RequestIo requestIo;
+
+ long startTime = clock.currentTimeMillis();
+ while (true) {
+ try {
+ allowingInterrupt.set(true);
+ Socket socket = serverSocket.accept();
+ long firstContactTime = clock.currentTimeMillis();
+ requestIo = new RequestIo(socket, firstContactTime);
+ break;
+ } catch (SocketTimeoutException e) {
+ long idleTime = clock.currentTimeMillis() - startTime;
+ if (lameDuck) {
+ closeServerSocket();
+ return;
+ } else if (idleTime > maxIdleMillis ||
+ (idleTime > statusCheckMillis && !idleChecker.continueProcessing(idleTime))) {
+ enterLameDuck();
+ }
+ }
+ }
+ idleChecker.busy();
+
+ try {
+ cmdNum.incrementAndGet();
+ inAction.set(true);
+ executeRequest(requestIo);
+ } finally {
+ inAction.set(false);
+ synchronized (interruptLock) {
+ allowingInterrupt.set(false);
+ Thread.interrupted(); // clears thread interrupted status
+ }
+ requestIo.shutdown();
+ if (rpcService.isShutdown()) {
+ return;
+ }
+ }
+ } catch (IOException e) {
+ if (e.getMessage().equals("Broken pipe")) {
+ LOG.info("Connection to the client lost: "
+ + e.getMessage());
+ } else {
+ // Other cases: print the stack for debugging.
+ printStack(e);
+ }
+ }
+ }
+ } finally {
+ rpcService.shutdown();
+ LOG.info("Logging finished");
+ sigintHandler.uninstall();
+ }
+ }
+
+ private void closeServerSocket() {
+ LOG.info("Closing serverSocket.");
+ try {
+ serverSocket.close();
+ } catch (IOException e) {
+ printStack(e);
+ }
+
+ if (!lameDuck) {
+ try {
+ getSocketPath().delete();
+ } catch (IOException e) {
+ printStack(e);
+ }
+ }
+ }
+
+ /**
+ * Allow one last request to be serviced.
+ */
+ private void enterLameDuck() {
+ lameDuck = true;
+ try {
+ getSocketPath().delete();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ serverSocket.setSoTimeout(1);
+ }
+
+ /**
+ * Returns the path of the socket file to be used.
+ */
+ public Path getSocketPath() {
+ return serverDirectory.getRelative("server.socket");
+ }
+
+ /**
+ * Ensures no other server is running for the current socket file. This
+ * guarantees that no two servers are running against the same output
+ * directory.
+ *
+ * @throws IOException if another server holds the lock for the socket file.
+ */
+ public static void ensureExclusiveAccess(Path socketFile) throws IOException {
+ LocalSocketAddress address =
+ new LocalSocketAddress(socketFile.getPathFile());
+ if (socketFile.exists()) {
+ try {
+ new LocalClientSocket(address).close();
+ } catch (IOException e) {
+ // The previous server process is dead--unlink the file:
+ socketFile.delete();
+ return;
+ }
+ // TODO(bazel-team): (2009) Read the previous server's pid from the "hello" message
+ // and add it to the message.
+ throw new IOException("Socket file " + socketFile.getPathString()
+ + " is locked by another server");
+ }
+ }
+
+ /**
+ * Schedule the specified file for (attempted) deletion at JVM exit.
+ */
+ private static void deleteAtExit(final Path socketFile, final boolean deleteParent) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ socketFile.delete();
+ if (deleteParent) {
+ socketFile.getParentDirectory().delete();
+ }
+ } catch (IOException e) {
+ printStack(e);
+ }
+ }
+ });
+ }
+
+ /**
+ * Opens a UNIX local server socket.
+ * @throws IOException if the socket file is used by another server or can
+ * not be made exclusive.
+ */
+ private LocalServerSocket openServerSocket() throws IOException {
+ // This is the "well known" socket path via which the server is found...
+ Path socketFile = getSocketPath();
+
+ // ...but it may have a name that's too long for AF_UNIX, in which case we
+ // make it a symlink to /tmp/something. This typically only happens in
+ // tests where the --output_base is beneath a very deep temp dir.
+ // (All this extra complexity is just used in tests... *sigh*).
+ if (socketFile.toString().length() >= 108) { // = UNIX_PATH_MAX
+ Path socketLink = socketFile;
+ String tmpDir = System.getProperty("blaze.rpcserver.tmpdir", "/tmp");
+ socketFile = createTempSocketDirectory(socketFile.getRelative(tmpDir)).
+ getRelative("server.socket");
+ LOG.info("Using symlinked socket at " + socketFile);
+
+ socketLink.delete(); // Remove stale symlink, if any.
+ socketLink.createSymbolicLink(socketFile);
+
+ deleteAtExit(socketLink, /*deleteParent=*/false);
+ deleteAtExit(socketFile, /*deleteParent=*/true);
+ } else {
+ deleteAtExit(socketFile, /*deleteParent=*/false);
+ }
+
+ ensureExclusiveAccess(socketFile);
+
+ LocalServerSocket serverSocket = new LocalServerSocket();
+ serverSocket.bind(new LocalSocketAddress(socketFile.getPathFile()));
+ FilesystemUtils.chmod(socketFile.getPathFile(), 0600); // Lock it down.
+ serverSocket.listen(/*backlog=*/50);
+ return serverSocket;
+ }
+
+ // Atomically create a new directory in the (assumed sticky) /tmp directory for use with a
+ // Unix domain socket. The directory will be mode 0700. Retries indefinitely until it
+ // succeeds.
+ private static Path createTempSocketDirectory(Path tempDir) {
+ Random random = new Random();
+ while (true) {
+ Path socketDir = tempDir.getRelative(String.format("blaze-%d", random.nextInt()));
+ try {
+ if (socketDir.createDirectory()) {
+ // Make sure it's private; unfortunately, createDirectory() doesn't take a mode
+ // argument.
+ socketDir.chmod(0700);
+ return socketDir; // Created.
+ }
+ // Already existed; try again.
+ } catch (IOException e) {
+ // Failed; try again.
+ }
+ }
+ }
+
+ /**
+ * Read a string in platform default encoding and split it into a list of
+ * NUL-separated words.
+ *
+ * <p>Blaze consistently uses the platform default encoding (defined in
+ * blaze.cc) to interface with Unix APIs.
+ */
+ private static List<String> readRequest(InputStream input) throws IOException {
+ byte[] inputBytes = ByteStreams.toByteArray(input);
+ if (inputBytes.length == 0) {
+ return null;
+ }
+ String s = new String(inputBytes, Charset.defaultCharset());
+ return ImmutableList.copyOf(NULLTERMINATOR_SPLITTER.split(s));
+ }
+
+ private void executeRequest(RequestIo requestIo) {
+ int exitStatus = 2;
+ try {
+ List<String> request = readRequest(requestIo.in);
+ if (request == null) {
+ LOG.info("Short-circuiting empty request");
+ return;
+ }
+ exitStatus = rpcService.executeRequest(request, requestIo.requestOutErr,
+ requestIo.firstContactTime);
+ LOG.info("Finished executing request");
+ } catch (UnknownCommandException e) {
+ requestIo.requestOutErr.printErrLn("SERVER ERROR: " + e.getMessage());
+ LOG.severe("SERVER ERROR: " + e.getMessage());
+ } catch (Exception e) {
+ // Stacktrace for unknown exception.
+ StringWriter trace = new StringWriter();
+ e.printStackTrace(new PrintWriter(trace, true));
+ requestIo.requestOutErr.printErr("SERVER ERROR: " + trace);
+ LOG.severe("SERVER ERROR: " + trace);
+ }
+
+ if (rpcService.isShutdown()) {
+ // In case of shutdown, disable the listening socket *before* we write
+ // the last part of the response. Otherwise, a sufficiently fast client
+ // could read the response and exit, and a new client could make a
+ // connection to this server, which is still in the listening state, even
+ // though it is about to shut down imminently.
+ closeServerSocket();
+ }
+
+ requestIo.writeExitStatus(exitStatus);
+ }
+
+ /**
+ * Because it's a little complicated, this class factors out all the IO Hook
+ * up we need per request, that is, in
+ * {@link RPCServer#executeRequest(RequestIo)}.
+ * It's unfortunately complicated, so it's explained here.
+ */
+ private static class RequestIo {
+
+ // Used by the client code
+ private final InputStream in;
+ private final OutErr requestOutErr;
+ private final OutputStream controlChannel;
+
+ // just used by this class to keep the state around
+ private final Socket requestSocket;
+ private final OutputStream requestOut;
+ private final long firstContactTime;
+
+ RequestIo(Socket requestSocket, long firstContactTime) throws IOException {
+ this.requestSocket = requestSocket;
+ this.firstContactTime = firstContactTime;
+ this.in = requestSocket.getInputStream();
+ this.requestOut = requestSocket.getOutputStream();
+
+ // We encode the response sent to the client with a multiplexer so
+ // we can send three streams (out / err / control) over one wire stream
+ // (requestOut).
+ StreamMultiplexer multiplexer = new StreamMultiplexer(requestOut);
+
+ // We'll be writing control messages (exit code + out of date message)
+ // to this control channel.
+ controlChannel = multiplexer.createControl();
+
+ // This is the outErr part of the multiplexed output.
+ requestOutErr = OutErr.create(multiplexer.createStdout(),
+ multiplexer.createStderr());
+ // We hook up System.out / System.err to our IO object. Stuff written to
+ // System.out / System.err will show up on the user's screen, prefixed
+ // with "System.out "/"System.err ".
+ requestOutErr.addSystemOutErrAsSource();
+ }
+
+ public void writeExitStatus(int exitStatus) {
+ // Make sure to flush the output / error streams prior to writing the exit status.
+ // The client may stop reading that direction of the socket immediately upon reading the
+ // exit code.
+ flushOutErr();
+ try {
+ controlChannel.write(("" + exitStatus + "\n").getBytes(UTF_8));
+ controlChannel.flush();
+ LOG.info("" + exitStatus);
+ } catch (IOException ignored) {
+ // This exception is historically ignored.
+ }
+ }
+
+ private void flushOutErr() {
+ try {
+ requestOutErr.getOutputStream().flush();
+ } catch (IOException e) {
+ printStack(e);
+ }
+ try {
+ requestOutErr.getErrorStream().flush();
+ } catch (IOException e) {
+ printStack(e);
+ }
+ }
+
+ public void shutdown() {
+ try {
+ requestOut.close();
+ } catch (IOException e) {
+ printStack(e);
+ }
+ try {
+ in.close();
+ } catch (IOException e) {
+ printStack(e);
+ }
+ try {
+ requestSocket.close();
+ } catch (IOException e) {
+ printStack(e);
+ }
+ }
+ }
+
+ /**
+ * Creates and returns a new RPC server.
+ * Use {@link RPCServer#serve()} to start the server.
+ *
+ * @param appCommand The application's ServerCommand implementation.
+ * @param serverDirectory The directory for server-related files. The caller
+ * must ensure the directory has been created.
+ * @param workspaceDir The workspace, used solely to ensure it persists.
+ * @param maxIdleSeconds The idle time in seconds after which the rpc
+ * server will die unless it receives a request.
+ */
+ public static RPCServer newServerWith(Clock clock,
+ ServerCommand appCommand,
+ Path serverDirectory,
+ Path workspaceDir,
+ int maxIdleSeconds)
+ throws IOException {
+ if (!serverDirectory.exists()) {
+ serverDirectory.createDirectory();
+ }
+
+ // Creates and starts the RPC server.
+ RPCService service = new RPCService(appCommand);
+
+ return new RPCServer(clock, service, maxIdleSeconds * 1000L,
+ serverDirectory, workspaceDir);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/server/RPCService.java b/src/main/java/com/google/devtools/build/lib/server/RPCService.java
new file mode 100644
index 0000000..379e83c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/server/RPCService.java
@@ -0,0 +1,95 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.server;
+
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * An RPCService is a Java object that can process RPC requests. Requests may
+ * be of the form:
+ * <pre>
+ * blaze <blaze-arguments>
+ * </pre>
+ * Requests are delegated to the ServerCommand instance provided
+ * to the constructor.
+ */
+public final class RPCService {
+
+ private boolean isShutdown;
+ private static final Logger LOG = Logger.getLogger(RPCService.class.getName());
+ private final ServerCommand appCommand;
+
+ public RPCService(ServerCommand appCommand) {
+ this.appCommand = appCommand;
+ }
+
+ /**
+ * The {@link #executeRequest(List, OutErr, long)} method may
+ * throw this exception if a command is unknown to the RPC service.
+ */
+ public static class UnknownCommandException extends Exception {
+ private static final long serialVersionUID = 1L;
+ UnknownCommandException(String command) {
+ super("Unknown command: " + command);
+ }
+ }
+
+ /**
+ * Executes the request; returns Unix like return codes (0 means success). May
+ * also throw arbitrary exceptions.
+ */
+ public int executeRequest(List<String> request,
+ OutErr outErr,
+ long firstContactTime) throws Exception {
+ if (isShutdown) {
+ throw new IllegalStateException("Received request after shutdown.");
+ }
+ String command = request.isEmpty() ? "" : request.get(0);
+ if (appCommand != null && command.equals("blaze")) { // an application request
+ int result = appCommand.exec(request.subList(1, request.size()), outErr, firstContactTime);
+ if (appCommand.shutdown()) { // an application shutdown request
+ shutdown();
+ }
+ return result;
+ } else {
+ throw new UnknownCommandException(command);
+ }
+ }
+
+ /**
+ * After executing this function, further requests will fail, and
+ * {@link #isShutdown()} will return true.
+ */
+ public void shutdown() {
+ if (isShutdown) {
+ return;
+ }
+ LOG.info("RPC Service: shutting down ...");
+ isShutdown = true;
+ }
+
+ /**
+ * Has this service been shutdown. If so, any call to
+ * {@link #executeRequest(List, OutErr, long)} will result in an
+ * {@link IllegalStateException}
+ */
+ public boolean isShutdown() {
+ return isShutdown;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/server/ServerCommand.java b/src/main/java/com/google/devtools/build/lib/server/ServerCommand.java
new file mode 100644
index 0000000..972753c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/server/ServerCommand.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.server;
+
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.util.List;
+
+/**
+ * The {@link RPCServer} calls an arbitrary command implementing this
+ * interface.
+ */
+public interface ServerCommand {
+
+ /**
+ * Executes the request, writing any output or error messages into err.
+ * Returns 0 on success; any other value or exception indicates an error.
+ */
+ int exec(List<String> args, OutErr outErr, long firstContactTime) throws Exception;
+
+ /**
+ * The implementation returns true from this method to initiate a shutdown.
+ * No further requests will be handled.
+ */
+ boolean shutdown();
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/server/ServerResponse.java b/src/main/java/com/google/devtools/build/lib/server/ServerResponse.java
new file mode 100644
index 0000000..e5ab930
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/server/ServerResponse.java
@@ -0,0 +1,114 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.server;
+
+import com.google.common.base.Preconditions;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+
+/**
+ * This class models a response from the {@link RPCServer}. This is a
+ * tuple of an error message and the exit status. The encoding of the response
+ * is extremely simple {@link #toString()}:
+ *
+ * <ul><li>Iff a message is present, the wire format is
+ * <pre>message + '\n' + exit code as string + '\n'</pre>
+ * </li>
+ * <li>Otherwise it's just the exit code as string + '\n'</li>
+ * </ul>
+ */
+final class ServerResponse {
+
+ /**
+ * Parses an input string into a {@link ServerResponse} object.
+ */
+ public static ServerResponse parseFrom(String input) {
+ if (input.charAt(input.length() - 1) != '\n') {
+ String msg = "Response must end with newline (" + input + ")";
+ throw new IllegalArgumentException(msg);
+ }
+ int newlineAt = input.lastIndexOf('\n', input.length() - 2);
+
+ final String exitStatusString;
+ final String errorMessage;
+ if (newlineAt == -1) {
+ errorMessage = "";
+ exitStatusString = input.substring(0, input.length() - 1);
+ } else {
+ errorMessage = input.substring(0, newlineAt);
+ exitStatusString = input.substring(newlineAt + 1, input.length() - 1);
+ }
+
+ return new ServerResponse(errorMessage, Integer.parseInt(exitStatusString));
+ }
+
+ /**
+ * Parses {@code bytes} into a {@link ServerResponse} instance, assuming
+ * Latin 1 encoding.
+ */
+ public static ServerResponse parseFrom(byte[] bytes) {
+ try {
+ return parseFrom(new String(bytes, "ISO-8859-1"));
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError(e); // Latin 1 is everywhere.
+ }
+ }
+
+ /**
+ * Parses {@code bytes} into a {@link ServerResponse} instance, assuming
+ * Latin 1 encoding.
+ */
+ public static ServerResponse parseFrom(ByteArrayOutputStream bytes) {
+ return parseFrom(bytes.toByteArray());
+ }
+
+ private final String errorMessage;
+ private final int exitStatus;
+
+ /**
+ * Construct a new instance given an error message and an exit status.
+ */
+ public ServerResponse(String errorMessage, int exitStatus) {
+ Preconditions.checkNotNull(errorMessage);
+ this.errorMessage = errorMessage;
+ this.exitStatus = exitStatus;
+ }
+
+ /**
+ * The wire representation of this response object.
+ */
+ @Override
+ public String toString() {
+ if (errorMessage.length() == 0) {
+ return Integer.toString(exitStatus) + '\n';
+ }
+ return errorMessage + '\n' + Integer.toString(exitStatus) + '\n';
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null || !(other instanceof ServerResponse)) return false;
+ ServerResponse otherResponse = (ServerResponse) other;
+ return exitStatus == otherResponse.exitStatus
+ && errorMessage.equals(otherResponse.errorMessage);
+ }
+
+ @Override
+ public int hashCode() {
+ return exitStatus * 31 ^ errorMessage.hashCode();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/server/signal/InterruptSignalHandler.java b/src/main/java/com/google/devtools/build/lib/server/signal/InterruptSignalHandler.java
new file mode 100644
index 0000000..521dcef
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/server/signal/InterruptSignalHandler.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.server.signal;
+
+
+import com.google.common.base.Preconditions;
+
+import sun.misc.Signal;
+import sun.misc.SignalHandler;
+
+/**
+ * A facade around sun.misc.Signal providing special-purpose SIGINT handling.
+ *
+ * We use this code in preference to using sun.misc directly since the latter
+ * is deprecated, and depending on it causes the jdk1.6 javac to emit an
+ * unsuppressable warning that sun.misc is "Sun proprietary API and may be
+ * removed in a future release".
+ */
+public abstract class InterruptSignalHandler implements Runnable {
+
+ private static final Signal SIGINT = new Signal("INT");
+
+ private SignalHandler oldHandler;
+
+ /**
+ * Constructs an InterruptSignalHandler instance. Until the uninstall()
+ * method is invoked, the delivery of a SIGINT signal to this process will
+ * cause the run() method to be invoked in another thread.
+ */
+ protected InterruptSignalHandler() {
+ this.oldHandler = Signal.handle(SIGINT, new SignalHandler() {
+ @Override
+ public void handle(Signal signal) {
+ run();
+ }
+ });
+ }
+
+ /**
+ * Disables SIGINT handling.
+ */
+ public synchronized final void uninstall() {
+ Preconditions.checkNotNull(oldHandler, "uninstall() already called");
+ Signal.handle(SIGINT, oldHandler);
+ oldHandler = null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/AbnormalTerminationException.java b/src/main/java/com/google/devtools/build/lib/shell/AbnormalTerminationException.java
new file mode 100644
index 0000000..30562c6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/AbnormalTerminationException.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Thrown when a command's execution terminates abnormally -- for example,
+ * if it is killed, or if it terminates with a non-zero exit status.
+ */
+public class AbnormalTerminationException extends CommandException {
+
+ private final CommandResult result;
+
+ public AbnormalTerminationException(final Command command,
+ final CommandResult result,
+ final String message) {
+ super(command, message);
+ this.result = result;
+ }
+
+ public AbnormalTerminationException(final Command command,
+ final CommandResult result,
+ final Throwable cause) {
+ super(command, cause);
+ this.result = result;
+ }
+
+ public AbnormalTerminationException(final Command command,
+ final CommandResult result,
+ final String message,
+ final Throwable cause) {
+ super(command, message, cause);
+ this.result = result;
+ }
+
+ public CommandResult getResult() {
+ return result;
+ }
+
+ private static final long serialVersionUID = 2L;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/BadExitStatusException.java b/src/main/java/com/google/devtools/build/lib/shell/BadExitStatusException.java
new file mode 100644
index 0000000..324007a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/BadExitStatusException.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Thrown when a command's execution terminates with a non-zero exit status.
+ */
+public final class BadExitStatusException extends AbnormalTerminationException {
+
+ public BadExitStatusException(final Command command,
+ final CommandResult result,
+ final String message) {
+ super(command, result, message);
+ }
+
+ public BadExitStatusException(final Command command,
+ final CommandResult result,
+ final String message,
+ final Throwable cause) {
+ super(command, result, message, cause);
+ }
+
+ private static final long serialVersionUID = 1L;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Command.java b/src/main/java/com/google/devtools/build/lib/shell/Command.java
new file mode 100644
index 0000000..ab4a7fc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/Command.java
@@ -0,0 +1,960 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * <p>Represents an executable command, including its arguments and
+ * runtime environment (environment variables, working directory). This class
+ * lets a caller execute a command, get its results, and optionally try to kill
+ * the task during execution.</p>
+ *
+ * <p>The use of "shell" in the full name of this class is a misnomer. In
+ * terms of the way its arguments are interpreted, this class is closer to
+ * {@code execve(2)} than to {@code system(3)}. No Bourne shell is executed.
+ *
+ * <p>The most basic use-case for this class is as follows:
+ * <pre>
+ * String[] args = { "/bin/du", "-s", directory };
+ * CommandResult result = new Command(args).execute();
+ * String output = new String(result.getStdout());
+ * </pre>
+ * which writes the output of the {@code du(1)} command into {@code output}.
+ * More complex cases might inspect the stderr stream, kill the subprocess
+ * asynchronously, feed input to its standard input, handle the exceptions
+ * thrown if the command fails, or print the termination status (exit code or
+ * signal name).
+ *
+ * <h4>Invoking the Bourne shell</h4>
+ *
+ * <p>Perhaps the most common command invoked programmatically is the UNIX
+ * shell, {@code /bin/sh}. Because the shell is a general-purpose programming
+ * language, care must be taken to ensure that variable parts of the shell
+ * command (e.g. strings entered by the user) do not contain shell
+ * metacharacters, as this poses a correctness and/or security risk.
+ *
+ * <p>To execute a shell command directly, use the following pattern:
+ * <pre>
+ * String[] args = { "/bin/sh", "-c", shellCommand };
+ * CommandResult result = new Command(args).execute();
+ * </pre>
+ * {@code shellCommand} is a complete Bourne shell program, possibly containing
+ * all kinds of unescaped metacharacters. For example, here's a shell command
+ * that enumerates the working directories of all processes named "foo":
+ * <pre>ps auxx | grep foo | awk '{print $1}' |
+ * while read pid; do readlink /proc/$pid/cwd; done</pre>
+ * It is the responsibility of the caller to ensure that this string means what
+ * they intend.
+ *
+ * <p>Consider the risk posed by allowing the "foo" part of the previous
+ * command to be some arbitrary (untrusted) string called {@code processName}:
+ * <pre>
+ * // WARNING: unsafe!
+ * String shellCommand = "ps auxx | grep " + processName + " | awk '{print $1}' | "
+ * + "while read pid; do readlink /proc/$pid/cwd; done";</pre>
+ * </pre>
+ * Passing this string to {@link Command} is unsafe because if the string
+ * {@processName} contains shell metacharacters, the meaning of the command can
+ * be arbitrarily changed; consider:
+ * <pre>String processName = ". ; rm -fr $HOME & ";</pre>
+ *
+ * <p>To defend against this possibility, it is essential to properly quote the
+ * variable portions of the shell command so that shell metacharacters are
+ * escaped. Use {@link ShellUtils#shellEscape} for this purpose:
+ * <pre>
+ * // Safe.
+ * String shellCommand = "ps auxx | grep " + ShellUtils.shellEscape(processName)
+ * + " | awk '{print $1}' | while read pid; do readlink /proc/$pid/cwd; done";
+ * </pre>
+ *
+ * <p>Tip: if you are only invoking a single known command, and no shell
+ * features (e.g. $PATH lookup, output redirection, pipelines, etc) are needed,
+ * call it directly without using a shell, as in the {@code du(1)} example
+ * above.
+ *
+ * <h4>Other features</h4>
+ *
+ * <p>A caller can optionally specify bytes to be written to the process's
+ * "stdin". The returned {@link CommandResult} object gives the caller access to
+ * the exit status, as well as output from "stdout" and "stderr". To use
+ * this class with processes that generate very large amounts of input/output,
+ * consider
+ * {@link #execute(InputStream, KillableObserver, OutputStream, OutputStream)}
+ * and
+ * {@link #execute(byte[], KillableObserver, OutputStream, OutputStream)}.
+ * </p>
+ *
+ * <p>This class ensures that stdout and stderr streams are read promptly,
+ * avoiding potential deadlock if the output is large. See <a
+ * href="http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html"> When
+ * <code>Runtime.exec()</code> won't</a>.</p>
+ *
+ * <p>This class is immutable and therefore thread-safe.</p>
+ */
+public final class Command {
+
+ private static final Logger log =
+ Logger.getLogger("com.google.devtools.build.lib.shell.Command");
+
+ /**
+ * Pass this value to {@link #execute(byte[])} to indicate that no input
+ * should be written to stdin.
+ */
+ public static final byte[] NO_INPUT = new byte[0];
+
+ private static final String[] EMPTY_STRING_ARRAY = new String[0];
+
+ /**
+ * Pass this to {@link #execute(byte[], KillableObserver, boolean)} to
+ * indicate that you do not wish to observe / kill the underlying
+ * process.
+ */
+ public static final KillableObserver NO_OBSERVER = new KillableObserver() {
+ @Override
+ public void startObserving(final Killable killable) {
+ // do nothing
+ }
+ @Override
+ public void stopObserving(final Killable killable) {
+ // do nothing
+ }
+ };
+
+ private final ProcessBuilder processBuilder;
+
+ // Start of public API -----------------------------------------------------
+
+ /**
+ * Creates a new {@link Command} that will execute a command line that
+ * is described by a {@link ProcessBuilder}. Command line elements,
+ * environment, and working directory are taken from this object. The
+ * command line is executed exactly as given, without a shell.
+ *
+ * @param processBuilder {@link ProcessBuilder} describing command line
+ * to execute
+ */
+ public Command(final ProcessBuilder processBuilder) {
+ this(processBuilder.command().toArray(EMPTY_STRING_ARRAY),
+ processBuilder.environment(),
+ processBuilder.directory());
+ }
+
+ /**
+ * Creates a new {@link Command} for the given command line elements. The
+ * command line is executed exactly as given, without a shell.
+ * Subsequent calls to {@link #execute()} will use the JVM's working
+ * directory and environment.
+ *
+ * @param commandLineElements elements of raw command line to execute
+ * @throws IllegalArgumentException if commandLine is null or empty
+ */
+ /* TODO(bazel-team): Use varargs here
+ */
+ public Command(final String[] commandLineElements) {
+ this(commandLineElements, null, null);
+ }
+
+ /**
+ * <p>Creates a new {@link Command} for the given command line elements.
+ * Subsequent calls to {@link #execute()} will use the JVM's working
+ * directory and environment.</p>
+ *
+ * <p>Note: be careful when setting useShell to <code>true</code>; you
+ * may inadvertently expose a security hole. See
+ * {@link #Command(String, Map, File)}.</p>
+ *
+ * @param commandLineElements elements of raw command line to execute
+ * @param useShell if true, command is executed using a shell interpreter
+ * (e.g. <code>/bin/sh</code> on Linux); if false, command is executed
+ * exactly as given
+ * @throws IllegalArgumentException if commandLine is null or empty
+ */
+ public Command(final String[] commandLineElements, final boolean useShell) {
+ this(commandLineElements, useShell, null, null);
+ }
+
+ /**
+ * Creates a new {@link Command} for the given command line elements. The
+ * command line is executed exactly as given, without a shell. The given
+ * environment variables and working directory are used in subsequent
+ * calls to {@link #execute()}.
+ *
+ * @param commandLineElements elements of raw command line to execute
+ * @param environmentVariables environment variables to replace JVM's
+ * environment variables; may be null
+ * @param workingDirectory working directory for execution; if null, current
+ * working directory is used
+ * @throws IllegalArgumentException if commandLine is null or empty
+ */
+ public Command(final String[] commandLineElements,
+ final Map<String, String> environmentVariables,
+ final File workingDirectory) {
+ this(commandLineElements, false, environmentVariables, workingDirectory);
+ }
+
+ /**
+ * <p>Creates a new {@link Command} for the given command line elements. The
+ * given environment variables and working directory are used in subsequent
+ * calls to {@link #execute()}.</p>
+ *
+ * <p>Note: be careful when setting useShell to <code>true</code>; you
+ * may inadvertently expose a security hole. See
+ * {@link #Command(String, Map, File)}.</p>
+ *
+ * @param commandLineElements elements of raw command line to execute
+ * @param useShell if true, command is executed using a shell interpreter
+ * (e.g. <code>/bin/sh</code> on Linux); if false, command is executed
+ * exactly as given
+ * @param environmentVariables environment variables to replace JVM's
+ * environment variables; may be null
+ * @param workingDirectory working directory for execution; if null, current
+ * working directory is used
+ * @throws IllegalArgumentException if commandLine is null or empty
+ */
+ public Command(final String[] commandLineElements,
+ final boolean useShell,
+ final Map<String, String> environmentVariables,
+ final File workingDirectory) {
+ if (commandLineElements == null || commandLineElements.length == 0) {
+ throw new IllegalArgumentException("command line is null or empty");
+ }
+ this.processBuilder =
+ new ProcessBuilder(maybeAddShell(commandLineElements, useShell));
+ if (environmentVariables != null) {
+ // TODO(bazel-team) remove next line eventually; it is here to mimic old
+ // Runtime.exec() behavior
+ this.processBuilder.environment().clear();
+ this.processBuilder.environment().putAll(environmentVariables);
+ }
+ this.processBuilder.directory(workingDirectory);
+ }
+
+ private static String[] maybeAddShell(final String[] commandLineElements,
+ final boolean useShell) {
+ if (useShell) {
+ final StringBuilder builder = new StringBuilder();
+ for (final String element : commandLineElements) {
+ if (builder.length() > 0) {
+ builder.append(' ');
+ }
+ builder.append(element);
+ }
+ return Shell.getPlatformShell().shellify(builder.toString());
+ } else {
+ return commandLineElements;
+ }
+ }
+
+ /**
+ * @return raw command line elements to be executed
+ */
+ public String[] getCommandLineElements() {
+ final List<String> elements = processBuilder.command();
+ return elements.toArray(new String[elements.size()]);
+ }
+
+ /**
+ * @return (unmodifiable) {@link Map} view of command's environment variables
+ */
+ public Map<String, String> getEnvironmentVariables() {
+ return Collections.unmodifiableMap(processBuilder.environment());
+ }
+
+ /**
+ * @return working directory used for execution, or null if the current
+ * working directory is used
+ */
+ public File getWorkingDirectory() {
+ return processBuilder.directory();
+ }
+
+ /**
+ * Execute this command with no input to stdin. This call will block until the
+ * process completes or an error occurs.
+ *
+ * @return {@link CommandResult} representing result of the execution
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if an {@link IOException} is
+ * encountered while reading from the process, or the process was terminated
+ * due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ */
+ public CommandResult execute() throws CommandException {
+ return execute(NO_INPUT);
+ }
+
+ /**
+ * Execute this command with given input to stdin. This call will block until
+ * the process completes or an error occurs.
+ *
+ * @param stdinInput bytes to be written to process's stdin
+ * @return {@link CommandResult} representing result of the execution
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if an {@link IOException} is
+ * encountered while reading from the process, or the process was terminated
+ * due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ * @throws NullPointerException if stdin is null
+ */
+ public CommandResult execute(final byte[] stdinInput)
+ throws CommandException {
+ nullCheck(stdinInput, "stdinInput");
+ return doExecute(new ByteArrayInputSource(stdinInput),
+ NO_OBSERVER,
+ Consumers.createAccumulatingConsumers(),
+ /*killSubprocess=*/false, /*closeOutput=*/false).get();
+ }
+
+ /**
+ * <p>Execute this command with given input to stdin. This call will block
+ * until the process completes or an error occurs. Caller may specify
+ * whether the method should ignore stdout/stderr output. If the
+ * given number of milliseconds elapses before the command has
+ * completed, this method will attempt to kill the command.</p>
+ *
+ * @param stdinInput bytes to be written to process's stdin, or
+ * {@link #NO_INPUT} if no bytes should be written
+ * @param timeout number of milliseconds to wait for command completion
+ * before attempting to kill the command
+ * @param ignoreOutput if true, method will ignore stdout/stderr output
+ * and return value will not contain this data
+ * @return {@link CommandResult} representing result of the execution
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if an {@link IOException} is
+ * encountered while reading from the process, or the process was terminated
+ * due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ * @throws NullPointerException if stdin is null
+ */
+ public CommandResult execute(final byte[] stdinInput,
+ final long timeout,
+ final boolean ignoreOutput)
+ throws CommandException {
+ return execute(stdinInput,
+ new TimeoutKillableObserver(timeout),
+ ignoreOutput);
+ }
+
+ /**
+ * <p>Execute this command with given input to stdin. This call will block
+ * until the process completes or an error occurs. Caller may specify
+ * whether the method should ignore stdout/stderr output. The given {@link
+ * KillableObserver} may also terminate the process early while running.</p>
+ *
+ * @param stdinInput bytes to be written to process's stdin, or
+ * {@link #NO_INPUT} if no bytes should be written
+ * @param observer {@link KillableObserver} that should observe the running
+ * process, or {@link #NO_OBSERVER} if caller does not wish to kill
+ * the process
+ * @param ignoreOutput if true, method will ignore stdout/stderr output
+ * and return value will not contain this data
+ * @return {@link CommandResult} representing result of the execution
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if the process is interrupted (or
+ * killed) before completion, if an {@link IOException} is encountered while
+ * reading from the process, or the process was terminated due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ * @throws NullPointerException if stdin is null
+ */
+ public CommandResult execute(final byte[] stdinInput,
+ final KillableObserver observer,
+ final boolean ignoreOutput)
+ throws CommandException {
+ // supporting "null" here for backwards compatibility
+ final KillableObserver theObserver =
+ observer == null ? NO_OBSERVER : observer;
+ return doExecute(new ByteArrayInputSource(stdinInput),
+ theObserver,
+ ignoreOutput ? Consumers.createDiscardingConsumers()
+ : Consumers.createAccumulatingConsumers(),
+ /*killSubprocess=*/false, /*closeOutput=*/false).get();
+ }
+
+ /**
+ * <p>Execute this command with given input to stdin. This call blocks
+ * until the process completes or an error occurs. The caller provides
+ * {@link OutputStream} instances into which the process writes its
+ * stdout/stderr output; these streams are <em>not</em> closed when the
+ * process terminates. The given {@link KillableObserver} may also
+ * terminate the process early while running.</p>
+ *
+ * <p>Note that stdout and stderr are written concurrently. If these are
+ * aliased to each other, it is the caller's duty to ensure thread safety.
+ * </p>
+ *
+ * @param stdinInput bytes to be written to process's stdin, or
+ * {@link #NO_INPUT} if no bytes should be written
+ * @param observer {@link KillableObserver} that should observe the running
+ * process, or {@link #NO_OBSERVER} if caller does not wish to kill the
+ * process
+ * @param stdOut the process will write its standard output into this stream.
+ * E.g., you could pass {@link System#out} as <code>stdOut</code>.
+ * @param stdErr the process will write its standard error into this stream.
+ * E.g., you could pass {@link System#err} as <code>stdErr</code>.
+ * @return {@link CommandResult} representing result of the execution. Note
+ * that {@link CommandResult#getStdout()} and
+ * {@link CommandResult#getStderr()} will yield {@link IllegalStateException}
+ * in this case, as the output is written to <code>stdOut/stdErr</code>
+ * instead.
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if the process is interrupted (or
+ * killed) before completion, if an {@link IOException} is encountered while
+ * reading from the process, or the process was terminated due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ * @throws NullPointerException if any argument is null.
+ */
+ public CommandResult execute(final byte[] stdinInput,
+ final KillableObserver observer,
+ final OutputStream stdOut,
+ final OutputStream stdErr)
+ throws CommandException {
+ return execute(stdinInput, observer, stdOut, stdErr, false);
+ }
+
+ /**
+ * Like {@link #execute(byte[], KillableObserver, OutputStream, OutputStream)}
+ * but enables setting of the killSubprocessOnInterrupt attribute.
+ *
+ * @param killSubprocessOnInterrupt if set to true, the execution of
+ * this command is <i>interruptible</i>: in other words, if this thread is
+ * interrupted during a call to execute, the subprocess will be terminated
+ * and the call will return in a timely manner. If false, the subprocess
+ * will run to completion; this is the default value use by all other
+ * constructors. The thread's interrupted status is preserved in all cases,
+ * however.
+ */
+ public CommandResult execute(final byte[] stdinInput,
+ final KillableObserver observer,
+ final OutputStream stdOut,
+ final OutputStream stdErr,
+ final boolean killSubprocessOnInterrupt)
+ throws CommandException {
+ nullCheck(stdinInput, "stdinInput");
+ nullCheck(observer, "observer");
+ nullCheck(stdOut, "stdOut");
+ nullCheck(stdErr, "stdErr");
+ return doExecute(new ByteArrayInputSource(stdinInput),
+ observer,
+ Consumers.createStreamingConsumers(stdOut, stdErr),
+ killSubprocessOnInterrupt, false).get();
+ }
+
+ /**
+ * <p>Execute this command with given input to stdin; this stream is closed
+ * when the process terminates, and exceptions raised when closing this
+ * stream are ignored. This call blocks
+ * until the process completes or an error occurs. The caller provides
+ * {@link OutputStream} instances into which the process writes its
+ * stdout/stderr output; these streams are <em>not</em> closed when the
+ * process terminates. The given {@link KillableObserver} may also
+ * terminate the process early while running.</p>
+ *
+ * @param stdinInput The input to this process's stdin
+ * @param observer {@link KillableObserver} that should observe the running
+ * process, or {@link #NO_OBSERVER} if caller does not wish to kill the
+ * process
+ * @param stdOut the process will write its standard output into this stream.
+ * E.g., you could pass {@link System#out} as <code>stdOut</code>.
+ * @param stdErr the process will write its standard error into this stream.
+ * E.g., you could pass {@link System#err} as <code>stdErr</code>.
+ * @return {@link CommandResult} representing result of the execution. Note
+ * that {@link CommandResult#getStdout()} and
+ * {@link CommandResult#getStderr()} will yield {@link IllegalStateException}
+ * in this case, as the output is written to <code>stdOut/stdErr</code>
+ * instead.
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if the process is interrupted (or
+ * killed) before completion, if an {@link IOException} is encountered while
+ * reading from the process, or the process was terminated due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ * @throws NullPointerException if any argument is null.
+ */
+ public CommandResult execute(final InputStream stdinInput,
+ final KillableObserver observer,
+ final OutputStream stdOut,
+ final OutputStream stdErr)
+ throws CommandException {
+ nullCheck(stdinInput, "stdinInput");
+ nullCheck(observer, "observer");
+ nullCheck(stdOut, "stdOut");
+ nullCheck(stdErr, "stdErr");
+ return doExecute(new InputStreamInputSource(stdinInput),
+ observer,
+ Consumers.createStreamingConsumers(stdOut, stdErr),
+ /*killSubprocess=*/false, /*closeOutput=*/false).get();
+ }
+
+ /**
+ * <p>Execute this command with given input to stdin; this stream is closed
+ * when the process terminates, and exceptions raised when closing this
+ * stream are ignored. This call blocks
+ * until the process completes or an error occurs. The caller provides
+ * {@link OutputStream} instances into which the process writes its
+ * stdout/stderr output; these streams are closed when the process terminates
+ * if closeOut is set. The given {@link KillableObserver} may also
+ * terminate the process early while running.</p>
+ *
+ * @param stdinInput The input to this process's stdin
+ * @param observer {@link KillableObserver} that should observe the running
+ * process, or {@link #NO_OBSERVER} if caller does not wish to kill the
+ * process
+ * @param stdOut the process will write its standard output into this stream.
+ * E.g., you could pass {@link System#out} as <code>stdOut</code>.
+ * @param stdErr the process will write its standard error into this stream.
+ * E.g., you could pass {@link System#err} as <code>stdErr</code>.
+ * @param closeOut whether to close the output streams when the subprocess
+ * terminates.
+ * @return {@link CommandResult} representing result of the execution. Note
+ * that {@link CommandResult#getStdout()} and
+ * {@link CommandResult#getStderr()} will yield {@link IllegalStateException}
+ * in this case, as the output is written to <code>stdOut/stdErr</code>
+ * instead.
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws AbnormalTerminationException if the process is interrupted (or
+ * killed) before completion, if an {@link IOException} is encountered while
+ * reading from the process, or the process was terminated due to a signal.
+ * @throws BadExitStatusException if the process exits with a
+ * non-zero status
+ * @throws NullPointerException if any argument is null.
+ */
+ public CommandResult execute(final InputStream stdinInput,
+ final KillableObserver observer,
+ final OutputStream stdOut,
+ final OutputStream stdErr,
+ boolean closeOut)
+ throws CommandException {
+ nullCheck(stdinInput, "stdinInput");
+ nullCheck(observer, "observer");
+ nullCheck(stdOut, "stdOut");
+ nullCheck(stdErr, "stdErr");
+ return doExecute(new InputStreamInputSource(stdinInput),
+ observer,
+ Consumers.createStreamingConsumers(stdOut, stdErr),
+ false, closeOut).get();
+ }
+
+ /**
+ * <p>Executes this command with the given stdinInput, but does not
+ * wait for it to complete. The caller may choose to observe the status
+ * of the launched process by calling methods on the returned object.
+ *
+ * @param stdinInput bytes to be written to process's stdin, or
+ * {@link #NO_INPUT} if no bytes should be written
+ * @return An object that can be used to check if the process terminated and
+ * obtain the process results.
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws NullPointerException if stdin is null
+ */
+ public FutureCommandResult executeAsynchronously(final byte[] stdinInput)
+ throws CommandException {
+ return executeAsynchronously(stdinInput, NO_OBSERVER);
+ }
+
+ /**
+ * <p>Executes this command with the given input to stdin, but does
+ * not wait for it to complete. The caller may choose to observe the
+ * status of the launched process by calling methods on the returned
+ * object. This method performs the minimum cleanup after the
+ * process terminates: It closes the input stream, and it ignores
+ * exceptions that result from closing it. The given {@link
+ * KillableObserver} may also terminate the process early while
+ * running.</p>
+ *
+ * <p>Note that in this case the {@link KillableObserver} will be assigned
+ * to start observing the process via
+ * {@link KillableObserver#startObserving(Killable)} but will only be
+ * unassigned via {@link KillableObserver#stopObserving(Killable)}, if
+ * {@link FutureCommandResult#get()} is called. If the
+ * {@link KillableObserver} implementation used with this method will
+ * not work correctly without calls to
+ * {@link KillableObserver#stopObserving(Killable)} then a new instance
+ * should be used for each call to this method.</p>
+ *
+ * @param stdinInput bytes to be written to process's stdin, or
+ * {@link #NO_INPUT} if no bytes should be written
+ * @param observer {@link KillableObserver} that should observe the running
+ * process, or {@link #NO_OBSERVER} if caller does not wish to kill
+ * the process
+ * @return An object that can be used to check if the process terminated and
+ * obtain the process results.
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws NullPointerException if stdin is null
+ */
+ public FutureCommandResult executeAsynchronously(final byte[] stdinInput,
+ final KillableObserver observer)
+ throws CommandException {
+ // supporting "null" here for backwards compatibility
+ final KillableObserver theObserver =
+ observer == null ? NO_OBSERVER : observer;
+ nullCheck(stdinInput, "stdinInput");
+ return doExecute(new ByteArrayInputSource(stdinInput),
+ theObserver,
+ Consumers.createDiscardingConsumers(),
+ /*killSubprocess=*/false, /*closeOutput=*/false);
+ }
+
+ /**
+ * <p>Executes this command with the given input to stdin, but does
+ * not wait for it to complete. The caller may choose to observe the
+ * status of the launched process by calling methods on the returned
+ * object. This method performs the minimum cleanup after the
+ * process terminates: It closes the input stream, and it ignores
+ * exceptions that result from closing it. The caller provides
+ * {@link OutputStream} instances into which the process writes its
+ * stdout/stderr output; these streams are <em>not</em> closed when
+ * the process terminates. The given {@link KillableObserver} may
+ * also terminate the process early while running.</p>
+ *
+ * <p>Note that stdout and stderr are written concurrently. If these are
+ * aliased to each other, or if the caller continues to write to these
+ * streams, it is the caller's duty to ensure thread safety.
+ * </p>
+ *
+ * <p>Note that in this case the {@link KillableObserver} will be assigned
+ * to start observing the process via
+ * {@link KillableObserver#startObserving(Killable)} but will only be
+ * unassigned via {@link KillableObserver#stopObserving(Killable)}, if
+ * {@link FutureCommandResult#get()} is called. If the
+ * {@link KillableObserver} implementation used with this method will
+ * not work correctly without calls to
+ * {@link KillableObserver#stopObserving(Killable)} then a new instance
+ * should be used for each call to this method.</p>
+ *
+ * @param stdinInput The input to this process's stdin
+ * @param observer {@link KillableObserver} that should observe the running
+ * process, or {@link #NO_OBSERVER} if caller does not wish to kill
+ * the process
+ * @param stdOut the process will write its standard output into this stream.
+ * E.g., you could pass {@link System#out} as <code>stdOut</code>.
+ * @param stdErr the process will write its standard error into this stream.
+ * E.g., you could pass {@link System#err} as <code>stdErr</code>.
+ * @return An object that can be used to check if the process terminated and
+ * obtain the process results.
+ * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any
+ * reason
+ * @throws NullPointerException if stdin is null
+ */
+ public FutureCommandResult executeAsynchronously(final InputStream stdinInput,
+ final KillableObserver observer,
+ final OutputStream stdOut,
+ final OutputStream stdErr)
+ throws CommandException {
+ // supporting "null" here for backwards compatibility
+ final KillableObserver theObserver =
+ observer == null ? NO_OBSERVER : observer;
+ nullCheck(stdinInput, "stdinInput");
+ return doExecute(new InputStreamInputSource(stdinInput),
+ theObserver,
+ Consumers.createStreamingConsumers(stdOut, stdErr),
+ /*killSubprocess=*/false, /*closeOutput=*/false);
+ }
+
+ // End of public API -------------------------------------------------------
+
+ private void nullCheck(Object argument, String argumentName) {
+ if (argument == null) {
+ String message = argumentName + " argument must not be null.";
+ throw new NullPointerException(message);
+ }
+ }
+
+ private FutureCommandResult doExecute(final InputSource stdinInput,
+ final KillableObserver observer,
+ final Consumers.OutErrConsumers outErrConsumers,
+ final boolean killSubprocessOnInterrupt,
+ final boolean closeOutputStreams)
+ throws CommandException {
+
+ logCommand();
+
+ final Process process = startProcess();
+
+ outErrConsumers.logConsumptionStrategy();
+
+ outErrConsumers.registerInputs(process.getInputStream(),
+ process.getErrorStream(),
+ closeOutputStreams);
+
+ processInput(stdinInput, process);
+
+ // TODO(bazel-team): if the input stream is unbounded, observers will not get start
+ // notification in a timely manner!
+ final Killable processKillable = observeProcess(process, observer);
+
+ return new FutureCommandResult() {
+ @Override
+ public CommandResult get() throws AbnormalTerminationException {
+ return waitForProcessToComplete(process,
+ observer,
+ processKillable,
+ outErrConsumers,
+ killSubprocessOnInterrupt);
+ }
+
+ @Override
+ public boolean isDone() {
+ try {
+ // exitValue seems to be the only non-blocking call for
+ // checking process liveness.
+ process.exitValue();
+ return true;
+ } catch (IllegalThreadStateException e) {
+ return false;
+ }
+ }
+ };
+ }
+
+ private Process startProcess()
+ throws ExecFailedException {
+ try {
+ return processBuilder.start();
+ } catch (IOException ioe) {
+ throw new ExecFailedException(this, ioe);
+ }
+ }
+
+ private static interface InputSource {
+ void copyTo(OutputStream out) throws IOException;
+ boolean isEmpty();
+ String toLogString(String sourceName);
+ }
+
+ private static class ByteArrayInputSource implements InputSource {
+ private byte[] bytes;
+ ByteArrayInputSource(byte[] bytes){
+ this.bytes = bytes;
+ }
+ @Override
+ public void copyTo(OutputStream out) throws IOException {
+ out.write(bytes);
+ out.flush();
+ }
+ @Override
+ public boolean isEmpty() {
+ return bytes.length == 0;
+ }
+ @Override
+ public String toLogString(String sourceName) {
+ if (isEmpty()) {
+ return "No input to " + sourceName;
+ } else {
+ return "Input to " + sourceName + ": " +
+ LogUtil.toTruncatedString(bytes);
+ }
+ }
+ }
+
+ private static class InputStreamInputSource implements InputSource {
+ private InputStream inputStream;
+ InputStreamInputSource(InputStream inputStream){
+ this.inputStream = inputStream;
+ }
+ @Override
+ public void copyTo(OutputStream out) throws IOException {
+ byte[] buf = new byte[4096];
+ int r;
+ while ((r = inputStream.read(buf)) != -1) {
+ out.write(buf, 0, r);
+ out.flush();
+ }
+ }
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+ @Override
+ public String toLogString(String sourceName) {
+ return "Input to " + sourceName + " is a stream.";
+ }
+ }
+
+ private static void processInput(final InputSource stdinInput,
+ final Process process) {
+ if (log.isLoggable(Level.FINER)) {
+ log.finer(stdinInput.toLogString("stdin"));
+ }
+ try {
+ if (stdinInput.isEmpty()) {
+ return;
+ }
+ stdinInput.copyTo(process.getOutputStream());
+ } catch (IOException ioe) {
+ // Note: this is not an error! Perhaps the command just isn't hungry for
+ // our input and exited with success. Process.waitFor (later) will tell
+ // us.
+ //
+ // (Unlike out/err streams, which are read asynchronously, the input stream is written
+ // synchronously, in its entirety, before processInput returns. If the input is
+ // infinite, and is passed through e.g. "cat" subprocess and back into the
+ // ByteArrayOutputStream, that will eventually run out of memory, causing the output stream
+ // to be closed, "cat" to terminate with SIGPIPE, and processInput to receive an IOException.
+ } finally {
+ // if this statement is ever deleted, the process's outputStream
+ // must be closed elsewhere -- it is not closed automatically
+ Command.silentClose(process.getOutputStream());
+ }
+ }
+
+ private static Killable observeProcess(final Process process,
+ final KillableObserver observer) {
+ final Killable processKillable = new ProcessKillable(process);
+ observer.startObserving(processKillable);
+ return processKillable;
+ }
+
+ private CommandResult waitForProcessToComplete(
+ final Process process,
+ final KillableObserver observer,
+ final Killable processKillable,
+ final Consumers.OutErrConsumers outErr,
+ final boolean killSubprocessOnInterrupt)
+ throws AbnormalTerminationException {
+
+ log.finer("Waiting for process...");
+
+ TerminationStatus status =
+ waitForProcess(process, killSubprocessOnInterrupt);
+
+ observer.stopObserving(processKillable);
+
+ log.finer(status.toString());
+
+ try {
+ outErr.waitForCompletion();
+ } catch (IOException ioe) {
+ CommandResult noOutputResult =
+ new CommandResult(CommandResult.EMPTY_OUTPUT,
+ CommandResult.EMPTY_OUTPUT,
+ status);
+ if (status.success()) {
+ // If command was otherwise successful, throw an exception about this
+ throw new AbnormalTerminationException(this, noOutputResult, ioe);
+ } else {
+ // Otherwise, throw the more important exception -- command
+ // was not successful
+ String message = status
+ + "; also encountered an error while attempting to retrieve output";
+ throw status.exited()
+ ? new BadExitStatusException(this, noOutputResult, message, ioe)
+ : new AbnormalTerminationException(this,
+ noOutputResult, message, ioe);
+ }
+ }
+
+ CommandResult result = new CommandResult(outErr.getAccumulatedOut(),
+ outErr.getAccumulatedErr(),
+ status);
+ result.logThis();
+ if (status.success()) {
+ return result;
+ } else if (status.exited()) {
+ throw new BadExitStatusException(this, result, status.toString());
+ } else {
+ throw new AbnormalTerminationException(this, result, status.toString());
+ }
+ }
+
+ private static TerminationStatus waitForProcess(Process process,
+ boolean killSubprocessOnInterrupt) {
+ boolean wasInterrupted = false;
+ try {
+ while (true) {
+ try {
+ return new TerminationStatus(process.waitFor());
+ } catch (InterruptedException ie) {
+ wasInterrupted = true;
+ if (killSubprocessOnInterrupt) {
+ process.destroy();
+ }
+ }
+ }
+ } finally {
+ // Read this for detailed explanation:
+ // http://www-128.ibm.com/developerworks/java/library/j-jtp05236.html
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt(); // preserve interrupted status
+ }
+ }
+ }
+
+ private void logCommand() {
+ if (!log.isLoggable(Level.FINE)) {
+ return;
+ }
+ log.fine(toDebugString());
+ }
+
+ /**
+ * A string representation of this command object which includes
+ * the arguments, the environment, and the working directory. Avoid
+ * relying on the specifics of this format. Note that the size
+ * of the result string will reflect the size of the command.
+ */
+ public String toDebugString() {
+ StringBuilder message = new StringBuilder(128);
+ message.append("Executing (without brackets):");
+ for (final String arg : processBuilder.command()) {
+ message.append(" [");
+ message.append(arg);
+ message.append(']');
+ }
+ message.append("; environment: ");
+ message.append(processBuilder.environment().toString());
+ final File workingDirectory = processBuilder.directory();
+ message.append("; working dir: ");
+ message.append(workingDirectory == null ?
+ "(current)" :
+ workingDirectory.toString());
+ return message.toString();
+ }
+
+ /**
+ * Close the <code>out</code> stream and log a warning if anything happens.
+ */
+ private static void silentClose(final OutputStream out) {
+ try {
+ out.close();
+ } catch (IOException ioe) {
+ String message = "Unexpected exception while closing output stream";
+ log.log(Level.WARNING, message, ioe);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/CommandException.java b/src/main/java/com/google/devtools/build/lib/shell/CommandException.java
new file mode 100644
index 0000000..a11be97
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/CommandException.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Superclass of all exceptions that may be thrown during command execution.
+ * It exists to unify them. It also provides access to the command name
+ * and arguments for the failing command.
+ */
+public class CommandException extends Exception {
+
+ private final Command command;
+
+ /** Returns the command that failed. */
+ public Command getCommand() {
+ return command;
+ }
+
+ public CommandException(Command command, final String message) {
+ super(message);
+ this.command = command;
+ }
+
+ public CommandException(Command command, final Throwable cause) {
+ super(cause);
+ this.command = command;
+ }
+
+ public CommandException(Command command, final String message,
+ final Throwable cause) {
+ super(message, cause);
+ this.command = command;
+ }
+
+ private static final long serialVersionUID = 2L;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/CommandResult.java b/src/main/java/com/google/devtools/build/lib/shell/CommandResult.java
new file mode 100644
index 0000000..185f91d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/CommandResult.java
@@ -0,0 +1,116 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.ByteArrayOutputStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Encapsulates the results of a command execution, including exit status
+ * and output to stdout and stderr.
+ */
+public final class CommandResult {
+
+ private static final Logger log =
+ Logger.getLogger("com.google.devtools.build.lib.shell.Command");
+
+ private static final byte[] NO_BYTES = new byte[0];
+
+ static final ByteArrayOutputStream EMPTY_OUTPUT =
+ new ByteArrayOutputStream() {
+
+ @Override
+ public byte[] toByteArray() {
+ return NO_BYTES;
+ }
+ };
+
+ static final ByteArrayOutputStream NO_OUTPUT_COLLECTED =
+ new ByteArrayOutputStream(){
+
+ @Override
+ public byte[] toByteArray() {
+ throw new IllegalStateException("Output was not collected");
+ }
+ };
+
+ private final ByteArrayOutputStream stdout;
+ private final ByteArrayOutputStream stderr;
+ private final TerminationStatus terminationStatus;
+
+ CommandResult(final ByteArrayOutputStream stdout,
+ final ByteArrayOutputStream stderr,
+ final TerminationStatus terminationStatus) {
+ checkNotNull(stdout);
+ checkNotNull(stderr);
+ checkNotNull(terminationStatus);
+ this.stdout = stdout;
+ this.stderr = stderr;
+ this.terminationStatus = terminationStatus;
+ }
+
+ /**
+ * @return raw bytes that were written to stdout by the command, or
+ * null if caller did chose to ignore output
+ * @throws IllegalStateException if output was not collected
+ */
+ public byte[] getStdout() {
+ return stdout.toByteArray();
+ }
+
+ /**
+ * @return raw bytes that were written to stderr by the command, or
+ * null if caller did chose to ignore output
+ * @throws IllegalStateException if output was not collected
+ */
+ public byte[] getStderr() {
+ return stderr.toByteArray();
+ }
+
+ /**
+ * @return the result of Process.waitFor for the subprocess.
+ * @deprecated this returns the result of Process.waitFor, which is not
+ * precisely defined, and is not to be confused with the value passed to
+ * exit(2) by the subprocess. Use getTerminationStatus() instead.
+ */
+ @Deprecated
+ public int getExitStatus() {
+ return terminationStatus.getRawResult();
+ }
+
+ /**
+ * @return the termination status of the subprocess.
+ */
+ public TerminationStatus getTerminationStatus() {
+ return terminationStatus;
+ }
+
+ void logThis() {
+ if (!log.isLoggable(Level.FINER)) {
+ return;
+ }
+ log.finer(terminationStatus.toString());
+
+ if (stdout == NO_OUTPUT_COLLECTED) {
+ return;
+ }
+ log.finer("Stdout: " + LogUtil.toTruncatedString(stdout.toByteArray()));
+ log.finer("Stderr: " + LogUtil.toTruncatedString(stderr.toByteArray()));
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Consumers.java b/src/main/java/com/google/devtools/build/lib/shell/Consumers.java
new file mode 100644
index 0000000..3ed5b7e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/Consumers.java
@@ -0,0 +1,359 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.shell;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This class provides convenience methods for consuming (actively reading)
+ * output and error streams with different consumption policies:
+ * discarding ({@link #createDiscardingConsumers()},
+ * accumulating ({@link #createAccumulatingConsumers()},
+ * and streaming ({@link #createStreamingConsumers(OutputStream, OutputStream)}).
+ */
+class Consumers {
+
+ private static final Logger log =
+ Logger.getLogger("com.google.devtools.build.lib.shell.Command");
+
+ private Consumers() {}
+
+ private static final ExecutorService pool =
+ Executors.newCachedThreadPool(new AccumulatorThreadFactory());
+
+ static OutErrConsumers createDiscardingConsumers() {
+ return new OutErrConsumers(new DiscardingConsumer(),
+ new DiscardingConsumer());
+ }
+
+ static OutErrConsumers createAccumulatingConsumers() {
+ return new OutErrConsumers(new AccumulatingConsumer(),
+ new AccumulatingConsumer());
+ }
+
+ static OutErrConsumers createStreamingConsumers(OutputStream out,
+ OutputStream err) {
+ return new OutErrConsumers(new StreamingConsumer(out),
+ new StreamingConsumer(err));
+ }
+
+ static class OutErrConsumers {
+
+ private final OutputConsumer out;
+ private final OutputConsumer err;
+
+ private OutErrConsumers(final OutputConsumer out, final OutputConsumer err){
+ this.out = out;
+ this.err = err;
+ }
+
+ void registerInputs(InputStream outInput, InputStream errInput, boolean closeStreams){
+ out.registerInput(outInput, closeStreams);
+ err.registerInput(errInput, closeStreams);
+ }
+
+ void cancel() {
+ out.cancel();
+ err.cancel();
+ }
+
+ void waitForCompletion() throws IOException {
+ out.waitForCompletion();
+ err.waitForCompletion();
+ }
+
+ ByteArrayOutputStream getAccumulatedOut(){
+ return out.getAccumulatedOut();
+ }
+
+ ByteArrayOutputStream getAccumulatedErr() {
+ return err.getAccumulatedOut();
+ }
+
+ void logConsumptionStrategy() {
+ // The creation methods guarantee that the consumption strategy is
+ // the same for out and err - doesn't matter whether we call out or err,
+ // let's pick out.
+ out.logConsumptionStrategy();
+ }
+
+ }
+
+ /**
+ * This interface describes just one consumer, which consumes the
+ * InputStream provided by {@link #registerInput(InputStream, boolean)}.
+ * Implementations implement different consumption strategies.
+ */
+ private static interface OutputConsumer {
+ /**
+ * Returns whatever the consumer accumulated internally, or
+ * {@link CommandResult#NO_OUTPUT_COLLECTED} if it doesn't accumulate
+ * any output.
+ *
+ * @see AccumulatingConsumer
+ */
+ ByteArrayOutputStream getAccumulatedOut();
+
+ void logConsumptionStrategy();
+
+ void registerInput(InputStream in, boolean closeConsumer);
+
+ void cancel();
+
+ void waitForCompletion() throws IOException;
+ }
+
+ /**
+ * This consumer sends the input to a stream while consuming it.
+ */
+ private static class StreamingConsumer extends FutureConsumption
+ implements OutputConsumer {
+ private OutputStream out;
+
+ StreamingConsumer(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public ByteArrayOutputStream getAccumulatedOut() {
+ return CommandResult.NO_OUTPUT_COLLECTED;
+ }
+
+ @Override
+ public void logConsumptionStrategy() {
+ log.finer("Output will be sent to streams provided by client");
+ }
+
+ @Override protected Runnable createConsumingAndClosingSink(InputStream in,
+ boolean closeConsumer) {
+ return new ClosingSink(in, out, closeConsumer);
+ }
+ }
+
+ /**
+ * This consumer sends the input to a {@link ByteArrayOutputStream}
+ * while consuming it. This accumulated stream can be obtained by
+ * calling {@link #getAccumulatedOut()}.
+ */
+ private static class AccumulatingConsumer extends FutureConsumption
+ implements OutputConsumer {
+ private ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ @Override
+ public ByteArrayOutputStream getAccumulatedOut() {
+ return out;
+ }
+
+ @Override
+ public void logConsumptionStrategy() {
+ log.finer("Output will be accumulated (promptly read off) and returned");
+ }
+
+ @Override public Runnable createConsumingAndClosingSink(InputStream in, boolean closeConsumer) {
+ return new ClosingSink(in, out);
+ }
+ }
+
+ /**
+ * This consumer just discards whatever it reads.
+ */
+ private static class DiscardingConsumer extends FutureConsumption
+ implements OutputConsumer {
+ private DiscardingConsumer() {
+ }
+
+ @Override
+ public ByteArrayOutputStream getAccumulatedOut() {
+ return CommandResult.NO_OUTPUT_COLLECTED;
+ }
+
+ @Override
+ public void logConsumptionStrategy() {
+ log.finer("Output will be ignored");
+ }
+
+ @Override public Runnable createConsumingAndClosingSink(InputStream in, boolean closeConsumer) {
+ return new ClosingSink(in);
+ }
+ }
+
+ /**
+ * A mixin that makes consumers active - this is where we kick of
+ * multithreading ({@link #registerInput(InputStream, boolean)}), cancel actions
+ * and wait for the consumers to complete.
+ */
+ private abstract static class FutureConsumption implements OutputConsumer {
+
+ private Future<?> future;
+
+ @Override
+ public void registerInput(InputStream in, boolean closeConsumer){
+ Runnable sink = createConsumingAndClosingSink(in, closeConsumer);
+ future = pool.submit(sink);
+ }
+
+ protected abstract Runnable createConsumingAndClosingSink(InputStream in, boolean close);
+
+ @Override
+ public void cancel() {
+ future.cancel(true);
+ }
+
+ @Override
+ public void waitForCompletion() throws IOException {
+ boolean wasInterrupted = false;
+ try {
+ while (true) {
+ try {
+ future.get();
+ break;
+ } catch (InterruptedException ie) {
+ wasInterrupted = true;
+ // continue waiting
+ } catch (ExecutionException ee) {
+ // Runnable threw a RuntimeException
+ Throwable nested = ee.getCause();
+ if (nested instanceof RuntimeException) {
+ final RuntimeException re = (RuntimeException) nested;
+ // The stream sink classes, unfortunately, tunnel IOExceptions
+ // out of run() in a RuntimeException. If that's the case,
+ // unpack and re-throw the IOException. Otherwise, re-throw
+ // this unexpected RuntimeException
+ final Throwable cause = re.getCause();
+ if (cause instanceof IOException) {
+ throw (IOException) cause;
+ } else {
+ throw re;
+ }
+ } else if (nested instanceof OutOfMemoryError) {
+ // OutOfMemoryError does not support exception chaining.
+ throw (OutOfMemoryError) nested;
+ } else if (nested instanceof Error) {
+ throw new Error("unhandled Error in worker thread", ee);
+ } else {
+ throw new RuntimeException("unknown execution problem", ee);
+ }
+ }
+ }
+ } finally {
+ // Read this for detailed explanation:
+ // http://www-128.ibm.com/developerworks/java/library/j-jtp05236.html
+ if (wasInterrupted) {
+ Thread.currentThread().interrupt(); // preserve interrupted status
+ }
+ }
+ }
+ }
+
+ /**
+ * Factory which produces threads with a 32K stack size.
+ */
+ private static class AccumulatorThreadFactory implements ThreadFactory {
+
+ private static final int THREAD_STACK_SIZE = 32 * 1024;
+
+ private static int threadInitNumber;
+
+ private static synchronized int nextThreadNum() {
+ return threadInitNumber++;
+ }
+
+ @Override
+ public Thread newThread(final Runnable runnable) {
+ final Thread t =
+ new Thread(null,
+ runnable,
+ "Command-Accumulator-Thread-" + nextThreadNum(),
+ THREAD_STACK_SIZE);
+ // Don't let this thread hold up JVM exit
+ t.setDaemon(true);
+ return t;
+ }
+
+ }
+
+ /**
+ * A sink that closes its input stream once its done.
+ */
+ private static class ClosingSink implements Runnable {
+
+ private final InputStream in;
+ private final OutputStream out;
+ private final Runnable sink;
+ private final boolean close;
+
+ /**
+ * Creates a sink that will pump InputStream <code>in</code>
+ * into OutputStream <code>out</code>.
+ */
+ ClosingSink(final InputStream in, OutputStream out) {
+ this(in, out, false);
+ }
+
+ /**
+ * Creates a sink that will read <code>in</code> and discard it.
+ */
+ ClosingSink(final InputStream in) {
+ this.sink = InputStreamSink.newRunnableSink(in);
+ this.in = in;
+ this.close = false;
+ this.out = null;
+ }
+
+ ClosingSink(final InputStream in, OutputStream out, boolean close){
+ this.sink = InputStreamSink.newRunnableSink(in, out);
+ this.in = in;
+ this.out = out;
+ this.close = close;
+ }
+
+
+ @Override
+ public void run() {
+ try {
+ sink.run();
+ } finally {
+ silentClose(in);
+ if (close && out != null) {
+ silentClose(out);
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Close the <code>in</code> stream and log a warning if anything happens.
+ */
+ private static void silentClose(final Closeable closeable) {
+ try {
+ closeable.close();
+ } catch (IOException ioe) {
+ String message = "Unexpected exception while closing input stream";
+ log.log(Level.WARNING, message, ioe);
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/ExecFailedException.java b/src/main/java/com/google/devtools/build/lib/shell/ExecFailedException.java
new file mode 100644
index 0000000..24f42a6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/ExecFailedException.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Thrown when a command could not even be executed by the JVM --
+ * in particular, when {@link Runtime#exec(String[])} fails.
+ */
+public final class ExecFailedException extends CommandException {
+
+ public ExecFailedException(Command command, final Throwable cause) {
+ super(command, cause);
+ }
+
+ private static final long serialVersionUID = 2L;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/FutureCommandResult.java b/src/main/java/com/google/devtools/build/lib/shell/FutureCommandResult.java
new file mode 100644
index 0000000..3e1f5c9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/FutureCommandResult.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Supplier of the command result which additionally allows to check if
+ * the command already terminated. Implementing full fledged Future would
+ * be a much harder undertaking, so a bare minimum that makes this class still
+ * useful for asynchronous command execution is implemented.
+ */
+public interface FutureCommandResult {
+ /**
+ * Returns the result of command execution. If the process is not finished
+ * yet (as reported by {@link #isDone()}, the call will block until that
+ * process terminates.
+ *
+ * @return non-null result of command execution
+ * @throws AbnormalTerminationException if command execution failed
+ */
+ CommandResult get() throws AbnormalTerminationException;
+
+ /**
+ * Returns true if the process terminated, the command result is available
+ * and the call to {@link #get()} will not block.
+ *
+ * @return true if the process terminated
+ */
+ boolean isDone();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/InputStreamSink.java b/src/main/java/com/google/devtools/build/lib/shell/InputStreamSink.java
new file mode 100644
index 0000000..c35552b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/InputStreamSink.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Provides sinks for input streams. Continuously read an input stream
+ * until the end-of-file is encountered. The stream may be redirected to
+ * an {@link OutputStream}, or discarded.
+ * <p>
+ * This class is useful for handing the {@code stdout} and {@code stderr}
+ * streams from a {@link Process} started with {@link Runtime#exec(String)}.
+ * If these streams are not consumed, the Process may block resulting in a
+ * deadlock.
+ *
+ * @see <a href="http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html">
+ * JavaWorld: When Runtime.exec() won't</a>
+ */
+public final class InputStreamSink {
+
+ /**
+ * Black hole into which bytes are sometimes discarded by {@link NullSink}.
+ * It is shared by all threads since the actual contents of the buffer
+ * are irrelevant.
+ */
+ private static final byte[] DISCARD = new byte[4096];
+
+ // Supresses default constructor; ensures non-instantiability
+ private InputStreamSink() {
+ }
+
+ /**
+ * A {@link Thread} which reads and discards data from an
+ * {@link InputStream}.
+ */
+ private static class NullSink implements Runnable {
+ private final InputStream in;
+
+ public NullSink(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public void run() {
+ try {
+ try {
+ // Attempt to just skip all input
+ do {
+ in.skip(Integer.MAX_VALUE);
+ } while (in.read() != -1); // Need to test for EOF
+ } catch (IOException ioe) {
+ // Some streams throw IOException when skip() is called;
+ // resort to reading off all input with read():
+ while (in.read(DISCARD) != -1) {
+ // no loop body
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * A {@link Thread} which reads data from an {@link InputStream},
+ * and translates it into an {@link OutputStream}.
+ */
+ private static class CopySink implements Runnable {
+
+ private final InputStream in;
+ private final OutputStream out;
+
+ public CopySink(InputStream in, OutputStream out) {
+ this.in = in;
+ this.out = out;
+ }
+
+ @Override
+ public void run() {
+ try {
+ byte[] buffer = new byte[2048];
+ int bytesRead;
+ while ((bytesRead = in.read(buffer)) >= 0) {
+ out.write(buffer, 0, bytesRead);
+ out.flush();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Creates a {@link Runnable} which consumes the provided
+ * {@link InputStream} 'in', discarding its contents.
+ */
+ public static Runnable newRunnableSink(InputStream in) {
+ if (in == null) {
+ throw new NullPointerException("in");
+ }
+ return new NullSink(in);
+ }
+
+ /**
+ * Creates a {@link Runnable} which copies everything from 'in'
+ * to 'out'. 'out' will be written to and flushed after each
+ * read from 'in'. However, 'out' will not be closed.
+ */
+ public static Runnable newRunnableSink(InputStream in, OutputStream out) {
+ if (in == null) {
+ throw new NullPointerException("in");
+ }
+ if (out == null) {
+ throw new NullPointerException("out");
+ }
+ return new CopySink(in, out);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Killable.java b/src/main/java/com/google/devtools/build/lib/shell/Killable.java
new file mode 100644
index 0000000..66d1146
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/Killable.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Implementations encapsulate a running process that can be killed.
+ * In particular, here, it is used to wrap up a {@link Process} object
+ * and expose it to a {@link KillableObserver}. It is wrapped in this way
+ * so that the actual {@link Process} object can't be altered by
+ * a {@link KillableObserver}.
+ */
+public interface Killable {
+
+ /**
+ * Kill this killable instance.
+ */
+ void kill();
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/KillableObserver.java b/src/main/java/com/google/devtools/build/lib/shell/KillableObserver.java
new file mode 100644
index 0000000..62d9aa0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/KillableObserver.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Implementations of this interface observe, and potentially kill,
+ * a {@link Killable} object. This is the mechanism by which "kill"
+ * functionality is exposed to callers in the
+ * {@link Command#execute(byte[], KillableObserver, boolean)} method.
+ *
+ */
+public interface KillableObserver {
+
+ /**
+ * <p>Begin observing the given {@link Killable}. This method must return
+ * promptly; until it returns, {@link Command#execute()} cannot complete.
+ * Implementations may wish to start a new {@link Thread} here to handle
+ * kill logic, and to interrupt or otherwise ask the thread to stop in the
+ * {@link #stopObserving(Killable)} method. See
+ * <a href="http://builder.com.com/5100-6370-5144546.html">
+ * Interrupting Java threads</a> for notes on how to implement this
+ * correctly.</p>
+ *
+ * <p>Implementations may or may not be able to observe more than
+ * one {@link Killable} at a time; see javadoc for details.</p>
+ *
+ * @param killable killable to observer
+ */
+ void startObserving(Killable killable);
+
+ /**
+ * Stop observing the given {@link Killable}, since it is
+ * no longer active.
+ */
+ void stopObserving(Killable killable);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/LogUtil.java b/src/main/java/com/google/devtools/build/lib/shell/LogUtil.java
new file mode 100644
index 0000000..ab646f6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/LogUtil.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Utilities for logging.
+ */
+class LogUtil {
+
+ private LogUtil() {}
+
+ private final static int TRUNCATE_STRINGS_AT = 150;
+
+ /**
+ * Make a string out of a byte array, and truncate it to a reasonable length.
+ * Useful for preventing logs from becoming excessively large.
+ */
+ static String toTruncatedString(final byte[] bytes) {
+ if(bytes == null || bytes.length == 0) {
+ return "";
+ }
+ /*
+ * Yes, we'll use the platform encoding here, and this is one of the rare
+ * cases where it makes sense. You want the logs to be encoded so that
+ * your platform tools (vi, emacs, cat) can render them, don't you?
+ * In practice, this means ISO-8859-1 or UTF-8, I guess.
+ */
+ try {
+ if (bytes.length > TRUNCATE_STRINGS_AT) {
+ return new String(bytes, 0, TRUNCATE_STRINGS_AT)
+ + "[... truncated. original size was " + bytes.length + " bytes.]";
+ }
+ return new String(bytes);
+ } catch (Exception e) {
+ /*
+ * In case encoding a binary string doesn't work for some reason, we
+ * don't want to bring a logging server down - do we? So we're paranoid.
+ */
+ return "IOUtil.toTruncatedString: " + e.getMessage();
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/ProcessKillable.java b/src/main/java/com/google/devtools/build/lib/shell/ProcessKillable.java
new file mode 100644
index 0000000..5d0cb8f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/ProcessKillable.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * {@link Killable} implementation which simply wraps a
+ * {@link Process} instance.
+ */
+final class ProcessKillable implements Killable {
+
+ private final Process process;
+
+ ProcessKillable(final Process process) {
+ this.process = process;
+ }
+
+ /**
+ * Calls {@link Process#destroy()}.
+ */
+ @Override
+ public void kill() {
+ process.destroy();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Shell.java b/src/main/java/com/google/devtools/build/lib/shell/Shell.java
new file mode 100644
index 0000000..2cae24e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/Shell.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+import java.util.logging.Logger;
+
+/**
+ * <p>Represents an OS shell, such as "cmd" on Windows or "sh" on Unix-like
+ * platforms. Currently, Linux and Windows XP are supported.</p>
+ *
+ * <p>This class encapsulates shell-specific logic, like how to
+ * create a command line that uses the shell to invoke another command.
+ */
+public abstract class Shell {
+
+ private static final Logger log =
+ Logger.getLogger("com.google.devtools.build.lib.shell.Shell");
+
+ private static final Shell platformShell;
+
+ static {
+ final String osName = System.getProperty("os.name");
+ if ("Linux".equals(osName)) {
+ platformShell = new SHShell();
+ } else if ("Windows XP".equals(osName)) {
+ platformShell = new WindowsCMDShell();
+ } else {
+ log.severe("OS not supported; will not be able to execute commands");
+ platformShell = null;
+ }
+ log.config("Loaded shell support '" + platformShell +
+ "' for OS '" + osName + "'");
+ }
+
+ private Shell() {
+ // do nothing
+ }
+
+ /**
+ * @return {@link Shell} subclass appropriate for the current platform
+ * @throws UnsupportedOperationException if no such subclass exists
+ */
+ public static Shell getPlatformShell() {
+ if (platformShell == null) {
+ throw new UnsupportedOperationException("OS is not supported");
+ }
+ return platformShell;
+ }
+
+ /**
+ * Creates a command line suitable for execution by
+ * {@link Runtime#exec(String[])} from the given command string,
+ * a command line which uses a shell appropriate for a particular
+ * platform to execute the command (e.g. "/bin/sh" on Linux).
+ *
+ * @param command command for which to create a command line
+ * @return String[] suitable for execution by
+ * {@link Runtime#exec(String[])}
+ */
+ public abstract String[] shellify(final String command);
+
+
+ /**
+ * Represents the <code>sh</code> shell commonly found on Unix-like
+ * operating systems, including Linux.
+ */
+ private static final class SHShell extends Shell {
+
+ /**
+ * <p>Returns a command line which uses <code>cmd</code> to execute
+ * the {@link Command}. Given the command <code>foo bar baz</code>,
+ * for example, this will return a String array corresponding
+ * to the command line:</p>
+ *
+ * <p><code>/bin/sh -c "foo bar baz"</code></p>
+ *
+ * <p>That is, it always returns a 3-element array.</p>
+ *
+ * @param command command for which to create a command line
+ * @return String[] suitable for execution by
+ * {@link Runtime#exec(String[])}
+ */
+ @Override public String[] shellify(final String command) {
+ if (command == null || command.length() == 0) {
+ throw new IllegalArgumentException("command is null or empty");
+ }
+ return new String[] { "/bin/sh", "-c", command };
+ }
+
+ }
+
+ /**
+ * Represents the Windows command shell <code>cmd</code>.
+ */
+ private static final class WindowsCMDShell extends Shell {
+
+ /**
+ * <p>Returns a command line which uses <code>cmd</code> to execute
+ * the {@link Command}. Given the command <code>foo bar baz</code>,
+ * for example, this will return a String array corresponding
+ * to the command line:</p>
+ *
+ * <p><code>cmd /S /C "foo bar baz"</code></p>
+ *
+ * <p>That is, it always returns a 4-element array.</p>
+ *
+ * @param command command for which to create a command line
+ * @return String[] suitable for execution by
+ * {@link Runtime#exec(String[])}
+ */
+ @Override public String[] shellify(final String command) {
+ if (command == null || command.length() == 0) {
+ throw new IllegalArgumentException("command is null or empty");
+ }
+ return new String[] { "cmd", "/S", "/C", command };
+ }
+
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/ShellUtils.java b/src/main/java/com/google/devtools/build/lib/shell/ShellUtils.java
new file mode 100644
index 0000000..5157f34
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/ShellUtils.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+import java.util.List;
+
+/**
+ * Utility functions for Bourne shell commands, including escaping and
+ * tokenizing.
+ */
+public abstract class ShellUtils {
+
+ private ShellUtils() {}
+
+ /**
+ * Characters that have no special meaning to the shell.
+ */
+ private static final String SAFE_PUNCTUATION = "@%-_+:,./";
+
+ /**
+ * Quotes a word so that it can be used, without further quoting,
+ * as an argument (or part of an argument) in a shell command.
+ */
+ public static String shellEscape(String word) {
+ int len = word.length();
+ if (len == 0) {
+ // Empty string is a special case: needs to be quoted to ensure that it gets
+ // treated as a separate argument.
+ return "''";
+ }
+ for (int ii = 0; ii < len; ii++) {
+ char c = word.charAt(ii);
+ // We do this positively so as to be sure we don't inadvertently forget
+ // any unsafe characters.
+ if (!Character.isLetterOrDigit(c) && SAFE_PUNCTUATION.indexOf(c) == -1) {
+ // replace() actually means "replace all".
+ return "'" + word.replace("'", "'\\''") + "'";
+ }
+ }
+ return word;
+ }
+
+ /**
+ * Given an argv array such as might be passed to execve(2), returns a string
+ * that can be copied and pasted into a Bourne shell for a similar effect.
+ */
+ public static String prettyPrintArgv(List<String> argv) {
+ StringBuilder buf = new StringBuilder();
+ for (String arg: argv) {
+ if (buf.length() > 0) {
+ buf.append(' ');
+ }
+ buf.append(shellEscape(arg));
+ }
+ return buf.toString();
+ }
+
+
+ /**
+ * Thrown by tokenize method if there is an error
+ */
+ public static class TokenizationException extends Exception {
+ TokenizationException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Populates the passed list of command-line options extracted from {@code
+ * optionString}, which is a string containing multiple options, delimited in
+ * a Bourne shell-like manner.
+ *
+ * @param options the list to be populated with tokens.
+ * @param optionString the string to be tokenized.
+ * @throws TokenizationException if there was an error (such as an
+ * unterminated quotation).
+ */
+ public static void tokenize(List<String> options, String optionString)
+ throws TokenizationException {
+ // See test suite for examples.
+ //
+ // Note: backslash escapes the following character, except within a
+ // single-quoted region where it is literal.
+
+ StringBuilder token = new StringBuilder();
+ boolean forceToken = false;
+ char quotation = '\0'; // NUL, '\'' or '"'
+ for (int ii = 0, len = optionString.length(); ii < len; ii++) {
+ char c = optionString.charAt(ii);
+ if (quotation != '\0') { // in quotation
+ if (c == quotation) { // end of quotation
+ quotation = '\0';
+ } else if (c == '\\' && quotation == '"') { // backslash in "-quotation
+ if (++ii == len) {
+ throw new TokenizationException("backslash at end of string");
+ }
+ c = optionString.charAt(ii);
+ if (c != '\\' && c != '"') {
+ token.append('\\');
+ }
+ token.append(c);
+ } else { // regular char, in quotation
+ token.append(c);
+ }
+ } else { // not in quotation
+ if (c == '\'' || c == '"') { // begin single/double quotation
+ quotation = c;
+ forceToken = true;
+ } else if (c == ' ' || c == '\t') { // space, not quoted
+ if (forceToken || token.length() > 0) {
+ options.add(token.toString());
+ token = new StringBuilder();
+ forceToken = false;
+ }
+ } else if (c == '\\') { // backslash, not quoted
+ if (++ii == len) {
+ throw new TokenizationException("backslash at end of string");
+ }
+ token.append(optionString.charAt(ii));
+ } else { // regular char, not quoted
+ token.append(c);
+ }
+ }
+ }
+ if (quotation != '\0') {
+ throw new TokenizationException("unterminated quotation");
+ }
+ if (forceToken || token.length() > 0) {
+ options.add(token.toString());
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/SimpleKillableObserver.java b/src/main/java/com/google/devtools/build/lib/shell/SimpleKillableObserver.java
new file mode 100644
index 0000000..85794b8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/SimpleKillableObserver.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * <p>A simple implementation of {@link KillableObserver} which can be told
+ * explicitly to kill its {@link Killable} by calling {@link #kill()}. This
+ * is the sort of functionality that callers might expect to find available
+ * on the {@link Command} class.</p>
+ *
+ * <p>Note that this class can only observe one {@link Killable} at a time;
+ * multiple instances should be used for concurrent calls to
+ * {@link Command#execute(byte[], KillableObserver, boolean)}.</p>
+ */
+public final class SimpleKillableObserver implements KillableObserver {
+
+ private Killable killable;
+
+ /**
+ * Does nothing except store a reference to the given {@link Killable}.
+ *
+ * @param killable {@link Killable} to kill
+ */
+ public synchronized void startObserving(final Killable killable) {
+ this.killable = killable;
+ }
+
+ /**
+ * Forgets reference to {@link Killable} provided to
+ * {@link #startObserving(Killable)}
+ */
+ public synchronized void stopObserving(final Killable killable) {
+ if (!this.killable.equals(killable)) {
+ throw new IllegalStateException("start/stopObservering called with " +
+ "different Killables");
+ }
+ this.killable = null;
+ }
+
+ /**
+ * Calls {@link Killable#kill()} on the saved {@link Killable}.
+ */
+ public synchronized void kill() {
+ if (killable != null) {
+ killable.kill();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/TerminationStatus.java b/src/main/java/com/google/devtools/build/lib/shell/TerminationStatus.java
new file mode 100644
index 0000000..73616c4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/TerminationStatus.java
@@ -0,0 +1,162 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+/**
+ * Represents the termination status of a command. {@link Process#waitFor} is
+ * not very precisely specified, so this class encapsulates the interpretation
+ * of values returned by it.
+ *
+ * Caveat: due to the lossy encoding, it's not always possible to accurately
+ * distinguish signal and exit cases. In particular, processes that exit with
+ * a value within the interval [129, 191] will be mistaken for having been
+ * terminated by a signal.
+ *
+ * Instances are immutable.
+ */
+public final class TerminationStatus {
+
+ private final int waitResult;
+
+ /**
+ * Values taken from the glibc strsignal(3) function.
+ */
+ private static final String[] SIGNAL_STRINGS = {
+ null,
+ "Hangup",
+ "Interrupt",
+ "Quit",
+ "Illegal instruction",
+ "Trace/breakpoint trap",
+ "Aborted",
+ "Bus error",
+ "Floating point exception",
+ "Killed",
+ "User defined signal 1",
+ "Segmentation fault",
+ "User defined signal 2",
+ "Broken pipe",
+ "Alarm clock",
+ "Terminated",
+ "Stack fault",
+ "Child exited",
+ "Continued",
+ "Stopped (signal)",
+ "Stopped",
+ "Stopped (tty input)",
+ "Stopped (tty output)",
+ "Urgent I/O condition",
+ "CPU time limit exceeded",
+ "File size limit exceeded",
+ "Virtual timer expired",
+ "Profiling timer expired",
+ "Window changed",
+ "I/O possible",
+ "Power failure",
+ "Bad system call",
+ };
+
+ private static String getSignalString(int signum) {
+ return signum > 0 && signum < SIGNAL_STRINGS.length
+ ? SIGNAL_STRINGS[signum]
+ : "Signal " + signum;
+ }
+
+ /**
+ * Construct a TerminationStatus instance from a Process waitFor code.
+ *
+ * @param waitResult the value returned by {@link java.lang.Process#waitFor}.
+ */
+ public TerminationStatus(int waitResult) {
+ this.waitResult = waitResult;
+ }
+
+ /**
+ * Returns the "raw" result returned by Process.waitFor.
+ */
+ int getRawResult() {
+ return waitResult;
+ }
+
+ /**
+ * Returns true iff the process exited with code 0.
+ */
+ public boolean success() {
+ return exited() && getExitCode() == 0;
+ }
+
+ // We're relying on undocumented behaviour of Process.waitFor, specifically
+ // that waitResult is the exit status when the process returns normally, or
+ // 128+signalnumber when the process is terminated by a signal. We further
+ // assume that value signal numbers fall in the interval [1, 63].
+ private static final int SIGNAL_1 = 128 + 1;
+ private static final int SIGNAL_63 = 128 + 63;
+
+ /**
+ * Returns true iff the process exited normally.
+ */
+ public boolean exited() {
+ return waitResult < SIGNAL_1 || waitResult > SIGNAL_63;
+ }
+
+ /**
+ * Returns the exit code of the subprocess. Undefined if exited() is false.
+ */
+ public int getExitCode() {
+ if (!exited()) {
+ throw new IllegalStateException("getExitCode() not defined");
+ }
+ return waitResult;
+ }
+
+ /**
+ * Returns the number of the signal that terminated the process. Undefined
+ * if exited() returns true.
+ */
+ public int getTerminatingSignal() {
+ if (exited()) {
+ throw new IllegalStateException("getTerminatingSignal() not defined");
+ }
+ return waitResult - SIGNAL_1 + 1;
+ }
+
+ /**
+ * Returns a short string describing the termination status.
+ * e.g. "Exit 1" or "Hangup".
+ */
+ public String toShortString() {
+ return exited()
+ ? ("Exit " + getExitCode())
+ : (getSignalString(getTerminatingSignal()));
+ }
+
+ @Override
+ public String toString() {
+ return exited()
+ ? ("Process exited with status " + getExitCode())
+ : ("Process terminated by signal " + getTerminatingSignal());
+ }
+
+ @Override
+ public int hashCode() {
+ return waitResult;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof TerminationStatus &&
+ ((TerminationStatus) other).waitResult == this.waitResult;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/TimeoutKillableObserver.java b/src/main/java/com/google/devtools/build/lib/shell/TimeoutKillableObserver.java
new file mode 100644
index 0000000..c2ed033
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/shell/TimeoutKillableObserver.java
@@ -0,0 +1,102 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.shell;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * <p>{@link KillableObserver} implementation which will kill its observed
+ * {@link Killable} if it is still being observed after a given amount
+ * of time has elapsed.</p>
+ *
+ * <p>Note that this class can only observe one {@link Killable} at a time;
+ * multiple instances should be used for concurrent calls to
+ * {@link Command#execute(byte[], KillableObserver, boolean)}.</p>
+ */
+public final class TimeoutKillableObserver implements KillableObserver {
+
+ private static final Logger log =
+ Logger.getLogger(TimeoutKillableObserver.class.getCanonicalName());
+
+ private final long timeoutMS;
+ private Killable killable;
+ private SleeperThread sleeperThread;
+ private boolean timedOut;
+
+ // TODO(bazel-team): I'd like to use ThreadPool2, but it doesn't currently
+ // provide a way to interrupt a thread
+
+ public TimeoutKillableObserver(final long timeoutMS) {
+ this.timeoutMS = timeoutMS;
+ }
+
+ /**
+ * Starts a new {@link Thread} to wait for the timeout period. This is
+ * interrupted by the {@link #stopObserving(Killable)} method.
+ *
+ * @param killable killable to kill when the timeout period expires
+ */
+ @Override
+ public synchronized void startObserving(final Killable killable) {
+ this.timedOut = false;
+ this.killable = killable;
+ this.sleeperThread = new SleeperThread();
+ this.sleeperThread.start();
+ }
+
+ @Override
+ public synchronized void stopObserving(final Killable killable) {
+ if (!this.killable.equals(killable)) {
+ throw new IllegalStateException("start/stopObservering called with " +
+ "different Killables");
+ }
+ if (sleeperThread.isAlive()) {
+ sleeperThread.interrupt();
+ }
+ this.killable = null;
+ sleeperThread = null;
+ }
+
+ private final class SleeperThread extends Thread {
+ @Override public void run() {
+ try {
+ if (log.isLoggable(Level.FINE)) {
+ log.fine("Waiting for " + timeoutMS + "ms to kill process");
+ }
+ Thread.sleep(timeoutMS);
+ // timeout expired; kill it
+ synchronized (TimeoutKillableObserver.this) {
+ if (killable != null) {
+ log.fine("Killing process");
+ killable.kill();
+ timedOut = true;
+ }
+ }
+ } catch (InterruptedException ie) {
+ // continue -- process finished before timeout
+ log.fine("Wait interrupted since process finished; continuing...");
+ }
+ }
+ }
+
+ /**
+ * Returns true if the observed process was killed by this observer.
+ */
+ public synchronized boolean hasTimedOut() {
+ // synchronized needed for memory model visibility.
+ return timedOut;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
new file mode 100644
index 0000000..6dcc224
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
@@ -0,0 +1,177 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * A SkyFunction for {@link ASTFileLookupValue}s. Tries to locate a file and load it as a
+ * syntax tree and cache the resulting {@link BuildFileAST}. If the file doesn't exist
+ * the function doesn't fail but returns a specific NO_FILE ASTLookupValue.
+ */
+public class ASTFileLookupFunction implements SkyFunction {
+
+ private abstract static class FileLookupResult {
+ /** Returns whether the file lookup was successful. */
+ public abstract boolean lookupSuccessful();
+
+ /** If {@code lookupSuccessful()}, returns the {@link RootedPath} to the file. */
+ public abstract RootedPath rootedPath();
+
+ static FileLookupResult noFile() {
+ return UnsuccessfulFileResult.INSTANCE;
+ }
+
+ static FileLookupResult file(RootedPath rootedPath) {
+ return new SuccessfulFileResult(rootedPath);
+ }
+
+ private static class SuccessfulFileResult extends FileLookupResult {
+ private final RootedPath rootedPath;
+
+ private SuccessfulFileResult(RootedPath rootedPath) {
+ this.rootedPath = rootedPath;
+ }
+
+ @Override
+ public boolean lookupSuccessful() {
+ return true;
+ }
+
+ @Override
+ public RootedPath rootedPath() {
+ return rootedPath;
+ }
+ }
+
+ private static class UnsuccessfulFileResult extends FileLookupResult {
+ private static final UnsuccessfulFileResult INSTANCE = new UnsuccessfulFileResult();
+ private UnsuccessfulFileResult() {
+ }
+
+ @Override
+ public boolean lookupSuccessful() {
+ return false;
+ }
+
+ @Override
+ public RootedPath rootedPath() {
+ throw new IllegalStateException("unsucessful lookup");
+ }
+ }
+ }
+
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final RuleClassProvider ruleClassProvider;
+ private final CachingPackageLocator packageManager;
+
+ public ASTFileLookupFunction(AtomicReference<PathPackageLocator> pkgLocator,
+ CachingPackageLocator packageManager,
+ RuleClassProvider ruleClassProvider) {
+ this.pkgLocator = pkgLocator;
+ this.packageManager = packageManager;
+ this.ruleClassProvider = ruleClassProvider;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ PathFragment astFilePathFragment = (PathFragment) skyKey.argument();
+ FileLookupResult lookupResult = getASTFile(env, astFilePathFragment);
+ if (lookupResult == null) {
+ return null;
+ }
+
+ BuildFileAST ast = null;
+ if (!lookupResult.lookupSuccessful()) {
+ // Return the specific NO_FILE ASTLookupValue instance if no file was found.
+ return ASTFileLookupValue.NO_FILE;
+ } else {
+ Path path = lookupResult.rootedPath().asPath();
+ // Skylark files end with bzl.
+ boolean parseAsSkylark = astFilePathFragment.getPathString().endsWith(".bzl");
+ try {
+ ast = parseAsSkylark
+ ? BuildFileAST.parseSkylarkFile(path, env.getListener(),
+ packageManager, ruleClassProvider.getSkylarkValidationEnvironment().clone())
+ : BuildFileAST.parseBuildFile(path, env.getListener(),
+ packageManager, false);
+ } catch (IOException e) {
+ throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException(
+ e.getMessage()), Transience.TRANSIENT);
+ }
+ }
+
+ return new ASTFileLookupValue(ast);
+ }
+
+ private FileLookupResult getASTFile(Environment env, PathFragment astFilePathFragment)
+ throws ASTLookupFunctionException {
+ for (Path packagePathEntry : pkgLocator.get().getPathEntries()) {
+ RootedPath rootedPath = RootedPath.toRootedPath(packagePathEntry, astFilePathFragment);
+ SkyKey fileSkyKey = FileValue.key(rootedPath);
+ FileValue fileValue = null;
+ try {
+ fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class);
+ } catch (IOException | FileSymlinkCycleException e) {
+ throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException(
+ e.getMessage()), Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ throw new ASTLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ if (fileValue == null) {
+ return null;
+ }
+ if (fileValue.isFile()) {
+ return FileLookupResult.file(rootedPath);
+ }
+ }
+ return FileLookupResult.noFile();
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static final class ASTLookupFunctionException extends SkyFunctionException {
+ private ASTLookupFunctionException(ErrorReadingSkylarkExtensionException e,
+ Transience transience) {
+ super(e, transience);
+ }
+
+ private ASTLookupFunctionException(InconsistentFilesystemException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java
new file mode 100644
index 0000000..1061c86
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value that represents an AST file lookup result.
+ */
+public class ASTFileLookupValue implements SkyValue {
+
+ static final ASTFileLookupValue NO_FILE = new ASTFileLookupValue(null);
+
+ @Nullable private final BuildFileAST ast;
+
+ public ASTFileLookupValue(@Nullable BuildFileAST ast) {
+ this.ast = ast;
+ }
+
+ /**
+ * Returns the original AST file.
+ */
+ @Nullable public BuildFileAST getAST() {
+ return ast;
+ }
+
+ static void checkInputArgument(PathFragment astFilePathFragment) throws ASTLookupInputException {
+ if (astFilePathFragment.isAbsolute()) {
+ throw new ASTLookupInputException(String.format(
+ "Input file '%s' cannot be an absolute path.", astFilePathFragment));
+ }
+ }
+
+ static SkyKey key(PathFragment astFilePathFragment) throws ASTLookupInputException {
+ checkInputArgument(astFilePathFragment);
+ return new SkyKey(SkyFunctions.AST_FILE_LOOKUP, astFilePathFragment);
+ }
+
+ static final class ASTLookupInputException extends Exception {
+ private ASTLookupInputException(String msg) {
+ super(msg);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java
new file mode 100644
index 0000000..797f158
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java
@@ -0,0 +1,130 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/** Reports cycles between skyframe values whose keys contains {@link Label}s. */
+abstract class AbstractLabelCycleReporter implements CyclesReporter.SingleCycleReporter {
+
+ private final LoadedPackageProvider loadedPackageProvider;
+
+ AbstractLabelCycleReporter(LoadedPackageProvider loadedPackageProvider) {
+ this.loadedPackageProvider = loadedPackageProvider;
+ }
+
+ /** Returns the String representation of the {@code SkyKey}. */
+ protected abstract String prettyPrint(SkyKey key);
+
+ /** Returns the associated Label of the SkyKey. */
+ protected abstract Label getLabel(SkyKey key);
+
+ protected abstract boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo);
+
+ protected String getAdditionalMessageAboutCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ return "";
+ }
+
+ @Override
+ public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo,
+ boolean alreadyReported, EventHandler eventHandler) {
+ Preconditions.checkNotNull(eventHandler);
+ if (!canReportCycle(topLevelKey, cycleInfo)) {
+ return false;
+ }
+
+ if (alreadyReported) {
+ Label label = getLabel(topLevelKey);
+ Target target = getTargetForLabel(label);
+ eventHandler.handle(Event.error(target.getLocation(),
+ "in " + target.getTargetKind() + " " + label +
+ ": cycle in dependency graph: target depends on an already-reported cycle"));
+ } else {
+ StringBuilder cycleMessage = new StringBuilder("cycle in dependency graph:");
+ ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle();
+ ImmutableList<SkyKey> cycle = cycleInfo.getCycle();
+ for (SkyKey value : pathToCycle) {
+ cycleMessage.append("\n ");
+ cycleMessage.append(prettyPrint(value));
+ }
+
+ SkyKey cycleValue = printCycle(cycle, cycleMessage, new Function<SkyKey, String>() {
+ @Override
+ public String apply(SkyKey input) {
+ return prettyPrint(input);
+ }
+ });
+
+ cycleMessage.append(getAdditionalMessageAboutCycle(topLevelKey, cycleInfo));
+
+ Label label = getLabel(cycleValue);
+ Target target = getTargetForLabel(label);
+ eventHandler.handle(
+ Event.error(target.getLocation(), "in " + target.getTargetKind() + " " + label
+ + ": " + cycleMessage.toString()));
+ }
+
+ return true;
+ }
+
+ /**
+ * Prints the SkyKey-s in cycle into cycleMessage using the print function.
+ */
+ static SkyKey printCycle(ImmutableList<SkyKey> cycle, StringBuilder cycleMessage,
+ Function<SkyKey, String> printFunction) {
+ Iterable<SkyKey> valuesToPrint = cycle.size() > 1
+ ? Iterables.concat(cycle, ImmutableList.of(cycle.get(0))) : cycle;
+ SkyKey cycleValue = null;
+ for (SkyKey value : valuesToPrint) {
+ if (cycleValue == null) {
+ cycleValue = value;
+ }
+ if (value == cycleValue) {
+ cycleMessage.append("\n * ");
+ } else {
+ cycleMessage.append("\n ");
+ }
+ cycleMessage.append(printFunction.apply(value));
+ }
+
+ if (cycle.size() == 1) {
+ cycleMessage.append(" [self-edge]");
+ }
+
+ return cycleValue;
+ }
+
+ protected final Target getTargetForLabel(Label label) {
+ try {
+ return loadedPackageProvider.getLoadedTarget(label);
+ } catch (NoSuchThingException e) {
+ // This method is used for getting the target from a label in a circular dependency.
+ // If we have a cycle that means that we need to have accessed the target (to get its
+ // dependencies). So all the labels in a dependency cycle need to exist.
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java
new file mode 100644
index 0000000..3105539
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Reports cycles between Actions and Artifacts. These indicates cycles within a rule.
+ */
+public class ActionArtifactCycleReporter extends AbstractLabelCycleReporter {
+
+ private static final Predicate<SkyKey> IS_ARTIFACT_OR_ACTION_SKY_KEY = Predicates.or(
+ SkyFunctions.isSkyFunction(SkyFunctions.ARTIFACT),
+ SkyFunctions.isSkyFunction(SkyFunctions.ACTION_EXECUTION),
+ SkyFunctions.isSkyFunction(SkyFunctions.TARGET_COMPLETION));
+
+ ActionArtifactCycleReporter(LoadedPackageProvider loadedPackageProvider) {
+ super(loadedPackageProvider);
+ }
+
+ @Override
+ protected String prettyPrint(SkyKey key) {
+ return prettyPrint(key.functionName(), key.argument());
+ }
+
+ private String prettyPrint(SkyFunctionName skyFunctionName, Object arg) {
+ if (arg instanceof OwnedArtifact) {
+ return "file: " + ((OwnedArtifact) arg).getArtifact().getRootRelativePathString();
+ } else if (arg instanceof Action) {
+ return "action: " + ((Action) arg).getMnemonic();
+ } else if (arg instanceof LabelAndConfiguration
+ && skyFunctionName == SkyFunctions.TARGET_COMPLETION) {
+ return "configured target: " + ((LabelAndConfiguration) arg).getLabel().toString();
+ }
+ throw new IllegalStateException(
+ "Argument is not Action, TargetCompletion, or OwnedArtifact: " + arg);
+ }
+
+ @Override
+ protected Label getLabel(SkyKey key) {
+ Object arg = key.argument();
+ if (arg instanceof OwnedArtifact) {
+ return ((OwnedArtifact) arg).getArtifact().getOwner();
+ } else if (arg instanceof Action) {
+ return ((Action) arg).getOwner().getLabel();
+ }
+ throw new IllegalStateException("Argument is not Action or OwnedArtifact: " + arg);
+ }
+
+ @Override
+ protected boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ return IS_ARTIFACT_OR_ACTION_SKY_KEY.apply(topLevelKey)
+ && Iterables.all(cycleInfo.getCycle(), IS_ARTIFACT_OR_ACTION_SKY_KEY);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
new file mode 100644
index 0000000..1420860
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
@@ -0,0 +1,338 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker.Token;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException2;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A builder for {@link ActionExecutionValue}s.
+ */
+public class ActionExecutionFunction implements SkyFunction {
+
+ private static final Predicate<Artifact> IS_SOURCE_ARTIFACT = new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact input) {
+ return input.isSourceArtifact();
+ }
+ };
+
+ private final SkyframeActionExecutor skyframeActionExecutor;
+ private final TimestampGranularityMonitor tsgm;
+
+ public ActionExecutionFunction(SkyframeActionExecutor skyframeActionExecutor,
+ TimestampGranularityMonitor tsgm) {
+ this.skyframeActionExecutor = skyframeActionExecutor;
+ this.tsgm = tsgm;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws ActionExecutionFunctionException,
+ InterruptedException {
+ Action action = (Action) skyKey.argument();
+ Map<Artifact, FileArtifactValue> inputArtifactData = null;
+ Map<Artifact, Collection<Artifact>> expandedMiddlemen = null;
+ boolean alreadyRan = skyframeActionExecutor.probeActionExecution(action);
+ try {
+ Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkedInputs =
+ checkInputs(env, action, alreadyRan); // Declare deps on known inputs to action.
+
+ if (checkedInputs != null) {
+ inputArtifactData = checkedInputs.first;
+ expandedMiddlemen = checkedInputs.second;
+ }
+ } catch (ActionExecutionException e) {
+ throw new ActionExecutionFunctionException(e);
+ }
+ // TODO(bazel-team): Non-volatile NotifyOnActionCacheHit actions perform worse in Skyframe than
+ // legacy when they are not at the top of the action graph. In legacy, they are stored
+ // separately, so notifying non-dirty actions is cheap. In Skyframe, they depend on the
+ // BUILD_ID, forcing invalidation of upward transitive closure on each build.
+ if (action.isVolatile() || action instanceof NotifyOnActionCacheHit) {
+ // Volatile build actions may need to execute even if none of their known inputs have changed.
+ // Depending on the buildID ensure that these actions have a chance to execute.
+ PrecomputedValue.BUILD_ID.get(env);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ ActionExecutionValue result;
+ try {
+ result = checkCacheAndExecuteIfNeeded(action, inputArtifactData, expandedMiddlemen, env);
+ } catch (ActionExecutionException e) {
+ // In this case we do not report the error to the action reporter because we have already
+ // done it in SkyframeExecutor.reportErrorIfNotAbortingMode() method. That method
+ // prints the error in the top-level reporter and also dumps the recorded StdErr for the
+ // action. Label can be null in the case of, e.g., the SystemActionOwner (for build-info.txt).
+ throw new ActionExecutionFunctionException(new AlreadyReportedActionExecutionException(e));
+ } finally {
+ declareAdditionalDependencies(env, action);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ return result;
+ }
+
+ private ActionExecutionValue checkCacheAndExecuteIfNeeded(
+ Action action,
+ Map<Artifact, FileArtifactValue> inputArtifactData,
+ Map<Artifact, Collection<Artifact>> expandedMiddlemen,
+ Environment env) throws ActionExecutionException, InterruptedException {
+ // Don't initialize the cache if the result has already been computed and this is just a
+ // rerun.
+ FileAndMetadataCache fileAndMetadataCache = null;
+ MetadataHandler metadataHandler = null;
+ Token token = null;
+ long actionStartTime = System.nanoTime();
+ // inputArtifactData is null exactly when we know that the execution result was already
+ // computed on a prior run of this SkyFunction. If it is null we don't need to initialize
+ // anything -- we will get the result directly from SkyframeActionExecutor's cache.
+ if (inputArtifactData != null) {
+ // Check action cache to see if we need to execute anything. Checking the action cache only
+ // needs to happen on the first run, since a cache hit means we'll return immediately, and
+ // there'll be no second run.
+ fileAndMetadataCache = new FileAndMetadataCache(
+ inputArtifactData,
+ expandedMiddlemen,
+ skyframeActionExecutor.getExecRoot(),
+ action.getOutputs(),
+ // Only give the metadata cache the ability to look up Skyframe values if the action
+ // might have undeclared inputs. If those undeclared inputs are generated, they are
+ // present in Skyframe, so we can save a stat by looking them up directly.
+ action.discoversInputs() ? env : null,
+ tsgm);
+ metadataHandler =
+ skyframeActionExecutor.constructMetadataHandler(fileAndMetadataCache);
+ token = skyframeActionExecutor.checkActionCache(action, metadataHandler, actionStartTime);
+ }
+ if (token == null && inputArtifactData != null) {
+ // We got a hit from the action cache -- no need to execute.
+ return new ActionExecutionValue(
+ fileAndMetadataCache.getOutputData(),
+ fileAndMetadataCache.getAdditionalOutputData());
+ } else {
+ ActionExecutionContext actionExecutionContext = null;
+ if (inputArtifactData != null) {
+ actionExecutionContext = skyframeActionExecutor.constructActionExecutionContext(
+ fileAndMetadataCache,
+ metadataHandler);
+ if (action.discoversInputs()) {
+ skyframeActionExecutor.discoverInputs(action, actionExecutionContext);
+ }
+ }
+ // If this is the second time we are here (because the action discovers inputs, and we had
+ // to restart the value builder after declaring our dependence on newly discovered inputs),
+ // the result returned here is the already-computed result from the first run.
+ // Similarly, if this is a shared action and the other action is the one that executed, we
+ // must use that other action's value, provided here, since it is populated with metadata
+ // for the outputs.
+ // If this action was not shared and this is the first run of the action, this returned
+ // result was computed during the call.
+ return skyframeActionExecutor.executeAction(action, fileAndMetadataCache, token,
+ actionStartTime, actionExecutionContext);
+ }
+ }
+
+ private static Iterable<SkyKey> toKeys(Iterable<Artifact> inputs,
+ Iterable<Artifact> mandatoryInputs) {
+ if (mandatoryInputs == null) {
+ // This is a non inputs-discovering action, so no need to distinguish mandatory from regular
+ // inputs.
+ return Iterables.transform(inputs, new Function<Artifact, SkyKey>() {
+ @Override
+ public SkyKey apply(Artifact artifact) {
+ return ArtifactValue.key(artifact, true);
+ }
+ });
+ } else {
+ Collection<SkyKey> discoveredArtifacts = new HashSet<>();
+ Set<Artifact> mandatory = Sets.newHashSet(mandatoryInputs);
+ for (Artifact artifact : inputs) {
+ discoveredArtifacts.add(ArtifactValue.key(artifact, mandatory.contains(artifact)));
+ }
+
+ // In case the action violates the invariant that getInputs() is a superset of
+ // getMandatoryInputs(), explicitly add the mandatory inputs. See bug about an
+ // "action not in canonical form" error message. Also note that we may add Skyframe edges on
+ // these potentially stale deps due to the way loading inputs from the action cache functions.
+ // In practice, this is safe since C++ actions (the only ones which discover inputs) only add
+ // possibly stale inputs on source artifacts, which we treat as non-mandatory.
+ for (Artifact artifact : mandatory) {
+ discoveredArtifacts.add(ArtifactValue.key(artifact, true));
+ }
+ return discoveredArtifacts;
+ }
+ }
+
+ /**
+ * Declare dependency on all known inputs of action. Throws exception if any are known to be
+ * missing. Some inputs may not yet be in the graph, in which case the builder should abort.
+ */
+ private Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkInputs(
+ Environment env, Action action, boolean alreadyRan) throws ActionExecutionException {
+ Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps =
+ env.getValuesOrThrow(toKeys(action.getInputs(), action.discoversInputs()
+ ? action.getMandatoryInputs() : null), MissingInputFileException.class,
+ ActionExecutionException.class);
+
+ // If the action was already run, then break out early. This avoids the cost of constructing the
+ // input map and expanded middlemen if they're not going to be used.
+ if (alreadyRan) {
+ return null;
+ }
+
+ int missingCount = 0;
+ int actionFailures = 0;
+ boolean catastrophe = false;
+ // Only populate input data if we have the input values, otherwise they'll just go unused.
+ // We still want to loop through the inputs to collect missing deps errors. During the
+ // evaluator "error bubbling", we may get one last chance at reporting errors even though
+ // some deps are stilling missing.
+ boolean populateInputData = !env.valuesMissing();
+ NestedSetBuilder<Label> rootCauses = NestedSetBuilder.stableOrder();
+ Map<Artifact, FileArtifactValue> inputArtifactData =
+ new HashMap<>(populateInputData ? inputDeps.size() : 0);
+ Map<Artifact, Collection<Artifact>> expandedMiddlemen =
+ new HashMap<>(populateInputData ? 128 : 0);
+
+ ActionExecutionException firstActionExecutionException = null;
+ for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException,
+ ActionExecutionException>> depsEntry : inputDeps.entrySet()) {
+ Artifact input = ArtifactValue.artifact(depsEntry.getKey());
+ try {
+ ArtifactValue value = (ArtifactValue) depsEntry.getValue().get();
+ if (populateInputData && value instanceof AggregatingArtifactValue) {
+ AggregatingArtifactValue aggregatingValue = (AggregatingArtifactValue) value;
+ for (Pair<Artifact, FileArtifactValue> entry : aggregatingValue.getInputs()) {
+ inputArtifactData.put(entry.first, entry.second);
+ }
+ // We have to cache the "digest" of the aggregating value itself, because the action cache
+ // checker may want it.
+ inputArtifactData.put(input, aggregatingValue.getSelfData());
+ expandedMiddlemen.put(input,
+ Collections2.transform(aggregatingValue.getInputs(),
+ Pair.<Artifact, FileArtifactValue>firstFunction()));
+ } else if (populateInputData && value instanceof FileArtifactValue) {
+ // TODO(bazel-team): Make sure middleman "virtual" artifact data is properly processed.
+ inputArtifactData.put(input, (FileArtifactValue) value);
+ }
+ } catch (MissingInputFileException e) {
+ missingCount++;
+ if (input.getOwner() != null) {
+ rootCauses.add(input.getOwner());
+ }
+ } catch (ActionExecutionException e) {
+ actionFailures++;
+ if (firstActionExecutionException == null) {
+ firstActionExecutionException = e;
+ }
+ catastrophe = catastrophe || e.isCatastrophe();
+ rootCauses.addTransitive(e.getRootCauses());
+ }
+ }
+ // We need to rethrow first exception because it can contain useful error message
+ if (firstActionExecutionException != null) {
+ if (missingCount == 0 && actionFailures == 1) {
+ // In the case a single action failed, just propagate the exception upward. This avoids
+ // having to copy the root causes to the upwards transitive closure.
+ throw firstActionExecutionException;
+ }
+ throw new ActionExecutionException(firstActionExecutionException.getMessage(),
+ firstActionExecutionException.getCause(), action, rootCauses.build(), catastrophe);
+ }
+
+ if (missingCount > 0) {
+ for (Label missingInput : rootCauses.build()) {
+ env.getListener().handle(Event.error(action.getOwner().getLocation(), String.format(
+ "%s: missing input file '%s'", action.getOwner().getLabel(), missingInput)));
+ }
+ throw new ActionExecutionException(missingCount + " input file(s) do not exist", action,
+ rootCauses.build(), /*catastrophe=*/false);
+ }
+ return Pair.of(
+ Collections.unmodifiableMap(inputArtifactData),
+ Collections.unmodifiableMap(expandedMiddlemen));
+ }
+
+ private static void declareAdditionalDependencies(Environment env, Action action) {
+ if (action.discoversInputs()) {
+ // TODO(bazel-team): Should this be all inputs, or just source files?
+ env.getValues(toKeys(Iterables.filter(action.getInputs(), IS_SOURCE_ARTIFACT),
+ action.getMandatoryInputs()));
+ }
+ }
+
+ /**
+ * All info/warning messages associated with actions should be always displayed.
+ */
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ActionExecutionFunction#compute}.
+ */
+ private static final class ActionExecutionFunctionException extends SkyFunctionException {
+
+ private final ActionExecutionException actionException;
+
+ public ActionExecutionFunctionException(ActionExecutionException e) {
+ // We conservatively assume that the error is transient. We don't have enough information to
+ // distinguish non-transient errors (e.g. compilation error from a deterministic compiler)
+ // from transient ones (e.g. IO error).
+ // TODO(bazel-team): Have ActionExecutionExceptions declare their transience.
+ super(e, Transience.TRANSIENT);
+ this.actionException = e;
+ }
+
+ @Override
+ public boolean isCatastrophic() {
+ return actionException.isCatastrophe();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java
new file mode 100644
index 0000000..87e3e0d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java
@@ -0,0 +1,180 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * An object that can monitor whether actions are getting completed in a timely manner.
+ *
+ * <p>If there's nothing happening for a while, a background thread will print (and update) the
+ * "Still waiting for N actions to complete..." message.
+ */
+public final class ActionExecutionInactivityWatchdog {
+
+ /** An object used in monitoring action execution inactivity. */
+ public interface InactivityMonitor {
+
+ /** Returns whether action execution has started. */
+ boolean hasStarted();
+
+ /** Returns the number of enqueued but not yet completed actions. */
+ int getPending();
+
+ /**
+ * Waits for any action to complete, or the timeout to elapse.
+ *
+ * <p>The thread must wait at least for the specified timeout, unless some action completes in
+ * the meantime. It's not allowed to return 0 too early.
+ *
+ * <p>Note that it's acceptable to return (any value) later than specified by the timeout.
+ *
+ * @return the number of actions completed during the wait
+ */
+ int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException;
+ }
+
+ /** An object that the watchdog can report inactivity to. */
+ public interface InactivityReporter {
+
+ /**
+ * Report that actions are not getting completed in a timely manner.
+ *
+ * <p>Inactivity is typically not reported if tests with streaming output are being run.
+ */
+ void maybeReportInactivity();
+ }
+
+ @VisibleForTesting
+ interface Sleep {
+ void sleep(int durationMilliseconds) throws InterruptedException;
+ }
+
+ private static final class WaitTime {
+ private final int progressIntervalFlagValue;
+ private int prev;
+
+ public WaitTime(int progressIntervalFlagValue) {
+ this.progressIntervalFlagValue = progressIntervalFlagValue;
+ }
+
+ public void reset() {
+ prev = 0;
+ }
+
+ public int next() {
+ prev = ActionExecutionStatusReporter.getWaitTime(progressIntervalFlagValue, prev);
+ return prev;
+ }
+ }
+
+ private final AtomicBoolean isRunning = new AtomicBoolean(false);
+ private final InactivityMonitor monitor;
+ private final InactivityReporter reporter;
+ private final Sleep sleeper;
+ private final Thread thread;
+ private final WaitTime waitTime;
+
+ public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter,
+ int progressIntervalFlagValue) {
+ this(monitor, reporter, progressIntervalFlagValue, new Sleep() {
+ @Override
+ public void sleep(int durationMilliseconds) throws InterruptedException {
+ Thread.sleep(durationMilliseconds);
+ }
+ });
+ }
+
+ @VisibleForTesting
+ public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter,
+ int progressIntervalFlagValue, Sleep sleeper) {
+ this.monitor = Preconditions.checkNotNull(monitor);
+ this.reporter = Preconditions.checkNotNull(reporter);
+ this.sleeper = Preconditions.checkNotNull(sleeper);
+ this.waitTime = new WaitTime(progressIntervalFlagValue);
+ this.thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ enterWatchdogLoop();
+ }
+ });
+ this.thread.setDaemon(true);
+ this.thread.setName("action-execution-watchdog");
+ }
+
+ /** Starts the watchdog thread. This method should only be called once. */
+ public void start() {
+ Preconditions.checkState(!isRunning.getAndSet(true));
+ thread.start();
+ }
+
+ /**
+ * Stops the watchdog thread. This method should only be called once.
+ *
+ * <p>The method waits for the thread to terminate. If the caller thread is interrupted
+ * in the meantime, the interrupted status will be set.
+ */
+ public void stop() {
+ Preconditions.checkState(isRunning.getAndSet(false));
+ thread.interrupt();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ // When Thread.join throws, the interrupted status is cleared. We need to set it again.
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void enterWatchdogLoop() {
+ while (isRunning.get()) {
+ try {
+ // Wait a while for any SkyFunction to finish. The returned number indicates how many
+ // actions completed during the wait. It's possible that this is more than 1, since
+ // this thread may not immediately regain control.
+ int completedActions = monitor.waitForNextCompletion(waitTime.next() * 1000);
+ if (!isRunning.get()) {
+ break;
+ }
+
+ int pending = monitor.getPending();
+ if (!monitor.hasStarted() || completedActions > 0 || pending == 0) {
+ // If no keys have been enqueued yet (execution hasn't started), or some actions
+ // were completed since this thread was notified (we are making visible progress),
+ // or there are currently no enqueued actions waiting to be processed (perhaps all
+ // have completed and we are about to stop monitoring), then there's no need to
+ // display any messages.
+ waitTime.reset();
+
+ // Sleep a while before checking again. Actions might be executing at a nice rate, no
+ // need to worry about inactivity. This extra sleep isn't required but it's nice to
+ // have: without it we would, at times of high action completion rate, unnecessarily
+ // put the monitor into a fast sleep-wake cycle --- not a big problem but wasteful.
+ sleeper.sleep(1000);
+ } else {
+ // If actions are executing but we haven't made any progress in a while (no new
+ // action completion), then reassure the user that we're still running. Next time
+ // wait a little longer.
+ reporter.maybeReportInactivity();
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java
new file mode 100644
index 0000000..de63c3b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java
@@ -0,0 +1,117 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value representing an executed action.
+ */
+@Immutable
+@ThreadSafe
+public class ActionExecutionValue implements SkyValue {
+ private final ImmutableMap<Artifact, FileValue> artifactData;
+ private final ImmutableMap<Artifact, FileArtifactValue> additionalOutputData;
+
+ /**
+ * @param artifactData Map from Artifacts to corresponding FileValues.
+ * @param additionalOutputData Map from Artifacts to values if the FileArtifactValue for this
+ * artifact cannot be derived from the corresponding FileValue (see {@link
+ * FileAndMetadataCache#getAdditionalOutputData} for when this is necessary).
+ */
+ ActionExecutionValue(Map<Artifact, FileValue> artifactData,
+ Map<Artifact, FileArtifactValue> additionalOutputData) {
+ this.artifactData = ImmutableMap.copyOf(artifactData);
+ this.additionalOutputData = ImmutableMap.copyOf(additionalOutputData);
+ }
+
+ /**
+ * Returns metadata for a given artifact, if that metadata cannot be inferred from the
+ * corresponding {@link #getData} call for that Artifact. See {@link
+ * FileAndMetadataCache#getAdditionalOutputData} for when that can happen.
+ */
+ @Nullable
+ FileArtifactValue getArtifactValue(Artifact artifact) {
+ return additionalOutputData.get(artifact);
+ }
+
+ /**
+ * @return The data for each non-middleman output of this action, in the form of the {@link
+ * FileValue} that would be created for the file if it were to be read from disk.
+ */
+ FileValue getData(Artifact artifact) {
+ Preconditions.checkState(!additionalOutputData.containsKey(artifact),
+ "Should not be requesting data for already-constructed FileArtifactValue: %s", artifact);
+ return artifactData.get(artifact);
+ }
+
+ /**
+ * @return The map from {@link Artifact} to the corresponding {@link FileValue} that would be
+ * returned by {@link #getData}. Should only be needed by {@link FilesystemValueChecker}.
+ */
+ ImmutableMap<Artifact, FileValue> getAllOutputArtifactData() {
+ return artifactData;
+ }
+
+ @ThreadSafe
+ @VisibleForTesting
+ public static SkyKey key(Action action) {
+ return new SkyKey(SkyFunctions.ACTION_EXECUTION, action);
+ }
+
+ /**
+ * Returns whether the key corresponds to a ActionExecutionValue worth reporting status about.
+ *
+ * <p>If an action can do real work, it's probably worth counting and reporting status about.
+ * Actions that don't really do any work (typically middleman actions) should not be counted
+ * towards enqueued and completed actions.
+ */
+ public static boolean isReportWorthyAction(SkyKey key) {
+ return key.functionName() == SkyFunctions.ACTION_EXECUTION
+ && isReportWorthyAction((Action) key.argument());
+ }
+
+ /**
+ * Returns whether the action is worth reporting status about.
+ *
+ * <p>If an action can do real work, it's probably worth counting and reporting status about.
+ * Actions that don't really do any work (typically middleman actions) should not be counted
+ * towards enqueued and completed actions.
+ */
+ public static boolean isReportWorthyAction(Action action) {
+ return action.getActionType() == MiddlemanType.NORMAL;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("artifactData", artifactData)
+ .add("additionalOutputData", additionalOutputData)
+ .toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java
new file mode 100644
index 0000000..1dfa722
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java
@@ -0,0 +1,106 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Base class for all values which can provide the generating action of an artifact. The primary
+ * instance of such lookup values is {@link ConfiguredTargetValue}. Values that hold the generating
+ * actions of target completion values and build info artifacts also fall into this category.
+ */
+public class ActionLookupValue implements SkyValue {
+ protected final ImmutableMap<Artifact, Action> generatingActionMap;
+
+ ActionLookupValue(Iterable<Action> actions) {
+ // Duplicate/shared actions get passed in all the time. Blaze is weird. We can't double-register
+ // the generated artifacts in an immutable map builder, so we double-register them in a more
+ // forgiving map, and then use that map to create the immutable one.
+ Map<Artifact, Action> generatingActions = new HashMap<>();
+ for (Action action : actions) {
+ for (Artifact artifact : action.getOutputs()) {
+ generatingActions.put(artifact, action);
+ }
+ }
+ generatingActionMap = ImmutableMap.copyOf(generatingActions);
+ }
+
+ ActionLookupValue(Action action) {
+ this(ImmutableList.of(action));
+ }
+
+ Action getGeneratingAction(Artifact artifact) {
+ return generatingActionMap.get(artifact);
+ }
+
+ /** To be used only when checking consistency of the action graph -- not by other values. */
+ ImmutableMap<Artifact, Action> getMapForConsistencyCheck() {
+ return generatingActionMap;
+ }
+
+ /**
+ * To be used only when setting the owners of deserialized artifacts whose owners were unknown at
+ * creation time -- not by other callers or values.
+ */
+ Iterable<Action> getActionsForFindingArtifactOwners() {
+ return generatingActionMap.values();
+ }
+
+ @VisibleForTesting
+ public static SkyKey key(ActionLookupKey ownerKey) {
+ return ownerKey.getSkyKey();
+ }
+
+ /**
+ * ArtifactOwner is not a SkyKey, but we wish to convert any ArtifactOwner into a SkyKey as
+ * simply as possible. To that end, all subclasses of ActionLookupValue "own" artifacts with
+ * ArtifactOwners that are subclasses of ActionLookupKey. This allows callers to easily find the
+ * value key, while remaining agnostic to what ActionLookupValues actually exist.
+ *
+ * <p>The methods of this class should only be called by {@link ActionLookupValue#key}.
+ */
+ protected abstract static class ActionLookupKey implements ArtifactOwner {
+ @Override
+ public Label getLabel() {
+ return null;
+ }
+
+ /**
+ * Subclasses must override this to specify their specific value type, unless they override
+ * {@link #getSkyKey}, in which case they are free not to implement this method.
+ */
+ abstract SkyFunctionName getType();
+
+ /**
+ * Prefer {@link ActionLookupValue#key} to calling this method directly.
+ *
+ * <p>Subclasses may override if the value key contents should not be the key itself.
+ */
+ SkyKey getSkyKey() {
+ return new SkyKey(getType(), this);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java
new file mode 100644
index 0000000..8374efe
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.Collection;
+
+/** Value for aggregating artifacts, which must be expanded to a set of other artifacts. */
+class AggregatingArtifactValue extends ArtifactValue {
+ private final FileArtifactValue selfData;
+ private final ImmutableList<Pair<Artifact, FileArtifactValue>> inputs;
+
+ AggregatingArtifactValue(ImmutableList<Pair<Artifact, FileArtifactValue>> inputs,
+ FileArtifactValue selfData) {
+ this.inputs = inputs;
+ this.selfData = selfData;
+ }
+
+ /** Returns the artifacts that this artifact expands to, together with their data. */
+ Collection<Pair<Artifact, FileArtifactValue>> getInputs() {
+ return inputs;
+ }
+
+ /** Returns the data of the artifact for this value, as computed by the action cache checker. */
+ FileArtifactValue getSelfData() {
+ return selfData;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
new file mode 100644
index 0000000..e277476
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
@@ -0,0 +1,230 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey;
+import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A builder for {@link ArtifactValue}s.
+ */
+class ArtifactFunction implements SkyFunction {
+
+ private final Predicate<PathFragment> allowedMissingInputs;
+
+ ArtifactFunction(Predicate<PathFragment> allowedMissingInputs) {
+ this.allowedMissingInputs = allowedMissingInputs;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws ArtifactFunctionException {
+ OwnedArtifact ownedArtifact = (OwnedArtifact) skyKey.argument();
+ Artifact artifact = ownedArtifact.getArtifact();
+ if (artifact.isSourceArtifact()) {
+ try {
+ return createSourceValue(artifact, ownedArtifact.isMandatory(), env);
+ } catch (MissingInputFileException e) {
+ // The error is not necessarily truly transient, but we mark it as such because we have
+ // the above side effect of posting an event to the EventBus. Importantly, that event
+ // is potentially used to report root causes.
+ throw new ArtifactFunctionException(e, Transience.TRANSIENT);
+ }
+ }
+
+ Action action = extractActionFromArtifact(artifact, env);
+ if (action == null) {
+ return null;
+ }
+
+ ActionExecutionValue actionValue =
+ (ActionExecutionValue) env.getValue(ActionExecutionValue.key(action));
+ if (actionValue == null) {
+ return null;
+ }
+
+ if (!isAggregatingValue(action)) {
+ try {
+ return createSimpleValue(artifact, actionValue);
+ } catch (IOException e) {
+ ActionExecutionException ex = new ActionExecutionException(e, action,
+ /*catastrophe=*/false);
+ env.getListener().handle(Event.error(ex.getLocation(), ex.getMessage()));
+ // This is a transient error since we did the work that led to the IOException.
+ throw new ArtifactFunctionException(ex, Transience.TRANSIENT);
+ }
+ } else {
+ return createAggregatingValue(artifact, action, actionValue.getArtifactValue(artifact), env);
+ }
+ }
+
+ private ArtifactValue createSourceValue(Artifact artifact, boolean mandatory, Environment env)
+ throws MissingInputFileException {
+ SkyKey fileSkyKey = FileValue.key(RootedPath.toRootedPath(artifact.getRoot().getPath(),
+ artifact.getPath()));
+ FileValue fileValue;
+ try {
+ fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
+ InconsistentFilesystemException.class, FileSymlinkCycleException.class);
+ } catch (IOException | InconsistentFilesystemException | FileSymlinkCycleException e) {
+ throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
+ }
+ if (fileValue == null) {
+ return null;
+ }
+ if (!fileValue.exists()) {
+ if (allowedMissingInputs.apply(((RootedPath) fileSkyKey.argument()).getRelativePath())) {
+ return FileArtifactValue.MISSING_FILE_MARKER;
+ } else {
+ return missingInputFile(artifact, mandatory, null, env.getListener());
+ }
+ }
+ try {
+ return FileArtifactValue.create(artifact, fileValue);
+ } catch (IOException e) {
+ throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
+ }
+ }
+
+ private static ArtifactValue missingInputFile(Artifact artifact, boolean mandatory,
+ Exception failure, EventHandler reporter) throws MissingInputFileException {
+ if (!mandatory) {
+ return FileArtifactValue.MISSING_FILE_MARKER;
+ }
+ throw makeMissingInputFileExn(artifact, mandatory, failure, reporter);
+ }
+
+ private static MissingInputFileException makeMissingInputFileExn(Artifact artifact,
+ boolean mandatory, Exception failure, EventHandler reporter) {
+ String extraMsg = (failure == null) ? "" : (":" + failure.getMessage());
+ MissingInputFileException ex = new MissingInputFileException(
+ constructErrorMessage(artifact) + extraMsg, null);
+ if (mandatory) {
+ reporter.handle(Event.error(ex.getLocation(), ex.getMessage()));
+ }
+ return ex;
+ }
+
+ // Non-aggregating artifact -- should contain at most one piece of artifact data.
+ // data may be null if and only if artifact is a middleman artifact.
+ private ArtifactValue createSimpleValue(Artifact artifact, ActionExecutionValue actionValue)
+ throws IOException {
+ ArtifactValue value = actionValue.getArtifactValue(artifact);
+ if (value != null) {
+ return value;
+ }
+ // Middleman artifacts have no corresponding files, so their ArtifactValues should have already
+ // been constructed during execution of the action.
+ Preconditions.checkState(!artifact.isMiddlemanArtifact(), artifact);
+ FileValue data = Preconditions.checkNotNull(actionValue.getData(artifact),
+ "%s %s", artifact, actionValue);
+ Preconditions.checkNotNull(data.getDigest(),
+ "Digest should already have been calculated for %s (%s)", artifact, data);
+ return FileArtifactValue.create(artifact, data);
+ }
+
+ private AggregatingArtifactValue createAggregatingValue(Artifact artifact, Action action,
+ FileArtifactValue value, SkyFunction.Environment env) {
+ // This artifact aggregates other artifacts. Keep track of them so callers can find them.
+ ImmutableList.Builder<Pair<Artifact, FileArtifactValue>> inputs = ImmutableList.builder();
+ for (Map.Entry<SkyKey, SkyValue> entry :
+ env.getValues(ArtifactValue.mandatoryKeys(action.getInputs())).entrySet()) {
+ Artifact input = ArtifactValue.artifact(entry.getKey());
+ ArtifactValue inputValue = (ArtifactValue) entry.getValue();
+ Preconditions.checkNotNull(inputValue, "%s has null dep %s", artifact, input);
+ if (!(inputValue instanceof FileArtifactValue)) {
+ // We do not recurse in aggregating middleman artifacts.
+ Preconditions.checkState(!(inputValue instanceof AggregatingArtifactValue),
+ "%s %s %s", artifact, action, inputValue);
+ continue;
+ }
+ inputs.add(Pair.of(input, (FileArtifactValue) inputValue));
+ }
+ return new AggregatingArtifactValue(inputs.build(), value);
+ }
+
+ /**
+ * Returns whether this value needs to contain the data of all its inputs. Currently only tests to
+ * see if the action is an aggregating middleman action. However, may include runfiles middleman
+ * actions and Fileset artifacts in the future.
+ */
+ private static boolean isAggregatingValue(Action action) {
+ return action.getActionType() == MiddlemanType.AGGREGATING_MIDDLEMAN;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((OwnedArtifact) skyKey.argument()).getArtifact().getOwner());
+ }
+
+ private Action extractActionFromArtifact(Artifact artifact, SkyFunction.Environment env) {
+ ArtifactOwner artifactOwner = artifact.getArtifactOwner();
+
+ Preconditions.checkState(artifactOwner instanceof ActionLookupKey, "", artifact, artifactOwner);
+ SkyKey actionLookupKey = ActionLookupValue.key((ActionLookupKey) artifactOwner);
+ ActionLookupValue value = (ActionLookupValue) env.getValue(actionLookupKey);
+ if (value == null) {
+ Preconditions.checkState(artifactOwner == CoverageReportValue.ARTIFACT_OWNER,
+ "Not-yet-present artifact owner: %s", artifactOwner);
+ return null;
+ }
+ // The value should already exist (except for the coverage report action output artifacts):
+ // ConfiguredTargetValues were created during the analysis phase, and BuildInfo*Values
+ // were created during the first analysis of a configured target.
+ Preconditions.checkNotNull(value,
+ "Owner %s of %s not in graph %s", artifactOwner, artifact, actionLookupKey);
+ return Preconditions.checkNotNull(value.getGeneratingAction(artifact),
+ "Value %s does not contain generating action of %s", value, artifact);
+ }
+
+ private static final class ArtifactFunctionException extends SkyFunctionException {
+ ArtifactFunctionException(MissingInputFileException e, Transience transience) {
+ super(e, transience);
+ }
+
+ ArtifactFunctionException(ActionExecutionException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+
+ private static String constructErrorMessage(Artifact artifact) {
+ if (artifact.getOwner() == null) {
+ return String.format("missing input file '%s'", artifact.getPath().getPathString());
+ } else {
+ return String.format("missing input file '%s'", artifact.getOwner());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java
new file mode 100644
index 0000000..6139d2e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java
@@ -0,0 +1,160 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+
+/**
+ * A value representing an artifact. Source artifacts are checked for existence, while output
+ * artifacts imply creation of the output file.
+ *
+ * <p>There are effectively two kinds of output artifact values. The first corresponds to an
+ * ordinary artifact {@link FileArtifactValue}. It stores the relevant data for the artifact --
+ * digest/mtime and size. The second corresponds to an "aggregating" artifact -- the output of an
+ * aggregating middleman action. It stores the relevant data of all its inputs.
+ */
+@Immutable
+@ThreadSafe
+public abstract class ArtifactValue implements SkyValue {
+
+ @ThreadSafe
+ static SkyKey key(Artifact artifact, boolean isMandatory) {
+ return new SkyKey(SkyFunctions.ARTIFACT, artifact.isSourceArtifact()
+ ? new OwnedArtifact(artifact, isMandatory)
+ : new OwnedArtifact(artifact));
+ }
+
+ private static final Function<Artifact, SkyKey> TO_MANDATORY_KEY =
+ new Function<Artifact, SkyKey>() {
+ @Override
+ public SkyKey apply(Artifact artifact) {
+ return key(artifact, true);
+ }
+ };
+
+ @ThreadSafe
+ public static Iterable<SkyKey> mandatoryKeys(Iterable<Artifact> artifacts) {
+ return Iterables.transform(artifacts, TO_MANDATORY_KEY);
+ }
+
+ private static final Function<OwnedArtifact, Artifact> TO_ARTIFACT =
+ new Function<OwnedArtifact, Artifact>() {
+ @Override
+ public Artifact apply(OwnedArtifact key) {
+ return key.getArtifact();
+ }
+ };
+
+ public static Collection<Artifact> artifacts(Collection<? extends OwnedArtifact> keys) {
+ return Collections2.transform(keys, TO_ARTIFACT);
+ }
+
+ public static Artifact artifact(SkyKey key) {
+ return TO_ARTIFACT.apply((OwnedArtifact) key.argument());
+ }
+
+ /**
+ * Artifacts are compared using just their paths, but in Skyframe, the configured target that owns
+ * an artifact must also be part of the comparison. For example, suppose we build //foo:foo in
+ * configurationA, yielding artifact foo.out. If we change the configuration to configurationB in
+ * such a way that the path to the artifact does not change, requesting foo.out from the graph
+ * will result in the value entry for foo.out under configurationA being returned. This would
+ * prevent caching the graph in different configurations, and also causes big problems with change
+ * pruning, which assumes the invariant that a value's first dependency will always be the same.
+ * In this case, the value entry's old dependency on //foo:foo in configurationA would cause it to
+ * request (//foo:foo, configurationA) from the graph, causing an undesired re-analysis of
+ * (//foo:foo, configurationA).
+ *
+ * <p>In order to prevent that, instead of using Artifacts as keys in the graph, we use
+ * OwnedArtifacts, which compare for equality using both the Artifact, and the owner. The effect
+ * is functionally that of making Artifact.equals() check the owner, but only within Skyframe,
+ * since outside of Skyframe it is quite crucial that Artifacts with different owners be able to
+ * compare equal.
+ */
+ public static class OwnedArtifact {
+ private final Artifact artifact;
+ // Always true for derived artifacts.
+ private final boolean isMandatory;
+
+ /** Constructs an OwnedArtifact wrapper for a source artifact. */
+ private OwnedArtifact(Artifact sourceArtifact, boolean mandatory) {
+ Preconditions.checkArgument(sourceArtifact.isSourceArtifact());
+ this.artifact = Preconditions.checkNotNull(sourceArtifact);
+ this.isMandatory = mandatory;
+ }
+
+ /**
+ * Constructs an OwnedArtifact wrapper for a derived artifact. The mandatory attribute is
+ * not needed because a derived artifact must be a mandatory input for some action in order to
+ * ensure that it is built in the first place. If it fails to build, then that fact is cached
+ * in the node, so any action that has it as a non-mandatory input can retrieve that
+ * information from the node.
+ */
+ private OwnedArtifact(Artifact derivedArtifact) {
+ this.artifact = Preconditions.checkNotNull(derivedArtifact);
+ Preconditions.checkArgument(!derivedArtifact.isSourceArtifact(), derivedArtifact);
+ this.isMandatory = true; // Unused.
+ }
+
+ @Override
+ public int hashCode() {
+ int initialHash = artifact.hashCode() + artifact.getArtifactOwner().hashCode();
+ return isMandatory ? initialHash : 47 * initialHash + 1;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+ if (!(that instanceof OwnedArtifact)) {
+ return false;
+ }
+ OwnedArtifact thatOwnedArtifact = ((OwnedArtifact) that);
+ Artifact thatArtifact = thatOwnedArtifact.artifact;
+ return artifact.equals(thatArtifact)
+ && artifact.getArtifactOwner().equals(thatArtifact.getArtifactOwner())
+ && isMandatory == thatOwnedArtifact.isMandatory;
+ }
+
+ Artifact getArtifact() {
+ return artifact;
+ }
+
+ /**
+ * Returns whether the artifact is a mandatory input of its requesting action. May only be
+ * called for source artifacts, since a derived artifact must be a mandatory input of some
+ * action in order to have been built in the first place.
+ */
+ public boolean isMandatory() {
+ Preconditions.checkState(artifact.isSourceArtifact(), artifact);
+ return isMandatory;
+ }
+
+ @Override
+ public String toString() {
+ return artifact.prettyPrint() + " " + artifact.getArtifactOwner();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
new file mode 100644
index 0000000..f1aa2f6e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
@@ -0,0 +1,187 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.AspectFactory;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.DependencyEvaluationException;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * The Skyframe function that generates aspects.
+ */
+public final class AspectFunction implements SkyFunction {
+ private final BuildViewProvider buildViewProvider;
+
+ public AspectFunction(BuildViewProvider buildViewProvider) {
+ this.buildViewProvider = buildViewProvider;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws AspectFunctionException {
+ SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
+ AspectKey key = (AspectKey) skyKey.argument();
+ ConfiguredAspectFactory aspectFactory =
+ (ConfiguredAspectFactory) AspectFactory.Util.create(key.getAspect());
+
+ PackageValue packageValue =
+ (PackageValue) env.getValue(PackageValue.key(key.getLabel().getPackageIdentifier()));
+ if (packageValue == null) {
+ return null;
+ }
+
+ Target target;
+ try {
+ target = packageValue.getPackage().getTarget(key.getLabel().getName());
+ } catch (NoSuchTargetException e) {
+ throw new AspectFunctionException(skyKey, e);
+ }
+
+ if (!(target instanceof Rule)) {
+ throw new AspectFunctionException(new AspectCreationException(
+ "aspects must be attached to rules"));
+ }
+
+ RuleConfiguredTarget associatedTarget = (RuleConfiguredTarget)
+ ((ConfiguredTargetValue) env.getValue(ConfiguredTargetValue.key(
+ key.getLabel(), key.getConfiguration()))).getConfiguredTarget();
+
+ if (associatedTarget == null) {
+ return null;
+ }
+
+ SkyframeDependencyResolver resolver = view.createDependencyResolver(env);
+ if (resolver == null) {
+ return null;
+ }
+
+ TargetAndConfiguration ctgValue =
+ new TargetAndConfiguration(target, key.getConfiguration());
+
+ try {
+ // Get the configuration targets that trigger this rule's configurable attributes.
+ Set<ConfigMatchingProvider> configConditions =
+ ConfiguredTargetFunction.getConfigConditions(target, env, resolver, ctgValue);
+ if (configConditions == null) {
+ // Those targets haven't yet been resolved.
+ return null;
+ }
+
+ ListMultimap<Attribute, ConfiguredTarget> depValueMap =
+ ConfiguredTargetFunction.computeDependencies(env, resolver, ctgValue,
+ aspectFactory.getDefinition(), configConditions);
+
+ return createAspect(env, key, associatedTarget, configConditions, depValueMap);
+ } catch (DependencyEvaluationException e) {
+ throw new AspectFunctionException(e.getRootCauseSkyKey(), e.getCause());
+ }
+ }
+
+ @Nullable
+ private AspectValue createAspect(Environment env, AspectKey key,
+ RuleConfiguredTarget associatedTarget, Set<ConfigMatchingProvider> configConditions,
+ ListMultimap<Attribute, ConfiguredTarget> directDeps)
+ throws AspectFunctionException {
+ SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
+ BuildConfiguration configuration = associatedTarget.getConfiguration();
+ boolean extendedSanityChecks = configuration != null && configuration.extendedSanityChecks();
+
+ StoredEventHandler events = new StoredEventHandler();
+ CachingAnalysisEnvironment analysisEnvironment = view.createAnalysisEnvironment(
+ key, false, extendedSanityChecks, events, env, true);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ ConfiguredAspectFactory aspectFactory =
+ (ConfiguredAspectFactory) AspectFactory.Util.create(key.getAspect());
+ Aspect aspect = view.createAspect(
+ analysisEnvironment, associatedTarget, aspectFactory, directDeps, configConditions);
+
+ events.replayOn(env.getListener());
+ if (events.hasErrors()) {
+ analysisEnvironment.disable(associatedTarget.getTarget());
+ throw new AspectFunctionException(new AspectCreationException(
+ "Analysis of target '" + associatedTarget.getLabel() + "' failed; build aborted"));
+ }
+ Preconditions.checkState(!analysisEnvironment.hasErrors(),
+ "Analysis environment hasError() but no errors reported");
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ analysisEnvironment.disable(associatedTarget.getTarget());
+ Preconditions.checkNotNull(aspect);
+
+ return new AspectValue(
+ aspect, ImmutableList.copyOf(analysisEnvironment.getRegisteredActions()));
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * An exception indicating that there was a problem creating an aspect.
+ */
+ public static final class AspectCreationException extends Exception {
+ public AspectCreationException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Used to indicate errors during the computation of an {@link AspectValue}.
+ */
+ private static final class AspectFunctionException extends SkyFunctionException {
+ public AspectFunctionException(Exception e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ /** Used to rethrow a child error that we cannot handle. */
+ public AspectFunctionException(SkyKey childKey, Exception transitiveError) {
+ super(transitiveError, childKey);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java
new file mode 100644
index 0000000..9b863bd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java
@@ -0,0 +1,109 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * An aspect in the context of the Skyframe graph.
+ */
+public final class AspectValue extends ActionLookupValue {
+ /**
+ * The key of an action that is generated by an aspect.
+ */
+ public static final class AspectKey extends ActionLookupKey {
+ private final Label label;
+ private final BuildConfiguration configuration;
+ // TODO(bazel-team): class objects are not really hashable or comparable for equality other than
+ // by reference. We should identify the aspect here in a way that does not rely on comparison
+ // by reference so that keys can be serialized and deserialized properly.
+ private final Class<? extends ConfiguredAspectFactory> aspectFactory;
+
+ private AspectKey(Label label, BuildConfiguration configuration,
+ Class<? extends ConfiguredAspectFactory> aspectFactory) {
+ this.label = label;
+ this.configuration = configuration;
+ this.aspectFactory = aspectFactory;
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ public Class<? extends ConfiguredAspectFactory> getAspect() {
+ return aspectFactory;
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ return SkyFunctions.ASPECT;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(label, configuration, aspectFactory);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (!(other instanceof AspectKey)) {
+ return false;
+ }
+
+ AspectKey that = (AspectKey) other;
+ return Objects.equal(label, that.label)
+ && Objects.equal(configuration, that.configuration)
+ && Objects.equal(aspectFactory, that.aspectFactory);
+ }
+
+ @Override
+ public String toString() {
+ return label + "#" + aspectFactory.getSimpleName() + " "
+ + (configuration == null ? "null" : configuration.shortCacheKey());
+ }
+ }
+
+ private final Aspect aspect;
+
+ public AspectValue(Aspect aspect, Iterable<Action> actions) {
+ super(actions);
+ this.aspect = aspect;
+ }
+
+ public Aspect get() {
+ return aspect;
+ }
+
+ public static SkyKey key(Label label, BuildConfiguration configuration,
+ Class<? extends ConfiguredAspectFactory> aspectFactory) {
+ return new SkyKey(SkyFunctions.ASPECT, new AspectKey(label, configuration, aspectFactory));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java b/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java
new file mode 100644
index 0000000..a5b0272
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thrown on {@link DiffAwareness#getDiff} to indicate that something is wrong with the
+ * {@link DiffAwareness} instance and it should not be used again.
+ */
+public class BrokenDiffAwarenessException extends Exception {
+
+ public BrokenDiffAwarenessException(String msg) {
+ super(Preconditions.checkNotNull(msg));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java
new file mode 100644
index 0000000..e717e51
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Supplier;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoContext;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoType;
+import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+
+/**
+ * Creates a {@link BuildInfoCollectionValue}. Only depends on the unique
+ * {@link WorkspaceStatusValue} and the constant {@link PrecomputedValue#BUILD_INFO_FACTORIES}
+ * injected value.
+ */
+public class BuildInfoCollectionFunction implements SkyFunction {
+ // Supplier only because the artifact factory has not yet been created at constructor time.
+ private final Supplier<ArtifactFactory> artifactFactory;
+ private final Root buildDataDirectory;
+
+ BuildInfoCollectionFunction(Supplier<ArtifactFactory> artifactFactory,
+ Root buildDataDirectory) {
+ this.artifactFactory = artifactFactory;
+ this.buildDataDirectory = buildDataDirectory;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ final BuildInfoKeyAndConfig keyAndConfig = (BuildInfoKeyAndConfig) skyKey.argument();
+ WorkspaceStatusValue infoArtifactValue =
+ (WorkspaceStatusValue) env.getValue(WorkspaceStatusValue.SKY_KEY);
+ if (infoArtifactValue == null) {
+ return null;
+ }
+ Map<BuildInfoKey, BuildInfoFactory> buildInfoFactories =
+ PrecomputedValue.BUILD_INFO_FACTORIES.get(env);
+ if (buildInfoFactories == null) {
+ return null;
+ }
+ final ArtifactFactory factory = artifactFactory.get();
+ BuildInfoContext context = new BuildInfoContext() {
+ @Override
+ public Artifact getBuildInfoArtifact(PathFragment rootRelativePath, Root root,
+ BuildInfoType type) {
+ return type == BuildInfoType.NO_REBUILD
+ ? factory.getConstantMetadataArtifact(rootRelativePath, root, keyAndConfig)
+ : factory.getDerivedArtifact(rootRelativePath, root, keyAndConfig);
+ }
+
+ @Override
+ public Root getBuildDataDirectory() {
+ return buildDataDirectory;
+ }
+ };
+
+ return new BuildInfoCollectionValue(buildInfoFactories.get(
+ keyAndConfig.getInfoKey()).create(context, keyAndConfig.getConfig(),
+ infoArtifactValue.getStableArtifact(), infoArtifactValue.getVolatileArtifact()));
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java
new file mode 100644
index 0000000..8958e1b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Objects;
+
+/**
+ * Value that stores {@link BuildInfoCollection}s generated by {@link BuildInfoFactory} instances.
+ * These collections are used during analysis (see {@code CachingAnalysisEnvironment}).
+ */
+public class BuildInfoCollectionValue extends ActionLookupValue {
+ private final BuildInfoCollection collection;
+
+ BuildInfoCollectionValue(BuildInfoCollection collection) {
+ super(collection.getActions());
+ this.collection = collection;
+ }
+
+ public BuildInfoCollection getCollection() {
+ return collection;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public String toString() {
+ return com.google.common.base.Objects.toStringHelper(getClass())
+ .add("collection", collection)
+ .add("generatingActionMap", generatingActionMap).toString();
+ }
+
+ /** Key for BuildInfoCollectionValues. */
+ public static class BuildInfoKeyAndConfig extends ActionLookupKey {
+ private final BuildInfoFactory.BuildInfoKey infoKey;
+ private final BuildConfiguration config;
+
+ public BuildInfoKeyAndConfig(BuildInfoFactory.BuildInfoKey key, BuildConfiguration config) {
+ this.infoKey = Preconditions.checkNotNull(key, config);
+ this.config = Preconditions.checkNotNull(config, key);
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ return SkyFunctions.BUILD_INFO_COLLECTION;
+ }
+
+ BuildInfoFactory.BuildInfoKey getInfoKey() {
+ return infoKey;
+ }
+
+ BuildConfiguration getConfig() {
+ return config;
+ }
+
+ @Override
+ public Label getLabel() {
+ return null;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(infoKey, config);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null) {
+ return false;
+ }
+ if (this.getClass() != other.getClass()) {
+ return false;
+ }
+ BuildInfoKeyAndConfig that = (BuildInfoKeyAndConfig) other;
+ return Objects.equals(this.infoKey, that.infoKey) && Objects.equals(this.config, that.config);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
new file mode 100644
index 0000000..7fdb55c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.util.AbruptExitException;
+
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * A Builder consumes top-level artifacts, targets, and tests,, and executes them in some
+ * topological order, possibly concurrently, using some dependency-checking policy.
+ *
+ * <p> The methods of the Builder interface are typically long-running, but honor the
+ * {@link java.lang.Thread#interrupt} contract: if an interrupt is delivered to the thread in which
+ * a call to buildTargets or buildArtifacts is active, the Builder attempts to terminate the call
+ * prematurely, throwing InterruptedException. No guarantee is made about the timeliness of such
+ * termination, as it depends on the ability of the Actions being executed to be interrupted, but
+ * typically any running subprocesses will be quickly killed.
+ */
+public interface Builder {
+
+ /**
+ * Transitively build all given artifacts, targets, and tests, and all necessary prerequisites
+ * thereof. For sequential implementations of this interface, the top-level requests will be
+ * built in the iteration order of the Set provided; for concurrent implementations, the order
+ * is undefined.
+ *
+ * <p>This method should not be invoked more than once concurrently on the same Builder instance.
+ *
+ * @param artifacts the set of Artifacts to build
+ * @param parallelTests tests to execute in parallel with the other top-level targetsToBuild and
+ * artifacts.
+ * @param exclusiveTests are executed one at a time, only after all other tasks have completed
+ * @param targetsToBuild Set of targets which will be built
+ * @param executor an opaque application-specific value that will be
+ * passed down to the execute() method of any Action executed during
+ * this call
+ * @param builtTargets (out) set of successfully built subset of targetsToBuild. This set is
+ * populated immediately upon confirmation that artifact is built so it will be
+ * valid even if a future action throws ActionExecutionException
+ * @throws BuildFailedException if there were problems establishing the action execution
+ * environment, if the the metadata of any file during the build could not be obtained,
+ * if any input files are missing, or if an action fails during execution
+ * @throws InterruptedException if there was an asynchronous stop request
+ * @throws TestExecException if any test fails
+ */
+ @ThreadCompatible
+ void buildArtifacts(Set<Artifact> artifacts,
+ Set<ConfiguredTarget> parallelTests,
+ Set<ConfiguredTarget> exclusiveTests,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Executor executor,
+ Set<ConfiguredTarget> builtTargets,
+ boolean explain)
+ throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java
new file mode 100644
index 0000000..89828c3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java
@@ -0,0 +1,164 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Supplier;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.skyframe.ConfigurationCollectionValue.ConfigurationCollectionKey;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A builder for {@link ConfigurationCollectionValue} instances.
+ */
+public class ConfigurationCollectionFunction implements SkyFunction {
+
+ private final Supplier<ConfigurationFactory> configurationFactory;
+ private final Supplier<Map<String, String>> clientEnv;
+ private final Supplier<Set<Package>> configurationPackages;
+
+ public ConfigurationCollectionFunction(
+ Supplier<ConfigurationFactory> configurationFactory,
+ Supplier<Map<String, String>> clientEnv,
+ Supplier<Set<Package>> configurationPackages) {
+ this.configurationFactory = configurationFactory;
+ this.clientEnv = clientEnv;
+ this.configurationPackages = configurationPackages;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException,
+ ConfigurationCollectionFunctionException {
+ ConfigurationCollectionKey collectionKey = (ConfigurationCollectionKey) skyKey.argument();
+ try {
+ // We are not using this value, because test_environment can be created from clientEnv. But
+ // we want ConfigurationCollection to be recomputed each time when test_environment changes.
+ PrecomputedValue.TEST_ENVIRONMENT_VARIABLES.get(env);
+ BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ BuildConfigurationCollection result =
+ getConfigurations(env.getListener(),
+ new SkyframePackageLoaderWithValueEnvironment(env, configurationPackages.get()),
+ new BuildConfigurationKey(collectionKey.getBuildOptions(), directories, clientEnv.get(),
+ collectionKey.getMultiCpu()));
+
+ // BuildConfigurationCollection can be created, but dependencies to some files might be
+ // missing. In that case we need to build configurationCollection again.
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ for (BuildConfiguration config : result.getTargetConfigurations()) {
+ config.declareSkyframeDependencies(env);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new ConfigurationCollectionValue(result, configurationPackages.get());
+ } catch (InvalidConfigurationException e) {
+ throw new ConfigurationCollectionFunctionException(e);
+ }
+ }
+
+ /** Create the build configurations with the given options. */
+ private BuildConfigurationCollection getConfigurations(EventHandler eventHandler,
+ PackageProviderForConfigurations loadedPackageProvider, BuildConfigurationKey key)
+ throws InvalidConfigurationException {
+ List<BuildConfiguration> targetConfigurations = new ArrayList<>();
+ if (!key.getMultiCpu().isEmpty()) {
+ for (String cpu : key.getMultiCpu()) {
+ BuildConfiguration targetConfiguration = createConfiguration(
+ eventHandler, loadedPackageProvider, key, cpu);
+ if (targetConfiguration == null || targetConfigurations.contains(targetConfiguration)) {
+ continue;
+ }
+ targetConfigurations.add(targetConfiguration);
+ }
+ if (loadedPackageProvider.valuesMissing()) {
+ return null;
+ }
+ } else {
+ BuildConfiguration targetConfiguration = createConfiguration(
+ eventHandler, loadedPackageProvider, key, null);
+ if (targetConfiguration == null) {
+ return null;
+ }
+ targetConfigurations.add(targetConfiguration);
+ }
+ return new BuildConfigurationCollection(targetConfigurations);
+ }
+
+ @Nullable
+ public BuildConfiguration createConfiguration(
+ EventHandler originalEventListener,
+ PackageProviderForConfigurations loadedPackageProvider,
+ BuildConfigurationKey key, String cpuOverride) throws InvalidConfigurationException {
+ StoredEventHandler errorEventListener = new StoredEventHandler();
+ BuildOptions buildOptions = key.getBuildOptions();
+ if (cpuOverride != null) {
+ // TODO(bazel-team): Options classes should be immutable. This is a bit of a hack.
+ buildOptions = buildOptions.clone();
+ buildOptions.get(BuildConfiguration.Options.class).cpu = cpuOverride;
+ }
+
+ BuildConfiguration targetConfig = configurationFactory.get().createConfiguration(
+ loadedPackageProvider, buildOptions, key, errorEventListener);
+ if (targetConfig == null) {
+ return null;
+ }
+ errorEventListener.replayOn(originalEventListener);
+ if (errorEventListener.hasErrors()) {
+ throw new InvalidConfigurationException("Build options are invalid");
+ }
+ return targetConfig;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ConfigurationCollectionFunction#compute}.
+ */
+ private static final class ConfigurationCollectionFunctionException extends
+ SkyFunctionException {
+ public ConfigurationCollectionFunctionException(InvalidConfigurationException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java
new file mode 100644
index 0000000..30e4fd7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java
@@ -0,0 +1,100 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A Skyframe value representing a build configuration collection.
+ */
+@Immutable
+@ThreadSafe
+public class ConfigurationCollectionValue implements SkyValue {
+
+ private final BuildConfigurationCollection configurationCollection;
+ private final ImmutableSet<Package> configurationPackages;
+
+ ConfigurationCollectionValue(BuildConfigurationCollection configurationCollection,
+ Set<Package> configurationPackages) {
+ this.configurationCollection = Preconditions.checkNotNull(configurationCollection);
+ this.configurationPackages = ImmutableSet.copyOf(configurationPackages);
+ }
+
+ public BuildConfigurationCollection getConfigurationCollection() {
+ return configurationCollection;
+ }
+
+ /**
+ * Returns set of packages required for configuration.
+ */
+ public Set<Package> getConfigurationPackages() {
+ return configurationPackages;
+ }
+
+ @ThreadSafe
+ public static SkyKey key(BuildOptions buildOptions, ImmutableSet<String> multiCpu) {
+ return new SkyKey(SkyFunctions.CONFIGURATION_COLLECTION,
+ new ConfigurationCollectionKey(buildOptions, multiCpu));
+ }
+
+ static final class ConfigurationCollectionKey implements Serializable {
+ private final BuildOptions buildOptions;
+ private final ImmutableSet<String> multiCpu;
+ private final int hashCode;
+
+ public ConfigurationCollectionKey(BuildOptions buildOptions, ImmutableSet<String> multiCpu) {
+ this.buildOptions = Preconditions.checkNotNull(buildOptions);
+ this.multiCpu = Preconditions.checkNotNull(multiCpu);
+ this.hashCode = Objects.hash(buildOptions, multiCpu);
+ }
+
+ public BuildOptions getBuildOptions() {
+ return buildOptions;
+ }
+
+ public ImmutableSet<String> getMultiCpu() {
+ return multiCpu;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ConfigurationCollectionKey)) {
+ return false;
+ }
+ ConfigurationCollectionKey confObject = (ConfigurationCollectionKey) o;
+ return Objects.equals(multiCpu, confObject.multiCpu)
+ && Objects.equals(buildOptions, confObject.buildOptions);
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java
new file mode 100644
index 0000000..0393b16
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.ConfigurationFragmentValue.ConfigurationFragmentKey;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * A builder for {@link ConfigurationFragmentValue}s.
+ */
+public class ConfigurationFragmentFunction implements SkyFunction {
+
+ private final Supplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments;
+ private final Supplier<Set<Package>> configurationPackages;
+
+ public ConfigurationFragmentFunction(
+ Supplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments,
+ Supplier<Set<Package>> configurationPackages) {
+ this.configurationFragments = configurationFragments;
+ this.configurationPackages = configurationPackages;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException,
+ ConfigurationFragmentFunctionException {
+ ConfigurationFragmentKey configurationFragmentKey =
+ (ConfigurationFragmentKey) skyKey.argument();
+ BuildOptions buildOptions = configurationFragmentKey.getBuildOptions();
+ ConfigurationFragmentFactory factory = getFactory(configurationFragmentKey.getFragmentType());
+ try {
+ PackageProviderForConfigurations loadedPackageProvider =
+ new SkyframePackageLoaderWithValueEnvironment(env, configurationPackages.get());
+ ConfigurationEnvironment confEnv = new ConfigurationBuilderEnvironment(loadedPackageProvider);
+ Fragment fragment = factory.create(confEnv, buildOptions);
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new ConfigurationFragmentValue(fragment);
+ } catch (InvalidConfigurationException e) {
+ // TODO(bazel-team): Rework the control-flow here so that we're not actually throwing this
+ // exception with missing Skyframe dependencies.
+ if (env.valuesMissing()) {
+ return null;
+ }
+ throw new ConfigurationFragmentFunctionException(e);
+ }
+ }
+
+ private ConfigurationFragmentFactory getFactory(Class<? extends Fragment> fragmentType) {
+ for (ConfigurationFragmentFactory factory : configurationFragments.get()) {
+ if (factory.creates().equals(fragmentType)) {
+ return factory;
+ }
+ }
+ throw new IllegalStateException(
+ "There is no factory for fragment: " + fragmentType.getSimpleName());
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * A {@link ConfigurationEnvironment} implementation that can create dependencies on files.
+ */
+ private final class ConfigurationBuilderEnvironment implements ConfigurationEnvironment {
+ private final PackageProviderForConfigurations loadedPackageProvider;
+
+ ConfigurationBuilderEnvironment(
+ PackageProviderForConfigurations loadedPackageProvider) {
+ this.loadedPackageProvider = loadedPackageProvider;
+ }
+
+ @Override
+ public Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException {
+ return loadedPackageProvider.getLoadedTarget(label);
+ }
+
+ @Override
+ public Path getPath(Package pkg, String fileName) {
+ Path result = pkg.getPackageDirectory().getRelative(fileName);
+ try {
+ loadedPackageProvider.addDependency(pkg, fileName);
+ } catch (IOException | SyntaxException e) {
+ return null;
+ }
+ return result;
+ }
+
+ @Override
+ public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType)
+ throws InvalidConfigurationException {
+ return loadedPackageProvider.getFragment(buildOptions, fragmentType);
+ }
+
+ @Override
+ public BlazeDirectories getBlazeDirectories() {
+ return loadedPackageProvider.getDirectories();
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ConfigurationFragmentFunction#compute}.
+ */
+ private static final class ConfigurationFragmentFunctionException extends SkyFunctionException {
+ public ConfigurationFragmentFunctionException(InvalidConfigurationException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java
new file mode 100644
index 0000000..cc07216
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A Skyframe node representing a build configuration fragment.
+ */
+@Immutable
+@ThreadSafe
+public class ConfigurationFragmentValue implements SkyValue {
+
+ @Nullable
+ private final BuildConfiguration.Fragment fragment;
+
+ ConfigurationFragmentValue(BuildConfiguration.Fragment fragment) {
+ this.fragment = fragment;
+ }
+
+ public BuildConfiguration.Fragment getFragment() {
+ return fragment;
+ }
+
+ @ThreadSafe
+ public static SkyKey key(BuildOptions buildOptions, Class<? extends Fragment> fragmentType) {
+ return new SkyKey(SkyFunctions.CONFIGURATION_FRAGMENT,
+ new ConfigurationFragmentKey(buildOptions, fragmentType));
+ }
+
+ static final class ConfigurationFragmentKey implements Serializable {
+ private final BuildOptions buildOptions;
+ private final Class<? extends Fragment> fragmentType;
+
+ public ConfigurationFragmentKey(BuildOptions buildOptions,
+ Class<? extends Fragment> fragmentType) {
+ this.buildOptions = Preconditions.checkNotNull(buildOptions);
+ this.fragmentType = Preconditions.checkNotNull(fragmentType);
+ }
+
+ public BuildOptions getBuildOptions() {
+ return buildOptions;
+ }
+
+ public Class<? extends Fragment> getFragmentType() {
+ return fragmentType;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ConfigurationFragmentKey)) {
+ return false;
+ }
+ ConfigurationFragmentKey confObject = (ConfigurationFragmentKey) o;
+ return Objects.equals(fragmentType, confObject.fragmentType)
+ && Objects.equals(buildOptions, confObject.buildOptions);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(buildOptions, fragmentType);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
new file mode 100644
index 0000000..9fc3df4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
@@ -0,0 +1,578 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.AspectDefinition;
+import com.google.devtools.build.lib.packages.AspectFactory;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.AspectFunction.AspectCreationException;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException2;
+import com.google.devtools.build.skyframe.ValueOrException3;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * SkyFunction for {@link ConfiguredTargetValue}s.
+ */
+final class ConfiguredTargetFunction implements SkyFunction {
+
+ /**
+ * Exception class that signals an error during the evaluation of a dependency.
+ */
+ public static class DependencyEvaluationException extends Exception {
+ private final SkyKey rootCauseSkyKey;
+
+ public DependencyEvaluationException(Exception cause) {
+ super(cause);
+ this.rootCauseSkyKey = null;
+ }
+
+ public DependencyEvaluationException(SkyKey rootCauseSkyKey, Exception cause) {
+ super(cause);
+ this.rootCauseSkyKey = rootCauseSkyKey;
+ }
+
+ /**
+ * Returns the key of the root cause or null if the problem was with this target.
+ */
+ public SkyKey getRootCauseSkyKey() {
+ return rootCauseSkyKey;
+ }
+
+ @Override
+ public Exception getCause() {
+ return (Exception) super.getCause();
+ }
+ }
+
+ private static final Function<Dependency, SkyKey> TO_KEYS =
+ new Function<Dependency, SkyKey>() {
+ @Override
+ public SkyKey apply(Dependency input) {
+ return ConfiguredTargetValue.key(input.getLabel(), input.getConfiguration());
+ }
+ };
+
+ private final BuildViewProvider buildViewProvider;
+
+ ConfiguredTargetFunction(BuildViewProvider buildViewProvider) {
+ this.buildViewProvider = buildViewProvider;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws ConfiguredTargetFunctionException,
+ InterruptedException {
+ SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
+
+ ConfiguredTargetKey configuredTargetKey = (ConfiguredTargetKey) key.argument();
+ LabelAndConfiguration lc = LabelAndConfiguration.of(
+ configuredTargetKey.getLabel(), configuredTargetKey.getConfiguration());
+
+ BuildConfiguration configuration = lc.getConfiguration();
+
+ PackageValue packageValue =
+ (PackageValue) env.getValue(PackageValue.key(lc.getLabel().getPackageIdentifier()));
+ if (packageValue == null) {
+ return null;
+ }
+
+ Target target;
+ try {
+ target = packageValue.getPackage().getTarget(lc.getLabel().getName());
+ } catch (NoSuchTargetException e1) {
+ throw new ConfiguredTargetFunctionException(new NoSuchTargetException(lc.getLabel(),
+ "No such target"));
+ }
+ // TODO(bazel-team): This is problematic - we create the right key, but then end up with a value
+ // that doesn't match; we can even have the same value multiple times. However, I think it's
+ // only triggered in tests (i.e., in normal operation, the configuration passed in is already
+ // null).
+ if (target instanceof InputFile) {
+ // InputFileConfiguredTarget expects its configuration to be null since it's not used.
+ configuration = null;
+ } else if (target instanceof PackageGroup) {
+ // Same for PackageGroupConfiguredTarget.
+ configuration = null;
+ }
+ TargetAndConfiguration ctgValue =
+ new TargetAndConfiguration(target, configuration);
+
+ SkyframeDependencyResolver resolver = view.createDependencyResolver(env);
+ if (resolver == null) {
+ return null;
+ }
+
+ try {
+ // Get the configuration targets that trigger this rule's configurable attributes.
+ Set<ConfigMatchingProvider> configConditions =
+ getConfigConditions(ctgValue.getTarget(), env, resolver, ctgValue);
+ if (configConditions == null) {
+ // Those targets haven't yet been resolved.
+ return null;
+ }
+
+ ListMultimap<Attribute, ConfiguredTarget> depValueMap =
+ computeDependencies(env, resolver, ctgValue, null, configConditions);
+ return createConfiguredTarget(
+ view, env, target, configuration, depValueMap, configConditions);
+ } catch (DependencyEvaluationException e) {
+ throw new ConfiguredTargetFunctionException(e.getRootCauseSkyKey(), e.getCause());
+ }
+ }
+
+ /**
+ * Computes the direct dependencies of a node in the configured target graph (a configured
+ * target or an aspect).
+ *
+ * <p>Returns null if Skyframe hasn't evaluated the required dependencies yet. In this case, the
+ * caller should also return null to Skyframe.
+ *
+ * @param env the Skyframe environment
+ * @param resolver The dependency resolver
+ * @param ctgValue The label and the configuration of the node
+ * @param aspectDefinition the aspect of the node (if null, the node is a configured target,
+ * otherwise it's an asect)
+ * @param configConditions the configuration conditions for evaluating the attributes of the node
+ * @return an attribute -> direct dependency multimap
+ * @throws ConfiguredTargetFunctionException
+ */
+ @Nullable
+ static ListMultimap<Attribute, ConfiguredTarget> computeDependencies(
+ Environment env, SkyframeDependencyResolver resolver, TargetAndConfiguration ctgValue,
+ AspectDefinition aspectDefinition, Set<ConfigMatchingProvider> configConditions)
+ throws DependencyEvaluationException {
+
+ // 1. Create the map from attributes to list of (target, configuration) pairs.
+ ListMultimap<Attribute, Dependency> depValueNames;
+ try {
+ depValueNames = resolver.dependentNodeMap(ctgValue, aspectDefinition, configConditions);
+ } catch (EvalException e) {
+ env.getListener().handle(Event.error(e.getLocation(), e.getMessage()));
+ throw new DependencyEvaluationException(new ConfiguredValueCreationException(e.print()));
+ }
+
+ // 2. Resolve configured target dependencies and handle errors.
+ Map<SkyKey, ConfiguredTarget> depValues =
+ resolveConfiguredTargetDependencies(env, depValueNames.values(), ctgValue.getTarget());
+ if (depValues == null) {
+ return null;
+ }
+
+ // 3. Resolve required aspects.
+ ListMultimap<SkyKey, Aspect> depAspects = resolveAspectDependencies(
+ env, depValues, depValueNames.values());
+ if (depAspects == null) {
+ return null;
+ }
+
+ // 3. Merge the dependent configured targets and aspects into a single map.
+ return mergeAspects(depValueNames, depValues, depAspects);
+ }
+
+ /**
+ * Merges the each direct dependency configured target with the aspects associated with it.
+ *
+ * <p>Note that the combination of a configured target and its associated aspects are not
+ * represented by a Skyframe node. This is because there can possibly be many different
+ * combinations of aspects for a particular configured target, so it would result in a
+ * combinatiorial explosion of Skyframe nodes.
+ */
+ private static ListMultimap<Attribute, ConfiguredTarget> mergeAspects(
+ ListMultimap<Attribute, Dependency> depValueNames,
+ Map<SkyKey, ConfiguredTarget> depConfiguredTargetMap,
+ ListMultimap<SkyKey, Aspect> depAspectMap) {
+ ListMultimap<Attribute, ConfiguredTarget> result = ArrayListMultimap.create();
+
+ for (Map.Entry<Attribute, Dependency> entry : depValueNames.entries()) {
+ Dependency dep = entry.getValue();
+ SkyKey depKey = TO_KEYS.apply(dep);
+ ConfiguredTarget depConfiguredTarget = depConfiguredTargetMap.get(depKey);
+ result.put(entry.getKey(),
+ RuleConfiguredTarget.mergeAspects(depConfiguredTarget, depAspectMap.get(depKey)));
+ }
+
+ return result;
+ }
+
+ /**
+ * Given a list of {@link Dependency} objects, returns a multimap from the {@link SkyKey} of the
+ * dependency to the {@link Aspect} instances that should be merged into it.
+ *
+ * <p>Returns null if the required aspects are not computed yet.
+ */
+ @Nullable
+ private static ListMultimap<SkyKey, Aspect> resolveAspectDependencies(Environment env,
+ Map<SkyKey, ConfiguredTarget> configuredTargetMap, Iterable<Dependency> deps)
+ throws DependencyEvaluationException {
+ ListMultimap<SkyKey, Aspect> result = ArrayListMultimap.create();
+ Set<SkyKey> aspectKeys = new HashSet<>();
+ for (Dependency dep : deps) {
+ for (Class<? extends ConfiguredAspectFactory> depAspect : dep.getAspects()) {
+ aspectKeys.add(AspectValue.key(dep.getLabel(), dep.getConfiguration(), depAspect));
+ }
+ }
+
+ Map<SkyKey, ValueOrException3<
+ AspectCreationException, NoSuchThingException, ConfiguredValueCreationException>>
+ depAspects = env.getValuesOrThrow(aspectKeys, AspectCreationException.class,
+ NoSuchThingException.class, ConfiguredValueCreationException.class);
+
+ for (Dependency dep : deps) {
+ SkyKey depKey = TO_KEYS.apply(dep);
+ ConfiguredTarget depConfiguredTarget = configuredTargetMap.get(depKey);
+ List<AspectValue> aspects = new ArrayList<>();
+ for (Class<? extends ConfiguredAspectFactory> depAspect : dep.getAspects()) {
+ if (!aspectMatchesConfiguredTarget(depConfiguredTarget, depAspect)) {
+ continue;
+ }
+
+ SkyKey aspectKey = AspectValue.key(dep.getLabel(), dep.getConfiguration(), depAspect);
+ AspectValue aspectValue = null;
+ try {
+ aspectValue = (AspectValue) depAspects.get(aspectKey).get();
+ } catch (ConfiguredValueCreationException e) {
+ // The configured target should have been created in resolveConfiguredTargetDependencies()
+ throw new IllegalStateException(e);
+ } catch (NoSuchThingException | AspectCreationException e) {
+ AspectFactory depAspectFactory = AspectFactory.Util.create(depAspect);
+ throw new DependencyEvaluationException(new ConfiguredValueCreationException(
+ String.format("Evaluation of aspect %s on %s failed: %s",
+ depAspectFactory.getDefinition().getName(), dep.getLabel(), e.toString())));
+ }
+
+ if (aspectValue == null) {
+ // Dependent aspect has either not been computed yet or is in error.
+ return null;
+ }
+ result.put(depKey, aspectValue.get());
+ }
+ }
+
+ return result;
+ }
+
+ private static boolean aspectMatchesConfiguredTarget(ConfiguredTarget dep,
+ Class<? extends ConfiguredAspectFactory> aspectFactory) {
+ AspectDefinition aspectDefinition = AspectFactory.Util.create(aspectFactory).getDefinition();
+ for (Class<?> provider : aspectDefinition.getRequiredProviders()) {
+ if (dep.getProvider((Class<? extends TransitiveInfoProvider>) provider) == null) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Returns which aspects are computable based on the precise set of providers direct dependencies
+ * publish (and not the upper estimate in their rule definition).
+ *
+ * <p>An aspect is computable for a particular configured target if the configured target supplies
+ * all the providers the aspect requires.
+ *
+ * @param upperEstimate a multimap from attribute to the upper estimates computed by
+ * {@link com.google.devtools.build.lib.analysis.DependencyResolver}.
+ * @param configuredTargetDeps a multimap from attribute to the directly dependent configured
+ * targets
+ * @return a multimap from attribute to the more precise {@link Dependency} objects
+ */
+ private static ListMultimap<Attribute, Dependency> getComputableAspects(
+ ListMultimap<Attribute, Dependency> upperEstimate,
+ Map<SkyKey, ConfiguredTarget> configuredTargetDeps) {
+ ListMultimap<Attribute, Dependency> result = ArrayListMultimap.create();
+ for (Map.Entry<Attribute, Dependency> entry : upperEstimate.entries()) {
+ ConfiguredTarget dep =
+ configuredTargetDeps.get(TO_KEYS.apply(entry.getValue()));
+ List<Class<? extends ConfiguredAspectFactory>> depAspects = new ArrayList<>();
+ for (Class<? extends ConfiguredAspectFactory> candidate : entry.getValue().getAspects()) {
+ boolean ok = true;
+ for (Class<?> requiredProvider :
+ AspectFactory.Util.create(candidate).getDefinition().getRequiredProviders()) {
+ if (dep.getProvider((Class<? extends TransitiveInfoProvider>) requiredProvider) == null) {
+ ok = false;
+ break;
+ }
+ }
+
+ if (ok) {
+ depAspects.add(candidate);
+ }
+ }
+
+ result.put(entry.getKey(), new Dependency(
+ entry.getValue().getLabel(), entry.getValue().getConfiguration(),
+ ImmutableSet.copyOf(depAspects)));
+ }
+
+ return result;
+ }
+
+ /**
+ * Returns the set of {@link ConfigMatchingProvider}s that key the configurable attributes
+ * used by this rule.
+ *
+ * <p>>If the configured targets supplying those providers aren't yet resolved by the
+ * dependency resolver, returns null.
+ */
+ @Nullable
+ static Set<ConfigMatchingProvider> getConfigConditions(Target target, Environment env,
+ SkyframeDependencyResolver resolver, TargetAndConfiguration ctgValue)
+ throws DependencyEvaluationException {
+ if (!(target instanceof Rule)) {
+ return ImmutableSet.of();
+ }
+
+ ImmutableSet.Builder<ConfigMatchingProvider> configConditions = ImmutableSet.builder();
+
+ // Collect the labels of the configured targets we need to resolve.
+ ListMultimap<Attribute, LabelAndConfiguration> configLabelMap = ArrayListMultimap.create();
+ RawAttributeMapper attributeMap = RawAttributeMapper.of(((Rule) target));
+ for (Attribute a : ((Rule) target).getAttributes()) {
+ for (Label configLabel : attributeMap.getConfigurabilityKeys(a.getName(), a.getType())) {
+ if (!Type.Selector.isReservedLabel(configLabel)) {
+ configLabelMap.put(a, LabelAndConfiguration.of(
+ configLabel, ctgValue.getConfiguration()));
+ }
+ }
+ }
+ if (configLabelMap.isEmpty()) {
+ return ImmutableSet.of();
+ }
+
+ // Collect the corresponding Skyframe configured target values. Abort early if they haven't
+ // been computed yet.
+ Collection<Dependency> configValueNames =
+ resolver.resolveRuleLabels(ctgValue, null, configLabelMap);
+ Map<SkyKey, ConfiguredTarget> configValues =
+ resolveConfiguredTargetDependencies(env, configValueNames, target);
+ if (configValues == null) {
+ return null;
+ }
+
+ // Get the configured targets as ConfigMatchingProvider interfaces.
+ for (Dependency entry : configValueNames) {
+ ConfiguredTarget value = configValues.get(TO_KEYS.apply(entry));
+ // The code above guarantees that value is non-null here.
+ ConfigMatchingProvider provider = value.getProvider(ConfigMatchingProvider.class);
+ if (provider != null) {
+ configConditions.add(provider);
+ } else {
+ // Not a valid provider for configuration conditions.
+ String message =
+ entry.getLabel() + " is not a valid configuration key for " + target.getLabel();
+ env.getListener().handle(Event.error(TargetUtils.getLocationMaybe(target), message));
+ throw new DependencyEvaluationException(new ConfiguredValueCreationException(message));
+ }
+ }
+
+ return configConditions.build();
+ }
+
+ /***
+ * Resolves the targets referenced in depValueNames and returns their ConfiguredTarget
+ * instances.
+ *
+ * <p>Returns null if not all instances are available yet.
+ *
+ */
+ @Nullable
+ private static Map<SkyKey, ConfiguredTarget> resolveConfiguredTargetDependencies(
+ Environment env, Collection<Dependency> deps, Target target)
+ throws DependencyEvaluationException {
+ boolean ok = !env.valuesMissing();
+ String message = null;
+ Iterable<SkyKey> depKeys = Iterables.transform(deps, TO_KEYS);
+ // TODO(bazel-team): maybe having a two-exception argument is better than typing a generic
+ // Exception here.
+ Map<SkyKey, ValueOrException2<NoSuchTargetException,
+ NoSuchPackageException>> depValuesOrExceptions = env.getValuesOrThrow(depKeys,
+ NoSuchTargetException.class, NoSuchPackageException.class);
+ Map<SkyKey, ConfiguredTarget> depValues = new HashMap<>(depValuesOrExceptions.size());
+ SkyKey childKey = null;
+ NoSuchThingException transitiveChildException = null;
+ for (Map.Entry<SkyKey, ValueOrException2<NoSuchTargetException, NoSuchPackageException>> entry
+ : depValuesOrExceptions.entrySet()) {
+ ConfiguredTargetKey depKey = (ConfiguredTargetKey) entry.getKey().argument();
+ LabelAndConfiguration depLabelAndConfiguration = LabelAndConfiguration.of(
+ depKey.getLabel(), depKey.getConfiguration());
+ Label depLabel = depLabelAndConfiguration.getLabel();
+ ConfiguredTargetValue depValue = null;
+ NoSuchThingException directChildException = null;
+ try {
+ depValue = (ConfiguredTargetValue) entry.getValue().get();
+ } catch (NoSuchTargetException e) {
+ if (depLabel.equals(e.getLabel())) {
+ directChildException = e;
+ } else {
+ childKey = entry.getKey();
+ transitiveChildException = e;
+ }
+ } catch (NoSuchPackageException e) {
+ if (depLabel.getPackageName().equals(e.getPackageName())) {
+ directChildException = e;
+ } else {
+ childKey = entry.getKey();
+ transitiveChildException = e;
+ }
+ }
+ // If an exception wasn't caused by a direct child target value, we'll treat it the same
+ // as any other missing dep by setting ok = false below, and returning null at the end.
+ if (directChildException != null) {
+ // Only update messages for missing targets we depend on directly.
+ message = TargetUtils.formatMissingEdge(target, depLabel, directChildException);
+ env.getListener().handle(Event.error(TargetUtils.getLocationMaybe(target), message));
+ }
+
+ if (depValue == null) {
+ ok = false;
+ } else {
+ depValues.put(entry.getKey(), depValue.getConfiguredTarget());
+ }
+ }
+ if (message != null) {
+ throw new DependencyEvaluationException(new NoSuchTargetException(message));
+ }
+ if (childKey != null) {
+ throw new DependencyEvaluationException(childKey, transitiveChildException);
+ }
+ if (!ok) {
+ return null;
+ } else {
+ return depValues;
+ }
+ }
+
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((ConfiguredTargetKey) skyKey.argument()).getLabel());
+ }
+
+ @Nullable
+ private ConfiguredTargetValue createConfiguredTarget(SkyframeBuildView view,
+ Environment env, Target target, BuildConfiguration configuration,
+ ListMultimap<Attribute, ConfiguredTarget> depValueMap,
+ Set<ConfigMatchingProvider> configConditions)
+ throws ConfiguredTargetFunctionException,
+ InterruptedException {
+ boolean extendedSanityChecks = configuration != null && configuration.extendedSanityChecks();
+
+ StoredEventHandler events = new StoredEventHandler();
+ BuildConfiguration ownerConfig = (configuration == null)
+ ? null : configuration.getArtifactOwnerConfiguration();
+ boolean allowRegisteringActions = configuration == null || configuration.isActionsEnabled();
+ CachingAnalysisEnvironment analysisEnvironment = view.createAnalysisEnvironment(
+ new ConfiguredTargetKey(target.getLabel(), ownerConfig), false,
+ extendedSanityChecks, events, env, allowRegisteringActions);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ ConfiguredTarget configuredTarget = view.createConfiguredTarget(target, configuration,
+ analysisEnvironment, depValueMap, configConditions);
+
+ events.replayOn(env.getListener());
+ if (events.hasErrors()) {
+ analysisEnvironment.disable(target);
+ throw new ConfiguredTargetFunctionException(new ConfiguredValueCreationException(
+ "Analysis of target '" + target.getLabel() + "' failed; build aborted"));
+ }
+ Preconditions.checkState(!analysisEnvironment.hasErrors(),
+ "Analysis environment hasError() but no errors reported");
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ analysisEnvironment.disable(target);
+ Preconditions.checkNotNull(configuredTarget, target);
+
+ return new ConfiguredTargetValue(configuredTarget,
+ ImmutableList.copyOf(analysisEnvironment.getRegisteredActions()));
+ }
+
+ /**
+ * An exception indicating that there was a problem during the construction of
+ * a ConfiguredTargetValue.
+ */
+ public static final class ConfiguredValueCreationException extends Exception {
+
+ public ConfiguredValueCreationException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link ConfiguredTargetFunction#compute}.
+ */
+ public static final class ConfiguredTargetFunctionException extends SkyFunctionException {
+ public ConfiguredTargetFunctionException(NoSuchTargetException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ private ConfiguredTargetFunctionException(ConfiguredValueCreationException error) {
+ super(error, Transience.PERSISTENT);
+ };
+
+ private ConfiguredTargetFunctionException(
+ @Nullable SkyKey childKey, Exception transitiveError) {
+ super(transitiveError, childKey);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java
new file mode 100644
index 0000000..ea744c1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java
@@ -0,0 +1,96 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A (Label, Configuration) pair. Note that this pair may be used to look up the generating action
+ * of an artifact. Callers may want to ensure that they have the correct configuration for this
+ * purpose by passing in {@link BuildConfiguration#getArtifactOwnerConfiguration} in preference to
+ * the raw configuration.
+ */
+public class ConfiguredTargetKey extends ActionLookupValue.ActionLookupKey {
+ private final Label label;
+ @Nullable
+ private final BuildConfiguration configuration;
+
+ public ConfiguredTargetKey(Label label, @Nullable BuildConfiguration configuration) {
+ this.label = Preconditions.checkNotNull(label);
+ this.configuration = configuration;
+ }
+
+ public ConfiguredTargetKey(ConfiguredTarget rule) {
+ this(rule.getTarget().getLabel(), rule.getConfiguration());
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ return SkyFunctions.CONFIGURED_TARGET;
+ }
+
+ @Nullable
+ public BuildConfiguration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public int hashCode() {
+ int configVal = configuration == null ? 79 : configuration.hashCode();
+ return 31 * label.hashCode() + configVal;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ConfiguredTargetKey)) {
+ return false;
+ }
+ ConfiguredTargetKey other = (ConfiguredTargetKey) obj;
+ return Objects.equals(label, other.label) && Objects.equals(configuration, other.configuration);
+ }
+
+ public String prettyPrint() {
+ if (label == null) {
+ return "null";
+ }
+ return (configuration != null && configuration.isHostConfiguration())
+ ? (label.toString() + " (host)") : label.toString();
+ }
+
+ @Override
+ public String toString() {
+ return label + " " + (configuration == null ? "null" : configuration.shortCacheKey());
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java
new file mode 100644
index 0000000..200e05f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java
@@ -0,0 +1,105 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import javax.annotation.Nullable;
+
+/**
+ * A configured target in the context of a Skyframe graph.
+ */
+@Immutable
+@ThreadSafe
+@VisibleForTesting
+public final class ConfiguredTargetValue extends ActionLookupValue {
+
+ // These variables are only non-final because they may be clear()ed to save memory. They are null
+ // only after they are cleared.
+ @Nullable private ConfiguredTarget configuredTarget;
+
+ // We overload this variable to check whether the value has been clear()ed. We don't use a
+ // separate variable in order to save memory.
+ @Nullable private volatile Iterable<Action> actions;
+
+ ConfiguredTargetValue(ConfiguredTarget configuredTarget, Iterable<Action> actions) {
+ super(actions);
+ this.configuredTarget = configuredTarget;
+ this.actions = actions;
+ }
+
+ @VisibleForTesting
+ public ConfiguredTarget getConfiguredTarget() {
+ Preconditions.checkNotNull(actions, configuredTarget);
+ return configuredTarget;
+ }
+
+ @VisibleForTesting
+ public Iterable<Action> getActions() {
+ return Preconditions.checkNotNull(actions, configuredTarget);
+ }
+
+ /**
+ * Clears configured target data from this value, leaving only the artifact->generating action
+ * map.
+ *
+ * <p>Should only be used when user specifies --discard_analysis_cache. Must be called at most
+ * once per value, after which {@link #getConfiguredTarget} and {@link #getActions} cannot be
+ * called.
+ */
+ public void clear() {
+ Preconditions.checkNotNull(actions, configuredTarget);
+ configuredTarget = null;
+ actions = null;
+ }
+
+ @VisibleForTesting
+ public static SkyKey key(Label label, BuildConfiguration configuration) {
+ return key(new ConfiguredTargetKey(label, configuration));
+ }
+
+ static ImmutableList<SkyKey> keys(Iterable<ConfiguredTargetKey> lacs) {
+ ImmutableList.Builder<SkyKey> keys = ImmutableList.builder();
+ for (ConfiguredTargetKey lac : lacs) {
+ keys.add(key(lac));
+ }
+ return keys.build();
+ }
+
+ /**
+ * Returns a label of ConfiguredTargetValue.
+ */
+ @ThreadSafe
+ static Label extractLabel(SkyKey value) {
+ Object valueName = value.argument();
+ Preconditions.checkState(valueName instanceof ConfiguredTargetKey, valueName);
+ return ((ConfiguredTargetKey) valueName).getLabel();
+ }
+
+ @Override
+ public String toString() {
+ return "ConfiguredTargetValue: "
+ + configuredTarget + ", actions: " + (actions == null ? null : Iterables.toString(actions));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java
new file mode 100644
index 0000000..58cb67d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * SkyFunction for {@link ContainingPackageLookupValue}s.
+ */
+public class ContainingPackageLookupFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ PackageIdentifier dir = (PackageIdentifier) skyKey.argument();
+ PackageLookupValue pkgLookupValue = null;
+ pkgLookupValue = (PackageLookupValue) env.getValue(PackageLookupValue.key(dir));
+ if (pkgLookupValue == null) {
+ return null;
+ }
+
+ if (pkgLookupValue.packageExists()) {
+ return ContainingPackageLookupValue.withContainingPackage(dir, pkgLookupValue.getRoot());
+ }
+
+ PathFragment parentDir = dir.getPackageFragment().getParentDirectory();
+ if (parentDir == null) {
+ return ContainingPackageLookupValue.noContainingPackage();
+ }
+ PackageIdentifier parentId = new PackageIdentifier(dir.getRepository(), parentDir);
+ return env.getValue(ContainingPackageLookupValue.key(parentId));
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java
new file mode 100644
index 0000000..16516b5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value that represents the result of looking for the existence of a package that owns a
+ * specific directory path. Compare with {@link PackageLookupValue}, which deals with existence of
+ * a specific package.
+ */
+public abstract class ContainingPackageLookupValue implements SkyValue {
+ /** Returns whether there is a containing package. */
+ public abstract boolean hasContainingPackage();
+
+ /** If there is a containing package, returns its name. */
+ abstract PackageIdentifier getContainingPackageName();
+
+ /** If there is a containing package, returns its package root */
+ public abstract Path getContainingPackageRoot();
+
+ public static SkyKey key(PackageIdentifier id) {
+ Preconditions.checkArgument(!id.getPackageFragment().isAbsolute(), id);
+ return new SkyKey(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, id);
+ }
+
+ static ContainingPackageLookupValue noContainingPackage() {
+ return NoContainingPackage.INSTANCE;
+ }
+
+ static ContainingPackageLookupValue withContainingPackage(PackageIdentifier pkgId, Path root) {
+ return new ContainingPackage(pkgId, root);
+ }
+
+ private static class NoContainingPackage extends ContainingPackageLookupValue {
+ private static final NoContainingPackage INSTANCE = new NoContainingPackage();
+
+ @Override
+ public boolean hasContainingPackage() {
+ return false;
+ }
+
+ @Override
+ public PackageIdentifier getContainingPackageName() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Path getContainingPackageRoot() {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static class ContainingPackage extends ContainingPackageLookupValue {
+ private final PackageIdentifier containingPackage;
+ private final Path containingPackageRoot;
+
+ private ContainingPackage(PackageIdentifier pkgId, Path containingPackageRoot) {
+ this.containingPackage = pkgId;
+ this.containingPackageRoot = containingPackageRoot;
+ }
+
+ @Override
+ public boolean hasContainingPackage() {
+ return true;
+ }
+
+ @Override
+ public PackageIdentifier getContainingPackageName() {
+ return containingPackage;
+ }
+
+ @Override
+ public Path getContainingPackageRoot() {
+ return containingPackageRoot;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ContainingPackage)) {
+ return false;
+ }
+ ContainingPackage other = (ContainingPackage) obj;
+ return containingPackage.equals(other.containingPackage)
+ && containingPackageRoot.equals(other.containingPackageRoot);
+ }
+
+ @Override
+ public int hashCode() {
+ return containingPackage.hashCode();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java
new file mode 100644
index 0000000..8d6fe3d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A Skyframe function to calculate the coverage report Action and Artifacts.
+ */
+public class CoverageReportFunction implements SkyFunction {
+ CoverageReportFunction() {}
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ Preconditions.checkState(
+ CoverageReportValue.SKY_KEY.equals(skyKey), String.format(
+ "Expected %s for SkyKey but got %s instead", CoverageReportValue.SKY_KEY, skyKey));
+
+ Action action = PrecomputedValue.COVERAGE_REPORT_KEY.get(env);
+ if (action == null) {
+ return null;
+ }
+
+ return new CoverageReportValue(
+ action.getOutputs(),
+ action);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java
new file mode 100644
index 0000000..862e381
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * A SkyValue to store the coverage report Action and Artifacts.
+ */
+public class CoverageReportValue extends ActionLookupValue {
+ private final ImmutableSet<Artifact> coverageReportArtifacts;
+
+ // There should only ever be one CoverageReportValue value in the graph.
+ public static final SkyKey SKY_KEY = new SkyKey(SkyFunctions.COVERAGE_REPORT, "COVERAGE_REPORT");
+ public static final ArtifactOwner ARTIFACT_OWNER = new CoverageReportKey();
+
+ public CoverageReportValue(ImmutableSet<Artifact> coverageReportArtifacts,
+ Action coverageReportAction) {
+ super(coverageReportAction);
+ this.coverageReportArtifacts = coverageReportArtifacts;
+ }
+
+ public ImmutableSet<Artifact> getCoverageReportArtifacts() {
+ return coverageReportArtifacts;
+ }
+
+ private static class CoverageReportKey extends ActionLookupKey {
+ @Override
+ SkyFunctionName getType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ SkyKey getSkyKey() {
+ return SKY_KEY;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java
new file mode 100644
index 0000000..d0f4c99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.Closeable;
+
+import javax.annotation.Nullable;
+
+/**
+ * Interface for computing modifications of files under a package path entry.
+ *
+ * <p> Skyframe has a {@link DiffAwareness} instance per package-path entry, and each instance is
+ * responsible for all files under its path entry. At the beginning of each incremental build,
+ * skyframe queries for changes using {@link #getDiff}. Ideally, {@link #getDiff} should be
+ * constant-time; if it were linear in the number of files of interest, we might as well just
+ * detect modifications manually.
+ */
+public interface DiffAwareness extends Closeable {
+
+ /** Factory for creating {@link DiffAwareness} instances. */
+ public interface Factory {
+ /**
+ * Returns a {@link DiffAwareness} instance suitable for managing changes to files under the
+ * given package path entry, or {@code null} if this factory cannot create such an instance.
+ *
+ * <p> Skyframe has a collection of factories, and will create a {@link DiffAwareness} instance
+ * per package path entry using one of the factories that returns a non-null value.
+ */
+ @Nullable
+ DiffAwareness maybeCreate(Path pathEntry);
+ }
+
+ /** Opaque view of the filesystem under a package path entry at a specific point in time. */
+ interface View {
+ }
+
+ /**
+ * Returns the live view of the filesystem under the package path entry.
+ *
+ * @throws BrokenDiffAwarenessException if something is wrong and the caller should discard this
+ * {@link DiffAwareness} instance. The {@link DiffAwareness} is expected to close itself in
+ * this case.
+ */
+ View getCurrentView() throws BrokenDiffAwarenessException;
+
+ /**
+ * Returns the set of files of interest that have been modified between the given two views.
+ *
+ * <p>The given views must have come from previous calls to {@link #getCurrentView} on the
+ * {@link DiffAwareness} instance (i.e. using a {@link View} from another instance is not
+ * supported).
+ *
+ * @throws IncompatibleViewException if the given views are not compatible with this
+ * {@link DiffAwareness} instance. This probably indicates a bug.
+ * @throws BrokenDiffAwarenessException if something is wrong and the caller should discard this
+ * {@link DiffAwareness} instance. The {@link DiffAwareness} is expected to close itself in
+ * this case.
+ */
+ ModifiedFileSet getDiff(View oldView, View newView)
+ throws IncompatibleViewException, BrokenDiffAwarenessException;
+
+ /**
+ * Must be called whenever the {@link DiffAwareness} object is to be discarded. Using a
+ * {@link DiffAwareness} instance after calling {@link #close} on it is unspecified behavior.
+ */
+ @Override
+ void close();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java
new file mode 100644
index 0000000..d1bb81e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java
@@ -0,0 +1,188 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.skyframe.DiffAwareness.View;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class to make it easier to correctly use the {@link DiffAwareness} interface in a
+ * sequential manner.
+ */
+public final class DiffAwarenessManager {
+
+ private final ImmutableSet<? extends DiffAwareness.Factory> diffAwarenessFactories;
+ private Map<Path, DiffAwarenessState> currentDiffAwarenessStates = Maps.newHashMap();
+ private final Reporter reporter;
+
+ public DiffAwarenessManager(Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Reporter reporter) {
+ this.diffAwarenessFactories = ImmutableSet.copyOf(diffAwarenessFactories);
+ this.reporter = reporter;
+ }
+
+ private static class DiffAwarenessState {
+ private final DiffAwareness diffAwareness;
+ /**
+ * The {@link View} that should be the baseline for the next {@link #getDiff} call, or
+ * {@code null} if the next {@link #getDiff} will be the first incremental one.
+ */
+ @Nullable
+ private View baselineView;
+
+ private DiffAwarenessState(DiffAwareness diffAwareness, @Nullable View baselineView) {
+ this.diffAwareness = diffAwareness;
+ this.baselineView = baselineView;
+ }
+ }
+
+ /** Reset internal {@link DiffAwareness} state. */
+ public void reset() {
+ for (DiffAwarenessState diffAwarenessState : currentDiffAwarenessStates.values()) {
+ diffAwarenessState.diffAwareness.close();
+ }
+ currentDiffAwarenessStates.clear();
+ }
+
+ /** A set of modified files that should be marked as processed. */
+ public interface ProcessableModifiedFileSet {
+ ModifiedFileSet getModifiedFileSet();
+
+ /**
+ * This should be called when the changes have been noted. Otherwise, the result from the next
+ * call to {@link #getDiff} will be from the baseline of the old, unprocessed, diff.
+ */
+ void markProcessed();
+ }
+
+ /**
+ * Gets the set of changed files since the last call with this path entry, or
+ * {@code ModifiedFileSet.EVERYTHING_MODIFIED} if this is the first such call.
+ */
+ public ProcessableModifiedFileSet getDiff(Path pathEntry) {
+ DiffAwarenessState diffAwarenessState = maybeGetDiffAwarenessState(pathEntry);
+ if (diffAwarenessState == null) {
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ }
+ DiffAwareness diffAwareness = diffAwarenessState.diffAwareness;
+ View newView;
+ try {
+ newView = diffAwareness.getCurrentView();
+ } catch (BrokenDiffAwarenessException e) {
+ handleBrokenDiffAwareness(pathEntry, e);
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ }
+
+ View baselineView = diffAwarenessState.baselineView;
+ if (baselineView == null) {
+ diffAwarenessState.baselineView = newView;
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ }
+
+ ModifiedFileSet diff;
+ try {
+ diff = diffAwareness.getDiff(baselineView, newView);
+ } catch (BrokenDiffAwarenessException e) {
+ handleBrokenDiffAwareness(pathEntry, e);
+ return BrokenProcessableModifiedFileSet.INSTANCE;
+ } catch (IncompatibleViewException e) {
+ throw new IllegalStateException(pathEntry + " " + baselineView + " " + newView, e);
+ }
+ ProcessableModifiedFileSet result = new ProcessableModifiedFileSetImpl(diff, pathEntry,
+ newView);
+ return result;
+ }
+
+ private void handleBrokenDiffAwareness(Path pathEntry, BrokenDiffAwarenessException e) {
+ currentDiffAwarenessStates.remove(pathEntry);
+ reporter.handle(Event.warn(e.getMessage() + "... temporarily falling back to manually "
+ + "checking files for changes"));
+ }
+
+ /**
+ * Returns the current diff awareness for the given path entry, or a fresh one if there is no
+ * current one, or otherwise {@code null} if no factory could make a fresh one.
+ */
+ @Nullable
+ private DiffAwarenessState maybeGetDiffAwarenessState(Path pathEntry) {
+ DiffAwarenessState diffAwarenessState = currentDiffAwarenessStates.get(pathEntry);
+ if (diffAwarenessState != null) {
+ return diffAwarenessState;
+ }
+ for (DiffAwareness.Factory factory : diffAwarenessFactories) {
+ DiffAwareness newDiffAwareness = factory.maybeCreate(pathEntry);
+ if (newDiffAwareness != null) {
+ diffAwarenessState = new DiffAwarenessState(newDiffAwareness, /*previousView=*/null);
+ currentDiffAwarenessStates.put(pathEntry, diffAwarenessState);
+ return diffAwarenessState;
+ }
+ }
+ return null;
+ }
+
+ private class ProcessableModifiedFileSetImpl implements ProcessableModifiedFileSet {
+
+ private final ModifiedFileSet modifiedFileSet;
+ private final Path pathEntry;
+ /**
+ * The {@link View} that should be the baseline on the next {@link #getDiff} call after
+ * {@link #markProcessed} is called.
+ */
+ private final View nextView;
+
+ private ProcessableModifiedFileSetImpl(ModifiedFileSet modifiedFileSet, Path pathEntry,
+ View nextView) {
+ this.modifiedFileSet = modifiedFileSet;
+ this.pathEntry = pathEntry;
+ this.nextView = nextView;
+ }
+
+ @Override
+ public ModifiedFileSet getModifiedFileSet() {
+ return modifiedFileSet;
+ }
+
+ @Override
+ public void markProcessed() {
+ DiffAwarenessState diffAwarenessState = currentDiffAwarenessStates.get(pathEntry);
+ if (diffAwarenessState != null) {
+ diffAwarenessState.baselineView = nextView;
+ }
+ }
+ }
+
+ private static class BrokenProcessableModifiedFileSet implements ProcessableModifiedFileSet {
+
+ private static final BrokenProcessableModifiedFileSet INSTANCE =
+ new BrokenProcessableModifiedFileSet();
+
+ @Override
+ public ModifiedFileSet getModifiedFileSet() {
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+
+ @Override
+ public void markProcessed() {
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java
new file mode 100644
index 0000000..93c3d75
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link SkyFunction} for {@link DirectoryListingValue}s.
+ */
+final class DirectoryListingFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws DirectoryListingFunctionException {
+ RootedPath dirRootedPath = (RootedPath) skyKey.argument();
+
+ FileValue dirFileValue = (FileValue) env.getValue(FileValue.key(dirRootedPath));
+ if (dirFileValue == null) {
+ return null;
+ }
+
+ RootedPath realDirRootedPath = dirFileValue.realRootedPath();
+ if (!dirFileValue.isDirectory()) {
+ // Recall that the directory is assumed to exist (see DirectoryListingValue#key).
+ throw new DirectoryListingFunctionException(new InconsistentFilesystemException(
+ dirRootedPath.asPath() + " is no longer an existing directory. Did you delete it during "
+ + "the build?"));
+ }
+
+ DirectoryListingStateValue directoryListingStateValue =
+ (DirectoryListingStateValue) env.getValue(DirectoryListingStateValue.key(
+ realDirRootedPath));
+ if (directoryListingStateValue == null) {
+ return null;
+ }
+
+ return DirectoryListingValue.value(dirRootedPath, dirFileValue, directoryListingStateValue);
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link DirectoryListingFunction#compute}.
+ */
+ private static final class DirectoryListingFunctionException extends SkyFunctionException {
+ public DirectoryListingFunctionException(InconsistentFilesystemException e) {
+ super(e, Transience.TRANSIENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
new file mode 100644
index 0000000..6e47a2d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * A {@link SkyFunction} for {@link DirectoryListingStateValue}s.
+ *
+ * <p>Merely calls DirectoryListingStateValue#create, but also has special handling for
+ * directories outside the package roots (see {@link ExternalFilesHelper}).
+ */
+public class DirectoryListingStateFunction implements SkyFunction {
+
+ private final ExternalFilesHelper externalFilesHelper;
+
+ public DirectoryListingStateFunction(ExternalFilesHelper externalFilesHelper) {
+ this.externalFilesHelper = externalFilesHelper;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws DirectoryListingStateFunctionException {
+ RootedPath dirRootedPath = (RootedPath) skyKey.argument();
+ externalFilesHelper.maybeAddDepOnBuildId(dirRootedPath, env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ try {
+ return DirectoryListingStateValue.create(dirRootedPath);
+ } catch (IOException e) {
+ throw new DirectoryListingStateFunctionException(e);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link DirectoryListingStateFunction#compute}.
+ */
+ private static final class DirectoryListingStateFunctionException
+ extends SkyFunctionException {
+ public DirectoryListingStateFunctionException(IOException e) {
+ super(e, Transience.TRANSIENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java
new file mode 100644
index 0000000..87b9748
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java
@@ -0,0 +1,214 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Objects;
+
+/**
+ * Encapsulates the filesystem operations needed to get the directory entries of a directory.
+ *
+ * <p>This class is an implementation detail of {@link DirectoryListingValue}.
+ */
+final class DirectoryListingStateValue implements SkyValue {
+
+ private final CompactSortedDirents compactSortedDirents;
+
+ private DirectoryListingStateValue(Collection<Dirent> dirents) {
+ this.compactSortedDirents = CompactSortedDirents.create(dirents);
+ }
+
+ @VisibleForTesting
+ public static DirectoryListingStateValue createForTesting(Collection<Dirent> dirents) {
+ return new DirectoryListingStateValue(dirents);
+ }
+
+ public static DirectoryListingStateValue create(RootedPath dirRootedPath) throws IOException {
+ Collection<Dirent> dirents = dirRootedPath.asPath().readdir(Symlinks.NOFOLLOW);
+ return new DirectoryListingStateValue(dirents);
+ }
+
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.DIRECTORY_LISTING_STATE, rootedPath);
+ }
+
+ /**
+ * Returns the directory entries for this directory, in a stable order.
+ *
+ * <p>Symlinks are not expanded.
+ */
+ public Iterable<Dirent> getDirents() {
+ return compactSortedDirents;
+ }
+
+ @Override
+ public int hashCode() {
+ return compactSortedDirents.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DirectoryListingStateValue)) {
+ return false;
+ }
+ DirectoryListingStateValue other = (DirectoryListingStateValue) obj;
+ return compactSortedDirents.equals(other.compactSortedDirents);
+ }
+
+ /** A space-efficient, sorted, immutable dirent structure. */
+ private static class CompactSortedDirents implements Iterable<Dirent>, Serializable {
+
+ private final String[] names;
+ private final BitSet packedTypes;
+
+ private CompactSortedDirents(String[] names, BitSet packedTypes) {
+ this.names = names;
+ this.packedTypes = packedTypes;
+ }
+
+ public static CompactSortedDirents create(Collection<Dirent> dirents) {
+ final Dirent[] direntArray = dirents.toArray(new Dirent[dirents.size()]);
+ Integer[] indices = new Integer[dirents.size()];
+ for (int i = 0; i < dirents.size(); i++) {
+ indices[i] = i;
+ }
+ Arrays.sort(indices,
+ new Comparator<Integer>() {
+ @Override
+ public int compare(Integer o1, Integer o2) {
+ return direntArray[o1].getName().compareTo(direntArray[o2].getName());
+ }
+ });
+ String[] names = new String[dirents.size()];
+ BitSet packedTypes = new BitSet(dirents.size() * 2);
+ for (int i = 0; i < dirents.size(); i++) {
+ Dirent dirent = direntArray[indices[i]];
+ names[i] = dirent.getName();
+ packType(packedTypes, dirent.getType(), i);
+ }
+ return new CompactSortedDirents(names, packedTypes);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof CompactSortedDirents)) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ CompactSortedDirents other = (CompactSortedDirents) obj;
+ return Arrays.equals(names, other.names) && packedTypes.equals(other.packedTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(Arrays.hashCode(names), packedTypes);
+ }
+
+ @Override
+ public Iterator<Dirent> iterator() {
+ return new Iterator<Dirent>() {
+
+ private int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < size();
+ }
+
+ @Override
+ public Dirent next() {
+ return direntAt(i++);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ private int size() {
+ return names.length;
+ }
+
+ /** Returns the type of the ith dirent. */
+ private Dirent.Type unpackType(int i) {
+ int start = i * 2;
+ boolean upper = packedTypes.get(start);
+ boolean lower = packedTypes.get(start + 1);
+ if (!upper && !lower) {
+ return Type.FILE;
+ } else if (!upper && lower){
+ return Type.DIRECTORY;
+ } else if (upper && !lower) {
+ return Type.SYMLINK;
+ } else {
+ return Type.UNKNOWN;
+ }
+ }
+
+ /** Sets the type of the ith dirent. */
+ private static void packType(BitSet bitSet, Dirent.Type type, int i) {
+ int start = i * 2;
+ switch (type) {
+ case FILE:
+ pack(bitSet, start, false, false);
+ break;
+ case DIRECTORY:
+ pack(bitSet, start, false, true);
+ break;
+ case SYMLINK:
+ pack(bitSet, start, true, false);
+ break;
+ case UNKNOWN:
+ pack(bitSet, start, true, true);
+ break;
+ default:
+ throw new IllegalStateException("Unknown dirent type: " + type);
+ }
+ }
+
+ private static void pack(BitSet bitSet, int start, boolean upper, boolean lower) {
+ bitSet.set(start, upper);
+ bitSet.set(start + 1, lower);
+ }
+
+ private Dirent direntAt(int i) {
+ Preconditions.checkState(i >= 0 && i < size(), "i: %s, size: %s", i, size());
+ return new Dirent(names[i], unpackType(i));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java
new file mode 100644
index 0000000..3fe6dba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java
@@ -0,0 +1,134 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Objects;
+
+/**
+ * A value that represents the list of files in a given directory under a given package path root.
+ * Anything in Skyframe that cares about the contents of a directory should have a dependency
+ * on the corresponding {@link DirectoryListingValue}.
+ *
+ * <p>This value only depends on the FileValue corresponding to the directory. In particular, note
+ * that it does not depend on any of its child entries.
+ *
+ * <p>Note that symlinks in dirents are <b>not</b> expanded. Dependents of the value are responsible
+ * for expanding the symlink entries by referring to FileValues that correspond to the symlinks.
+ * This is a little onerous, but correct: we do not need to reread the directory when a symlink
+ * inside it changes, therefore this value should not be invalidated in that case.
+ */
+@Immutable
+@ThreadSafe
+abstract class DirectoryListingValue implements SkyValue {
+
+ /**
+ * Returns the directory entries for this directory, in a stable order.
+ *
+ * <p>Symlinks are not expanded.
+ */
+ public abstract Iterable<Dirent> getDirents();
+
+ /**
+ * Returns a {@link SkyKey} for getting the directory entries of the given directory. The
+ * given path is assumed to be an existing directory (e.g. via {@link FileValue#isDirectory} or
+ * from a directory listing on its parent directory).
+ */
+ @ThreadSafe
+ static SkyKey key(RootedPath directoryUnderRoot) {
+ return new SkyKey(SkyFunctions.DIRECTORY_LISTING, directoryUnderRoot);
+ }
+
+ static DirectoryListingValue value(RootedPath dirRootedPath, FileValue dirFileValue,
+ DirectoryListingStateValue realDirectoryListingStateValue) {
+ return dirFileValue.realRootedPath().equals(dirRootedPath)
+ ? new RegularDirectoryListingValue(realDirectoryListingStateValue)
+ : new DifferentRealPathDirectoryListingValue(dirFileValue.realRootedPath(),
+ realDirectoryListingStateValue);
+ }
+
+ @ThreadSafe
+ private static final class RegularDirectoryListingValue extends DirectoryListingValue {
+
+ private final DirectoryListingStateValue directoryListingStateValue;
+
+ private RegularDirectoryListingValue(DirectoryListingStateValue directoryListingStateValue) {
+ this.directoryListingStateValue = directoryListingStateValue;
+ }
+
+ @Override
+ public Iterable<Dirent> getDirents() {
+ return directoryListingStateValue.getDirents();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RegularDirectoryListingValue)) {
+ return false;
+ }
+ RegularDirectoryListingValue other = (RegularDirectoryListingValue) obj;
+ return directoryListingStateValue.equals(other.directoryListingStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return directoryListingStateValue.hashCode();
+ }
+ }
+
+ @ThreadSafe
+ private static final class DifferentRealPathDirectoryListingValue extends DirectoryListingValue {
+
+ private final RootedPath realDirRootedPath;
+ private final DirectoryListingStateValue directoryListingStateValue;
+
+ private DifferentRealPathDirectoryListingValue(RootedPath realDirRootedPath,
+ DirectoryListingStateValue directoryListingStateValue) {
+ this.realDirRootedPath = realDirRootedPath;
+ this.directoryListingStateValue = directoryListingStateValue;
+ }
+
+ @Override
+ public Iterable<Dirent> getDirents() {
+ return directoryListingStateValue.getDirents();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DifferentRealPathDirectoryListingValue)) {
+ return false;
+ }
+ DifferentRealPathDirectoryListingValue other = (DifferentRealPathDirectoryListingValue) obj;
+ return realDirRootedPath.equals(other.realDirRootedPath)
+ && directoryListingStateValue.equals(other.directoryListingStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(realDirRootedPath, directoryListingStateValue);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java b/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java
new file mode 100644
index 0000000..8593afb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java
@@ -0,0 +1,21 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+/** Indicates some sort of IO error while dealing with a Skylark extension. */
+public class ErrorReadingSkylarkExtensionException extends Exception {
+ public ErrorReadingSkylarkExtensionException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
new file mode 100644
index 0000000..ce858de
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+/** Common utilities for dealing with files outside the package roots. */
+class ExternalFilesHelper {
+
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final Set<Path> immutableDirs;
+
+ ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator) {
+ this(pkgLocator, ImmutableSet.<Path>of());
+ }
+
+ ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator, Set<Path> immutableDirs) {
+ this.pkgLocator = pkgLocator;
+ this.immutableDirs = immutableDirs;
+ }
+
+ private enum FileType {
+ // A file inside the package roots.
+ INTERNAL_FILE,
+
+ // A file outside the package roots that we may pretends is immutable.
+ EXTERNAL_IMMUTABLE_FILE,
+
+ // A file outside the package roots about which we may make no other assumptions.
+ EXTERNAL_MUTABLE_FILE,
+ }
+
+ private FileType getFileType(RootedPath rootedPath) {
+ // TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a
+ // lot of immutable directories. Consider either explicitly preventing this case or using a more
+ // efficient approach here (e.g. use a trie for determing if a file is under an immutable
+ // directory).
+ if (!pkgLocator.get().getPathEntries().contains(rootedPath.getRoot())) {
+ Path path = rootedPath.asPath();
+ for (Path immutableDir : immutableDirs) {
+ if (path.startsWith(immutableDir)) {
+ return FileType.EXTERNAL_IMMUTABLE_FILE;
+ }
+ }
+ return FileType.EXTERNAL_MUTABLE_FILE;
+ }
+ return FileType.INTERNAL_FILE;
+ }
+
+ public boolean shouldAssumeImmutable(RootedPath rootedPath) {
+ return getFileType(rootedPath) == FileType.EXTERNAL_IMMUTABLE_FILE;
+ }
+
+ public void maybeAddDepOnBuildId(RootedPath rootedPath, SkyFunction.Environment env) {
+ if (getFileType(rootedPath) == FileType.EXTERNAL_MUTABLE_FILE) {
+ // For files outside the package roots that are not assumed to be immutable, add a dependency
+ // on the build_id. This is sufficient for correctness; all other files will be handled by
+ // diff awareness of their respective package path, but these files need to be addressed
+ // separately.
+ //
+ // Using the build_id here seems to introduce a performance concern because the upward
+ // transitive closure of these external files will get eagerly invalidated on each
+ // incremental build (e.g. if every file had a transitive dependency on the filesystem root,
+ // then we'd have a big performance problem). But this a non-issue by design:
+ // - We don't add a dependency on the parent directory at the package root boundary, so the
+ // only transitive dependencies from files inside the package roots to external files are
+ // through symlinks. So the upwards transitive closure of external files is small.
+ // - The only way external source files get into the skyframe graph in the first place is
+ // through symlinks outside the package roots, which we neither want to encourage nor
+ // optimize for since it is not common. So the set of external files is small.
+ //
+ // The above reasoning doesn't hold for bazel, because external repositories
+ // (e.g. new_local_repository) cause lots of external symlinks to be present in the build.
+ // So bazel pretends that these external repositories are immutable to avoid the performance
+ // penalty described above.
+ PrecomputedValue.dependOnBuildId(env);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java
new file mode 100644
index 0000000..2a0de78
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java
@@ -0,0 +1,466 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+import com.google.common.collect.Sets;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.cache.Digest;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.actions.cache.Metadata;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * Cache provided by an {@link ActionExecutionFunction}, allowing Blaze to obtain data from the
+ * graph and to inject data (e.g. file digests) back into the graph.
+ *
+ * <p>Data for the action's inputs is injected into this cache on construction, using the graph as
+ * the source of truth.
+ *
+ * <p>As well, this cache collects data about the action's output files, which is used in three
+ * ways. First, it is served as requested during action execution, primarily by the {@code
+ * ActionCacheChecker} when determining if the action must be rerun, and then after the action is
+ * run, to gather information about the outputs. Second, it is accessed by {@link
+ * ArtifactFunction}s in order to construct {@link ArtifactValue}s. Third, the {@link
+ * FilesystemValueChecker} uses it to determine the set of output files to check for inter-build
+ * modifications. Because all these use cases are slightly different, we must occasionally store two
+ * versions of the data for a value (see {@link #getAdditionalOutputData} for more.
+ */
+@VisibleForTesting
+public class FileAndMetadataCache implements ActionInputFileCache, MetadataHandler {
+ /** This should never be read directly. Use {@link #getInputFileArtifactValue} instead. */
+ private final Map<Artifact, FileArtifactValue> inputArtifactData;
+ private final Map<Artifact, Collection<Artifact>> expandedInputMiddlemen;
+ private final File execRoot;
+ private final Map<ByteString, Artifact> reverseMap = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Artifact, FileValue> outputArtifactData =
+ new ConcurrentHashMap<>();
+ // See #getAdditionalOutputData for documentation of this field.
+ private final ConcurrentMap<Artifact, FileArtifactValue> additionalOutputData =
+ new ConcurrentHashMap<>();
+ private final Set<Artifact> injectedArtifacts = Sets.newConcurrentHashSet();
+ private final ImmutableSet<Artifact> outputs;
+ @Nullable private final SkyFunction.Environment env;
+ private final TimestampGranularityMonitor tsgm;
+
+ private static final Interner<ByteString> BYTE_INTERNER = Interners.newWeakInterner();
+
+ public FileAndMetadataCache(Map<Artifact, FileArtifactValue> inputArtifactData,
+ Map<Artifact, Collection<Artifact>> expandedInputMiddlemen, File execRoot,
+ Iterable<Artifact> outputs, @Nullable SkyFunction.Environment env,
+ TimestampGranularityMonitor tsgm) {
+ this.inputArtifactData = Preconditions.checkNotNull(inputArtifactData);
+ this.expandedInputMiddlemen = Preconditions.checkNotNull(expandedInputMiddlemen);
+ this.execRoot = Preconditions.checkNotNull(execRoot);
+ this.outputs = ImmutableSet.copyOf(outputs);
+ this.env = env;
+ this.tsgm = tsgm;
+ }
+
+ @Override
+ public Metadata getMetadataMaybe(Artifact artifact) {
+ try {
+ return getMetadata(artifact);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ private static Metadata metadataFromValue(FileArtifactValue value) throws FileNotFoundException {
+ if (value == FileArtifactValue.MISSING_FILE_MARKER) {
+ throw new FileNotFoundException();
+ }
+ // If the file is empty or a directory, we need to return the mtime because the action cache
+ // uses mtime to determine if this artifact has changed. We do not optimize for this code
+ // path (by storing the mtime somewhere) because we eventually may be switching to use digests
+ // for empty files. We want this code path to go away somehow too for directories (maybe by
+ // implementing FileSet in Skyframe).
+ return value.getSize() > 0
+ ? new Metadata(value.getDigest())
+ : new Metadata(value.getModifiedTime());
+ }
+
+ @Override
+ public Metadata getMetadata(Artifact artifact) throws IOException {
+ Metadata metadata = getRealMetadata(artifact);
+ return artifact.isConstantMetadata() ? Metadata.CONSTANT_METADATA : metadata;
+ }
+
+ @Nullable
+ private FileArtifactValue getInputFileArtifactValue(ActionInput input) {
+ FileArtifactValue value = inputArtifactData.get(input);
+ if (value != null) {
+ return value;
+ }
+ if (outputs.contains(input)) {
+ // When this method is called to calculate the metadata of an artifact, the artifact may be an
+ // output artifact. Don't try to do anything then.
+ return null;
+ }
+ if (!(input instanceof Artifact)) {
+ // Maybe we're being asked for some strange constructed ActionInput coming from runfiles or
+ // similar. We have no information about such things.
+ return null;
+ }
+ // TODO(bazel-team): Remove this codepath once Skyframe has native input discovery, so all
+ // inputs will already have metadata known.
+ // ActionExecutionFunction may have passed in null environment if this action does not
+ // discover inputs. In which case we should not have gotten here.
+ Preconditions.checkNotNull(env, input);
+ Artifact artifact = (Artifact) input;
+ if (artifact.isSourceArtifact()) {
+ // We might have no artifact data for discovered source inputs, and it's not worth storing
+ // it in this cache, because it won't be reused across actions -- while we could request an
+ // artifact from the graph, we would have to be tolerant to it not yet being present in the
+ // graph yet, which adds complexity. Instead, we let the undeclared inputs handler stat it, so
+ // it can be reused.
+ return null;
+ } else {
+ // This getValue call is not expected to return null, because if the artifact is a
+ // transitive dependency of this action (as it should be), it will already have been built,
+ // so this call will return a value.
+ // This getValue call is theoretically less efficient for a subsequent incremental build
+ // than it would be to do a bulk getValues call after action execution, as is done for
+ // source inputs. However, since almost all nodes requested here are transitive deps of an
+ // already-declared dependency, change pruning will request these values serially, but they
+ // will already have been built. So the only penalty is restarting ParallelEvaluator#run, as
+ // opposed to traversing the entire downward transitive closure on a single thread.
+ value = (FileArtifactValue) env.getValue(
+ FileArtifactValue.key(artifact, /*argument ignored for derived artifacts*/false));
+ return value;
+ }
+ }
+
+ /**
+ * We cache data for constant-metadata artifacts, even though it is technically unnecessary,
+ * because the data stored in this cache is consumed by various parts of Blaze via the {@link
+ * ActionExecutionValue} (for now, {@link FilesystemValueChecker} and {@link ArtifactFunction}).
+ * It is simpler for those parts if every output of the action is present in the cache. However,
+ * we must not return the actual metadata for a constant-metadata artifact.
+ */
+ private Metadata getRealMetadata(Artifact artifact) throws IOException {
+ FileArtifactValue value = getInputFileArtifactValue(artifact);
+ if (value != null) {
+ return metadataFromValue(value);
+ }
+ if (artifact.isSourceArtifact()) {
+ // A discovered input we didn't have data for.
+ // TODO(bazel-team): Change this to an assertion once Skyframe has native input discovery, so
+ // all inputs will already have metadata known.
+ return null;
+ } else if (artifact.isMiddlemanArtifact()) {
+ // A middleman artifact's data was either already injected from the action cache checker using
+ // #setDigestForVirtualArtifact, or it has the default middleman value.
+ value = additionalOutputData.get(artifact);
+ if (value != null) {
+ return metadataFromValue(value);
+ }
+ value = FileArtifactValue.DEFAULT_MIDDLEMAN;
+ FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
+ checkInconsistentData(artifact, oldValue, value);
+ return metadataFromValue(value);
+ }
+ FileValue fileValue = outputArtifactData.get(artifact);
+ if (fileValue != null) {
+ // Non-middleman artifacts should only have additionalOutputData if they have
+ // outputArtifactData. We don't assert this because of concurrency possibilities, but at least
+ // we don't check additionalOutputData unless we expect that we might see the artifact there.
+ value = additionalOutputData.get(artifact);
+ // If additional output data is present for this artifact, we use it in preference to the
+ // usual calculation.
+ if (value != null) {
+ return metadataFromValue(value);
+ }
+ if (!fileValue.exists()) {
+ throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
+ }
+ return new Metadata(Preconditions.checkNotNull(fileValue.getDigest(), artifact));
+ }
+ // We do not cache exceptions besides nonexistence here, because it is unlikely that the file
+ // will be requested from this cache too many times.
+ fileValue = fileValueFromArtifact(artifact, null, tsgm);
+ FileValue oldFileValue = outputArtifactData.putIfAbsent(artifact, fileValue);
+ checkInconsistentData(artifact, oldFileValue, value);
+ return maybeStoreAdditionalData(artifact, fileValue, null);
+ }
+
+ /** Expands one of the input middlemen artifacts of the corresponding action. */
+ public Collection<Artifact> expandInputMiddleman(Artifact middlemanArtifact) {
+ Preconditions.checkState(middlemanArtifact.isMiddlemanArtifact(), middlemanArtifact);
+ Collection<Artifact> result = expandedInputMiddlemen.get(middlemanArtifact);
+ // Note that result may be null for non-aggregating middlemen.
+ return result == null ? ImmutableSet.<Artifact>of() : result;
+ }
+
+ /**
+ * Check that the new {@code data} we just calculated for an {@code artifact} agrees with the
+ * {@code oldData} (presumably calculated concurrently), if it was present.
+ */
+ // Not private only because used by SkyframeActionExecutor's metadata handler.
+ static void checkInconsistentData(Artifact artifact,
+ @Nullable Object oldData, Object data) throws IOException {
+ if (oldData != null && !oldData.equals(data)) {
+ // Another thread checked this file since we looked at the map, and got a different answer
+ // than we did. Presumably the user modified the file between reads.
+ throw new IOException("Data for " + artifact.prettyPrint() + " changed to " + data
+ + " after it was calculated as " + oldData);
+ }
+ }
+
+ /**
+ * See {@link #getAdditionalOutputData} for why we sometimes need to store additional data, even
+ * for normal (non-middleman) artifacts.
+ */
+ @Nullable
+ private Metadata maybeStoreAdditionalData(Artifact artifact, FileValue data,
+ @Nullable byte[] injectedDigest) throws IOException {
+ if (!data.exists()) {
+ // Nonexistent files should only occur before executing an action.
+ throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
+ }
+ boolean isFile = data.isFile();
+ boolean useDigest = DigestUtils.useFileDigest(artifact, isFile, isFile ? data.getSize() : 0);
+ if (useDigest && data.getDigest() != null) {
+ // We do not need to store the FileArtifactValue separately -- the digest is in the file value
+ // and that is all that is needed for this file's metadata.
+ return new Metadata(data.getDigest());
+ }
+ // Unfortunately, the FileValue does not contain enough information for us to calculate the
+ // corresponding FileArtifactValue -- either the metadata must use the modified time, which we
+ // do not expose in the FileValue, or the FileValue didn't store the digest So we store the
+ // metadata separately.
+ // Use the FileValue's digest if no digest was injected, or if the file can't be digested.
+ injectedDigest = injectedDigest != null || !isFile ? injectedDigest : data.getDigest();
+ FileArtifactValue value =
+ FileArtifactValue.create(artifact, isFile, isFile ? data.getSize() : 0, injectedDigest);
+ FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value);
+ checkInconsistentData(artifact, oldValue, value);
+ return metadataFromValue(value);
+ }
+
+ @Override
+ public void setDigestForVirtualArtifact(Artifact artifact, Digest digest) {
+ Preconditions.checkState(artifact.isMiddlemanArtifact(), artifact);
+ Preconditions.checkNotNull(digest, artifact);
+ additionalOutputData.put(artifact,
+ FileArtifactValue.createMiddleman(digest.asMetadata().digest));
+ }
+
+ @Override
+ public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
+ if (output instanceof Artifact) {
+ Artifact artifact = (Artifact) output;
+ Preconditions.checkState(injectedArtifacts.add(artifact), artifact);
+ FileValue fileValue;
+ try {
+ // This call may do an unnecessary call to Path#getFastDigest to see if the digest is
+ // readily available. We cannot pass the digest in, though, because if it is not available
+ // from the filesystem, this FileValue will not compare equal to another one created for the
+ // same file, because the other one will be missing its digest.
+ fileValue = fileValueFromArtifact(artifact, FileStatusWithDigestAdapter.adapt(statNoFollow),
+ tsgm);
+ byte[] fileDigest = fileValue.getDigest();
+ Preconditions.checkState(fileDigest == null || Arrays.equals(digest, fileDigest),
+ "%s %s %s", artifact, digest, fileDigest);
+ outputArtifactData.put(artifact, fileValue);
+ } catch (IOException e) {
+ // Do nothing - we just failed to inject metadata. Real error handling will be done later,
+ // when somebody will try to access that file.
+ return;
+ }
+ // If needed, insert additional data. Note that this can only be true if the file is empty or
+ // the filesystem does not support fast digests. Since we usually only inject digests when
+ // running with a filesystem that supports fast digests, this is fairly unlikely.
+ try {
+ maybeStoreAdditionalData(artifact, fileValue, digest);
+ } catch (IOException e) {
+ if (fileValue.getSize() != 0) {
+ // Empty files currently have their mtimes examined, and so could throw. No other files
+ // should throw, since all filesystem access has already been done.
+ throw new IllegalStateException(
+ "Filesystem should not have been accessed while injecting data for "
+ + artifact.prettyPrint(), e);
+ }
+ // Ignore exceptions for empty files, as above.
+ }
+ }
+ }
+
+ @Override
+ public void discardMetadata(Collection<Artifact> artifactList) {
+ Preconditions.checkState(injectedArtifacts.isEmpty(),
+ "Artifacts cannot be injected before action execution: %s", injectedArtifacts);
+ outputArtifactData.keySet().removeAll(artifactList);
+ additionalOutputData.keySet().removeAll(artifactList);
+ }
+
+ @Override
+ public boolean artifactExists(Artifact artifact) {
+ return getMetadataMaybe(artifact) != null;
+ }
+
+ @Override
+ public boolean isRegularFile(Artifact artifact) {
+ // Currently this method is used only for genrule input directory checks. If we need to call
+ // this on output artifacts too, this could be more efficient.
+ FileArtifactValue value = getInputFileArtifactValue(artifact);
+ if (value != null && value.getDigest() != null) {
+ return true;
+ }
+ return artifact.getPath().isFile();
+ }
+
+ @Override
+ public boolean isInjected(Artifact artifact) {
+ return injectedArtifacts.contains(artifact);
+ }
+
+ /**
+ * @return data for output files that was computed during execution. Should include data for all
+ * non-middleman artifacts.
+ */
+ Map<Artifact, FileValue> getOutputData() {
+ return outputArtifactData;
+ }
+
+ /**
+ * Returns data for any output files whose metadata was not computable from the corresponding
+ * entry in {@link #getOutputData}.
+ *
+ * <p>There are three reasons why we might not be able to compute metadata for an artifact from
+ * the FileValue. First, middleman artifacts have no corresponding FileValues. Second, if
+ * computing a file's digest is not fast, the FileValue does not do so, so a file on a filesystem
+ * without fast digests has to have its metadata stored separately. Third, some files' metadata
+ * (directories, empty files) contain their mtimes, which the FileValue does not expose, so that
+ * has to be stored separately.
+ *
+ * <p>Note that for files that need digests, we can't easily inject the digest in the FileValue
+ * because it would complicate equality-checking on subsequent builds -- if our filesystem doesn't
+ * do fast digests, the comparison value would not have a digest.
+ */
+ Map<Artifact, FileArtifactValue> getAdditionalOutputData() {
+ return additionalOutputData;
+ }
+
+ @Override
+ public long getSizeInBytes(ActionInput input) throws IOException {
+ FileArtifactValue metadata = getInputFileArtifactValue(input);
+ if (metadata != null) {
+ return metadata.getSize();
+ }
+ return -1;
+ }
+
+ @Nullable
+ @Override
+ public File getFileFromDigest(ByteString digest) throws IOException {
+ Artifact artifact = reverseMap.get(digest);
+ if (artifact != null) {
+ String relPath = artifact.getExecPathString();
+ return relPath.startsWith("/") ? new File(relPath) : new File(execRoot, relPath);
+ }
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public ByteString getDigest(ActionInput input) throws IOException {
+ FileArtifactValue value = getInputFileArtifactValue(input);
+ if (value != null) {
+ byte[] bytes = value.getDigest();
+ if (bytes != null) {
+ ByteString digest = ByteString.copyFrom(BaseEncoding.base16().lowerCase().encode(bytes)
+ .getBytes(StandardCharsets.US_ASCII));
+ reverseMap.put(BYTE_INTERNER.intern(digest), (Artifact) input);
+ return digest;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean contentsAvailableLocally(ByteString digest) {
+ return reverseMap.containsKey(digest);
+ }
+
+ static FileValue fileValueFromArtifact(Artifact artifact,
+ @Nullable FileStatusWithDigest statNoFollow, TimestampGranularityMonitor tsgm)
+ throws IOException {
+ Path path = artifact.getPath();
+ RootedPath rootedPath =
+ RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath());
+ if (statNoFollow == null) {
+ statNoFollow = FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW));
+ if (statNoFollow == null) {
+ return FileValue.value(rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE,
+ rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE);
+ }
+ }
+ Path realPath = path;
+ // We use FileStatus#isSymbolicLink over Path#isSymbolicLink to avoid the unnecessary stat
+ // done by the latter.
+ if (statNoFollow.isSymbolicLink()) {
+ realPath = path.resolveSymbolicLinks();
+ // We need to protect against symlink cycles since FileValue#value assumes it's dealing with a
+ // file that's not in a symlink cycle.
+ if (realPath.equals(path)) {
+ throw new IOException("symlink cycle");
+ }
+ }
+ RootedPath realRootedPath = RootedPath.toRootedPathMaybeUnderRoot(realPath,
+ ImmutableList.of(artifact.getRoot().getPath()));
+ FileStateValue fileStateValue;
+ FileStateValue realFileStateValue;
+ try {
+ fileStateValue = FileStateValue.createWithStatNoFollow(rootedPath, statNoFollow, tsgm);
+ // TODO(bazel-team): consider avoiding a 'stat' here when the symlink target hasn't changed
+ // and is a source file (since changes to those are checked separately).
+ realFileStateValue = realPath.equals(path) ? fileStateValue
+ : FileStateValue.create(realRootedPath, tsgm);
+ } catch (InconsistentFilesystemException e) {
+ throw new IOException(e);
+ }
+ return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
new file mode 100644
index 0000000..7acc38c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
@@ -0,0 +1,148 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import javax.annotation.Nullable;
+
+/**
+ * Stores the data of an artifact corresponding to a file. This file may be an ordinary file, in
+ * which case we would expect to see a digest and size; a directory, in which case we would expect
+ * to see an mtime; or an empty file, where we would expect to see a size (=0), mtime, and digest
+ */
+public class FileArtifactValue extends ArtifactValue {
+ /** Data for Middleman artifacts that did not have data specified. */
+ static final FileArtifactValue DEFAULT_MIDDLEMAN = new FileArtifactValue(null, 0, 0);
+ /** Data that marks that a file is not present on the filesystem. */
+ static final FileArtifactValue MISSING_FILE_MARKER = new FileArtifactValue(null, 1, 0);
+
+ @Nullable private final byte[] digest;
+ private final long mtime;
+ private final long size;
+
+ private FileArtifactValue(byte[] digest, long size) {
+ Preconditions.checkState(size >= 0, "size must be non-negative: %s %s", digest, size);
+ this.digest = Preconditions.checkNotNull(digest, size);
+ this.size = size;
+ this.mtime = -1;
+ }
+
+ // Only used by empty files (non-null digest) and directories (null digest).
+ private FileArtifactValue(byte[] digest, long mtime, long size) {
+ Preconditions.checkState(mtime >= 0, "mtime must be non-negative: %s %s", mtime, size);
+ Preconditions.checkState(size == 0, "size must be zero: %s %s", mtime, size);
+ this.digest = digest;
+ this.size = size;
+ this.mtime = mtime;
+ }
+
+ static FileArtifactValue create(Artifact artifact) throws IOException {
+ Path path = artifact.getPath();
+ FileStatus stat = path.stat();
+ boolean isFile = stat.isFile();
+ return create(artifact, isFile, isFile ? stat.getSize() : 0, null);
+ }
+
+ static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException {
+ boolean isFile = fileValue.isFile();
+ return create(artifact, isFile, isFile ? fileValue.getSize() : 0,
+ isFile ? fileValue.getDigest() : null);
+ }
+
+ static FileArtifactValue create(Artifact artifact, boolean isFile, long size,
+ @Nullable byte[] digest) throws IOException {
+ if (isFile && digest == null) {
+ digest = DigestUtils.getDigestOrFail(artifact.getPath(), size);
+ }
+ if (!DigestUtils.useFileDigest(artifact, isFile, size)) {
+ // In this case, we need to store the mtime because the action cache uses mtime to determine
+ // if this artifact has changed. This is currently true for empty files and directories. We
+ // do not optimize for this code path (by storing the mtime in a FileValue) because we do not
+ // like it and may remove this special-casing for empty files in the future. We want this code
+ // path to go away somehow too for directories (maybe by implementing FileSet
+ // in Skyframe)
+ return new FileArtifactValue(digest, artifact.getPath().getLastModifiedTime(), size);
+ }
+ Preconditions.checkState(digest != null, artifact);
+ return new FileArtifactValue(digest, size);
+ }
+
+ static FileArtifactValue createMiddleman(byte[] digest) {
+ Preconditions.checkNotNull(digest);
+ // The Middleman artifact values have size 1 because we want their digests to be used. This hack
+ // can be removed once empty files are digested.
+ return new FileArtifactValue(digest, /*size=*/1);
+ }
+
+ @Nullable
+ byte[] getDigest() {
+ return digest;
+ }
+
+ /** Gets the size of the file. Directories have size 0. */
+ long getSize() {
+ return size;
+ }
+
+ /**
+ * Gets last modified time of file. Should only be called if {@link DigestUtils#useFileDigest} was
+ * false for this artifact -- namely, either it is a directory or an empty file. Note that since
+ * we store directory sizes as 0, all files for which this method can be called have size 0.
+ */
+ long getModifiedTime() {
+ Preconditions.checkState(size == 0, "%s %s %s", digest, mtime, size);
+ return mtime;
+ }
+
+ @Override
+ public int hashCode() {
+ // Hash digest by content, not reference. Note that digest is the only array in this array.
+ return Arrays.deepHashCode(new Object[] {size, mtime, digest});
+ }
+
+ /**
+ * Two FileArtifactValues will only compare equal if they have the same content. This differs
+ * from the {@code Metadata#equivalence} method, which allows for comparison using mtime if
+ * one object does not have a digest available.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof FileArtifactValue)) {
+ return false;
+ }
+ FileArtifactValue that = (FileArtifactValue) other;
+ return this.mtime == that.mtime && this.size == that.size
+ && Arrays.equals(this.digest, that.digest);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(FileArtifactValue.class)
+ .add("digest", digest)
+ .add("mtime", mtime)
+ .add("size", size).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java
new file mode 100644
index 0000000..344c364
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * In case we can't get a fast digest from the filesystem, we store this metadata as a proxy to
+ * the file contents. Currently it is a pair of the mtime and "value id" (which is right now just
+ * the ivalue number). We may wish to augment this object with the following data:
+ * a. the device number
+ * b. the ctime, which cannot be tampered with in userspace
+ *
+ * <p>For an example of why mtime alone is insufficient, note that 'mv' preserves timestamps. So if
+ * files 'a' and 'b' initially have the same timestamp, then we would think 'b' is unchanged after
+ * the user executes `mv a b` between two builds.
+ */
+public final class FileContentsProxy implements Serializable {
+ private final long mtime;
+ private final long valueId;
+
+ private FileContentsProxy(long mtime, long valueId) {
+ this.mtime = mtime;
+ this.valueId = valueId;
+ }
+
+ public static FileContentsProxy create(long mtime, long valueId) {
+ return new FileContentsProxy(mtime, valueId);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof FileContentsProxy)) {
+ return false;
+ }
+
+ FileContentsProxy that = (FileContentsProxy) other;
+ return mtime == that.mtime && valueId == that.valueId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(mtime, valueId);
+ }
+
+ @Override
+ public String toString() {
+ return "mtime: " + mtime + " valueId: " + valueId;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java
new file mode 100644
index 0000000..ad3cb74
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java
@@ -0,0 +1,217 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.LinkedHashSet;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link SkyFunction} for {@link FileValue}s.
+ *
+ * <p>Most of the complexity in the implementation is associated to handling symlinks. Namely,
+ * this class makes sure that {@code FileValue}s corresponding to symlinks are correctly invalidated
+ * if the destination of the symlink is invalidated. Directory symlinks are also covered.
+ */
+public class FileFunction implements SkyFunction {
+
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final ExternalFilesHelper externalFilesHelper;
+
+ public FileFunction(AtomicReference<PathPackageLocator> pkgLocator,
+ ExternalFilesHelper externalFilesHelper) {
+ this.pkgLocator = pkgLocator;
+ this.externalFilesHelper = externalFilesHelper;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws FileFunctionException {
+ RootedPath rootedPath = (RootedPath) skyKey.argument();
+ RootedPath realRootedPath = rootedPath;
+ FileStateValue realFileStateValue = null;
+ PathFragment relativePath = rootedPath.getRelativePath();
+
+ // Resolve ancestor symlinks, but only if the current file is not the filesystem root (has no
+ // parent) or a package path root (treated opaquely and handled by skyframe's DiffAwareness
+ // interface) or otherwise assumed to be immutable (handling ancestors would add dependencies
+ // too aggressively). Note that this is the first thing we do - if an ancestor is part of a
+ // symlink cycle, we want to detect that quickly as it gives a more informative error message
+ // than we'd get doing bogus filesystem operations.
+ if (!relativePath.equals(PathFragment.EMPTY_FRAGMENT)
+ && !externalFilesHelper.shouldAssumeImmutable(rootedPath)) {
+ Pair<RootedPath, FileStateValue> resolvedState =
+ resolveFromAncestors(rootedPath, env);
+ if (resolvedState == null) {
+ return null;
+ }
+ realRootedPath = resolvedState.getFirst();
+ realFileStateValue = resolvedState.getSecond();
+ }
+
+ FileStateValue fileStateValue = (FileStateValue) env.getValue(FileStateValue.key(rootedPath));
+ if (fileStateValue == null) {
+ return null;
+ }
+ if (realFileStateValue == null) {
+ realFileStateValue = fileStateValue;
+ }
+
+ LinkedHashSet<RootedPath> seenPaths = Sets.newLinkedHashSet();
+ while (realFileStateValue.getType().equals(FileStateValue.Type.SYMLINK)) {
+ if (!seenPaths.add(realRootedPath)) {
+ FileSymlinkCycleException fileSymlinkCycleException =
+ makeFileSymlinkCycleException(realRootedPath, seenPaths);
+ if (env.getValue(FileSymlinkCycleUniquenessValue.key(fileSymlinkCycleException.getCycle()))
+ == null) {
+ // Note that this dependency is merely to ensure that each unique cycle gets reported
+ // exactly once.
+ return null;
+ }
+ throw new FileFunctionException(fileSymlinkCycleException);
+ }
+ Pair<RootedPath, FileStateValue> resolvedState = getSymlinkTargetRootedPath(realRootedPath,
+ realFileStateValue.getSymlinkTarget(), env);
+ if (resolvedState == null) {
+ return null;
+ }
+ realRootedPath = resolvedState.getFirst();
+ realFileStateValue = resolvedState.getSecond();
+ }
+ return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue);
+ }
+
+ /**
+ * Returns the path and file state of {@code rootedPath}, accounting for ancestor symlinks, or
+ * {@code null} if there was a missing dep.
+ */
+ @Nullable
+ private Pair<RootedPath, FileStateValue> resolveFromAncestors(RootedPath rootedPath,
+ Environment env) throws FileFunctionException {
+ PathFragment relativePath = rootedPath.getRelativePath();
+ RootedPath realRootedPath = rootedPath;
+ FileValue parentFileValue = null;
+ if (!relativePath.equals(PathFragment.EMPTY_FRAGMENT)) {
+ RootedPath parentRootedPath = RootedPath.toRootedPath(rootedPath.getRoot(),
+ relativePath.getParentDirectory());
+ parentFileValue = (FileValue) env.getValue(FileValue.key(parentRootedPath));
+ if (parentFileValue == null) {
+ return null;
+ }
+ PathFragment baseName = new PathFragment(relativePath.getBaseName());
+ RootedPath parentRealRootedPath = parentFileValue.realRootedPath();
+ realRootedPath = RootedPath.toRootedPath(parentRealRootedPath.getRoot(),
+ parentRealRootedPath.getRelativePath().getRelative(baseName));
+ }
+ FileStateValue realFileStateValue =
+ (FileStateValue) env.getValue(FileStateValue.key(realRootedPath));
+ if (realFileStateValue == null) {
+ return null;
+ }
+ if (realFileStateValue.getType() != FileStateValue.Type.NONEXISTENT
+ && parentFileValue != null && !parentFileValue.isDirectory()) {
+ String type = realFileStateValue.getType().toString().toLowerCase();
+ String message = type + " " + rootedPath.asPath() + " exists but its parent "
+ + "directory " + parentFileValue.realRootedPath().asPath() + " doesn't exist.";
+ throw new FileFunctionException(new InconsistentFilesystemException(message),
+ Transience.TRANSIENT);
+ }
+ return Pair.of(realRootedPath, realFileStateValue);
+ }
+
+ /**
+ * Returns the symlink target and file state of {@code rootedPath}'s symlink to
+ * {@code symlinkTarget}, accounting for ancestor symlinks, or {@code null} if there was a
+ * missing dep.
+ */
+ @Nullable
+ private Pair<RootedPath, FileStateValue> getSymlinkTargetRootedPath(RootedPath rootedPath,
+ PathFragment symlinkTarget, Environment env) throws FileFunctionException {
+ if (symlinkTarget.isAbsolute()) {
+ Path path = rootedPath.asPath().getFileSystem().getRootDirectory().getRelative(
+ symlinkTarget);
+ return resolveFromAncestors(
+ RootedPath.toRootedPathMaybeUnderRoot(path, pkgLocator.get().getPathEntries()), env);
+ }
+ Path path = rootedPath.asPath();
+ Path symlinkTargetPath;
+ if (path.getParentDirectory() != null) {
+ RootedPath parentRootedPath = RootedPath.toRootedPathMaybeUnderRoot(
+ path.getParentDirectory(), pkgLocator.get().getPathEntries());
+ FileValue parentFileValue = (FileValue) env.getValue(FileValue.key(parentRootedPath));
+ if (parentFileValue == null) {
+ return null;
+ }
+ symlinkTargetPath = parentFileValue.realRootedPath().asPath().getRelative(symlinkTarget);
+ } else {
+ // This means '/' is a symlink to 'symlinkTarget'.
+ symlinkTargetPath = path.getRelative(symlinkTarget);
+ }
+ RootedPath symlinkTargetRootedPath = RootedPath.toRootedPathMaybeUnderRoot(symlinkTargetPath,
+ pkgLocator.get().getPathEntries());
+ return resolveFromAncestors(symlinkTargetRootedPath, env);
+ }
+
+ private FileSymlinkCycleException makeFileSymlinkCycleException(RootedPath startOfCycle,
+ Iterable<RootedPath> symlinkPaths) {
+ boolean inPathToCycle = true;
+ ImmutableList.Builder<RootedPath> pathToCycleBuilder = ImmutableList.builder();
+ ImmutableList.Builder<RootedPath> cycleBuilder = ImmutableList.builder();
+ for (RootedPath path : symlinkPaths) {
+ if (path.equals(startOfCycle)) {
+ inPathToCycle = false;
+ }
+ if (inPathToCycle) {
+ pathToCycleBuilder.add(path);
+ } else {
+ cycleBuilder.add(path);
+ }
+ }
+ return new FileSymlinkCycleException(pathToCycleBuilder.build(), cycleBuilder.build());
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link FileFunction#compute}.
+ */
+ private static final class FileFunctionException extends SkyFunctionException {
+
+ public FileFunctionException(InconsistentFilesystemException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public FileFunctionException(FileSymlinkCycleException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
new file mode 100644
index 0000000..ec2e871
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * A {@link SkyFunction} for {@link FileStateValue}s.
+ *
+ * <p>Merely calls FileStateValue#create, but also has special handling for files outside the
+ * package roots (see {@link ExternalFilesHelper}).
+ */
+public class FileStateFunction implements SkyFunction {
+
+ private final TimestampGranularityMonitor tsgm;
+ private final ExternalFilesHelper externalFilesHelper;
+
+ public FileStateFunction(TimestampGranularityMonitor tsgm,
+ ExternalFilesHelper externalFilesHelper) {
+ this.tsgm = tsgm;
+ this.externalFilesHelper = externalFilesHelper;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws FileStateFunctionException {
+ RootedPath rootedPath = (RootedPath) skyKey.argument();
+ externalFilesHelper.maybeAddDepOnBuildId(rootedPath, env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ try {
+ return FileStateValue.create(rootedPath, tsgm);
+ } catch (IOException e) {
+ throw new FileStateFunctionException(e);
+ } catch (InconsistentFilesystemException e) {
+ throw new FileStateFunctionException(e);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link FileStateFunction#compute}.
+ */
+ private static final class FileStateFunctionException extends SkyFunctionException {
+ public FileStateFunctionException(IOException e) {
+ super(e, Transience.TRANSIENT);
+ }
+
+ public FileStateFunctionException(InconsistentFilesystemException e) {
+ super(e, Transience.TRANSIENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
new file mode 100644
index 0000000..8631ff3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
@@ -0,0 +1,317 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Encapsulates the filesystem operations needed to get state for a path. This is at least a
+ * 'lstat' to determine what type of file the path is.
+ * <ul>
+ * <li> For a non-existent file, the non existence is noted.
+ * <li> For a symlink, the symlink target is noted.
+ * <li> For a directory, the existence is noted.
+ * <li> For a file, the existence is noted, along with metadata about the file (e.g.
+ * file digest). See {@link FileFileStateValue}.
+ * <ul>
+ *
+ * <p>This class is an implementation detail of {@link FileValue} and should not be used outside of
+ * {@link FileFunction}. Instead, {@link FileValue} should be used by consumers that care about
+ * files.
+ *
+ * <p>All subclasses must implement {@link #equals} and {@link #hashCode} properly.
+ */
+abstract class FileStateValue implements SkyValue {
+
+ public static final FileStateValue DIRECTORY_FILE_STATE_NODE = DirectoryFileStateValue.INSTANCE;
+ public static final FileStateValue NONEXISTENT_FILE_STATE_NODE =
+ NonexistentFileStateValue.INSTANCE;
+
+ public enum Type {
+ FILE,
+ DIRECTORY,
+ SYMLINK,
+ NONEXISTENT,
+ }
+
+ protected FileStateValue() {
+ }
+
+ public static FileStateValue create(RootedPath rootedPath,
+ @Nullable TimestampGranularityMonitor tsgm) throws InconsistentFilesystemException,
+ IOException {
+ Path path = rootedPath.asPath();
+ // Stat, but don't throw an exception for the common case of a nonexistent file. This still
+ // throws an IOException in case any other IO error is encountered.
+ FileStatus stat = path.statIfFound(Symlinks.NOFOLLOW);
+ if (stat == null) {
+ return NONEXISTENT_FILE_STATE_NODE;
+ }
+ return createWithStatNoFollow(rootedPath, FileStatusWithDigestAdapter.adapt(stat), tsgm);
+ }
+
+ public static FileStateValue createWithStatNoFollow(RootedPath rootedPath,
+ FileStatusWithDigest statNoFollow, @Nullable TimestampGranularityMonitor tsgm)
+ throws InconsistentFilesystemException, IOException {
+ Path path = rootedPath.asPath();
+ if (statNoFollow.isFile()) {
+ return FileFileStateValue.fromPath(path, statNoFollow, tsgm);
+ } else if (statNoFollow.isDirectory()) {
+ return DIRECTORY_FILE_STATE_NODE;
+ } else if (statNoFollow.isSymbolicLink()) {
+ return new SymlinkFileStateValue(path.readSymbolicLink());
+ }
+ throw new InconsistentFilesystemException("according to stat, existing path " + path + " is "
+ + "neither a file nor directory nor symlink.");
+ }
+
+ @ThreadSafe
+ static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.FILE_STATE, rootedPath);
+ }
+
+ abstract Type getType();
+
+ PathFragment getSymlinkTarget() {
+ throw new IllegalStateException();
+ }
+
+ long getSize() {
+ throw new IllegalStateException();
+ }
+
+ @Nullable
+ byte[] getDigest() {
+ throw new IllegalStateException();
+ }
+
+ /**
+ * Implementation of {@link FileStateValue} for files that exist.
+ *
+ * <p>A union of (digest, mtime). We use digests only if a fast digest lookup is available from
+ * the filesystem. If not, we fall back to mtime-based digests. This avoids the case where Blaze
+ * must read all files involved in the build in order to check for modifications in the case
+ * where fast digest lookups are not available.
+ */
+ @ThreadSafe
+ private static final class FileFileStateValue extends FileStateValue {
+ private final long size;
+ // Only needed for empty-file equality-checking. Otherwise is always -1.
+ // TODO(bazel-team): Consider getting rid of this special case for empty files.
+ private final long mtime;
+ @Nullable private final byte[] digest;
+ @Nullable private final FileContentsProxy contentsProxy;
+
+ private FileFileStateValue(long size, long mtime, byte[] digest,
+ FileContentsProxy contentsProxy) {
+ Preconditions.checkState((digest == null) != (contentsProxy == null));
+ this.size = size;
+ // mtime is forced to be -1 so that we do not accidentally depend on it for non-empty files,
+ // which should only be compared using digests.
+ this.mtime = size == 0 ? mtime : -1;
+ this.digest = digest;
+ this.contentsProxy = contentsProxy;
+ }
+
+ /**
+ * Create a FileFileStateValue instance corresponding to the given existing file.
+ * @param stat must be of type "File". (Not a symlink).
+ */
+ public static FileFileStateValue fromPath(Path path, FileStatusWithDigest stat,
+ @Nullable TimestampGranularityMonitor tsgm)
+ throws InconsistentFilesystemException {
+ Preconditions.checkState(stat.isFile(), path);
+ try {
+ byte[] digest = stat.getDigest();
+ if (digest == null) {
+ digest = path.getFastDigest();
+ }
+ if (digest == null) {
+ long mtime = stat.getLastModifiedTime();
+ // Note that TimestampGranularityMonitor#notifyDependenceOnFileTime is a thread-safe
+ // method.
+ if (tsgm != null) {
+ tsgm.notifyDependenceOnFileTime(mtime);
+ }
+ return new FileFileStateValue(stat.getSize(), stat.getLastModifiedTime(), null,
+ FileContentsProxy.create(mtime, stat.getNodeId()));
+ } else {
+ // We are careful here to avoid putting the value ID into FileMetadata if we already have
+ // a digest. Arbitrary filesystems may do weird things with the value ID; a digest is more
+ // robust.
+ return new FileFileStateValue(stat.getSize(), stat.getLastModifiedTime(), digest, null);
+ }
+ } catch (IOException e) {
+ String errorMessage = e.getMessage() != null
+ ? "error '" + e.getMessage() + "'" : "an error";
+ throw new InconsistentFilesystemException("'stat' said " + path + " is a file but then we "
+ + "later encountered " + errorMessage + " which indicates that " + path + " no longer "
+ + "exists. Did you delete it during the build?");
+ }
+ }
+
+ @Override
+ public Type getType() {
+ return Type.FILE;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ @Nullable
+ public byte[] getDigest() {
+ return digest;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof FileFileStateValue) {
+ FileFileStateValue other = (FileFileStateValue) obj;
+ return size == other.size && mtime == other.mtime && Arrays.equals(digest, other.digest)
+ && Objects.equals(contentsProxy, other.contentsProxy);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(size, mtime, Arrays.hashCode(digest), contentsProxy);
+ }
+
+ @Override
+ public String toString() {
+ return "[size: " + size + " " + (mtime != -1 ? "mtime: " + mtime : "")
+ + (digest != null ? "digest: " + Arrays.toString(digest) : contentsProxy) + "]";
+ }
+ }
+
+ /** Implementation of {@link FileStateValue} for directories that exist. */
+ private static final class DirectoryFileStateValue extends FileStateValue {
+
+ public static final DirectoryFileStateValue INSTANCE = new DirectoryFileStateValue();
+
+ private DirectoryFileStateValue() {
+ }
+
+ @Override
+ public Type getType() {
+ return Type.DIRECTORY;
+ }
+
+ @Override
+ public String toString() {
+ return "directory";
+ }
+
+ // This object is normally a singleton, but deserialization produces copies.
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof DirectoryFileStateValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return 7654321;
+ }
+ }
+
+ /** Implementation of {@link FileStateValue} for symlinks. */
+ private static final class SymlinkFileStateValue extends FileStateValue {
+
+ private final PathFragment symlinkTarget;
+
+ private SymlinkFileStateValue(PathFragment symlinkTarget) {
+ this.symlinkTarget = symlinkTarget;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.SYMLINK;
+ }
+
+ @Override
+ public PathFragment getSymlinkTarget() {
+ return symlinkTarget;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof SymlinkFileStateValue)) {
+ return false;
+ }
+ SymlinkFileStateValue other = (SymlinkFileStateValue) obj;
+ return symlinkTarget.equals(other.symlinkTarget);
+ }
+
+ @Override
+ public int hashCode() {
+ return symlinkTarget.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "symlink to " + symlinkTarget;
+ }
+ }
+
+ /** Implementation of {@link FileStateValue} for nonexistent files. */
+ private static final class NonexistentFileStateValue extends FileStateValue {
+
+ public static final NonexistentFileStateValue INSTANCE = new NonexistentFileStateValue();
+
+ private NonexistentFileStateValue() {
+ }
+
+ @Override
+ public Type getType() {
+ return Type.NONEXISTENT;
+ }
+
+ @Override
+ public String toString() {
+ return "nonexistent";
+ }
+
+ // This object is normally a singleton, but deserialization produces copies.
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof NonexistentFileStateValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return 8765432;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java
new file mode 100644
index 0000000..d57fc42
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.RootedPath;
+
+/** Exception indicating that a cycle was found in the filesystem. */
+public class FileSymlinkCycleException extends Exception {
+
+ private final ImmutableList<RootedPath> pathToCycle;
+ private final ImmutableList<RootedPath> cycle;
+
+ FileSymlinkCycleException(ImmutableList<RootedPath> pathToCycle,
+ ImmutableList<RootedPath> cycle) {
+ // The cycle itself has already been reported by FileSymlinkCycleUniquenessValue, but we still
+ // want to have a readable #getMessage.
+ super("Symlink cycle");
+ this.pathToCycle = pathToCycle;
+ this.cycle = cycle;
+ }
+
+ /**
+ * The symlink path to the symlink cycle. For example, suppose 'a' -> 'b' -> 'c' -> 'd' -> 'c'.
+ * The path to the cycle is 'a', 'b'.
+ */
+ ImmutableList<RootedPath> getPathToCycle() {
+ return pathToCycle;
+ }
+
+ /**
+ * The symlink cycle. For example, suppose 'a' -> 'b' -> 'c' -> 'd' -> 'c'.
+ * The cycle is 'c', 'd'.
+ */
+ ImmutableList<RootedPath> getCycle() {
+ return cycle;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java
new file mode 100644
index 0000000..a0604b5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/** A value builder that has the side effect of reporting a file symlink cycle. */
+public class FileSymlinkCycleUniquenessFunction implements SkyFunction {
+
+ @SuppressWarnings("unchecked") // Cast from Object to ImmutableList<RootedPath>.
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ StringBuilder cycleMessage = new StringBuilder("circular symlinks detected\n");
+ cycleMessage.append("[start of symlink cycle]\n");
+ for (RootedPath rootedPath : (ImmutableList<RootedPath>) skyKey.argument()) {
+ cycleMessage.append(rootedPath.asPath() + "\n");
+ }
+ cycleMessage.append("[end of symlink cycle]");
+ // The purpose of this value builder is the side effect of emitting an error message exactly
+ // once per build per unique cycle.
+ env.getListener().handle(Event.error(cycleMessage.toString()));
+ return FileSymlinkCycleUniquenessValue.INSTANCE;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java
new file mode 100644
index 0000000..627276d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value for ensuring that a file symlink cycle is reported exactly once. This is achieved by
+ * forcing the same value key for two logically equivalent cycles (e.g. ['a' -> 'b' -> 'c' -> 'a']
+ * and ['b' -> 'c' -> 'a' -> 'b'], and letting Skyframe do its magic.
+ */
+class FileSymlinkCycleUniquenessValue implements SkyValue {
+
+ public static final FileSymlinkCycleUniquenessValue INSTANCE =
+ new FileSymlinkCycleUniquenessValue();
+
+ private FileSymlinkCycleUniquenessValue() {
+ }
+
+ static SkyKey key(ImmutableList<RootedPath> cycle) {
+ Preconditions.checkState(!cycle.isEmpty());
+ return new SkyKey(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, canonicalize(cycle));
+ }
+
+ private static ImmutableList<RootedPath> canonicalize(ImmutableList<RootedPath> cycle) {
+ int minPos = 0;
+ String minString = cycle.get(0).toString();
+ for (int i = 1; i < cycle.size(); i++) {
+ String candidateString = cycle.get(i).toString();
+ if (candidateString.compareTo(minString) < 0) {
+ minPos = i;
+ minString = candidateString;
+ }
+ }
+ ImmutableList.Builder<RootedPath> builder = ImmutableList.builder();
+ for (int i = 0; i < cycle.size(); i++) {
+ int pos = (minPos + i) % cycle.size();
+ builder.add(cycle.get(pos));
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java
new file mode 100644
index 0000000..1850fd9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java
@@ -0,0 +1,279 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.skyframe.FileStateValue.Type;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value that corresponds to a file (or directory or symlink or non-existent file), fully
+ * accounting for symlinks (e.g. proper dependencies on ancestor symlinks so as to be incrementally
+ * correct). Anything in Skyframe that cares about the fully resolved path of a file (e.g. anything
+ * that cares about the contents of a file) should have a dependency on the corresponding
+ * {@link FileValue}.
+ *
+ * <p>
+ * Note that the existence of a file value does not imply that the file exists on the filesystem.
+ * File values for missing files will be created on purpose in order to facilitate incremental
+ * builds in the case those files have reappeared.
+ *
+ * <p>
+ * This class contains the relevant metadata for a file, although not the contents. Note that
+ * since a FileValue doesn't store its corresponding SkyKey, it's possible for the FileValues for
+ * two different paths to be the same.
+ *
+ * <p>
+ * This should not be used for build outputs; use {@link ArtifactValue} for those.
+ */
+@Immutable
+@ThreadSafe
+public abstract class FileValue implements SkyValue {
+
+ boolean exists() {
+ return realFileStateValue().getType() != Type.NONEXISTENT;
+ }
+
+ public boolean isSymlink() {
+ return false;
+ }
+
+ /**
+ * Returns true if this value corresponds to a file or symlink to an existing file. If so, its
+ * parent directory is guaranteed to exist.
+ */
+ public boolean isFile() {
+ return realFileStateValue().getType() == Type.FILE;
+ }
+
+ /**
+ * Returns true if the file is a directory or a symlink to an existing directory. If so, its
+ * parent directory is guaranteed to exist.
+ */
+ public boolean isDirectory() {
+ return realFileStateValue().getType() == Type.DIRECTORY;
+ }
+
+ /**
+ * Returns the real rooted path of the file, taking ancestor symlinks into account. For example,
+ * the rooted path ['root']/['a/b'] is really ['root']/['c/b'] if 'a' is a symlink to 'b'. Note
+ * that ancestor symlinks outside the root boundary are not taken into consideration.
+ */
+ public abstract RootedPath realRootedPath();
+
+ abstract FileStateValue realFileStateValue();
+
+ /**
+ * Returns the unresolved link target if {@link #isSymlink()}.
+ *
+ * <p>This is useful if the caller wants to, for example, duplicate a relative symlink. An actual
+ * example could be a build rule that copies a set of input files to the output directory, but
+ * upon encountering symbolic links it can decide between copying or following them.
+ */
+ PathFragment getUnresolvedLinkTarget() {
+ throw new IllegalStateException(this.toString());
+ }
+
+ long getSize() {
+ Preconditions.checkState(isFile(), this);
+ return realFileStateValue().getSize();
+ }
+
+ @Nullable
+ byte[] getDigest() {
+ Preconditions.checkState(isFile(), this);
+ return realFileStateValue().getDigest();
+ }
+
+ /**
+ * Returns a key for building a file value for the given root-relative path.
+ */
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.FILE, rootedPath);
+ }
+
+ /**
+ * Only intended to be used by {@link FileFunction}. Should not be used for symlink cycles.
+ */
+ static FileValue value(RootedPath rootedPath, FileStateValue fileStateValue,
+ RootedPath realRootedPath, FileStateValue realFileStateValue) {
+ if (rootedPath.equals(realRootedPath)) {
+ Preconditions.checkState(fileStateValue.getType() != FileStateValue.Type.SYMLINK,
+ "rootedPath: %s, fileStateValue: %s, realRootedPath: %s, realFileStateValue: %s",
+ rootedPath, fileStateValue, realRootedPath, realFileStateValue);
+ return new RegularFileValue(rootedPath, fileStateValue);
+ } else {
+ if (fileStateValue.getType() == FileStateValue.Type.SYMLINK) {
+ return new SymlinkFileValue(realRootedPath, realFileStateValue,
+ fileStateValue.getSymlinkTarget());
+ } else {
+ return new DifferentRealPathFileValue(realRootedPath, realFileStateValue);
+ }
+ }
+ }
+
+ /**
+ * Implementation of {@link FileValue} for files whose fully resolved path is the same as the
+ * requested path. For example, this is the case for the path "foo/bar/baz" if neither 'foo' nor
+ * 'foo/bar' nor 'foo/bar/baz' are symlinks.
+ */
+ private static final class RegularFileValue extends FileValue {
+
+ private final RootedPath rootedPath;
+ private final FileStateValue fileStateValue;
+
+ private RegularFileValue(RootedPath rootedPath, FileStateValue fileState) {
+ this.rootedPath = Preconditions.checkNotNull(rootedPath);
+ this.fileStateValue = Preconditions.checkNotNull(fileState);
+ }
+
+ @Override
+ public RootedPath realRootedPath() {
+ return rootedPath;
+ }
+
+ @Override
+ FileStateValue realFileStateValue() {
+ return fileStateValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj.getClass() != RegularFileValue.class) {
+ return false;
+ }
+ RegularFileValue other = (RegularFileValue) obj;
+ return rootedPath.equals(other.rootedPath) && fileStateValue.equals(other.fileStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rootedPath, fileStateValue);
+ }
+
+ @Override
+ public String toString() {
+ return rootedPath + ", " + fileStateValue;
+ }
+ }
+
+ /**
+ * Base class for {@link FileValue}s for files whose fully resolved path is different than the
+ * requested path. For example, this is the case for the path "foo/bar/baz" if at least one of
+ * 'foo', 'foo/bar', or 'foo/bar/baz' is a symlink.
+ */
+ private static class DifferentRealPathFileValue extends FileValue {
+
+ protected final RootedPath realRootedPath;
+ protected final FileStateValue realFileStateValue;
+
+ private DifferentRealPathFileValue(RootedPath realRootedPath,
+ FileStateValue realFileStateValue) {
+ this.realRootedPath = Preconditions.checkNotNull(realRootedPath);
+ this.realFileStateValue = Preconditions.checkNotNull(realFileStateValue);
+ }
+
+ @Override
+ public RootedPath realRootedPath() {
+ return realRootedPath;
+ }
+
+ @Override
+ FileStateValue realFileStateValue() {
+ return realFileStateValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj.getClass() != DifferentRealPathFileValue.class) {
+ return false;
+ }
+ DifferentRealPathFileValue other = (DifferentRealPathFileValue) obj;
+ return realRootedPath.equals(other.realRootedPath)
+ && realFileStateValue.equals(other.realFileStateValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(realRootedPath, realFileStateValue);
+ }
+
+ @Override
+ public String toString() {
+ return realRootedPath + ", " + realFileStateValue + " (symlink ancestor)";
+ }
+ }
+
+ /** Implementation of {@link FileValue} for files that are symlinks. */
+ private static final class SymlinkFileValue extends DifferentRealPathFileValue {
+ private final PathFragment linkValue;
+
+ private SymlinkFileValue(RootedPath realRootedPath, FileStateValue realFileState,
+ PathFragment linkTarget) {
+ super(realRootedPath, realFileState);
+ this.linkValue = linkTarget;
+ }
+
+ @Override
+ public boolean isSymlink() {
+ return true;
+ }
+
+ @Override
+ public PathFragment getUnresolvedLinkTarget() {
+ return linkValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj.getClass() != SymlinkFileValue.class) {
+ return false;
+ }
+ SymlinkFileValue other = (SymlinkFileValue) obj;
+ return realRootedPath.equals(other.realRootedPath)
+ && realFileStateValue.equals(other.realFileStateValue)
+ && linkValue.equals(other.linkValue);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(realRootedPath, realFileStateValue, linkValue, Boolean.TRUE);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("symlink (real_path=%s, real_state=%s, link_value=%s)",
+ realRootedPath, realFileStateValue, linkValue);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
new file mode 100644
index 0000000..a559206
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
@@ -0,0 +1,320 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversal;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.DanglingSymlinkException;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.RecursiveFilesystemTraversalException;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+/** SkyFunction for {@link FilesetEntryValue}. */
+public final class FilesetEntryFunction implements SkyFunction {
+
+ private static final class MissingDepException extends Exception {}
+
+ private static final class FilesetEntryFunctionException extends SkyFunctionException {
+ FilesetEntryFunctionException(RecursiveFilesystemTraversalException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws FilesetEntryFunctionException {
+ FilesetTraversalParams t = (FilesetTraversalParams) key.argument();
+ Preconditions.checkState(
+ t.getNestedTraversal().isPresent() != t.getDirectTraversal().isPresent(),
+ "Exactly one of the nested and direct traversals must be specified: %s", t);
+
+ // Create the set of excluded files. Only top-level files can be excluded, i.e. ones that are
+ // directly under the root if the root is a directory.
+ Set<String> exclusions = createExclusionSet(t.getExcludedFiles());
+
+ // The map of output symlinks. Each key is the path of a output symlink that the Fileset must
+ // create, relative to the Fileset.out directory, and each value specifies extra information
+ // about the link (its target, associated metadata and again its name).
+ Map<PathFragment, FilesetOutputSymlink> outputSymlinks = new LinkedHashMap<>();
+
+ if (t.getNestedTraversal().isPresent()) {
+ // The "nested" traversal parameters are present if and only if FilesetEntry.srcdir specifies
+ // another Fileset (a "nested" one).
+ FilesetEntryValue nested = (FilesetEntryValue) env.getValue(
+ FilesetEntryValue.key(t.getNestedTraversal().get()));
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ for (FilesetOutputSymlink s : nested.getSymlinks()) {
+ maybeStoreSymlink(s, t.getDestPath(), exclusions, outputSymlinks);
+ }
+ } else {
+ // The "nested" traversal params are absent if and only if the "direct" traversal params are
+ // present, which is the case when the FilesetEntry specifies a package's BUILD file, a
+ // directory or a list of files.
+
+ // The root of the direct traversal is defined as follows.
+ //
+ // If FilesetEntry.files is specified, then a TraversalRequest is created for each entry, the
+ // root being the respective entry itself. These are all traversed for they may be
+ // directories or symlinks to directories, and we need to establish Skyframe dependencies on
+ // their contents for incremental correctness. If an entry is indeed a directory (but not when
+ // it's a symlink to one) then we have to create symlinks to each of their childen.
+ // (NB: there seems to be no good reason for this, it's just how legacy Fileset works. We may
+ // want to consider creating a symlink just for the directory and not for its child elements.)
+ //
+ // If FilesetEntry.files is not specified, then srcdir refers to either a BUILD file or a
+ // directory. For the former, the root will be the parent of the BUILD file. For the latter,
+ // the root will be srcdir itself.
+ DirectTraversal direct = t.getDirectTraversal().get();
+
+ RecursiveFilesystemTraversalValue rftv;
+ try {
+ // Traverse the filesystem to establish skyframe dependencies.
+ rftv = traverse(env, createErrorInfo(t), direct);
+ } catch (MissingDepException e) {
+ return null;
+ }
+
+ // The root can only be absent for the EMPTY rftv instance.
+ if (!rftv.getResolvedRoot().isPresent()) {
+ return FilesetEntryValue.EMPTY;
+ }
+
+ ResolvedFile resolvedRoot = rftv.getResolvedRoot().get();
+
+ // Handle dangling symlinks gracefully be returning empty results.
+ if (!resolvedRoot.type.exists()) {
+ return FilesetEntryValue.EMPTY;
+ }
+
+ // The prefix to remove is the entire path of the root. This is OK:
+ // - when the root is a file, this removes the entire path, but the traversal's destination
+ // path is actually the name of the output symlink, so this works out correctly
+ // - when the root is a directory or a symlink to one then we need to strip off the
+ // directory's path from every result (this is how the output symlinks must be created)
+ // before making them relative to the destination path
+ PathFragment prefixToRemove = direct.getRoot().getRelativePart();
+
+ Iterable<ResolvedFile> results = null;
+
+ if (direct.isRecursive()
+ || (resolvedRoot.type.isDirectory() && !resolvedRoot.type.isSymlink())) {
+ // The traversal is recursive (requested for an entire FilesetEntry.srcdir) or it was
+ // requested for a FilesetEntry.files entry which turned out to be a directory. We need to
+ // create an output symlink for every file in it and all of its subdirectories. Only
+ // exception is when the subdirectory is really a symlink to a directory -- no output
+ // shall be created for the contents of those.
+ // Now we create Dir objects to model the filesystem tree. The object employs a trick to
+ // find directory symlinks: directory symlinks have corresponding ResolvedFile entries and
+ // are added as files too, while their children, also added as files, contain the path of
+ // the parent. Finding and discarding the children is easy if we traverse the tree from
+ // root to leaf.
+ DirectoryTree root = new DirectoryTree();
+ for (ResolvedFile f : rftv.getTransitiveFiles().toCollection()) {
+ PathFragment path = f.getNameInSymlinkTree().relativeTo(prefixToRemove);
+ if (path.segmentCount() > 0) {
+ path = t.getDestPath().getRelative(path);
+ DirectoryTree dir = root;
+ for (int i = 0; i < path.segmentCount() - 1; ++i) {
+ dir = dir.addOrGetSubdir(path.getSegment(i));
+ }
+ dir.maybeAddFile(f);
+ }
+ }
+ // Here's where the magic happens. The returned iterable will yield all files in the
+ // directory that are not under symlinked directories, as well as all directory symlinks.
+ results = root.iterateFiles();
+ } else {
+ // If we're on this branch then the traversal was done for just one entry in
+ // FilesetEntry.files (which was not a directory, so it was either a file, a symlink to one
+ // or a symlink to a directory), meaning we'll have only one output symlink.
+ results = ImmutableList.of(resolvedRoot);
+ }
+
+ // Create one output symlink for each entry in the results.
+ for (ResolvedFile f : results) {
+ PathFragment targetName;
+ try {
+ targetName = f.getTargetInSymlinkTree(direct.isFollowingSymlinks());
+ } catch (DanglingSymlinkException e) {
+ throw new FilesetEntryFunctionException(e);
+ }
+
+ // Metadata field must be present. It can only be absent when stripped by tests.
+ String metadata = Integer.toHexString(f.metadata.get().hashCode());
+
+ // The linkName has to be under the traversal's root, which is also the prefix to remove.
+ PathFragment linkName = f.getNameInSymlinkTree().relativeTo(prefixToRemove);
+ maybeStoreSymlink(linkName, targetName, metadata, t.getDestPath(), exclusions,
+ outputSymlinks);
+ }
+ }
+
+ return FilesetEntryValue.of(ImmutableSet.copyOf(outputSymlinks.values()));
+ }
+
+ /** Stores an output symlink unless it's excluded or would overwrite an existing one. */
+ private static void maybeStoreSymlink(FilesetOutputSymlink nestedLink, PathFragment destPath,
+ Set<String> exclusions, Map<PathFragment, FilesetOutputSymlink> result) {
+ maybeStoreSymlink(nestedLink.name, nestedLink.target, nestedLink.metadata, destPath,
+ exclusions, result);
+ }
+
+ /** Stores an output symlink unless it's excluded or would overwrite an existing one. */
+ private static void maybeStoreSymlink(PathFragment linkName, PathFragment linkTarget,
+ String metadata, PathFragment destPath, Set<String> exclusions,
+ Map<PathFragment, FilesetOutputSymlink> result) {
+ if (!exclusions.contains(linkName.getPathString())) {
+ linkName = destPath.getRelative(linkName);
+ if (!result.containsKey(linkName)) {
+ result.put(linkName, new FilesetOutputSymlink(linkName, linkTarget, metadata));
+ }
+ }
+ }
+
+ private static Set<String> createExclusionSet(Set<String> input) {
+ return Sets.filter(input, new Predicate<String>() {
+ @Override
+ public boolean apply(String e) {
+ // Keep the top-level exclusions only. Do not look for "/" but count the path segments
+ // instead, in anticipation of future Windows support.
+ return new PathFragment(e).segmentCount() == 1;
+ }
+ });
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private RecursiveFilesystemTraversalValue traverse(Environment env, String errorInfo,
+ DirectTraversal traversal) throws MissingDepException {
+ SkyKey depKey = RecursiveFilesystemTraversalValue.key(
+ new RecursiveFilesystemTraversalValue.TraversalRequest(traversal.getRoot().asRootedPath(),
+ traversal.isGenerated(), traversal.getCrossPackageBoundary(), traversal.isPackage(),
+ errorInfo));
+ RecursiveFilesystemTraversalValue v = (RecursiveFilesystemTraversalValue) env.getValue(depKey);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return v;
+ }
+
+ private static String createErrorInfo(FilesetTraversalParams t) {
+ if (t.getDirectTraversal().isPresent()) {
+ DirectTraversal direct = t.getDirectTraversal().get();
+ return String.format("Fileset '%s' traversing %s %s", t.getOwnerLabel(),
+ direct.isPackage() ? "package" : "file (or directory)",
+ direct.getRoot().getRelativePart().getPathString());
+ } else {
+ return String.format("Fileset '%s' traversing another Fileset", t.getOwnerLabel());
+ }
+ }
+
+ /**
+ * Models a FilesetEntryFunction's portion of the symlink output tree created by a Fileset rule.
+ *
+ * <p>A Fileset rule's output is computed by zero or more {@link FilesetEntryFunction}s, resulting
+ * in one {@link FilesetEntryValue} for each. Each of those represents a portion of the grand
+ * output tree of the Fileset. These portions are later merged and written to the fileset manifest
+ * file, which is then consumed by a tool that ultimately creates the symlinks in the filesystem.
+ *
+ * <p>Because the Fileset doesn't process the lists in the FilesetEntryValues any further than
+ * merging them, they have to adhere to the conventions of the manifest file. One of these is that
+ * files are alphabetically ordered (enables the consumer of the manifest to work faster than
+ * otherwise) and another is that the contents of regular directories are listed, but contents
+ * of directory symlinks are not, only the symlinks are. (Other details of the manifest file are
+ * not relevant here.)
+ *
+ * <p>See {@link DirectoryTree#iterateFiles()} for more details.
+ */
+ private static final class DirectoryTree {
+ // Use TreeMaps for the benefit of alphabetically ordered iteration.
+ public final Map<String, ResolvedFile> files = new TreeMap<>();
+ public final Map<String, DirectoryTree> dirs = new TreeMap<>();
+
+ DirectoryTree addOrGetSubdir(String name) {
+ DirectoryTree result = dirs.get(name);
+ if (result == null) {
+ result = new DirectoryTree();
+ dirs.put(name, result);
+ }
+ return result;
+ }
+
+ void maybeAddFile(ResolvedFile r) {
+ String name = r.getNameInSymlinkTree().getBaseName();
+ if (!files.containsKey(name)) {
+ files.put(name, r);
+ }
+ }
+
+ /**
+ * Lazily yields all files in this directory and all of its subdirectories.
+ *
+ * <p>The function first yields all the files directly under this directory, in alphabetical
+ * order. Then come the contents of subdirectories, processed recursively in the same fashion
+ * as this directory, and also in alphabetical order.
+ *
+ * <p>If a directory symlink is encountered its contents are not listed, only the symlink is.
+ */
+ Iterable<ResolvedFile> iterateFiles() {
+ // 1. Filter directory symlinks. If the symlink target contains files, those were added
+ // as normal files so their parent directory (the symlink) would show up in the dirs map
+ // (as a directory) as well as in the files map (as a symlink to a directory).
+ final Set<String> fileNames = files.keySet();
+ Iterable<Map.Entry<String, DirectoryTree>> noDirSymlinkes = Iterables.filter(dirs.entrySet(),
+ new Predicate<Map.Entry<String, DirectoryTree>>() {
+ @Override
+ public boolean apply(Map.Entry<String, DirectoryTree> input) {
+ return !fileNames.contains(input.getKey());
+ }
+ });
+
+ // 2. Extract the iterables of the true subdirectories.
+ Iterable<Iterable<ResolvedFile>> subdirIters = Iterables.transform(noDirSymlinkes,
+ new Function<Map.Entry<String, DirectoryTree>, Iterable<ResolvedFile>>() {
+ @Override
+ public Iterable<ResolvedFile> apply(Entry<String, DirectoryTree> input) {
+ return input.getValue().iterateFiles();
+ }
+ });
+
+ // 3. Just concat all subdirectory iterations for one, seamless iteration.
+ Iterable<ResolvedFile> dirsIter = Iterables.concat(subdirIters);
+
+ return Iterables.concat(files.values(), dirsIter);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java
new file mode 100644
index 0000000..e7b6580
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/** Output symlinks produced by a whole FilesetEntry or by a single file in FilesetEntry.files. */
+public final class FilesetEntryValue implements SkyValue {
+ static final FilesetEntryValue EMPTY =
+ new FilesetEntryValue(ImmutableSet.<FilesetOutputSymlink>of());
+
+ private final ImmutableSet<FilesetOutputSymlink> symlinks;
+
+ private FilesetEntryValue(ImmutableSet<FilesetOutputSymlink> symlinks) {
+ this.symlinks = symlinks;
+ }
+
+ static FilesetEntryValue of(ImmutableSet<FilesetOutputSymlink> symlinks) {
+ if (symlinks.isEmpty()) {
+ return EMPTY;
+ } else {
+ return new FilesetEntryValue(symlinks);
+ }
+ }
+
+ /** Returns the list of output symlinks. */
+ public ImmutableSet<FilesetOutputSymlink> getSymlinks() {
+ return symlinks;
+ }
+
+ public static SkyKey key(FilesetTraversalParams params) {
+ return new SkyKey(SkyFunctions.FILESET_ENTRY, params);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
new file mode 100644
index 0000000..be4f4e8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
@@ -0,0 +1,398 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil;
+import com.google.devtools.build.lib.concurrent.Sharder;
+import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class to find dirty values by accessing the filesystem directly (contrast with
+ * {@link DiffAwareness}).
+ */
+class FilesystemValueChecker {
+
+ private static final int DIRTINESS_CHECK_THREADS = 50;
+ private static final Logger LOG = Logger.getLogger(FilesystemValueChecker.class.getName());
+
+ private static final Predicate<SkyKey> FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER =
+ SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.FILE_STATE,
+ SkyFunctions.DIRECTORY_LISTING_STATE));
+ private static final Predicate<SkyKey> ACTION_FILTER =
+ SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION);
+
+ private final TimestampGranularityMonitor tsgm;
+ private final Supplier<Map<SkyKey, SkyValue>> valuesSupplier;
+ private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0);
+
+ FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm) {
+ this.tsgm = tsgm;
+
+ // Construct the full map view of the entire graph at most once ("memoized"), lazily. If
+ // getDirtyFilesystemValues(Iterable<SkyKey>) is called on an empty Iterable, we avoid having
+ // to create the Map of value keys to values. This is useful in the case where the graph
+ // getValues() method could be slow.
+ this.valuesSupplier = Suppliers.memoize(new Supplier<Map<SkyKey, SkyValue>>() {
+ @Override
+ public Map<SkyKey, SkyValue> get() {
+ return evaluator.getValues();
+ }
+ });
+ }
+
+ Iterable<SkyKey> getFilesystemSkyKeys() {
+ return Iterables.filter(valuesSupplier.get().keySet(),
+ FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER);
+ }
+
+ Differencer.Diff getDirtyFilesystemSkyKeys() throws InterruptedException {
+ return getDirtyFilesystemValues(getFilesystemSkyKeys());
+ }
+
+ /**
+ * Check the given file and directory values for modifications. {@code values} is assumed to only
+ * have {@link FileValue}s and {@link DirectoryListingStateValue}s.
+ */
+ Differencer.Diff getDirtyFilesystemValues(Iterable<SkyKey> values)
+ throws InterruptedException {
+ return getDirtyValues(values, FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER,
+ new DirtyChecker() {
+ @Override
+ public DirtyResult check(SkyKey key, SkyValue oldValue, TimestampGranularityMonitor tsgm) {
+ if (key.functionName() == SkyFunctions.FILE_STATE) {
+ return checkFileStateValue((RootedPath) key.argument(), (FileStateValue) oldValue,
+ tsgm);
+ } else if (key.functionName() == SkyFunctions.DIRECTORY_LISTING_STATE) {
+ return checkDirectoryListingStateValue((RootedPath) key.argument(),
+ (DirectoryListingStateValue) oldValue);
+ } else {
+ throw new IllegalStateException("Unexpected key type " + key);
+ }
+ }
+ });
+ }
+
+ /**
+ * Return a collection of action values which have output files that are not in-sync with
+ * the on-disk file value (were modified externally).
+ */
+ public Collection<SkyKey> getDirtyActionValues(@Nullable final BatchStat batchStatter)
+ throws InterruptedException {
+ // CPU-bound (usually) stat() calls, plus a fudge factor.
+ LOG.info("Accumulating dirty actions");
+ final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4;
+ final Set<SkyKey> actionSkyKeys =
+ Sets.filter(valuesSupplier.get().keySet(), ACTION_FILTER);
+ final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards =
+ new Sharder<>(numOutputJobs, actionSkyKeys.size());
+
+ for (SkyKey key : actionSkyKeys) {
+ outputShards.add(Pair.of(key, (ActionExecutionValue) valuesSupplier.get().get(key)));
+ }
+ LOG.info("Sharded action values for batching");
+
+ ExecutorService executor = Executors.newFixedThreadPool(
+ numOutputJobs,
+ new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build());
+
+ Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
+ ThrowableRecordingRunnableWrapper wrapper =
+ new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
+
+ modifiedOutputFilesCounter.set(0);
+ for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
+ Runnable job = (batchStatter == null)
+ ? outputStatJob(dirtyKeys, shard)
+ : batchStatJob(dirtyKeys, shard, batchStatter);
+ executor.submit(wrapper.wrap(job));
+ }
+
+ boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ LOG.info("Completed output file stat checks");
+ if (interrupted) {
+ throw new InterruptedException();
+ }
+ return dirtyKeys;
+ }
+
+ private Runnable batchStatJob(final Collection<SkyKey> dirtyKeys,
+ final List<Pair<SkyKey, ActionExecutionValue>> shard,
+ final BatchStat batchStatter) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ Map<Artifact, Pair<SkyKey, ActionExecutionValue>> artifactToKeyAndValue = new HashMap<>();
+ for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
+ ActionExecutionValue actionValue = keyAndValue.getSecond();
+ if (actionValue == null) {
+ dirtyKeys.add(keyAndValue.getFirst());
+ } else {
+ for (Artifact artifact : actionValue.getAllOutputArtifactData().keySet()) {
+ artifactToKeyAndValue.put(artifact, keyAndValue);
+ }
+ }
+ }
+
+ List<Artifact> artifacts = ImmutableList.copyOf(artifactToKeyAndValue.keySet());
+ List<FileStatusWithDigest> stats;
+ try {
+ stats = batchStatter.batchStat(/*includeDigest=*/true, /*includeLinks=*/true,
+ Artifact.asPathFragments(artifacts));
+ } catch (IOException e) {
+ // Batch stat did not work. Log an exception and fall back on system calls.
+ LoggingUtil.logToRemote(Level.WARNING, "Unable to process batch stat", e);
+ outputStatJob(dirtyKeys, shard).run();
+ return;
+ } catch (InterruptedException e) {
+ // We handle interrupt in the main thread.
+ return;
+ }
+
+ Preconditions.checkState(artifacts.size() == stats.size(),
+ "artifacts.size() == %s stats.size() == %s", artifacts.size(), stats.size());
+ for (int i = 0; i < artifacts.size(); i++) {
+ Artifact artifact = artifacts.get(i);
+ FileStatusWithDigest stat = stats.get(i);
+ Pair<SkyKey, ActionExecutionValue> keyAndValue = artifactToKeyAndValue.get(artifact);
+ ActionExecutionValue actionValue = keyAndValue.getSecond();
+ SkyKey key = keyAndValue.getFirst();
+ FileValue lastKnownData = actionValue.getAllOutputArtifactData().get(artifact);
+ try {
+ FileValue newData = FileAndMetadataCache.fileValueFromArtifact(artifact, stat, tsgm);
+ if (!newData.equals(lastKnownData)) {
+ modifiedOutputFilesCounter.getAndIncrement();
+ dirtyKeys.add(key);
+ }
+ } catch (IOException e) {
+ // This is an unexpected failure getting a digest or symlink target.
+ modifiedOutputFilesCounter.getAndIncrement();
+ dirtyKeys.add(key);
+ }
+ }
+ }
+ };
+ }
+
+ private Runnable outputStatJob(final Collection<SkyKey> dirtyKeys,
+ final List<Pair<SkyKey, ActionExecutionValue>> shard) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) {
+ ActionExecutionValue value = keyAndValue.getSecond();
+ if (value == null || actionValueIsDirtyWithDirectSystemCalls(value)) {
+ dirtyKeys.add(keyAndValue.getFirst());
+ }
+ }
+ }
+ };
+ }
+
+ /**
+ * Returns number of modified output files inside of dirty actions.
+ */
+ int getNumberOfModifiedOutputFiles() {
+ return modifiedOutputFilesCounter.get();
+ }
+
+ private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue) {
+ boolean isDirty = false;
+ for (Map.Entry<Artifact, FileValue> entry :
+ actionValue.getAllOutputArtifactData().entrySet()) {
+ Artifact artifact = entry.getKey();
+ FileValue lastKnownData = entry.getValue();
+ try {
+ if (!FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm).equals(
+ lastKnownData)) {
+ modifiedOutputFilesCounter.getAndIncrement();
+ isDirty = true;
+ }
+ } catch (IOException e) {
+ // This is an unexpected failure getting a digest or symlink target.
+ modifiedOutputFilesCounter.getAndIncrement();
+ isDirty = true;
+ }
+ }
+ return isDirty;
+ }
+
+ private BatchDirtyResult getDirtyValues(Iterable<SkyKey> values,
+ Predicate<SkyKey> keyFilter,
+ final DirtyChecker checker) throws InterruptedException {
+ ExecutorService executor = Executors.newFixedThreadPool(DIRTINESS_CHECK_THREADS,
+ new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build());
+
+ final BatchDirtyResult batchResult = new BatchDirtyResult();
+ ThrowableRecordingRunnableWrapper wrapper =
+ new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues");
+ for (final SkyKey key : values) {
+ Preconditions.checkState(keyFilter.apply(key), key);
+ final SkyValue value = valuesSupplier.get().get(key);
+ executor.execute(wrapper.wrap(new Runnable() {
+ @Override
+ public void run() {
+ if (value == null) {
+ // value will be null if the value is in error or part of a cycle.
+ // TODO(bazel-team): This is overly conservative.
+ batchResult.add(key, /*newValue=*/null);
+ return;
+ }
+ DirtyResult result = checker.check(key, value, tsgm);
+ if (result.isDirty()) {
+ batchResult.add(key, result.getNewValue());
+ }
+ }
+ }));
+ }
+
+ boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ if (interrupted) {
+ throw new InterruptedException();
+ }
+ return batchResult;
+ }
+
+ private static DirtyResult checkFileStateValue(RootedPath rootedPath,
+ FileStateValue fileStateValue, TimestampGranularityMonitor tsgm) {
+ try {
+ FileStateValue newValue = FileStateValue.create(rootedPath, tsgm);
+ return newValue.equals(fileStateValue)
+ ? DirtyResult.NOT_DIRTY : DirtyResult.dirtyWithNewValue(newValue);
+ } catch (InconsistentFilesystemException | IOException e) {
+ // TODO(bazel-team): An IOException indicates a failure to get a file digest or a symlink
+ // target, not a missing file. Such a failure really shouldn't happen, so failing early
+ // may be better here.
+ return DirtyResult.DIRTY;
+ }
+ }
+
+ private static DirtyResult checkDirectoryListingStateValue(RootedPath dirRootedPath,
+ DirectoryListingStateValue directoryListingStateValue) {
+ try {
+ DirectoryListingStateValue newValue = DirectoryListingStateValue.create(dirRootedPath);
+ return newValue.equals(directoryListingStateValue)
+ ? DirtyResult.NOT_DIRTY : DirtyResult.dirtyWithNewValue(newValue);
+ } catch (IOException e) {
+ return DirtyResult.DIRTY;
+ }
+ }
+
+ /**
+ * Result of a batch call to {@link DirtyChecker#check}. Partitions the dirty values based on
+ * whether we have a new value available for them or not.
+ */
+ private static class BatchDirtyResult implements Differencer.Diff {
+
+ private final Set<SkyKey> concurrentDirtyKeysWithoutNewValues =
+ Collections.newSetFromMap(new ConcurrentHashMap<SkyKey, Boolean>());
+ private final ConcurrentHashMap<SkyKey, SkyValue> concurrentDirtyKeysWithNewValues =
+ new ConcurrentHashMap<>();
+
+ private void add(SkyKey key, @Nullable SkyValue newValue) {
+ if (newValue == null) {
+ concurrentDirtyKeysWithoutNewValues.add(key);
+ } else {
+ concurrentDirtyKeysWithNewValues.put(key, newValue);
+ }
+ }
+
+ @Override
+ public Iterable<SkyKey> changedKeysWithoutNewValues() {
+ return concurrentDirtyKeysWithoutNewValues;
+ }
+
+ @Override
+ public Map<SkyKey, ? extends SkyValue> changedKeysWithNewValues() {
+ return concurrentDirtyKeysWithNewValues;
+ }
+ }
+
+ private static class DirtyResult {
+
+ static final DirtyResult NOT_DIRTY = new DirtyResult(false, null);
+ static final DirtyResult DIRTY = new DirtyResult(true, null);
+
+ private final boolean isDirty;
+ @Nullable private final SkyValue newValue;
+
+ private DirtyResult(boolean isDirty, @Nullable SkyValue newValue) {
+ this.isDirty = isDirty;
+ this.newValue = newValue;
+ }
+
+ boolean isDirty() {
+ return isDirty;
+ }
+
+ /**
+ * If {@code isDirty()}, then either returns the new value for the value or {@code null} if
+ * the new value wasn't computed. In the case where the value is dirty and a new value is
+ * available, then the new value can be injected into the skyframe graph. Otherwise, the value
+ * should simply be invalidated.
+ */
+ @Nullable
+ SkyValue getNewValue() {
+ Preconditions.checkState(isDirty());
+ return newValue;
+ }
+
+ static DirtyResult dirtyWithNewValue(SkyValue newValue) {
+ return new DirtyResult(true, newValue);
+ }
+ }
+
+ private static interface DirtyChecker {
+ DirtyResult check(SkyKey key, SkyValue oldValue, TimestampGranularityMonitor tsgm);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java
new file mode 100644
index 0000000..5baeae8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * A descriptor for a glob request.
+ *
+ * <p>{@code subdir} must be empty or point to an existing directory.</p>
+ *
+ * <p>{@code pattern} must be valid, as indicated by {@code UnixGlob#checkPatternForError}.
+ */
+@ThreadSafe
+public final class GlobDescriptor implements Serializable {
+ final PackageIdentifier packageId;
+ final PathFragment subdir;
+ final String pattern;
+ final boolean excludeDirs;
+
+ /**
+ * Constructs a GlobDescriptor.
+ *
+ * @param packageId the name of the owner package (must be an existing package)
+ * @param subdir the subdirectory being looked at (must exist and must be a directory. It's
+ * assumed that there are no other packages between {@code packageName} and
+ * {@code subdir}.
+ * @param pattern a valid glob pattern
+ * @param excludeDirs true if directories should be excluded from results
+ */
+ GlobDescriptor(PackageIdentifier packageId, PathFragment subdir, String pattern,
+ boolean excludeDirs) {
+ this.packageId = Preconditions.checkNotNull(packageId);
+ this.subdir = Preconditions.checkNotNull(subdir);
+ this.pattern = Preconditions.checkNotNull(StringCanonicalizer.intern(pattern));
+ this.excludeDirs = excludeDirs;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("<GlobDescriptor packageName=%s subdir=%s pattern=%s excludeDirs=%s>",
+ packageId, subdir, pattern, excludeDirs);
+ }
+
+ /**
+ * Returns the package that "owns" this glob.
+ *
+ * <p>The glob evaluation code ensures that the boundaries of this package are not crossed.
+ */
+ public PackageIdentifier getPackageId() {
+ return packageId;
+ }
+
+ /**
+ * Returns the subdirectory of the package under consideration.
+ */
+ PathFragment getSubdir() {
+ return subdir;
+ }
+
+ /**
+ * Returns the glob pattern under consideration. May contain wildcards.
+ *
+ * <p>As the glob evaluator traverses deeper into the file tree, components are added at the
+ * beginning of {@code subdir} and removed from the beginning of {@code pattern}.
+ */
+ String getPattern() {
+ return pattern;
+ }
+
+ /**
+ * Returns true if directories should be excluded from results.
+ */
+ boolean excludeDirs() {
+ return excludeDirs;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(packageId, subdir, pattern, excludeDirs);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof GlobDescriptor)) {
+ return false;
+ }
+ GlobDescriptor other = (GlobDescriptor) obj;
+ return packageId.equals(other.packageId) && subdir.equals(other.subdir)
+ && pattern.equals(other.pattern) && excludeDirs == other.excludeDirs;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java
new file mode 100644
index 0000000..a1fdcb2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java
@@ -0,0 +1,251 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link SkyFunction} for {@link GlobValue}s.
+ *
+ * <p>This code drives the glob matching process.
+ */
+final class GlobFunction implements SkyFunction {
+
+ private final Cache<String, Pattern> regexPatternCache =
+ CacheBuilder.newBuilder().concurrencyLevel(4).build();
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws GlobFunctionException {
+ GlobDescriptor glob = (GlobDescriptor) skyKey.argument();
+
+ PackageLookupValue globPkgLookupValue = (PackageLookupValue)
+ env.getValue(PackageLookupValue.key(glob.getPackageId()));
+ if (globPkgLookupValue == null) {
+ return null;
+ }
+ Preconditions.checkState(globPkgLookupValue.packageExists(), "%s isn't an existing package",
+ glob.getPackageId());
+ // Note that this implies that the package's BUILD file exists which implies that the
+ // package's directory exists.
+
+ PathFragment globSubdir = glob.getSubdir();
+ if (!globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) {
+ PackageLookupValue globSubdirPkgLookupValue = (PackageLookupValue) env.getValue(
+ PackageLookupValue.key(glob.getPackageId().getPackageFragment()
+ .getRelative(globSubdir)));
+ if (globSubdirPkgLookupValue == null) {
+ return null;
+ }
+ if (globSubdirPkgLookupValue.packageExists()) {
+ // We crossed the package boundary, that is, pkg/subdir contains a BUILD file and thus
+ // defines another package, so glob expansion should not descend into that subdir.
+ return GlobValue.EMPTY;
+ }
+ }
+
+ String pattern = glob.getPattern();
+ // Split off the first path component of the pattern.
+ int slashPos = pattern.indexOf("/");
+ String patternHead;
+ String patternTail;
+ if (slashPos == -1) {
+ patternHead = pattern;
+ patternTail = null;
+ } else {
+ // Substrings will share the backing array of the original glob string. That should be fine.
+ patternHead = pattern.substring(0, slashPos);
+ patternTail = pattern.substring(slashPos + 1);
+ }
+
+ NestedSetBuilder<PathFragment> matches = NestedSetBuilder.stableOrder();
+
+ // "**" also matches an empty segment, so try the case where it is not present.
+ if ("**".equals(patternHead)) {
+ if (patternTail == null) {
+ if (!glob.excludeDirs()) {
+ matches.add(globSubdir);
+ }
+ } else {
+ SkyKey globKey = GlobValue.internalKey(
+ glob.getPackageId(), globSubdir, patternTail, glob.excludeDirs());
+ GlobValue globValue = (GlobValue) env.getValue(globKey);
+ if (globValue == null) {
+ return null;
+ }
+ matches.addTransitive(globValue.getMatches());
+ }
+ }
+
+ PathFragment dirPathFragment = glob.getPackageId().getPackageFragment().getRelative(globSubdir);
+ RootedPath dirRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(),
+ dirPathFragment);
+ if (containsGlobs(patternHead)) {
+ // Pattern contains globs, so a directory listing is required.
+ //
+ // Note that we have good reason to believe the directory exists: if this is the
+ // top-level directory of the package, the package's existence implies the directory's
+ // existence; if this is a lower-level directory in the package, then we got here from
+ // previous directory listings. Filesystem operations concurrent with build could mean the
+ // directory no longer exists, but DirectoryListingFunction handles that gracefully.
+ DirectoryListingValue listingValue = (DirectoryListingValue)
+ env.getValue(DirectoryListingValue.key(dirRootedPath));
+ if (listingValue == null) {
+ return null;
+ }
+
+ for (Dirent dirent : listingValue.getDirents()) {
+ Type direntType = dirent.getType();
+ String fileName = dirent.getName();
+
+ boolean isDirectory = (direntType == Dirent.Type.DIRECTORY);
+
+ if (!UnixGlob.matches(patternHead, fileName, regexPatternCache)) {
+ continue;
+ }
+
+ if (direntType == Dirent.Type.SYMLINK) {
+ // TODO(bazel-team): Consider extracting the symlink resolution logic.
+ // For symlinks, look up the corresponding FileValue. This ensures that if the symlink
+ // changes and "switches types" (say, from a file to a directory), this value will be
+ // invalidated.
+ RootedPath symlinkRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(),
+ dirPathFragment.getRelative(fileName));
+ FileValue symlinkFileValue = (FileValue) env.getValue(FileValue.key(symlinkRootedPath));
+ if (symlinkFileValue == null) {
+ continue;
+ }
+ if (!symlinkFileValue.isSymlink()) {
+ throw new GlobFunctionException(new InconsistentFilesystemException(
+ "readdir and stat disagree about whether " + symlinkRootedPath.asPath()
+ + " is a symlink."), Transience.TRANSIENT);
+ }
+ isDirectory = symlinkFileValue.isDirectory();
+ }
+
+ String subdirPattern = "**".equals(patternHead) ? glob.getPattern() : patternTail;
+ addFile(fileName, glob, subdirPattern, patternTail == null, isDirectory,
+ matches, env);
+ }
+ } else {
+ // Pattern does not contain globs, so a direct stat is enough.
+ String fileName = patternHead;
+ RootedPath fileRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(),
+ dirPathFragment.getRelative(fileName));
+ FileValue fileValue = (FileValue) env.getValue(FileValue.key(fileRootedPath));
+ if (fileValue == null) {
+ return null;
+ }
+ if (fileValue.exists()) {
+ addFile(fileName, glob, patternTail, patternTail == null,
+ fileValue.isDirectory(), matches, env);
+ }
+ }
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ NestedSet<PathFragment> matchesBuilt = matches.build();
+ // Use the same value to represent that we did not match anything.
+ if (matchesBuilt.isEmpty()) {
+ return GlobValue.EMPTY;
+ }
+ return new GlobValue(matchesBuilt);
+ }
+
+ /**
+ * Returns true if the given pattern contains globs.
+ */
+ private boolean containsGlobs(String pattern) {
+ return pattern.contains("*") || pattern.contains("?");
+ }
+
+ /**
+ * Includes the given file/directory in the glob.
+ *
+ * <p>{@code fileName} must exist.
+ *
+ * <p>{@code isDirectory} must be true iff the file is a directory.
+ *
+ * <p>{@code directResult} must be set if the file should be included in the result set
+ * directly rather than recursed into if it is a directory.
+ */
+ private void addFile(String fileName, GlobDescriptor glob, String subdirPattern,
+ boolean directResult, boolean isDirectory, NestedSetBuilder<PathFragment> matches,
+ Environment env) {
+ if (isDirectory && subdirPattern != null) {
+ // This is a directory, and the pattern covers files under that directory.
+ SkyKey subdirGlobKey = GlobValue.internalKey(glob.getPackageId(),
+ glob.getSubdir().getRelative(fileName), subdirPattern, glob.excludeDirs());
+ GlobValue subdirGlob = (GlobValue) env.getValue(subdirGlobKey);
+ if (subdirGlob == null) {
+ return;
+ }
+ matches.addTransitive(subdirGlob.getMatches());
+ }
+
+ if (directResult && !(isDirectory && glob.excludeDirs())) {
+ if (isDirectory) {
+ // TODO(bazel-team): Refactor. This is basically inlined code from the next recursion level.
+ // Ensure that subdirectories that contain other packages are not picked up.
+ PathFragment directory = glob.getPackageId().getPackageFragment()
+ .getRelative(glob.getSubdir()).getRelative(fileName);
+ PackageLookupValue pkgLookupValue = (PackageLookupValue)
+ env.getValue(PackageLookupValue.key(directory));
+ if (pkgLookupValue == null) {
+ return;
+ }
+ if (pkgLookupValue.packageExists()) {
+ // The file is a directory and contains another package.
+ return;
+ }
+ }
+ matches.add(glob.getSubdir().getRelative(fileName));
+ }
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link GlobFunction#compute}.
+ */
+ private static final class GlobFunctionException extends SkyFunctionException {
+ public GlobFunctionException(InconsistentFilesystemException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java
new file mode 100644
index 0000000..6de0fbd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value corresponding to a glob.
+ */
+@Immutable
+@ThreadSafe
+final class GlobValue implements SkyValue {
+
+ static final GlobValue EMPTY = new GlobValue(
+ NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER));
+
+ private final NestedSet<PathFragment> matches;
+
+ GlobValue(NestedSet<PathFragment> matches) {
+ this.matches = Preconditions.checkNotNull(matches);
+ }
+
+ /**
+ * Returns glob matches.
+ */
+ NestedSet<PathFragment> getMatches() {
+ return matches;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (!(other instanceof GlobValue)) {
+ return false;
+ }
+ // shallowEquals() may fail to detect that two equivalent (according to toString())
+ // NestedSets are equal, but will always detect when two NestedSets are different.
+ // This makes this implementation of equals() overly strict, but we only call this
+ // method when doing change pruning, which can accept false negatives.
+ return getMatches().shallowEquals(((GlobValue) other).getMatches());
+ }
+
+ @Override
+ public int hashCode() {
+ return matches.shallowHashCode();
+ }
+
+ /**
+ * Constructs a {@link SkyKey} for a glob lookup. {@code packageName} is assumed to be an
+ * existing package. Trying to glob into a non-package is undefined behavior.
+ *
+ * @throws InvalidGlobPatternException if the pattern is not valid.
+ */
+ @ThreadSafe
+ static SkyKey key(PackageIdentifier packageId, String pattern, boolean excludeDirs)
+ throws InvalidGlobPatternException {
+ if (pattern.indexOf('?') != -1) {
+ throw new InvalidGlobPatternException(pattern, "wildcard ? forbidden");
+ }
+
+ String error = UnixGlob.checkPatternForError(pattern);
+ if (error != null) {
+ throw new InvalidGlobPatternException(pattern, error);
+ }
+
+ return internalKey(packageId, PathFragment.EMPTY_FRAGMENT, pattern, excludeDirs);
+ }
+
+ /**
+ * Constructs a {@link SkyKey} for a glob lookup.
+ *
+ * <p>Do not use outside {@code GlobFunction}.
+ */
+ @ThreadSafe
+ static SkyKey internalKey(PackageIdentifier packageId, PathFragment subdir, String pattern,
+ boolean excludeDirs) {
+ return new SkyKey(SkyFunctions.GLOB,
+ new GlobDescriptor(packageId, subdir, pattern, excludeDirs));
+ }
+
+ /**
+ * Constructs a {@link SkyKey} for a glob lookup.
+ *
+ * <p>Do not use outside {@code GlobFunction}.
+ */
+ @ThreadSafe
+ static SkyKey internalKey(GlobDescriptor glob, String subdirName) {
+ return internalKey(glob.packageId, glob.subdir.getRelative(subdirName),
+ glob.pattern, glob.excludeDirs);
+ }
+
+ /**
+ * An exception that indicates that a glob pattern is syntactically invalid.
+ */
+ @ThreadSafe
+ static final class InvalidGlobPatternException extends Exception {
+ private final String pattern;
+
+ InvalidGlobPatternException(String pattern, String error) {
+ super(error);
+ this.pattern = pattern;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("invalid glob pattern '%s': %s", pattern, getMessage());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java b/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java
new file mode 100644
index 0000000..9d6f550
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thrown on {@link DiffAwareness#getDiff} to indicate that the given {@link DiffAwareness.View}s
+ * are incompatible with the {@link DiffAwareness} instance.
+ */
+public class IncompatibleViewException extends Exception {
+ public IncompatibleViewException(String msg) {
+ super(Preconditions.checkNotNull(msg));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java b/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java
new file mode 100644
index 0000000..26cb02f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+/**
+ * Used to indicate a filesystem inconsistency, e.g. file 'a/b' exists but directory 'a' doesn't
+ * exist. This generally means the result of the build is undefined but we shouldn't crash hard.
+ */
+public class InconsistentFilesystemException extends Exception {
+ public InconsistentFilesystemException(String inconsistencyMessage) {
+ super("Inconsistent filesystem operations. " + inconsistencyMessage + " The results of the "
+ + "build are not guaranteed to be correct. You should probably run 'blaze clean' and "
+ + "investigate the filesystem inconsistency (likely due to filesytem updates concurrent "
+ + "with the build)");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java
new file mode 100644
index 0000000..861f89ac
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java
@@ -0,0 +1,329 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashBiMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.nio.file.ClosedWatchServiceException;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardWatchEventKinds;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchEvent.Kind;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * File system watcher for local filesystems. It's able to provide a list of changed
+ * files between two consecutive calls. Uses the standard Java WatchService, which uses
+ * 'inotify' on Linux.
+ */
+public class LocalDiffAwareness implements DiffAwareness {
+
+ /** Factory for creating {@link LocalDiffAwareness} instances. */
+ public static class Factory implements DiffAwareness.Factory {
+ @Override
+ public DiffAwareness maybeCreate(com.google.devtools.build.lib.vfs.Path pathEntry) {
+ com.google.devtools.build.lib.vfs.Path resolvedPathEntry;
+ try {
+ resolvedPathEntry = pathEntry.resolveSymbolicLinks();
+ } catch (IOException e) {
+ return null;
+ }
+ PathFragment resolvedPathEntryFragment = resolvedPathEntry.asFragment();
+ // There's no good way to automatically detect network file systems. We rely on a blacklist
+ // for now (and maybe add a command-line option in the future?).
+ for (String prefix : Constants.WATCHFS_BLACKLIST) {
+ if (resolvedPathEntryFragment.startsWith(new PathFragment(prefix))) {
+ return null;
+ }
+ }
+
+ WatchService watchService;
+ try {
+ watchService = FileSystems.getDefault().newWatchService();
+ } catch (IOException e) {
+ return null;
+ }
+ return new LocalDiffAwareness(resolvedPathEntryFragment.toString(),
+ watchService);
+ }
+ }
+
+ private int numGetCurrentViewCalls = 0;
+
+ /**
+ * Bijection from WatchKey to the (absolute) Path being watched. WatchKeys don't have this
+ * functionality built-in so we do it ourselves.
+ */
+ private final HashBiMap<WatchKey, Path> watchKeyToDirBiMap = HashBiMap.create();
+
+ /** Root directory to watch. This is an absolute path. */
+ private final Path watchRootPath;
+
+ /** Every directory is registered under this watch service. */
+ private WatchService watchService;
+
+ private LocalDiffAwareness(String watchRoot, WatchService watchService) {
+ this.watchRootPath = FileSystems.getDefault().getPath(watchRoot);
+ this.watchService = watchService;
+ }
+
+ /**
+ * The WatchService is inherently sequential and side-effectful, so we enforce this by only
+ * supporting {@link #getDiff} calls that happen to be sequential.
+ */
+ private static class SequentialView implements DiffAwareness.View {
+ private final LocalDiffAwareness owner;
+ private final int position;
+ private final Set<Path> modifiedAbsolutePaths;
+
+ public SequentialView(LocalDiffAwareness owner, int position, Set<Path> modifiedAbsolutePaths) {
+ this.owner = owner;
+ this.position = position;
+ this.modifiedAbsolutePaths = modifiedAbsolutePaths;
+ }
+
+ public static boolean areInSequence(SequentialView oldView, SequentialView newView) {
+ return oldView.owner == newView.owner && (oldView.position + 1) == newView.position;
+ }
+ }
+
+ @Override
+ public SequentialView getCurrentView() throws BrokenDiffAwarenessException {
+ Set<Path> modifiedAbsolutePaths;
+ if (numGetCurrentViewCalls++ == 0) {
+ try {
+ registerSubDirectoriesAndReturnContents(watchRootPath);
+ } catch (IOException e) {
+ close();
+ throw new BrokenDiffAwarenessException(
+ "Error encountered with local file system watcher " + e);
+ }
+ modifiedAbsolutePaths = ImmutableSet.of();
+ } else {
+ try {
+ modifiedAbsolutePaths = collectChanges();
+ } catch (BrokenDiffAwarenessException e) {
+ close();
+ throw e;
+ } catch (IOException e) {
+ close();
+ throw new BrokenDiffAwarenessException(
+ "Error encountered with local file system watcher " + e);
+ } catch (ClosedWatchServiceException e) {
+ throw new BrokenDiffAwarenessException(
+ "Internal error with the local file system watcher " + e);
+ }
+ }
+ return new SequentialView(this, numGetCurrentViewCalls, modifiedAbsolutePaths);
+ }
+
+ @Override
+ public ModifiedFileSet getDiff(View oldView, View newView)
+ throws IncompatibleViewException, BrokenDiffAwarenessException {
+ SequentialView oldSequentialView;
+ SequentialView newSequentialView;
+ try {
+ oldSequentialView = (SequentialView) oldView;
+ newSequentialView = (SequentialView) newView;
+ } catch (ClassCastException e) {
+ throw new IncompatibleViewException("Given views are not from LocalDiffAwareness");
+ }
+ if (!SequentialView.areInSequence(oldSequentialView, newSequentialView)) {
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+ return ModifiedFileSet.builder()
+ .modifyAll(Iterables.transform(newSequentialView.modifiedAbsolutePaths,
+ nioAbsolutePathToPathFragment))
+ .build();
+ }
+
+ @Override
+ public void close() {
+ try {
+ watchService.close();
+ } catch (IOException ignored) {
+ // Nothing we can do here.
+ }
+ }
+
+ /** Converts java.nio.file.Path objects to vfs.PathFragment. */
+ private final Function<Path, PathFragment> nioAbsolutePathToPathFragment =
+ new Function<Path, PathFragment>() {
+ @Override
+ public PathFragment apply(Path input) {
+ Preconditions.checkArgument(input.startsWith(watchRootPath), "%s %s", input,
+ watchRootPath);
+ return new PathFragment(watchRootPath.relativize(input).toString());
+ }
+ };
+
+ /** Returns the changed files caught by the watch service. */
+ private Set<Path> collectChanges() throws BrokenDiffAwarenessException, IOException {
+ Set<Path> createdFilesAndDirectories = new HashSet<Path>();
+ Set<Path> deletedOrModifiedFilesAndDirectories = new HashSet<Path>();
+ Set<Path> deletedTrackedDirectories = new HashSet<Path>();
+
+ WatchKey watchKey;
+ while ((watchKey = watchService.poll()) != null) {
+ Path dir = watchKeyToDirBiMap.get(watchKey);
+ Preconditions.checkArgument(dir != null);
+
+ // We replay all the events for this watched directory in chronological order and
+ // construct the diff of this directory since the last #collectChanges call.
+ for (WatchEvent<?> event : watchKey.pollEvents()) {
+ Kind<?> kind = event.kind();
+ if (kind == StandardWatchEventKinds.OVERFLOW) {
+ // TODO(bazel-team): find out when an overflow might happen, and maybe handle it more
+ // gently.
+ throw new BrokenDiffAwarenessException("Overflow when watching local filesystem for "
+ + "changes");
+ }
+ if (event.context() == null) {
+ // The WatchService documentation mentions that WatchEvent#context may return null, but
+ // doesn't explain how/why it would do so. Looking at the implementation, it only
+ // happens on an overflow event. But we make no assumptions about that implementation
+ // detail here.
+ throw new BrokenDiffAwarenessException("Insufficient information from local file system "
+ + "watcher");
+ }
+ // For the events we've registered, the context given is a relative path.
+ Path relativePath = (Path) event.context();
+ Path path = dir.resolve(relativePath);
+ Preconditions.checkState(path.isAbsolute(), path);
+ if (kind == StandardWatchEventKinds.ENTRY_CREATE) {
+ createdFilesAndDirectories.add(path);
+ deletedOrModifiedFilesAndDirectories.remove(path);
+ } else if (kind == StandardWatchEventKinds.ENTRY_DELETE) {
+ createdFilesAndDirectories.remove(path);
+ deletedOrModifiedFilesAndDirectories.add(path);
+ if (watchKeyToDirBiMap.containsValue(path)) {
+ // If the deleted directory has children, then there will also be events for the
+ // WatchKey of the directory itself. WatchService#poll doesn't specify the order in
+ // which WatchKeys are returned, so the key for the directory itself may be processed
+ // *after* the current key (the parent of the deleted directory), and so we don't want
+ // to remove the deleted directory from our bimap just yet.
+ //
+ // For example, suppose we have the file '/root/a/foo.txt' and are watching the
+ // directories '/root' and '/root/a'. If the directory '/root/a' gets deleted then the
+ // following is a valid sequence of events by key.
+ //
+ // WatchKey '/root/'
+ // WatchEvent EVENT_MODIFY 'a'
+ // WatchEvent EVENT_DELETE 'a'
+ // WatchKey '/root/a'
+ // WatchEvent EVENT_DELETE 'foo.txt'
+ deletedTrackedDirectories.add(path);
+ }
+ } else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) {
+ // If a file was created and then modified, then the net diff is that it was
+ // created.
+ if (!createdFilesAndDirectories.contains(path)) {
+ deletedOrModifiedFilesAndDirectories.add(path);
+ }
+ }
+ }
+
+ if (!watchKey.reset()) {
+ // Watcher got deleted, directory no longer valid.
+ watchKeyToDirBiMap.remove(watchKey);
+ }
+ }
+
+ for (Path path : deletedTrackedDirectories) {
+ WatchKey staleKey = watchKeyToDirBiMap.inverse().get(path);
+ watchKeyToDirBiMap.remove(staleKey);
+ }
+ if (watchKeyToDirBiMap.isEmpty()) {
+ // No more directories to watch, something happened the root directory being watched.
+ throw new IOException("Root directory " + watchRootPath + " became inaccessible.");
+ }
+
+ Set<Path> changedPaths = new HashSet<Path>();
+ for (Path path : createdFilesAndDirectories) {
+ if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) {
+ // This is a new directory, so changes to it since its creation have not been watched.
+ // We manually traverse the directory tree to register all the new subdirectories and find
+ // all the new subdirectories and files.
+ changedPaths.addAll(registerSubDirectoriesAndReturnContents(path));
+ } else {
+ changedPaths.add(path);
+ }
+ }
+ changedPaths.addAll(deletedOrModifiedFilesAndDirectories);
+ return changedPaths;
+ }
+
+ /**
+ * Traverses directory tree to register subdirectories. Returns all paths traversed (as absolute
+ * paths).
+ */
+ private Set<Path> registerSubDirectoriesAndReturnContents(Path rootDir) throws IOException {
+ Set<Path> visitedAbsolutePaths = new HashSet<Path>();
+ // Note that this does not follow symlinks.
+ Files.walkFileTree(rootDir, new WatcherFileVisitor(visitedAbsolutePaths));
+ return visitedAbsolutePaths;
+ }
+
+ /** File visitor used by Files.walkFileTree() upon traversing subdirectories. */
+ private class WatcherFileVisitor extends SimpleFileVisitor<Path> {
+
+ private final Set<Path> visitedAbsolutePaths;
+
+ private WatcherFileVisitor(Set<Path> visitedPaths) {
+ this.visitedAbsolutePaths = visitedPaths;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
+ Preconditions.checkState(path.isAbsolute(), path);
+ visitedAbsolutePaths.add(path);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
+ throws IOException {
+ // It's important that we register the directory before we visit its children. This way we
+ // are guaranteed to see new files/directories either on this #getDiff or the next one.
+ // Otherwise, e.g., an intra-build creation of a child directory will be forever missed if it
+ // happens before the directory is listed as part of the visitation.
+ WatchKey key = path.register(watchService,
+ StandardWatchEventKinds.ENTRY_CREATE,
+ StandardWatchEventKinds.ENTRY_MODIFY,
+ StandardWatchEventKinds.ENTRY_DELETE);
+ Preconditions.checkState(path.isAbsolute(), path);
+ visitedAbsolutePaths.add(path);
+ watchKeyToDirBiMap.put(key, path);
+ return FileVisitResult.CONTINUE;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java b/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java
new file mode 100644
index 0000000..86de11d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Supplier;
+
+/**
+ * Supplier whose value can be changed by clients who have a reference to it as a MutableSupplier.
+ * Unlike an {@code AtomicReference}, clients who are passed a MutableSupplier as a Supplier cannot
+ * change its value without a reckless cast.
+ */
+public class MutableSupplier<T> implements Supplier<T> {
+ private T val;
+
+ @Override
+ public T get() {
+ return val;
+ }
+
+ /**
+ * Sets the value of the object supplied. Do not cast a Supplier to a MutableSupplier in order to
+ * call this method!
+ */
+ public void set(T newVal) {
+ val = newVal;
+ }
+
+ @SuppressWarnings("deprecation") // MoreObjects.toStringHelper() is not in Guava
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(getClass())
+ .add("val", val).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
new file mode 100644
index 0000000..2404b99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
@@ -0,0 +1,809 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.packages.InvalidPackageNameException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.Globber;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.PackageLoadedEvent;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException;
+import com.google.devtools.build.lib.skyframe.GlobValue.InvalidGlobPatternException;
+import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction.SkylarkImportFailedException;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.syntax.Statement;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.JavaClock;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException3;
+import com.google.devtools.build.skyframe.ValueOrException4;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * A SkyFunction for {@link PackageValue}s.
+ */
+public class PackageFunction implements SkyFunction {
+
+ private final EventHandler reporter;
+ private final PackageFactory packageFactory;
+ private final CachingPackageLocator packageLocator;
+ private final ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache;
+ private final AtomicBoolean showLoadingProgress;
+ private final AtomicReference<EventBus> eventBus;
+ private final AtomicInteger numPackagesLoaded;
+ private final Profiler profiler = Profiler.instance();
+
+ private static final PathFragment PRELUDE_FILE_FRAGMENT =
+ new PathFragment(Constants.PRELUDE_FILE_DEPOT_RELATIVE_PATH);
+
+ static final String DEFAULTS_PACKAGE_NAME = "tools/defaults";
+ public static final String EXTERNAL_PACKAGE_NAME = "external";
+
+ static {
+ Preconditions.checkArgument(!PRELUDE_FILE_FRAGMENT.isAbsolute());
+ }
+
+ public PackageFunction(Reporter reporter, PackageFactory packageFactory,
+ CachingPackageLocator pkgLocator, AtomicBoolean showLoadingProgress,
+ ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache,
+ AtomicReference<EventBus> eventBus, AtomicInteger numPackagesLoaded) {
+ this.reporter = reporter;
+
+ this.packageFactory = packageFactory;
+ this.packageLocator = pkgLocator;
+ this.showLoadingProgress = showLoadingProgress;
+ this.packageFunctionCache = packageFunctionCache;
+ this.eventBus = eventBus;
+ this.numPackagesLoaded = numPackagesLoaded;
+ }
+
+ private static void maybeThrowFilesystemInconsistency(String packageName,
+ Exception skyframeException, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ if (!packageWasInError) {
+ throw new InternalInconsistentFilesystemException(packageName, "Encountered error '"
+ + skyframeException.getMessage() + "' but didn't encounter it when doing the same thing "
+ + "earlier in the build");
+ }
+ }
+
+ /**
+ * Marks the given dependencies, and returns those already present. Ignores any exception
+ * thrown while building the dependency, except for filesystem inconsistencies.
+ *
+ * <p>We need to mark dependencies implicitly used by the legacy package loading code, but we
+ * don't care about any skyframe errors since the package knows whether it's in error or not.
+ */
+ private static Pair<? extends Map<PathFragment, PackageLookupValue>, Boolean>
+ getPackageLookupDepsAndPropagateInconsistentFilesystemExceptions(String packageName,
+ Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ Preconditions.checkState(
+ Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE_LOOKUP)), depKeys);
+ boolean packageShouldBeInError = packageWasInError;
+ ImmutableMap.Builder<PathFragment, PackageLookupValue> builder = ImmutableMap.builder();
+ for (Map.Entry<SkyKey, ValueOrException3<BuildFileNotFoundException,
+ InconsistentFilesystemException, FileSymlinkCycleException>> entry :
+ env.getValuesOrThrow(depKeys, BuildFileNotFoundException.class,
+ InconsistentFilesystemException.class,
+ FileSymlinkCycleException.class).entrySet()) {
+ PathFragment pkgName = ((PackageIdentifier) entry.getKey().argument()).getPackageFragment();
+ try {
+ PackageLookupValue value = (PackageLookupValue) entry.getValue().get();
+ if (value != null) {
+ builder.put(pkgName, value);
+ }
+ } catch (BuildFileNotFoundException e) {
+ maybeThrowFilesystemInconsistency(packageName, e, packageWasInError);
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ } catch (FileSymlinkCycleException e) {
+ // Legacy doesn't detect symlink cycles.
+ packageShouldBeInError = true;
+ }
+ }
+ return Pair.of(builder.build(), packageShouldBeInError);
+ }
+
+ private static boolean markFileDepsAndPropagateInconsistentFilesystemExceptions(
+ String packageName, Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ Preconditions.checkState(
+ Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.FILE)), depKeys);
+ boolean packageShouldBeInError = packageWasInError;
+ for (Map.Entry<SkyKey, ValueOrException3<IOException, FileSymlinkCycleException,
+ InconsistentFilesystemException>> entry : env.getValuesOrThrow(depKeys, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class).entrySet()) {
+ try {
+ entry.getValue().get();
+ } catch (IOException e) {
+ maybeThrowFilesystemInconsistency(packageName, e, packageWasInError);
+ } catch (FileSymlinkCycleException e) {
+ // Legacy doesn't detect symlink cycles.
+ packageShouldBeInError = true;
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ }
+ }
+ return packageShouldBeInError;
+ }
+
+ private static boolean markGlobDepsAndPropagateInconsistentFilesystemExceptions(
+ String packageName, Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError)
+ throws InternalInconsistentFilesystemException {
+ Preconditions.checkState(
+ Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.GLOB)), depKeys);
+ boolean packageShouldBeInError = packageWasInError;
+ for (Map.Entry<SkyKey, ValueOrException4<IOException, BuildFileNotFoundException,
+ FileSymlinkCycleException, InconsistentFilesystemException>> entry :
+ env.getValuesOrThrow(depKeys, IOException.class, BuildFileNotFoundException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class).entrySet()) {
+ try {
+ entry.getValue().get();
+ } catch (IOException | BuildFileNotFoundException e) {
+ maybeThrowFilesystemInconsistency(packageName, e, packageWasInError);
+ } catch (FileSymlinkCycleException e) {
+ // Legacy doesn't detect symlink cycles.
+ packageShouldBeInError = true;
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ }
+ }
+ return packageShouldBeInError;
+ }
+
+ /**
+ * Marks dependencies implicitly used by legacy package loading code, after the fact. Note that
+ * the given package might already be in error.
+ *
+ * <p>Any skyframe exceptions encountered here are ignored, as similar errors should have
+ * already been encountered by legacy package loading (if not, then the filesystem is
+ * inconsistent).
+ */
+ private static boolean markDependenciesAndPropagateInconsistentFilesystemExceptions(
+ Package pkg, Environment env, Collection<Pair<String, Boolean>> globPatterns,
+ Map<Label, Path> subincludes) throws InternalInconsistentFilesystemException {
+ boolean packageShouldBeInError = pkg.containsErrors();
+
+ // TODO(bazel-team): This means that many packages will have to be preprocessed twice. Ouch!
+ // We need a better continuation mechanism to avoid repeating work. [skyframe-loading]
+
+ // TODO(bazel-team): It would be preferable to perform I/O from the package preprocessor via
+ // Skyframe rather than add (potentially incomplete) dependencies after the fact.
+ // [skyframe-loading]
+
+ Set<SkyKey> subincludePackageLookupDepKeys = Sets.newHashSet();
+ for (Label label : pkg.getSubincludeLabels()) {
+ // Declare a dependency on the package lookup for the package giving access to the label.
+ subincludePackageLookupDepKeys.add(PackageLookupValue.key(label.getPackageFragment()));
+ }
+ Pair<? extends Map<PathFragment, PackageLookupValue>, Boolean> subincludePackageLookupResult =
+ getPackageLookupDepsAndPropagateInconsistentFilesystemExceptions(pkg.getName(),
+ subincludePackageLookupDepKeys, env, pkg.containsErrors());
+ Map<PathFragment, PackageLookupValue> subincludePackageLookupDeps =
+ subincludePackageLookupResult.getFirst();
+ packageShouldBeInError = subincludePackageLookupResult.getSecond();
+ List<SkyKey> subincludeFileDepKeys = Lists.newArrayList();
+ for (Entry<Label, Path> subincludeEntry : subincludes.entrySet()) {
+ // Ideally, we would have a direct dependency on the target with the given label, but then
+ // subincluding a file from the same package will cause a dependency cycle, since targets
+ // depend on their containing packages.
+ Label label = subincludeEntry.getKey();
+ PackageLookupValue subincludePackageLookupValue =
+ subincludePackageLookupDeps.get(label.getPackageFragment());
+ if (subincludePackageLookupValue != null) {
+ // Declare a dependency on the actual file that was subincluded.
+ Path subincludeFilePath = subincludeEntry.getValue();
+ if (subincludeFilePath != null) {
+ if (!subincludePackageLookupValue.packageExists()) {
+ // Legacy blaze puts a non-null path when only when the package does indeed exist.
+ throw new InternalInconsistentFilesystemException(pkg.getName(), String.format(
+ "Unexpected package in %s. Was it modified during the build?", subincludeFilePath));
+ }
+ // Sanity check for consistency of Skyframe and legacy blaze.
+ Path subincludeFilePathSkyframe =
+ subincludePackageLookupValue.getRoot().getRelative(label.toPathFragment());
+ if (!subincludeFilePathSkyframe.equals(subincludeFilePath)) {
+ throw new InternalInconsistentFilesystemException(pkg.getName(), String.format(
+ "Inconsistent package location for %s: '%s' vs '%s'. "
+ + "Was the source tree modified during the build?",
+ label.getPackageFragment(), subincludeFilePathSkyframe, subincludeFilePath));
+ }
+ // The actual file may be under a different package root than the package being
+ // constructed.
+ SkyKey subincludeSkyKey =
+ FileValue.key(RootedPath.toRootedPath(subincludePackageLookupValue.getRoot(),
+ subincludeFilePath));
+ subincludeFileDepKeys.add(subincludeSkyKey);
+ }
+ }
+ }
+ packageShouldBeInError = markFileDepsAndPropagateInconsistentFilesystemExceptions(
+ pkg.getName(), subincludeFileDepKeys, env, pkg.containsErrors());
+ // Another concern is a subpackage cutting off the subinclude label, but this is already
+ // handled by the legacy package loading code which calls into our SkyframePackageLocator.
+
+ // TODO(bazel-team): In the long term, we want to actually resolve the glob patterns within
+ // Skyframe. For now, just logging the glob requests provides correct incrementality and
+ // adequate performance.
+ PackageIdentifier packageId = pkg.getPackageIdentifier();
+ List<SkyKey> globDepKeys = Lists.newArrayList();
+ for (Pair<String, Boolean> globPattern : globPatterns) {
+ String pattern = globPattern.getFirst();
+ boolean excludeDirs = globPattern.getSecond();
+ SkyKey globSkyKey;
+ try {
+ globSkyKey = GlobValue.key(packageId, pattern, excludeDirs);
+ } catch (InvalidGlobPatternException e) {
+ // Globs that make it to pkg.getGlobPatterns() should already be filtered for errors.
+ throw new IllegalStateException(e);
+ }
+ globDepKeys.add(globSkyKey);
+ }
+ packageShouldBeInError = markGlobDepsAndPropagateInconsistentFilesystemExceptions(
+ pkg.getName(), globDepKeys, env, pkg.containsErrors());
+ return packageShouldBeInError;
+ }
+
+ /**
+ * Adds a dependency on the WORKSPACE file, representing it as a special type of package.
+ * @throws PackageFunctionException if there is an error computing the workspace file or adding
+ * its rules to the //external package.
+ */
+ private SkyValue getExternalPackage(Environment env, Path packageLookupPath)
+ throws PackageFunctionException {
+ RootedPath workspacePath = RootedPath.toRootedPath(
+ packageLookupPath, new PathFragment("WORKSPACE"));
+ SkyKey workspaceKey = WorkspaceFileValue.key(workspacePath);
+ WorkspaceFileValue workspace = null;
+ try {
+ workspace = (WorkspaceFileValue) env.getValueOrThrow(workspaceKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class,
+ EvalException.class);
+ } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException
+ | EvalException e) {
+ throw new PackageFunctionException(new BadWorkspaceFileException(e.getMessage()),
+ Transience.PERSISTENT);
+ }
+ if (workspace == null) {
+ return null;
+ }
+
+ Package pkg = workspace.getPackage();
+ Event.replayEventsOn(env.getListener(), pkg.getEvents());
+ if (pkg.containsErrors()) {
+ throw new PackageFunctionException(new BuildFileContainsErrorsException("external",
+ "Package 'external' contains errors"),
+ pkg.containsTemporaryErrors() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+
+ return new PackageValue(pkg);
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws PackageFunctionException,
+ InterruptedException {
+ PackageIdentifier packageId = (PackageIdentifier) key.argument();
+ PathFragment packageNameFragment = packageId.getPackageFragment();
+ String packageName = packageNameFragment.getPathString();
+
+ SkyKey packageLookupKey = PackageLookupValue.key(packageId);
+ PackageLookupValue packageLookupValue;
+ try {
+ packageLookupValue = (PackageLookupValue)
+ env.getValueOrThrow(packageLookupKey, BuildFileNotFoundException.class,
+ InconsistentFilesystemException.class);
+ } catch (BuildFileNotFoundException e) {
+ throw new PackageFunctionException(e, Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ // This error is not transient from the perspective of the PackageFunction.
+ throw new PackageFunctionException(
+ new InternalInconsistentFilesystemException(packageName, e), Transience.PERSISTENT);
+ }
+ if (packageLookupValue == null) {
+ return null;
+ }
+
+ if (!packageLookupValue.packageExists()) {
+ switch (packageLookupValue.getErrorReason()) {
+ case NO_BUILD_FILE:
+ case DELETED_PACKAGE:
+ case NO_EXTERNAL_PACKAGE:
+ throw new PackageFunctionException(new BuildFileNotFoundException(packageName,
+ packageLookupValue.getErrorMsg()), Transience.PERSISTENT);
+ case INVALID_PACKAGE_NAME:
+ throw new PackageFunctionException(new InvalidPackageNameException(packageName,
+ packageLookupValue.getErrorMsg()), Transience.PERSISTENT);
+ default:
+ // We should never get here.
+ Preconditions.checkState(false);
+ }
+ }
+
+ if (packageName.equals(EXTERNAL_PACKAGE_NAME)) {
+ return getExternalPackage(env, packageLookupValue.getRoot());
+ }
+
+ RootedPath buildFileRootedPath = RootedPath.toRootedPath(packageLookupValue.getRoot(),
+ packageNameFragment.getChild("BUILD"));
+ FileValue buildFileValue;
+ try {
+ buildFileValue = (FileValue) env.getValueOrThrow(FileValue.key(buildFileRootedPath),
+ IOException.class, FileSymlinkCycleException.class,
+ InconsistentFilesystemException.class);
+ } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException e) {
+ throw new IllegalStateException("Package lookup succeeded but encountered error when "
+ + "getting FileValue for BUILD file directly.", e);
+ }
+ if (buildFileValue == null) {
+ return null;
+ }
+ Preconditions.checkState(buildFileValue.exists(),
+ "Package lookup succeeded but BUILD file doesn't exist");
+ Path buildFilePath = buildFileRootedPath.asPath();
+
+ String replacementContents = null;
+ if (packageName.equals(DEFAULTS_PACKAGE_NAME)) {
+ replacementContents = PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.get(env);
+ if (replacementContents == null) {
+ return null;
+ }
+ }
+
+ RuleVisibility defaultVisibility = PrecomputedValue.DEFAULT_VISIBILITY.get(env);
+ if (defaultVisibility == null) {
+ return null;
+ }
+
+ ASTFileLookupValue astLookupValue = null;
+ SkyKey astLookupKey = null;
+ try {
+ astLookupKey = ASTFileLookupValue.key(PRELUDE_FILE_FRAGMENT);
+ } catch (ASTLookupInputException e) {
+ // There's a static check ensuring that PRELUDE_FILE_FRAGMENT is relative.
+ throw new IllegalStateException(e);
+ }
+ try {
+ astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey,
+ ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class);
+ } catch (ErrorReadingSkylarkExtensionException | InconsistentFilesystemException e) {
+ throw new PackageFunctionException(new BadPreludeFileException(packageName, e.getMessage()),
+ Transience.PERSISTENT);
+ }
+ if (astLookupValue == null) {
+ return null;
+ }
+ List<Statement> preludeStatements = astLookupValue == ASTFileLookupValue.NO_FILE
+ ? ImmutableList.<Statement>of() : astLookupValue.getAST().getStatements();
+
+ // Load the BUILD file AST and handle Skylark dependencies. This way BUILD files are
+ // only loaded twice if there are unavailable Skylark or package dependencies or an
+ // IOException occurs. Note that the BUILD files are still parsed two times.
+ ParserInputSource inputSource;
+ try {
+ if (showLoadingProgress.get() && !packageFunctionCache.containsKey(packageId)) {
+ // TODO(bazel-team): don't duplicate the loading message if there are unavailable
+ // Skylark dependencies.
+ reporter.handle(Event.progress("Loading package: " + packageName));
+ }
+ inputSource = ParserInputSource.create(buildFilePath);
+ } catch (IOException e) {
+ env.getListener().handle(Event.error(Location.fromFile(buildFilePath), e.getMessage()));
+ // Note that we did this work, so we should conservatively report this error as transient.
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(
+ packageName, e.getMessage()), Transience.TRANSIENT);
+ }
+ SkylarkImportResult importResult = fetchImportsFromBuildFile(
+ buildFilePath, packageId.getRepository(), preludeStatements, inputSource, packageName, env);
+ if (importResult == null) {
+ return null;
+ }
+
+ Package.LegacyBuilder legacyPkgBuilder = loadPackage(inputSource, replacementContents,
+ packageId, buildFilePath, defaultVisibility, preludeStatements, importResult);
+ legacyPkgBuilder.buildPartial();
+ try {
+ handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions(
+ packageLookupValue.getRoot(), packageId, legacyPkgBuilder, env);
+ } catch (InternalInconsistentFilesystemException e) {
+ packageFunctionCache.remove(packageId);
+ throw new PackageFunctionException(e,
+ e.isTransient() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+ if (env.valuesMissing()) {
+ // The package we just loaded will be in the {@code packageFunctionCache} next when this
+ // SkyFunction is called again.
+ return null;
+ }
+ Collection<Pair<String, Boolean>> globPatterns = legacyPkgBuilder.getGlobPatterns();
+ Map<Label, Path> subincludes = legacyPkgBuilder.getSubincludes();
+ Package pkg = legacyPkgBuilder.finishBuild();
+ Event.replayEventsOn(env.getListener(), pkg.getEvents());
+ boolean packageShouldBeConsideredInError = pkg.containsErrors();
+ try {
+ packageShouldBeConsideredInError =
+ markDependenciesAndPropagateInconsistentFilesystemExceptions(pkg, env,
+ globPatterns, subincludes);
+ } catch (InternalInconsistentFilesystemException e) {
+ packageFunctionCache.remove(packageId);
+ throw new PackageFunctionException(e,
+ e.isTransient() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+ // We know this SkyFunction will not be called again, so we can remove the cache entry.
+ packageFunctionCache.remove(packageId);
+
+ if (packageShouldBeConsideredInError) {
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(pkg,
+ "Package '" + packageName + "' contains errors"),
+ pkg.containsTemporaryErrors() ? Transience.TRANSIENT : Transience.PERSISTENT);
+ }
+ return new PackageValue(pkg);
+ }
+
+ private SkylarkImportResult fetchImportsFromBuildFile(Path buildFilePath, RepositoryName repo,
+ List<Statement> preludeStatements, ParserInputSource inputSource,
+ String packageName, Environment env) throws PackageFunctionException {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ BuildFileAST buildFileAST = BuildFileAST.parseBuildFile(
+ inputSource, preludeStatements, eventHandler, null, true);
+
+ if (eventHandler.hasErrors()) {
+ // In case of Python preprocessing, errors have already been reported (see checkSyntax).
+ // In other cases, errors will be reported later.
+ // TODO(bazel-team): maybe we could get rid of checkSyntax and always report errors here?
+ return new SkylarkImportResult(
+ ImmutableMap.<PathFragment, SkylarkEnvironment>of(),
+ ImmutableList.<Label>of());
+ }
+
+ ImmutableCollection<PathFragment> imports = buildFileAST.getImports();
+ Map<PathFragment, SkylarkEnvironment> importMap = new HashMap<>();
+ ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
+ try {
+ for (PathFragment importFile : imports) {
+ SkyKey importsLookupKey = SkylarkImportLookupValue.key(repo, importFile);
+ SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue)
+ env.getValueOrThrow(importsLookupKey, SkylarkImportFailedException.class,
+ InconsistentFilesystemException.class, ASTLookupInputException.class,
+ BuildFileNotFoundException.class);
+ if (importLookupValue != null) {
+ importMap.put(importFile, importLookupValue.getImportedEnvironment());
+ fileDependencies.add(importLookupValue.getDependency());
+ }
+ }
+ } catch (SkylarkImportFailedException e) {
+ env.getListener().handle(Event.error(Location.fromFile(buildFilePath), e.getMessage()));
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(packageName,
+ e.getMessage()), Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ throw new PackageFunctionException(new InternalInconsistentFilesystemException(packageName,
+ e), Transience.PERSISTENT);
+ } catch (ASTLookupInputException e) {
+ // The load syntax is bad in the BUILD file so BuildFileContainsErrorsException is OK.
+ throw new PackageFunctionException(new BuildFileContainsErrorsException(packageName,
+ e.getMessage()), Transience.PERSISTENT);
+ } catch (BuildFileNotFoundException e) {
+ throw new PackageFunctionException(e, Transience.PERSISTENT);
+ }
+ if (env.valuesMissing()) {
+ // There are unavailable Skylark dependencies.
+ return null;
+ }
+ return new SkylarkImportResult(importMap, transitiveClosureOfLabels(fileDependencies.build()));
+ }
+
+ private ImmutableList<Label> transitiveClosureOfLabels(
+ ImmutableList<SkylarkFileDependency> immediateDeps) {
+ Set<Label> transitiveClosure = Sets.newHashSet();
+ transitiveClosureOfLabels(immediateDeps, transitiveClosure);
+ return ImmutableList.copyOf(transitiveClosure);
+ }
+
+ private void transitiveClosureOfLabels(
+ ImmutableList<SkylarkFileDependency> immediateDeps, Set<Label> transitiveClosure) {
+ for (SkylarkFileDependency dep : immediateDeps) {
+ if (!transitiveClosure.contains(dep.getLabel())) {
+ transitiveClosure.add(dep.getLabel());
+ transitiveClosureOfLabels(dep.getDependencies(), transitiveClosure);
+ }
+ }
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static void handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions(
+ Path pkgRoot, PackageIdentifier pkgId, Package.LegacyBuilder pkgBuilder, Environment env)
+ throws InternalInconsistentFilesystemException {
+ Set<SkyKey> containingPkgLookupKeys = Sets.newHashSet();
+ Map<Target, SkyKey> targetToKey = new HashMap<>();
+ for (Target target : pkgBuilder.getTargets()) {
+ PathFragment dir = target.getLabel().toPathFragment().getParentDirectory();
+ PackageIdentifier dirId = new PackageIdentifier(pkgId.getRepository(), dir);
+ if (dir.equals(pkgId.getPackageFragment())) {
+ continue;
+ }
+ SkyKey key = ContainingPackageLookupValue.key(dirId);
+ targetToKey.put(target, key);
+ containingPkgLookupKeys.add(key);
+ }
+ Map<Label, SkyKey> subincludeToKey = new HashMap<>();
+ for (Label subincludeLabel : pkgBuilder.getSubincludeLabels()) {
+ PathFragment dir = subincludeLabel.toPathFragment().getParentDirectory();
+ PackageIdentifier dirId = new PackageIdentifier(pkgId.getRepository(), dir);
+ if (dir.equals(pkgId.getPackageFragment())) {
+ continue;
+ }
+ SkyKey key = ContainingPackageLookupValue.key(dirId);
+ subincludeToKey.put(subincludeLabel, key);
+ containingPkgLookupKeys.add(ContainingPackageLookupValue.key(dirId));
+ }
+ Map<SkyKey, ValueOrException3<BuildFileNotFoundException, InconsistentFilesystemException,
+ FileSymlinkCycleException>> containingPkgLookupValues = env.getValuesOrThrow(
+ containingPkgLookupKeys, BuildFileNotFoundException.class,
+ InconsistentFilesystemException.class, FileSymlinkCycleException.class);
+ if (env.valuesMissing()) {
+ return;
+ }
+ for (Target target : ImmutableSet.copyOf(pkgBuilder.getTargets())) {
+ SkyKey key = targetToKey.get(target);
+ if (!containingPkgLookupValues.containsKey(key)) {
+ continue;
+ }
+ ContainingPackageLookupValue containingPackageLookupValue =
+ getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(
+ pkgId.getPackageFragment().getPathString(), containingPkgLookupValues.get(key), env);
+ if (maybeAddEventAboutLabelCrossingSubpackage(pkgBuilder, pkgRoot, target.getLabel(),
+ target.getLocation(), containingPackageLookupValue)) {
+ pkgBuilder.removeTarget(target);
+ pkgBuilder.setContainsErrors();
+ }
+ }
+ for (Label subincludeLabel : pkgBuilder.getSubincludeLabels()) {
+ SkyKey key = subincludeToKey.get(subincludeLabel);
+ if (!containingPkgLookupValues.containsKey(key)) {
+ continue;
+ }
+ ContainingPackageLookupValue containingPackageLookupValue =
+ getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(
+ pkgId.getPackageFragment().getPathString(), containingPkgLookupValues.get(key), env);
+ if (maybeAddEventAboutLabelCrossingSubpackage(pkgBuilder, pkgRoot, subincludeLabel,
+ /*location=*/null, containingPackageLookupValue)) {
+ pkgBuilder.setContainsErrors();
+ }
+ }
+ }
+
+ @Nullable
+ private static ContainingPackageLookupValue
+ getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(String packageName,
+ ValueOrException3<BuildFileNotFoundException, InconsistentFilesystemException,
+ FileSymlinkCycleException> containingPkgLookupValueOrException, Environment env)
+ throws InternalInconsistentFilesystemException {
+ try {
+ return (ContainingPackageLookupValue) containingPkgLookupValueOrException.get();
+ } catch (BuildFileNotFoundException | FileSymlinkCycleException e) {
+ env.getListener().handle(Event.error(null, e.getMessage()));
+ return null;
+ } catch (InconsistentFilesystemException e) {
+ throw new InternalInconsistentFilesystemException(packageName, e);
+ }
+ }
+
+ private static boolean maybeAddEventAboutLabelCrossingSubpackage(
+ Package.LegacyBuilder pkgBuilder, Path pkgRoot, Label label, @Nullable Location location,
+ @Nullable ContainingPackageLookupValue containingPkgLookupValue) {
+ if (containingPkgLookupValue == null) {
+ return true;
+ }
+ if (!containingPkgLookupValue.hasContainingPackage()) {
+ // The missing package here is a problem, but it's not an error from the perspective of
+ // PackageFunction.
+ return false;
+ }
+ PackageIdentifier containingPkg = containingPkgLookupValue.getContainingPackageName();
+ if (containingPkg.equals(label.getPackageIdentifier())) {
+ // The label does not cross a subpackage boundary.
+ return false;
+ }
+ if (!containingPkg.getPackageFragment().startsWith(label.getPackageFragment())) {
+ // This label is referencing an imaginary package, because the containing package should
+ // extend the label's package: if the label is //a/b:c/d, the containing package could be
+ // //a/b/c or //a/b, but should never be //a. Usually such errors will be caught earlier, but
+ // in some exceptional cases (such as a Python-aware BUILD file catching its own io
+ // exceptions), it reaches here, and we tolerate it.
+ return false;
+ }
+ PathFragment labelNameFragment = new PathFragment(label.getName());
+ String message = String.format("Label '%s' crosses boundary of subpackage '%s'",
+ label, containingPkg);
+ Path containingRoot = containingPkgLookupValue.getContainingPackageRoot();
+ if (pkgRoot.equals(containingRoot)) {
+ PathFragment labelNameInContainingPackage = labelNameFragment.subFragment(
+ containingPkg.getPackageFragment().segmentCount()
+ - label.getPackageFragment().segmentCount(),
+ labelNameFragment.segmentCount());
+ message += " (perhaps you meant to put the colon here: "
+ + "'//" + containingPkg + ":" + labelNameInContainingPackage + "'?)";
+ } else {
+ message += " (have you deleted " + containingPkg + "/BUILD? "
+ + "If so, use the --deleted_packages=" + containingPkg + " option)";
+ }
+ pkgBuilder.addEvent(Event.error(location, message));
+ return true;
+ }
+
+ /**
+ * Constructs a {@link Package} object for the given package using legacy package loading.
+ * Note that the returned package may be in error.
+ */
+ private Package.LegacyBuilder loadPackage(ParserInputSource inputSource,
+ @Nullable String replacementContents,
+ PackageIdentifier packageId, Path buildFilePath, RuleVisibility defaultVisibility,
+ List<Statement> preludeStatements, SkylarkImportResult importResult)
+ throws InterruptedException {
+ ParserInputSource replacementSource = replacementContents == null ? null
+ : ParserInputSource.create(replacementContents, buildFilePath);
+ Package.LegacyBuilder pkgBuilder = packageFunctionCache.get(packageId);
+ if (pkgBuilder == null) {
+ Clock clock = new JavaClock();
+ long startTime = clock.nanoTime();
+ profiler.startTask(ProfilerTask.CREATE_PACKAGE, packageId.toString());
+ try {
+ Globber globber = packageFactory.createLegacyGlobber(buildFilePath.getParentDirectory(),
+ packageId, packageLocator);
+ StoredEventHandler localReporter = new StoredEventHandler();
+ Preprocessor.Result preprocessingResult = replacementSource == null
+ ? packageFactory.preprocess(packageId, buildFilePath, inputSource, globber,
+ localReporter)
+ : Preprocessor.Result.noPreprocessing(replacementSource);
+ pkgBuilder = packageFactory.createPackageFromPreprocessingResult(packageId, buildFilePath,
+ preprocessingResult, localReporter.getEvents(), preludeStatements,
+ importResult.importMap, importResult.fileDependencies, packageLocator,
+ defaultVisibility, globber);
+ if (eventBus.get() != null) {
+ eventBus.get().post(new PackageLoadedEvent(packageId.toString(),
+ (clock.nanoTime() - startTime) / (1000 * 1000),
+ // It's impossible to tell if the package was loaded before, so we always pass false.
+ /*reloading=*/false,
+ // This isn't completely correct since we may encounter errors later (e.g. filesystem
+ // inconsistencies)
+ !pkgBuilder.containsErrors()));
+ }
+ numPackagesLoaded.incrementAndGet();
+ packageFunctionCache.put(packageId, pkgBuilder);
+ } finally {
+ profiler.completeTask(ProfilerTask.CREATE_PACKAGE);
+ }
+ }
+ return pkgBuilder;
+ }
+
+ private static class InternalInconsistentFilesystemException extends NoSuchPackageException {
+ private boolean isTransient;
+
+ /**
+ * Used to represent a filesystem inconsistency discovered outside the
+ * {@link PackageFunction}.
+ */
+ public InternalInconsistentFilesystemException(String packageName,
+ InconsistentFilesystemException e) {
+ super(packageName, e.getMessage(), e);
+ // This is not a transient error from the perspective of the PackageFunction.
+ this.isTransient = false;
+ }
+
+ /** Used to represent a filesystem inconsistency discovered by the {@link PackageFunction}. */
+ public InternalInconsistentFilesystemException(String packageName,
+ String inconsistencyMessage) {
+ this(packageName, new InconsistentFilesystemException(inconsistencyMessage));
+ this.isTransient = true;
+ }
+
+ public boolean isTransient() {
+ return isTransient;
+ }
+ }
+
+ private static class BadWorkspaceFileException extends NoSuchPackageException {
+ private BadWorkspaceFileException(String message) {
+ super("external", "Error encountered while dealing with the WORKSPACE file: " + message);
+ }
+ }
+
+ private static class BadPreludeFileException extends NoSuchPackageException {
+ private BadPreludeFileException(String packageName, String message) {
+ super(packageName, "Error encountered while reading the prelude file: " + message);
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link PackageFunction#compute}.
+ */
+ private static class PackageFunctionException extends SkyFunctionException {
+ public PackageFunctionException(NoSuchPackageException e, Transience transience) {
+ super(e, transience);
+ }
+ }
+
+ /** A simple value class to store the result of the Skylark imports.*/
+ private static final class SkylarkImportResult {
+ private final Map<PathFragment, SkylarkEnvironment> importMap;
+ private final ImmutableList<Label> fileDependencies;
+ private SkylarkImportResult(Map<PathFragment, SkylarkEnvironment> importMap,
+ ImmutableList<Label> fileDependencies) {
+ this.importMap = importMap;
+ this.fileDependencies = fileDependencies;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java
new file mode 100644
index 0000000..ae4ee55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java
@@ -0,0 +1,180 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * SkyFunction for {@link PackageLookupValue}s.
+ */
+class PackageLookupFunction implements SkyFunction {
+
+ private final AtomicReference<ImmutableSet<String>> deletedPackages;
+
+ PackageLookupFunction(AtomicReference<ImmutableSet<String>> deletedPackages) {
+ this.deletedPackages = deletedPackages;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws PackageLookupFunctionException {
+ PathPackageLocator pkgLocator = PrecomputedValue.PATH_PACKAGE_LOCATOR.get(env);
+ PackageIdentifier packageKey = (PackageIdentifier) skyKey.argument();
+ if (!packageKey.getRepository().isDefault()) {
+ return computeExternalPackageLookupValue(skyKey, env);
+ }
+ PathFragment pkg = packageKey.getPackageFragment();
+
+ // This represents a package lookup at the package root.
+ if (pkg.equals(PathFragment.EMPTY_FRAGMENT)) {
+ return PackageLookupValue.invalidPackageName("The empty package name is invalid");
+ }
+
+ String pkgName = pkg.getPathString();
+ String packageNameErrorMsg = LabelValidator.validatePackageName(pkgName);
+ if (packageNameErrorMsg != null) {
+ return PackageLookupValue.invalidPackageName("Invalid package name '" + pkgName + "': "
+ + packageNameErrorMsg);
+ }
+
+ if (deletedPackages.get().contains(pkg.getPathString())) {
+ return PackageLookupValue.deletedPackage();
+ }
+
+ // TODO(bazel-team): The following is O(n^2) on the number of elements on the package path due
+ // to having restart the SkyFunction after every new dependency. However, if we try to batch
+ // the missing value keys, more dependencies than necessary will be declared. This wart can be
+ // fixed once we have nicer continuation support [skyframe-loading]
+ for (Path packagePathEntry : pkgLocator.getPathEntries()) {
+ PackageLookupValue value = getPackageLookupValue(env, packagePathEntry, pkg);
+ if (value == null || value.packageExists()) {
+ return value;
+ }
+ }
+ return PackageLookupValue.noBuildFile();
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private PackageLookupValue getPackageLookupValue(Environment env, Path packagePathEntry,
+ PathFragment pkgFragment) throws PackageLookupFunctionException {
+ PathFragment buildFileFragment;
+ if (pkgFragment.getPathString().equals(PackageFunction.EXTERNAL_PACKAGE_NAME)) {
+ buildFileFragment = new PathFragment("WORKSPACE");
+ } else {
+ buildFileFragment = pkgFragment.getChild("BUILD");
+ }
+ RootedPath buildFileRootedPath = RootedPath.toRootedPath(packagePathEntry,
+ buildFileFragment);
+ String basename = buildFileRootedPath.asPath().getBaseName();
+ SkyKey fileSkyKey = FileValue.key(buildFileRootedPath);
+ FileValue fileValue = null;
+ try {
+ fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
+ FileSymlinkCycleException.class, InconsistentFilesystemException.class);
+ } catch (IOException e) {
+ String pkgName = pkgFragment.getPathString();
+ // TODO(bazel-team): throw an IOException here and let PackageFunction wrap that into a
+ // BuildFileNotFoundException.
+ throw new PackageLookupFunctionException(new BuildFileNotFoundException(pkgName,
+ "IO errors while looking for " + basename + " file reading "
+ + buildFileRootedPath.asPath() + ": " + e.getMessage(), e),
+ Transience.PERSISTENT);
+ } catch (FileSymlinkCycleException e) {
+ String pkgName = buildFileRootedPath.asPath().getPathString();
+ throw new PackageLookupFunctionException(new BuildFileNotFoundException(pkgName,
+ "Symlink cycle detected while trying to find " + basename + " file "
+ + buildFileRootedPath.asPath()),
+ Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ // This error is not transient from the perspective of the PackageLookupFunction.
+ throw new PackageLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ if (fileValue == null) {
+ return null;
+ }
+ if (fileValue.isFile()) {
+ return PackageLookupValue.success(buildFileRootedPath.getRoot());
+ }
+ return PackageLookupValue.noBuildFile();
+ }
+
+ /**
+ * Gets a PackageLookupValue from a different Bazel repository.
+ *
+ * To do this, it looks up the "external" package and finds a path mapping for the repository
+ * name.
+ */
+ private PackageLookupValue computeExternalPackageLookupValue(
+ SkyKey skyKey, Environment env) throws PackageLookupFunctionException {
+ PackageIdentifier id = (PackageIdentifier) skyKey.argument();
+ SkyKey repositoryKey = RepositoryValue.key(id.getRepository());
+ RepositoryValue repositoryValue = null;
+ try {
+ repositoryValue = (RepositoryValue) env.getValueOrThrow(
+ repositoryKey, NoSuchPackageException.class, IOException.class, EvalException.class);
+ if (repositoryValue == null) {
+ return null;
+ }
+ } catch (NoSuchPackageException e) {
+ throw new PackageLookupFunctionException(e, Transience.PERSISTENT);
+ } catch (IOException e) {
+ throw new PackageLookupFunctionException(new BuildFileContainsErrorsException(
+ PackageFunction.EXTERNAL_PACKAGE_NAME, e.getMessage()), Transience.PERSISTENT);
+ } catch (EvalException e) {
+ throw new PackageLookupFunctionException(new BuildFileContainsErrorsException(
+ PackageFunction.EXTERNAL_PACKAGE_NAME, e.getMessage()), Transience.PERSISTENT);
+ }
+
+ return getPackageLookupValue(env, repositoryValue.getPath(), id.getPackageFragment());
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link PackageLookupFunction#compute}.
+ */
+ private static final class PackageLookupFunctionException extends SkyFunctionException {
+ public PackageLookupFunctionException(NoSuchPackageException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public PackageLookupFunctionException(InconsistentFilesystemException e,
+ Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java
new file mode 100644
index 0000000..c877d38
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java
@@ -0,0 +1,249 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value that represents a package lookup result.
+ *
+ * <p>Package lookups will always produce a value. On success, the {@code #getRoot} returns the
+ * package path root under which the package resides and the package's BUILD file is guaranteed to
+ * exist; on failure, {@code #getErrorReason} and {@code #getErrorMsg} describe why the package
+ * doesn't exist.
+ *
+ * <p>Implementation detail: we use inheritance here to optimize for memory usage.
+ */
+abstract class PackageLookupValue implements SkyValue {
+
+ enum ErrorReason {
+ // There is no BUILD file.
+ NO_BUILD_FILE,
+
+ // The package name is invalid.
+ INVALID_PACKAGE_NAME,
+
+ // The package is considered deleted because of --deleted_packages.
+ DELETED_PACKAGE,
+
+ // The //external package could not be loaded, either because the WORKSPACE file could not be
+ // parsed or the packages it references cannot be loaded.
+ NO_EXTERNAL_PACKAGE
+ }
+
+ protected PackageLookupValue() {
+ }
+
+ public static PackageLookupValue success(Path root) {
+ return new SuccessfulPackageLookupValue(root);
+ }
+
+ public static PackageLookupValue noBuildFile() {
+ return NoBuildFilePackageLookupValue.INSTANCE;
+ }
+
+ public static PackageLookupValue noExternalPackage() {
+ return NoExternalPackageLookupValue.INSTANCE;
+ }
+
+ public static PackageLookupValue invalidPackageName(String errorMsg) {
+ return new InvalidNamePackageLookupValue(errorMsg);
+ }
+
+ public static PackageLookupValue deletedPackage() {
+ return DeletedPackageLookupValue.INSTANCE;
+ }
+
+ /**
+ * For a successful package lookup, returns the root (package path entry) that the package
+ * resides in.
+ */
+ public abstract Path getRoot();
+
+ /**
+ * Returns whether the package lookup was successful.
+ */
+ public abstract boolean packageExists();
+
+ /**
+ * For an unsuccessful package lookup, gets the reason why {@link #packageExists} returns
+ * {@code false}.
+ */
+ abstract ErrorReason getErrorReason();
+
+ /**
+ * For an unsuccessful package lookup, gets a detailed error message for {@link #getErrorReason}
+ * that is suitable for reporting to a user.
+ */
+ abstract String getErrorMsg();
+
+ static SkyKey key(PathFragment directory) {
+ Preconditions.checkArgument(!directory.isAbsolute(), directory);
+ return key(PackageIdentifier.createInDefaultRepo(directory));
+ }
+
+ static SkyKey key(PackageIdentifier pkgIdentifier) {
+ return new SkyKey(SkyFunctions.PACKAGE_LOOKUP, pkgIdentifier);
+ }
+
+ private static class SuccessfulPackageLookupValue extends PackageLookupValue {
+
+ private final Path root;
+
+ private SuccessfulPackageLookupValue(Path root) {
+ this.root = root;
+ }
+
+ @Override
+ public boolean packageExists() {
+ return true;
+ }
+
+ @Override
+ public Path getRoot() {
+ return root;
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ String getErrorMsg() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof SuccessfulPackageLookupValue)) {
+ return false;
+ }
+ SuccessfulPackageLookupValue other = (SuccessfulPackageLookupValue) obj;
+ return root.equals(other.root);
+ }
+
+ @Override
+ public int hashCode() {
+ return root.hashCode();
+ }
+ }
+
+ private abstract static class UnsuccessfulPackageLookupValue extends PackageLookupValue {
+
+ @Override
+ public boolean packageExists() {
+ return false;
+ }
+
+ @Override
+ public Path getRoot() {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static class NoBuildFilePackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ public static final NoBuildFilePackageLookupValue INSTANCE =
+ new NoBuildFilePackageLookupValue();
+
+ private NoBuildFilePackageLookupValue() {
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.NO_BUILD_FILE;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return "BUILD file not found on package path";
+ }
+ }
+
+ private static class NoExternalPackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ public static final NoExternalPackageLookupValue INSTANCE =
+ new NoExternalPackageLookupValue();
+
+ private NoExternalPackageLookupValue() {
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.NO_EXTERNAL_PACKAGE;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return "Error loading the //external package";
+ }
+ }
+
+ private static class InvalidNamePackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ private final String errorMsg;
+
+ private InvalidNamePackageLookupValue(String errorMsg) {
+ this.errorMsg = errorMsg;
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.INVALID_PACKAGE_NAME;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return errorMsg;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof InvalidNamePackageLookupValue)) {
+ return false;
+ }
+ InvalidNamePackageLookupValue other = (InvalidNamePackageLookupValue) obj;
+ return errorMsg.equals(other.errorMsg);
+ }
+
+ @Override
+ public int hashCode() {
+ return errorMsg.hashCode();
+ }
+ }
+
+ private static class DeletedPackageLookupValue extends UnsuccessfulPackageLookupValue {
+
+ public static final DeletedPackageLookupValue INSTANCE = new DeletedPackageLookupValue();
+
+ private DeletedPackageLookupValue() {
+ }
+
+ @Override
+ ErrorReason getErrorReason() {
+ return ErrorReason.DELETED_PACKAGE;
+ }
+
+ @Override
+ String getErrorMsg() {
+ return "Package is considered deleted due to --deleted_packages";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java
new file mode 100644
index 0000000..65fd2af
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A Skyframe value representing a package.
+ */
+@Immutable
+@ThreadSafe
+public class PackageValue implements SkyValue {
+
+ private final Package pkg;
+
+ PackageValue(Package pkg) {
+ this.pkg = Preconditions.checkNotNull(pkg);
+ }
+
+ public Package getPackage() {
+ return pkg;
+ }
+
+ @Override
+ public String toString() {
+ return "<PackageValue name=" + pkg.getName() + ">";
+ }
+
+ @ThreadSafe
+ public static SkyKey key(PathFragment pkgName) {
+ return key(PackageIdentifier.createInDefaultRepo(pkgName));
+ }
+
+ public static SkyKey key(PackageIdentifier pkgIdentifier) {
+ return new SkyKey(SkyFunctions.PACKAGE, pkgIdentifier);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
new file mode 100644
index 0000000..5116a6f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
@@ -0,0 +1,131 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * A per-build cache of filesystem operations for Skyframe invocations of legacy package loading.
+ */
+class PerBuildSyscallCache implements UnixGlob.FilesystemCalls {
+
+ private final LoadingCache<Pair<Path, Symlinks>, FileStatus> statCache =
+ newStatMap();
+ private final LoadingCache<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>>
+ readdirCache = newReaddirMap();
+
+ private static final FileStatus NO_STATUS = new FakeFileStatus();
+
+ @Override
+ public Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException {
+ Pair<Collection<Dirent>, IOException> result =
+ readdirCache.getUnchecked(Pair.of(path, symlinks));
+ Collection<Dirent> entries = result.getFirst();
+ if (entries != null) {
+ return entries;
+ }
+ throw result.getSecond();
+ }
+
+ @Override
+ public FileStatus statNullable(Path path, Symlinks symlinks) {
+ FileStatus status = statCache.getUnchecked(Pair.of(path, symlinks));
+ return (status == NO_STATUS) ? null : status;
+ }
+
+ // This is used because the cache implementations don't allow null.
+ private static final class FakeFileStatus implements FileStatus {
+ @Override
+ public long getLastChangeTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getNodeId() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getLastModifiedTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isFile() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * A cache of stat calls.
+ * Input: (path, following_symlinks)
+ * Output: FileStatus
+ */
+ private static LoadingCache<Pair<Path, Symlinks>, FileStatus> newStatMap() {
+ return CacheBuilder.newBuilder().build(
+ new CacheLoader<Pair<Path, Symlinks>, FileStatus>() {
+ @Override
+ public FileStatus load(Pair<Path, Symlinks> p) {
+ FileStatus f = p.first.statNullable(p.second);
+ return (f == null) ? NO_STATUS : f;
+ }
+ });
+ }
+
+ /**
+ * A cache of readdir calls.
+ * Input: (path, following_symlinks)
+ * Output: A union of (Dirents, IOException).
+ */
+ private static
+ LoadingCache<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>> newReaddirMap() {
+ return CacheBuilder.newBuilder().build(
+ new CacheLoader<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>>() {
+ @Override
+ public Pair<Collection<Dirent>, IOException> load(Pair<Path, Symlinks> p) {
+ try {
+ return Pair.of(p.first.readdir(p.second), null);
+ } catch (IOException e) {
+ return Pair.of(null, e);
+ }
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java
new file mode 100644
index 0000000..03920b7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.RawAttributeMapper;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Build a post-processed ConfiguredTarget, vetting it for action conflict issues.
+ */
+public class PostConfiguredTargetFunction implements SkyFunction {
+ private static final Function<Dependency, SkyKey> TO_KEYS =
+ new Function<Dependency, SkyKey>() {
+ @Override
+ public SkyKey apply(Dependency input) {
+ return PostConfiguredTargetValue.key(
+ new ConfiguredTargetKey(input.getLabel(), input.getConfiguration()));
+ }
+ };
+
+ private final SkyframeExecutor.BuildViewProvider buildViewProvider;
+
+ public PostConfiguredTargetFunction(
+ SkyframeExecutor.BuildViewProvider buildViewProvider) {
+ this.buildViewProvider = Preconditions.checkNotNull(buildViewProvider);
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ ImmutableMap<Action, ConflictException> badActions = PrecomputedValue.BAD_ACTIONS.get(env);
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue)
+ env.getValue(ConfiguredTargetValue.key((ConfiguredTargetKey) skyKey.argument()));
+ SkyframeDependencyResolver resolver =
+ buildViewProvider.getSkyframeBuildView().createDependencyResolver(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ for (Action action : ctValue.getActions()) {
+ if (badActions.containsKey(action)) {
+ throw new ActionConflictFunctionException(badActions.get(action));
+ }
+ }
+
+ ConfiguredTarget ct = ctValue.getConfiguredTarget();
+ TargetAndConfiguration ctgValue =
+ new TargetAndConfiguration(ct.getTarget(), ct.getConfiguration());
+
+ Set<ConfigMatchingProvider> configConditions =
+ getConfigurableAttributeConditions(ctgValue, env);
+ if (configConditions == null) {
+ return null;
+ }
+
+ Collection<Dependency> deps = resolver.dependentNodes(ctgValue, configConditions);
+ env.getValues(Iterables.transform(deps, TO_KEYS));
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ return new PostConfiguredTargetValue(ct);
+ }
+
+ /**
+ * Returns the configurable attribute conditions necessary to evaluate the given configured
+ * target, or null if not all dependencies have yet been SkyFrame-evaluated.
+ */
+ @Nullable
+ private Set<ConfigMatchingProvider> getConfigurableAttributeConditions(
+ TargetAndConfiguration ctg, Environment env) {
+ if (!(ctg.getTarget() instanceof Rule)) {
+ return ImmutableSet.of();
+ }
+ Rule rule = (Rule) ctg.getTarget();
+ RawAttributeMapper mapper = RawAttributeMapper.of(rule);
+ Set<SkyKey> depKeys = new LinkedHashSet<>();
+ for (Attribute attribute : rule.getAttributes()) {
+ for (Label label : mapper.getConfigurabilityKeys(attribute.getName(), attribute.getType())) {
+ if (!Type.Selector.isReservedLabel(label)) {
+ depKeys.add(ConfiguredTargetValue.key(label, ctg.getConfiguration()));
+ }
+ }
+ }
+ Map<SkyKey, SkyValue> cts = env.getValues(depKeys);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ ImmutableSet.Builder<ConfigMatchingProvider> conditions = ImmutableSet.builder();
+ for (SkyValue ctValue : cts.values()) {
+ ConfiguredTarget ct = ((ConfiguredTargetValue) ctValue).getConfiguredTarget();
+ conditions.add(Preconditions.checkNotNull(ct.getProvider(ConfigMatchingProvider.class)));
+ }
+ return conditions.build();
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel());
+ }
+
+ private static class ActionConflictFunctionException extends SkyFunctionException {
+ public ActionConflictFunctionException(ConflictException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java
new file mode 100644
index 0000000..42d2b38
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A post-processed ConfiguredTarget which is known to be transitively error-free from action
+ * conflict issues.
+ */
+class PostConfiguredTargetValue implements SkyValue {
+
+ private final ConfiguredTarget ct;
+
+ public PostConfiguredTargetValue(ConfiguredTarget ct) {
+ this.ct = Preconditions.checkNotNull(ct);
+ }
+
+ public static ImmutableList<SkyKey> keys(Iterable<ConfiguredTargetKey> lacs) {
+ ImmutableList.Builder<SkyKey> keys = ImmutableList.builder();
+ for (ConfiguredTargetKey lac : lacs) {
+ keys.add(key(lac));
+ }
+ return keys.build();
+ }
+
+ public static SkyKey key(ConfiguredTargetKey lac) {
+ return new SkyKey(SkyFunctions.POST_CONFIGURED_TARGET, lac);
+ }
+
+ public ConfiguredTarget getCt() {
+ return ct;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java
new file mode 100644
index 0000000..8254987
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * Builder for {@link PrecomputedValue}s.
+ *
+ * <p>Always throws an error, because the values aren't computed inside the skyframe framework.
+ */
+public class PrecomputedFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ throw new IllegalStateException(skyKey + " not set");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java
new file mode 100644
index 0000000..bb2656d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java
@@ -0,0 +1,182 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Map;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * A value that represents something computed outside of the skyframe framework. These values are
+ * "precomputed" from skyframe's perspective and so the graph needs to be prepopulated with them
+ * (e.g. via injection).
+ */
+public class PrecomputedValue implements SkyValue {
+ /**
+ * An externally-injected precomputed value. Exists so that modules can inject precomputed values
+ * into Skyframe's graph.
+ *
+ * <p>{@see com.google.devtools.build.lib.blaze.BlazeModule#getPrecomputedValues}.
+ */
+ public static final class Injected {
+ private final Precomputed<?> precomputed;
+ private final Supplier<? extends Object> supplier;
+
+ private Injected(Precomputed<?> precomputed, Supplier<? extends Object> supplier) {
+ this.precomputed = precomputed;
+ this.supplier = supplier;
+ }
+
+ void inject(Injectable injectable) {
+ injectable.inject(ImmutableMap.of(precomputed.key, new PrecomputedValue(supplier.get())));
+ }
+ }
+
+ public static <T> Injected injected(Precomputed<T> precomputed, Supplier<T> value) {
+ return new Injected(precomputed, value);
+ }
+
+ public static <T> Injected injected(Precomputed<T> precomputed, T value) {
+ return new Injected(precomputed, Suppliers.ofInstance(value));
+ }
+
+ static final Precomputed<String> DEFAULTS_PACKAGE_CONTENTS =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "default_pkg"));
+
+ static final Precomputed<RuleVisibility> DEFAULT_VISIBILITY =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "default_visibility"));
+
+ static final Precomputed<UUID> BUILD_ID =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "build_id"));
+
+ static final Precomputed<WorkspaceStatusAction> WORKSPACE_STATUS_KEY =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "workspace_status_action"));
+
+ static final Precomputed<Action> COVERAGE_REPORT_KEY =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "coverage_report_action"));
+
+ static final Precomputed<TopLevelArtifactContext> TOP_LEVEL_CONTEXT =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "top_level_context"));
+
+ static final Precomputed<Map<BuildInfoKey, BuildInfoFactory>> BUILD_INFO_FACTORIES =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "build_info_factories"));
+
+ static final Precomputed<Map<String, String>> TEST_ENVIRONMENT_VARIABLES =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "test_environment"));
+
+ static final Precomputed<BlazeDirectories> BLAZE_DIRECTORIES =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "blaze_directories"));
+
+ static final Precomputed<ImmutableMap<Action, ConflictException>> BAD_ACTIONS =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "bad_actions"));
+
+ public static final Precomputed<PathPackageLocator> PATH_PACKAGE_LOCATOR =
+ new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "path_package_locator"));
+
+ private final Object value;
+
+ public PrecomputedValue(Object value) {
+ this.value = Preconditions.checkNotNull(value);
+ }
+
+ /**
+ * Returns the value of the variable.
+ */
+ public Object get() {
+ return value;
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof PrecomputedValue)) {
+ return false;
+ }
+ PrecomputedValue other = (PrecomputedValue) obj;
+ return value.equals(other.value);
+ }
+
+ @Override
+ public String toString() {
+ return "<BuildVariable " + value + ">";
+ }
+
+ public static final void dependOnBuildId(SkyFunction.Environment env) {
+ BUILD_ID.get(env);
+ }
+
+ /**
+ * A helper object corresponding to a variable in Skyframe.
+ *
+ * <p>Instances do not have internal state.
+ */
+ public static final class Precomputed<T> {
+ private final SkyKey key;
+
+ public Precomputed(SkyKey key) {
+ this.key = key;
+ }
+
+ @VisibleForTesting
+ SkyKey getKeyForTesting() {
+ return key;
+ }
+
+ /**
+ * Retrieves the value of this variable from Skyframe.
+ *
+ * <p>If the value was not set, an exception will be raised.
+ */
+ @Nullable
+ @SuppressWarnings("unchecked")
+ public T get(SkyFunction.Environment env) {
+ PrecomputedValue value = (PrecomputedValue) env.getValue(key);
+ if (value == null) {
+ return null;
+ }
+ return (T) value.get();
+ }
+
+ /**
+ * Injects a new variable value.
+ */
+ void set(Injectable injectable, T value) {
+ injectable.inject(ImmutableMap.of(key, new PrecomputedValue(value)));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
new file mode 100644
index 0000000..d54e98f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
@@ -0,0 +1,454 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Verify;
+import com.google.common.collect.Collections2;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.TraversalRequest;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */
+public final class RecursiveFilesystemTraversalFunction implements SkyFunction {
+
+ private static final class MissingDepException extends Exception {}
+
+ /** Base class for exceptions that {@link RecursiveFilesystemTraversalFunctionException} wraps. */
+ public abstract static class RecursiveFilesystemTraversalException extends Exception {
+ protected RecursiveFilesystemTraversalException(String message) {
+ super(message);
+ }
+ }
+
+ /** Thrown when a generated directory's root-relative path conflicts with a package's path. */
+ public static final class GeneratedPathConflictException extends
+ RecursiveFilesystemTraversalException {
+ GeneratedPathConflictException(TraversalRequest traversal) {
+ super(String.format(
+ "Generated directory %s conflicts with package under the same path. Additional info: %s",
+ traversal.path.getRelativePath().getPathString(),
+ traversal.errorInfo != null ? traversal.errorInfo : traversal.toString()));
+ }
+ }
+
+ /**
+ * Thrown when the traversal encounters a subdirectory with a BUILD file but is not allowed to
+ * recurse into it.
+ */
+ public static final class CannotCrossPackageBoundaryException extends
+ RecursiveFilesystemTraversalException {
+ CannotCrossPackageBoundaryException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Thrown when a dangling symlink is attempted to be dereferenced.
+ *
+ * <p>Note: this class is not identical to the one in com.google.devtools.build.lib.view.fileset
+ * and it's not easy to merge the two because of the dependency structure. The other one will
+ * probably be removed along with the rest of the legacy Fileset code.
+ */
+ public static final class DanglingSymlinkException extends RecursiveFilesystemTraversalException {
+ public final String path;
+ public final String unresolvedLink;
+
+ public DanglingSymlinkException(String path, String unresolvedLink) {
+ super("Found dangling symlink: " + path + ", unresolved path: ");
+ Preconditions.checkArgument(path != null && !path.isEmpty());
+ Preconditions.checkArgument(unresolvedLink != null && !unresolvedLink.isEmpty());
+ this.path = path;
+ this.unresolvedLink = unresolvedLink;
+ }
+
+ public String getPath() {
+ return path;
+ }
+ }
+
+ /** Exception type thrown by {@link RecursiveFilesystemTraversalFunction#compute}. */
+ private static final class RecursiveFilesystemTraversalFunctionException extends
+ SkyFunctionException {
+ RecursiveFilesystemTraversalFunctionException(RecursiveFilesystemTraversalException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws RecursiveFilesystemTraversalFunctionException {
+ TraversalRequest traversal = (TraversalRequest) skyKey.argument();
+ try {
+ // Stat the traversal root.
+ FileInfo rootInfo = lookUpFileInfo(env, traversal);
+
+ if (!rootInfo.type.exists()) {
+ // May be a dangling symlink or a non-existent file. Handle gracefully.
+ if (rootInfo.type.isSymlink()) {
+ return resultForDanglingSymlink(traversal.path, rootInfo);
+ } else {
+ return RecursiveFilesystemTraversalValue.EMPTY;
+ }
+ }
+
+ if (rootInfo.type.isFile()) {
+ // The root is a file or a symlink to one.
+ return resultForFileRoot(traversal.path, rootInfo);
+ }
+
+ // Otherwise the root is a directory or a symlink to one.
+ PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo);
+ traversal = pkgLookupResult.traversal;
+
+ if (pkgLookupResult.isConflicting()) {
+ // The traversal was requested for an output directory whose root-relative path conflicts
+ // with a source package. We can't handle that, bail out.
+ throw new RecursiveFilesystemTraversalFunctionException(
+ new GeneratedPathConflictException(traversal));
+ } else if (pkgLookupResult.isPackage() && !traversal.skipTestingForSubpackage) {
+ // The traversal was requested for a directory that defines a package.
+ if (traversal.crossPkgBoundaries) {
+ // We are free to traverse the subpackage but we need to display a warning.
+ String msg = traversal.errorInfo + " crosses package boundary into package rooted at "
+ + traversal.path.getRelativePath().getPathString();
+ env.getListener().handle(new Event(EventKind.WARNING, null, msg));
+ } else {
+ // We cannot traverse the subpackage and should skip it silently. Return empty results.
+ return RecursiveFilesystemTraversalValue.EMPTY;
+ }
+ }
+
+ // We are free to traverse this directory.
+ Collection<SkyKey> dependentKeys = createRecursiveTraversalKeys(env, traversal);
+ return resultForDirectory(traversal, rootInfo, traverseChildren(env, dependentKeys));
+ } catch (MissingDepException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static final class FileInfo {
+ final FileType type;
+ final FileStateValue metadata;
+ @Nullable final RootedPath realPath;
+ @Nullable final PathFragment unresolvedSymlinkTarget;
+
+ FileInfo(FileType type, FileStateValue metadata, @Nullable RootedPath realPath,
+ @Nullable PathFragment unresolvedSymlinkTarget) {
+ this.type = Preconditions.checkNotNull(type);
+ this.metadata = Preconditions.checkNotNull(metadata);
+ this.realPath = realPath;
+ this.unresolvedSymlinkTarget = unresolvedSymlinkTarget;
+ }
+
+ @Override
+ public String toString() {
+ if (type.isSymlink()) {
+ return String.format("(%s: link_value=%s, real_path=%s)", type,
+ unresolvedSymlinkTarget.getPathString(), realPath);
+ } else {
+ return String.format("(%s: real_path=%s)", type, realPath);
+ }
+ }
+ }
+
+ private static FileInfo lookUpFileInfo(Environment env, TraversalRequest traversal)
+ throws MissingDepException {
+ // Stat the file.
+ FileValue fileValue = (FileValue) getDependentSkyValue(env, FileValue.key(traversal.path));
+ if (fileValue.exists()) {
+ // If it exists, it may either be a symlink or a file/directory.
+ PathFragment unresolvedLinkTarget = null;
+ FileType type = null;
+ if (fileValue.isSymlink()) {
+ unresolvedLinkTarget = fileValue.getUnresolvedLinkTarget();
+ type = fileValue.isDirectory() ? FileType.SYMLINK_TO_DIRECTORY : FileType.SYMLINK_TO_FILE;
+ } else {
+ type = fileValue.isDirectory() ? FileType.DIRECTORY : FileType.FILE;
+ }
+ return new FileInfo(type, fileValue.realFileStateValue(),
+ fileValue.realRootedPath(), unresolvedLinkTarget);
+ } else {
+ // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully.
+ return new FileInfo(
+ fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT,
+ fileValue.realFileStateValue(), null,
+ fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null);
+ }
+ }
+
+ private static final class PkgLookupResult {
+ private enum Type {
+ CONFLICT, DIRECTORY, PKG
+ }
+
+ private final Type type;
+ final TraversalRequest traversal;
+ final FileInfo rootInfo;
+
+ /** Result for a generated directory that conflicts with a source package. */
+ static PkgLookupResult conflict(TraversalRequest traversal, FileInfo rootInfo) {
+ return new PkgLookupResult(Type.CONFLICT, traversal, rootInfo);
+ }
+
+ /** Result for a source or generated directory (not a package). */
+ static PkgLookupResult directory(TraversalRequest traversal, FileInfo rootInfo) {
+ return new PkgLookupResult(Type.DIRECTORY, traversal, rootInfo);
+ }
+
+ /** Result for a package, i.e. a directory with a BUILD file. */
+ static PkgLookupResult pkg(TraversalRequest traversal, FileInfo rootInfo) {
+ return new PkgLookupResult(Type.PKG, traversal, rootInfo);
+ }
+
+ private PkgLookupResult(Type type, TraversalRequest traversal, FileInfo rootInfo) {
+ this.type = Preconditions.checkNotNull(type);
+ this.traversal = Preconditions.checkNotNull(traversal);
+ this.rootInfo = Preconditions.checkNotNull(rootInfo);
+ }
+
+ boolean isPackage() {
+ return type == Type.PKG;
+ }
+
+ boolean isConflicting() {
+ return type == Type.CONFLICT;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("(%s: info=%s, traversal=%s)", type, rootInfo, traversal);
+ }
+ }
+
+ /**
+ * Checks whether the {@code traversal}'s path refers to a package directory.
+ *
+ * @return the result of the lookup; it contains potentially new {@link TraversalRequest} and
+ * {@link FileInfo} so the caller should use these instead of the old ones (this happens when
+ * a package is found, but under a different root than expected)
+ */
+ private static PkgLookupResult checkIfPackage(Environment env, TraversalRequest traversal,
+ FileInfo rootInfo) throws MissingDepException {
+ Preconditions.checkArgument(rootInfo.type.exists() && !rootInfo.type.isFile(),
+ "{%s} {%s}", traversal, rootInfo);
+ PackageLookupValue pkgLookup = (PackageLookupValue) getDependentSkyValue(env,
+ PackageLookupValue.key(traversal.path.getRelativePath()));
+
+ if (pkgLookup.packageExists()) {
+ if (traversal.isGenerated) {
+ // The traversal's root was a generated directory, but its root-relative path conflicts with
+ // an existing package.
+ return PkgLookupResult.conflict(traversal, rootInfo);
+ } else {
+ // The traversal's root was a source directory and it defines a package.
+ Path pkgRoot = pkgLookup.getRoot();
+ if (!pkgRoot.equals(traversal.path.getRoot())) {
+ // However the root of this package is different from what we expected. stat() the real
+ // BUILD file of that package.
+ traversal = traversal.forChangedRootPath(pkgRoot);
+ rootInfo = lookUpFileInfo(env, traversal);
+ Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo);
+ }
+ return PkgLookupResult.pkg(traversal, rootInfo);
+ }
+ } else {
+ // The traversal's root was a directory (source or generated one), no package exists under the
+ // same root-relative path.
+ return PkgLookupResult.directory(traversal, rootInfo);
+ }
+ }
+
+ /**
+ * List the directory and create {@code SkyKey}s to request contents of its children recursively.
+ *
+ * <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}.
+ */
+ private static Collection<SkyKey> createRecursiveTraversalKeys(Environment env,
+ TraversalRequest traversal) throws MissingDepException {
+ // Use the traversal's path, even if it's a symlink. The contents of the directory, as listed
+ // in the result, must be relative to it.
+ DirectoryListingValue dirListing = (DirectoryListingValue) getDependentSkyValue(env,
+ DirectoryListingValue.key(traversal.path));
+
+ List<SkyKey> result = new ArrayList<>();
+ for (Dirent dirent : dirListing.getDirents()) {
+ RootedPath childPath = RootedPath.toRootedPath(traversal.path.getRoot(),
+ traversal.path.getRelativePath().getRelative(dirent.getName()));
+ TraversalRequest childTraversal = traversal.forChildEntry(childPath);
+ result.add(RecursiveFilesystemTraversalValue.key(childTraversal));
+ }
+ return result;
+ }
+
+ /**
+ * Creates result for a dangling symlink.
+ *
+ * @param linkName path to the symbolic link
+ * @param info the {@link FileInfo} associated with the link file
+ */
+ private static RecursiveFilesystemTraversalValue resultForDanglingSymlink(RootedPath linkName,
+ FileInfo info) {
+ Preconditions.checkState(info.type.isSymlink() && !info.type.exists(), "{%s} {%s}", linkName,
+ info.type);
+ return RecursiveFilesystemTraversalValue.of(
+ ResolvedFile.danglingSymlink(linkName, info.unresolvedSymlinkTarget, info.metadata));
+ }
+
+ /**
+ * Creates results for a file or for a symlink that points to one.
+ *
+ * <p>A symlink may be direct (points to a file) or transitive (points at a direct or transitive
+ * symlink).
+ */
+ private static RecursiveFilesystemTraversalValue resultForFileRoot(RootedPath path,
+ FileInfo info) {
+ Preconditions.checkState(info.type.isFile() && info.type.exists(), "{%s} {%s}", path,
+ info.type);
+ if (info.type.isSymlink()) {
+ return RecursiveFilesystemTraversalValue.of(ResolvedFile.symlinkToFile(info.realPath, path,
+ info.unresolvedSymlinkTarget, info.metadata));
+ } else {
+ return RecursiveFilesystemTraversalValue.of(ResolvedFile.regularFile(path, info.metadata));
+ }
+ }
+
+ private static RecursiveFilesystemTraversalValue resultForDirectory(TraversalRequest traversal,
+ FileInfo rootInfo, Collection<RecursiveFilesystemTraversalValue> subdirTraversals) {
+ // Collect transitive closure of files in subdirectories.
+ NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder();
+ for (RecursiveFilesystemTraversalValue child : subdirTraversals) {
+ paths.addTransitive(child.getTransitiveFiles());
+ }
+ ResolvedFile root;
+ if (rootInfo.type.isSymlink()) {
+ root = ResolvedFile.symlinkToDirectory(rootInfo.realPath, traversal.path,
+ rootInfo.unresolvedSymlinkTarget, rootInfo.metadata);
+ paths.add(root);
+ } else {
+ root = ResolvedFile.directory(rootInfo.realPath);
+ }
+ return RecursiveFilesystemTraversalValue.of(root, paths.build());
+ }
+
+ private static SkyValue getDependentSkyValue(Environment env, SkyKey key)
+ throws MissingDepException {
+ SkyValue value = env.getValue(key);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return value;
+ }
+
+ /**
+ * Requests Skyframe to compute the dependent values and returns them.
+ *
+ * <p>The keys must all be {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL} keys.
+ */
+ private static Collection<RecursiveFilesystemTraversalValue> traverseChildren(
+ Environment env, Iterable<SkyKey> keys)
+ throws MissingDepException {
+ Map<SkyKey, SkyValue> values = env.getValues(keys);
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+ return Collections2.transform(values.values(),
+ new Function<SkyValue, RecursiveFilesystemTraversalValue>() {
+ @Override
+ public RecursiveFilesystemTraversalValue apply(SkyValue input) {
+ return (RecursiveFilesystemTraversalValue) input;
+ }
+ });
+ }
+
+ /** Type information about the filesystem entry residing at a path. */
+ enum FileType {
+ /** A regular file. */
+ FILE {
+ @Override boolean isFile() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<f>"; }
+ },
+ /**
+ * A symlink to a regular file.
+ *
+ * <p>The symlink may be direct (points to a non-symlink (here a file)) or it may be transitive
+ * (points to a direct or transitive symlink).
+ */
+ SYMLINK_TO_FILE {
+ @Override boolean isFile() { return true; }
+ @Override boolean isSymlink() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<lf>"; }
+ },
+ /** A directory. */
+ DIRECTORY {
+ @Override boolean isDirectory() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<d>"; }
+ },
+ /**
+ * A symlink to a directory.
+ *
+ * <p>The symlink may be direct (points to a non-symlink (here a directory)) or it may be
+ * transitive (points to a direct or transitive symlink).
+ */
+ SYMLINK_TO_DIRECTORY {
+ @Override boolean isDirectory() { return true; }
+ @Override boolean isSymlink() { return true; }
+ @Override boolean exists() { return true; }
+ @Override public String toString() { return "<ld>"; }
+ },
+ /** A dangling symlink, i.e. one whose target is known not to exist. */
+ DANGLING_SYMLINK {
+ @Override boolean isFile() { throw new UnsupportedOperationException(); }
+ @Override boolean isDirectory() { throw new UnsupportedOperationException(); }
+ @Override boolean isSymlink() { return true; }
+ @Override public String toString() { return "<l?>"; }
+ },
+ /** A path that does not exist or should be ignored. */
+ NONEXISTENT {
+ @Override public String toString() { return "<?>"; }
+ };
+
+ boolean isFile() { return false; }
+ boolean isDirectory() { return false; }
+ boolean isSymlink() { return false; }
+ boolean exists() { return false; }
+ @Override public abstract String toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
new file mode 100644
index 0000000..023b1cf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
@@ -0,0 +1,597 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.DanglingSymlinkException;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.FileType;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * Collection of files found while recursively traversing a path.
+ *
+ * <p>The path may refer to files, symlinks or directories that may or may not exist.
+ *
+ * <p>Traversing a file or a symlink results in a single {@link ResolvedFile} corresponding to the
+ * file or symlink.
+ *
+ * <p>Traversing a directory results in a collection of {@link ResolvedFile}s for all files and
+ * symlinks under it, and in all of its subdirectories. The {@link TraversalRequest} can specify
+ * whether to traverse source subdirectories that are packages (have BUILD files in them).
+ *
+ * <p>Traversing a symlink that points to a directory is the same as traversing a normal directory.
+ * The paths in the result will not be resolved; the files will be listed under the symlink, as if
+ * it was the actual directory they reside in.
+ *
+ * <p>Editing a file that is part of this traversal, or adding or removing a file in a directory
+ * that is part of this traversal, will invalidate this {@link SkyValue}. This also applies to
+ * directories that are symlinked to.
+ */
+public final class RecursiveFilesystemTraversalValue implements SkyValue {
+ static final RecursiveFilesystemTraversalValue EMPTY = new RecursiveFilesystemTraversalValue(
+ Optional.<ResolvedFile>absent(),
+ NestedSetBuilder.<ResolvedFile>emptySet(Order.STABLE_ORDER));
+
+ /** The root of the traversal. May only be absent for the {@link #EMPTY} instance. */
+ private final Optional<ResolvedFile> resolvedRoot;
+
+ /** The transitive closure of {@link ResolvedFile}s. */
+ private final NestedSet<ResolvedFile> resolvedPaths;
+
+ private RecursiveFilesystemTraversalValue(Optional<ResolvedFile> resolvedRoot,
+ NestedSet<ResolvedFile> resolvedPaths) {
+ this.resolvedRoot = Preconditions.checkNotNull(resolvedRoot);
+ this.resolvedPaths = Preconditions.checkNotNull(resolvedPaths);
+ }
+
+ static RecursiveFilesystemTraversalValue of(ResolvedFile resolvedRoot,
+ NestedSet<ResolvedFile> resolvedPaths) {
+ if (resolvedPaths.isEmpty()) {
+ return EMPTY;
+ } else {
+ return new RecursiveFilesystemTraversalValue(Optional.of(resolvedRoot), resolvedPaths);
+ }
+ }
+
+ static RecursiveFilesystemTraversalValue of(ResolvedFile singleMember) {
+ return new RecursiveFilesystemTraversalValue(Optional.of(singleMember),
+ NestedSetBuilder.<ResolvedFile>create(Order.STABLE_ORDER, singleMember));
+ }
+
+ /** Returns the root of the traversal; absent only for the {@link #EMPTY} instance. */
+ public Optional<ResolvedFile> getResolvedRoot() {
+ return resolvedRoot;
+ }
+
+ /**
+ * Retrieves the set of {@link ResolvedFile}s that were found by this traversal.
+ *
+ * <p>The returned set may be empty if no files were found, or the ones found were to be
+ * considered non-existent. Unless it's empty, the returned set always includes the
+ * {@link #getResolvedRoot() resolved root}.
+ *
+ * <p>The returned set also includes symlinks. If a symlink points to a directory, its contents
+ * are also included in this set, and their path will start with the symlink's path, just like on
+ * a usual Unix file system.
+ */
+ public NestedSet<ResolvedFile> getTransitiveFiles() {
+ return resolvedPaths;
+ }
+
+ public static SkyKey key(TraversalRequest traversal) {
+ return new SkyKey(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, traversal);
+ }
+
+ /** The parameters of a file or directory traversal. */
+ public static final class TraversalRequest {
+
+ /** The path to start the traversal from; may be a file, a directory or a symlink. */
+ final RootedPath path;
+
+ /**
+ * Whether the path is in the output tree.
+ *
+ * <p>Such paths and all their subdirectories are assumed not to define packages, so package
+ * lookup for them is skipped.
+ */
+ final boolean isGenerated;
+
+ /** Whether traversal should descend into directories that are roots of subpackages. */
+ final boolean crossPkgBoundaries;
+
+ /**
+ * Whether to skip checking if the root (if it's a directory) contains a BUILD file.
+ *
+ * <p>Such directories are not considered to be packages when this flag is true. This needs to
+ * be true in order to traverse directories of packages, but should be false for <i>their</i>
+ * subdirectories.
+ */
+ final boolean skipTestingForSubpackage;
+
+ /** Information to be attached to any error messages that may be reported. */
+ @Nullable final String errorInfo;
+
+ public TraversalRequest(RootedPath path, boolean isRootGenerated,
+ boolean crossPkgBoundaries, boolean skipTestingForSubpackage,
+ @Nullable String errorInfo) {
+ this.path = path;
+ this.isGenerated = isRootGenerated;
+ this.crossPkgBoundaries = crossPkgBoundaries;
+ this.skipTestingForSubpackage = skipTestingForSubpackage;
+ this.errorInfo = errorInfo;
+ }
+
+ private TraversalRequest duplicate(RootedPath newRoot, boolean newSkipTestingForSubpackage) {
+ return new TraversalRequest(newRoot, isGenerated, crossPkgBoundaries,
+ newSkipTestingForSubpackage, errorInfo);
+ }
+
+ /** Creates a new request to traverse a child element in the current directory (the root). */
+ TraversalRequest forChildEntry(RootedPath newPath) {
+ return duplicate(newPath, false);
+ }
+
+ /**
+ * Creates a new request for a changed root.
+ *
+ * <p>This method can be used when a package is found out to be under a different root path than
+ * originally assumed.
+ */
+ TraversalRequest forChangedRootPath(Path newRoot) {
+ return duplicate(RootedPath.toRootedPath(newRoot, path.getRelativePath()),
+ skipTestingForSubpackage);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TraversalRequest)) {
+ return false;
+ }
+ TraversalRequest o = (TraversalRequest) obj;
+ return path.equals(o.path) && isGenerated == o.isGenerated
+ && crossPkgBoundaries == o.crossPkgBoundaries
+ && skipTestingForSubpackage == o.skipTestingForSubpackage;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(path, isGenerated, crossPkgBoundaries, skipTestingForSubpackage);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TraversalParams(root=%s, is_generated=%d, skip_testing_for_subpkg=%d,"
+ + " pkg_boundaries=%d)", path, isGenerated ? 1 : 0, skipTestingForSubpackage ? 1 : 0,
+ crossPkgBoundaries ? 1 : 0);
+ }
+ }
+
+ /**
+ * Path and type information about a single file or symlink.
+ *
+ * <p>The object stores things such as the absolute path of the file or symlink, its exact type
+ * and, if it's a symlink, the resolved and unresolved link target paths.
+ */
+ public abstract static class ResolvedFile {
+ private static final class Symlink {
+ private final RootedPath linkName;
+ private final PathFragment unresolvedLinkTarget;
+ // The resolved link target is stored in ResolvedFile.path
+
+ private Symlink(RootedPath linkName, PathFragment unresolvedLinkTarget) {
+ this.linkName = Preconditions.checkNotNull(linkName);
+ this.unresolvedLinkTarget = Preconditions.checkNotNull(unresolvedLinkTarget);
+ }
+
+ PathFragment getNameInSymlinkTree() {
+ return linkName.getRelativePath();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof Symlink)) {
+ return false;
+ }
+ Symlink o = (Symlink) obj;
+ return linkName.equals(o.linkName) && unresolvedLinkTarget.equals(o.unresolvedLinkTarget);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(linkName, unresolvedLinkTarget);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Symlink(link_name=%s, unresolved_target=%s)",
+ linkName, unresolvedLinkTarget);
+ }
+ }
+
+ private static final class RegularFile extends ResolvedFile {
+ private RegularFile(RootedPath path) {
+ super(FileType.FILE, Optional.of(path), Optional.<FileStateValue>absent());
+ }
+
+ RegularFile(RootedPath path, FileStateValue metadata) {
+ super(FileType.FILE, Optional.of(path), Optional.of(metadata));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RegularFile)) {
+ return false;
+ }
+ return super.isEqualTo((RegularFile) obj);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("RegularFile(%s)", super.toString());
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new RegularFile(path.get());
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return path.get().getRelativePath();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return path.get().asPath().asFragment();
+ }
+ }
+
+ private static final class Directory extends ResolvedFile {
+ Directory(RootedPath path) {
+ super(FileType.DIRECTORY, Optional.of(path), Optional.of(
+ FileStateValue.DIRECTORY_FILE_STATE_NODE));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof Directory)) {
+ return false;
+ }
+ return super.isEqualTo((Directory) obj);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Directory(%s)", super.toString());
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return this;
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return path.get().getRelativePath();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return path.get().asPath().asFragment();
+ }
+ }
+
+ private static final class DanglingSymlink extends ResolvedFile {
+ private final Symlink symlink;
+
+ private DanglingSymlink(Symlink symlink) {
+ super(FileType.DANGLING_SYMLINK, Optional.<RootedPath>absent(),
+ Optional.<FileStateValue>absent());
+ this.symlink = symlink;
+ }
+
+ DanglingSymlink(RootedPath linkNamePath, PathFragment linkTargetPath,
+ FileStateValue metadata) {
+ super(FileType.DANGLING_SYMLINK, Optional.<RootedPath>absent(), Optional.of(metadata));
+ this.symlink = new Symlink(linkNamePath, linkTargetPath);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof DanglingSymlink)) {
+ return false;
+ }
+ DanglingSymlink o = (DanglingSymlink) obj;
+ return super.isEqualTo(o) && symlink.equals(o.symlink);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(super.hashCode(), symlink);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("DanglingSymlink(%s, %s)", super.toString(), symlink);
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new DanglingSymlink(symlink);
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return symlink.getNameInSymlinkTree();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks)
+ throws DanglingSymlinkException {
+ if (followSymlinks) {
+ throw new DanglingSymlinkException(symlink.linkName.asPath().getPathString(),
+ symlink.unresolvedLinkTarget.getPathString());
+ } else {
+ return symlink.unresolvedLinkTarget;
+ }
+ }
+ }
+
+ private static final class SymlinkToFile extends ResolvedFile {
+ private final Symlink symlink;
+
+ private SymlinkToFile(RootedPath targetPath, Symlink symlink) {
+ super(FileType.SYMLINK_TO_FILE, Optional.of(targetPath), Optional.<FileStateValue>absent());
+ this.symlink = symlink;
+ }
+
+ SymlinkToFile(RootedPath targetPath, RootedPath linkNamePath,
+ PathFragment linkTargetPath, FileStateValue metadata) {
+ super(FileType.SYMLINK_TO_FILE, Optional.of(targetPath), Optional.of(metadata));
+ this.symlink = new Symlink(linkNamePath, linkTargetPath);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SymlinkToFile)) {
+ return false;
+ }
+ SymlinkToFile o = (SymlinkToFile) obj;
+ return super.isEqualTo(o) && symlink.equals(o.symlink);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(super.hashCode(), symlink);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("SymlinkToFile(%s, %s)", super.toString(), symlink);
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new SymlinkToFile(path.get(), symlink);
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return symlink.getNameInSymlinkTree();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return followSymlinks ? path.get().asPath().asFragment() : symlink.unresolvedLinkTarget;
+ }
+ }
+
+ private static final class SymlinkToDirectory extends ResolvedFile {
+ private final Symlink symlink;
+
+ private SymlinkToDirectory(RootedPath targetPath, Symlink symlink) {
+ super(FileType.SYMLINK_TO_DIRECTORY, Optional.of(targetPath),
+ Optional.<FileStateValue>absent());
+ this.symlink = symlink;
+ }
+
+ SymlinkToDirectory(RootedPath targetPath, RootedPath linkNamePath,
+ PathFragment linkValue, FileStateValue metadata) {
+ super(FileType.SYMLINK_TO_DIRECTORY, Optional.of(targetPath), Optional.of(metadata));
+ this.symlink = new Symlink(linkNamePath, linkValue);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SymlinkToDirectory)) {
+ return false;
+ }
+ SymlinkToDirectory o = (SymlinkToDirectory) obj;
+ return super.isEqualTo(o) && symlink.equals(o.symlink);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(super.hashCode(), symlink);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("SymlinkToDirectory(%s, %s)", super.toString(), symlink);
+ }
+
+ @Override
+ ResolvedFile stripMetadataForTesting() {
+ return new SymlinkToDirectory(path.get(), symlink);
+ }
+
+ @Override
+ public PathFragment getNameInSymlinkTree() {
+ return symlink.getNameInSymlinkTree();
+ }
+
+ @Override
+ public PathFragment getTargetInSymlinkTree(boolean followSymlinks) {
+ return followSymlinks ? path.get().asPath().asFragment() : symlink.unresolvedLinkTarget;
+ }
+ }
+
+ /** Type of the entity under {@link #path}. */
+ final FileType type;
+
+ /**
+ * Path of the file, directory or resolved target of the symlink.
+ *
+ * <p>May only be absent for dangling symlinks.
+ */
+ protected final Optional<RootedPath> path;
+
+ /**
+ * Associated metadata.
+ *
+ * <p>This field must be stored so that this {@link ResolvedFile} is (also) the function of the
+ * stat() of the file, but otherwise it is likely not something the consumer of the
+ * {@link ResolvedFile} is directly interested in.
+ *
+ * <p>May only be absent if stripped for tests.
+ */
+ final Optional<FileStateValue> metadata;
+
+ private ResolvedFile(FileType type, Optional<RootedPath> path,
+ Optional<FileStateValue> metadata) {
+ this.type = Preconditions.checkNotNull(type);
+ this.path = Preconditions.checkNotNull(path);
+ this.metadata = Preconditions.checkNotNull(metadata);
+ }
+
+ static ResolvedFile regularFile(RootedPath path, FileStateValue metadata) {
+ return new RegularFile(path, metadata);
+ }
+
+ static ResolvedFile directory(RootedPath path) {
+ return new Directory(path);
+ }
+
+ static ResolvedFile symlinkToFile(RootedPath targetPath, RootedPath linkNamePath,
+ PathFragment linkTargetPath, FileStateValue metadata) {
+ return new SymlinkToFile(targetPath, linkNamePath, linkTargetPath, metadata);
+ }
+
+ static ResolvedFile symlinkToDirectory(RootedPath targetPath,
+ RootedPath linkNamePath, PathFragment linkValue, FileStateValue metadata) {
+ return new SymlinkToDirectory(targetPath, linkNamePath, linkValue, metadata);
+ }
+
+ static ResolvedFile danglingSymlink(RootedPath linkNamePath, PathFragment linkValue,
+ FileStateValue metadata) {
+ return new DanglingSymlink(linkNamePath, linkValue, metadata);
+ }
+
+ private boolean isEqualTo(ResolvedFile o) {
+ return type.equals(o.type) && path.equals(o.path) && metadata.equals(o.metadata);
+ }
+
+ @Override
+ public abstract boolean equals(Object obj);
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(type, path, metadata);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("type=%s, path=%s, metadata=%s", type, path,
+ metadata.isPresent() ? Integer.toHexString(metadata.get().hashCode()) : "(stripped)");
+ }
+
+ /**
+ * Returns the path of the Fileset-output symlink relative to the output directory.
+ *
+ * <p>The path should contain the FilesetEntry-specific destination directory (if any) and
+ * should have necessary prefixes stripped (if any).
+ */
+ public abstract PathFragment getNameInSymlinkTree();
+
+ /**
+ * Returns the path of the symlink target.
+ *
+ * @throws DanglingSymlinkException if the target cannot be resolved because the symlink is
+ * dangling
+ */
+ public abstract PathFragment getTargetInSymlinkTree(boolean followSymlinks)
+ throws DanglingSymlinkException;
+
+ /**
+ * Returns a copy of this object with the metadata stripped away.
+ *
+ * <p>This method should only be used by tests that wish to assert that this
+ * {@link ResolvedFile} refers to the expected absolute path and has the expected type, without
+ * asserting its actual contents (which the metadata is a function of).
+ */
+ @VisibleForTesting
+ abstract ResolvedFile stripMetadataForTesting();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RecursiveFilesystemTraversalValue)) {
+ return false;
+ }
+ RecursiveFilesystemTraversalValue o = (RecursiveFilesystemTraversalValue) obj;
+ return resolvedRoot.equals(o.resolvedRoot) && resolvedPaths.equals(o.resolvedPaths);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(resolvedRoot, resolvedPaths);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java
new file mode 100644
index 0000000..11ed3be
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java
@@ -0,0 +1,151 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * RecursivePkgFunction builds up the set of packages underneath a given directory
+ * transitively.
+ *
+ * <p>Example: foo/BUILD, foo/sub/x, foo/subpkg/BUILD would yield transitive packages "foo" and
+ * "foo/subpkg".
+ */
+public class RecursivePkgFunction implements SkyFunction {
+
+ private static final Order ORDER = Order.STABLE_ORDER;
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ RootedPath rootedPath = (RootedPath) skyKey.argument();
+ Path root = rootedPath.getRoot();
+ PathFragment rootRelativePath = rootedPath.getRelativePath();
+
+ SkyKey fileKey = FileValue.key(rootedPath);
+ FileValue fileValue = (FileValue) env.getValue(fileKey);
+ if (fileValue == null) {
+ return null;
+ }
+
+ if (!fileValue.isDirectory()) {
+ return new RecursivePkgValue(NestedSetBuilder.<String>emptySet(ORDER));
+ }
+
+ if (fileValue.isSymlink()) {
+ // We do not follow directory symlinks when we look recursively for packages. It also
+ // prevents symlink loops.
+ return new RecursivePkgValue(NestedSetBuilder.<String>emptySet(ORDER));
+ }
+
+ PackageIdentifier packageId = PackageIdentifier.createInDefaultRepo(
+ rootRelativePath.getPathString());
+ PackageLookupValue pkgLookupValue =
+ (PackageLookupValue) env.getValue(PackageLookupValue.key(packageId));
+ if (pkgLookupValue == null) {
+ return null;
+ }
+
+ NestedSetBuilder<String> packages = new NestedSetBuilder<>(ORDER);
+
+ if (pkgLookupValue.packageExists()) {
+ if (pkgLookupValue.getRoot().equals(root)) {
+ try {
+ PackageValue pkgValue = (PackageValue)
+ env.getValueOrThrow(PackageValue.key(packageId),
+ NoSuchPackageException.class);
+ if (pkgValue == null) {
+ return null;
+ }
+ packages.add(pkgValue.getPackage().getName());
+ } catch (NoSuchPackageException e) {
+ // The package had errors, but don't fail-fast as there might subpackages below the
+ // current directory.
+ env.getListener().handle(Event.error(
+ "package contains errors: " + rootRelativePath.getPathString()));
+ if (e.getPackage() != null) {
+ packages.add(e.getPackage().getName());
+ }
+ }
+ }
+ // The package lookup succeeded, but was under a different root. We still, however, need to
+ // recursively consider subdirectories. For example:
+ //
+ // Pretend --package_path=rootA/workspace:rootB/workspace and these are the only files:
+ // rootA/workspace/foo/
+ // rootA/workspace/foo/bar/BUILD
+ // rootB/workspace/foo/BUILD
+ // If we're doing a recursive package lookup under 'rootA/workspace' starting at 'foo', note
+ // that even though the package 'foo' is under 'rootB/workspace', there is still a package
+ // 'foo/bar' under 'rootA/workspace'.
+ }
+
+ DirectoryListingValue dirValue = (DirectoryListingValue)
+ env.getValue(DirectoryListingValue.key(rootedPath));
+ if (dirValue == null) {
+ return null;
+ }
+
+ List<SkyKey> childDeps = Lists.newArrayList();
+ for (Dirent dirent : dirValue.getDirents()) {
+ if (dirent.getType() != Type.DIRECTORY) {
+ // Non-directories can never host packages, and we do not follow symlinks (see above).
+ continue;
+ }
+ String basename = dirent.getName();
+ if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
+ && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
+ continue;
+ }
+ SkyKey req = RecursivePkgValue.key(RootedPath.toRootedPath(root,
+ rootRelativePath.getRelative(basename)));
+ childDeps.add(req);
+ }
+ Map<SkyKey, SkyValue> childValueMap = env.getValues(childDeps);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ // Aggregate the transitive subpackages.
+ for (SkyValue childValue : childValueMap.values()) {
+ if (childValue != null) {
+ packages.addTransitive(((RecursivePkgValue) childValue).getPackages());
+ }
+ }
+ return new RecursivePkgValue(packages.build());
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java
new file mode 100644
index 0000000..4013b90
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * This value represents the result of looking up all the packages under a given package path root,
+ * starting at a given directory.
+ */
+@Immutable
+@ThreadSafe
+public class RecursivePkgValue implements SkyValue {
+
+ private final NestedSet<String> packages;
+
+ public RecursivePkgValue(NestedSet<String> packages) {
+ this.packages = packages;
+ }
+
+ /**
+ * Create a transitive package lookup request.
+ */
+ @ThreadSafe
+ public static SkyKey key(RootedPath rootedPath) {
+ return new SkyKey(SkyFunctions.RECURSIVE_PKG, rootedPath);
+ }
+
+ public NestedSet<String> getPackages() {
+ return packages;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
new file mode 100644
index 0000000..3183953
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A local view of an external repository.
+ */
+public class RepositoryValue implements SkyValue {
+ private final Path path;
+
+ /**
+ * If path is a symlink, this will keep track of what the symlink actually points to (for
+ * checking equality).
+ */
+ private final FileValue details;
+
+ public RepositoryValue(Path path, FileValue repositoryDirectory) {
+ this.path = path;
+ this.details = repositoryDirectory;
+ }
+
+ /**
+ * Returns the path to the directory containing the repository's contents. This directory is
+ * guaranteed to exist. It may contain a full Bazel repository (with a WORKSPACE file,
+ * directories, and BUILD files) or simply contain a file (or set of files) for, say, a jar from
+ * Maven.
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof RepositoryValue) {
+ RepositoryValue otherValue = (RepositoryValue) other;
+ return path.equals(otherValue.path) && details.equals(otherValue.details);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(path, details);
+ }
+
+ /**
+ * Creates a key from the given repository name.
+ */
+ public static SkyKey key(RepositoryName repository) {
+ return new SkyKey(SkyFunctions.REPOSITORY, repository);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
new file mode 100644
index 0000000..e547166
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -0,0 +1,580 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.ResourceUsage;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.BuildDriver;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.ImmutableDiff;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+
+/**
+ * A SkyframeExecutor that implicitly assumes that builds can be done incrementally from the most
+ * recent build. In other words, builds are "sequenced".
+ */
+public final class SequencedSkyframeExecutor extends SkyframeExecutor {
+ /** Lower limit for number of loaded packages to consider clearing CT values. */
+ private int valueCacheEvictionLimit = -1;
+
+ /** Union of labels of loaded packages since the last eviction of CT values. */
+ private Set<PackageIdentifier> allLoadedPackages = ImmutableSet.of();
+ private boolean lastAnalysisDiscarded = false;
+
+ // Can only be set once (to false) over the lifetime of this object. If false, the graph will not
+ // store edges, saving memory but making incremental builds impossible.
+ private boolean keepGraphEdges = true;
+
+ private RecordingDifferencer recordingDiffer;
+ private final DiffAwarenessManager diffAwarenessManager;
+
+ private SequencedSkyframeExecutor(Reporter reporter, EvaluatorSupplier evaluatorSupplier,
+ PackageFactory pkgFactory, TimestampGranularityMonitor tsgm,
+ BlazeDirectories directories, Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ super(reporter, evaluatorSupplier, pkgFactory, tsgm, directories,
+ workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ this.diffAwarenessManager = new DiffAwarenessManager(diffAwarenessFactories, reporter);
+ }
+
+ private SequencedSkyframeExecutor(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ this(reporter, InMemoryMemoizingEvaluator.SUPPLIER, pkgFactory, tsgm,
+ directories, workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ }
+
+ private static SequencedSkyframeExecutor create(Reporter reporter,
+ EvaluatorSupplier evaluatorSupplier, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory, ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ SequencedSkyframeExecutor skyframeExecutor = new SequencedSkyframeExecutor(reporter,
+ evaluatorSupplier, pkgFactory, tsgm, directories, workspaceStatusActionFactory,
+ buildInfoFactories, immutableDirectories, diffAwarenessFactories, allowedMissingInputs,
+ preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ skyframeExecutor.init();
+ return skyframeExecutor;
+ }
+
+ public static SequencedSkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ return create(reporter, InMemoryMemoizingEvaluator.SUPPLIER, pkgFactory, tsgm,
+ directories, workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ }
+
+ @VisibleForTesting
+ public static SequencedSkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ WorkspaceStatusAction.Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories) {
+ return create(reporter, pkgFactory, tsgm, directories, workspaceStatusActionFactory,
+ buildInfoFactories, immutableDirectories, diffAwarenessFactories,
+ Predicates.<PathFragment>alwaysFalse(),
+ Preprocessor.Factory.Supplier.NullSupplier.INSTANCE,
+ ImmutableMap.<SkyFunctionName, SkyFunction>of(),
+ ImmutableList.<PrecomputedValue.Injected>of());
+ }
+
+ @Override
+ protected BuildDriver newBuildDriver() {
+ return new SequentialBuildDriver(memoizingEvaluator);
+ }
+
+ @Override
+ protected void init() {
+ // Note that we need to set recordingDiffer first since SkyframeExecutor#init calls
+ // SkyframeExecutor#evaluatorDiffer.
+ recordingDiffer = new RecordingDifferencer();
+ super.init();
+ }
+
+ @Override
+ public void resetEvaluator() {
+ super.resetEvaluator();
+ diffAwarenessManager.reset();
+ }
+
+ @Override
+ protected Differencer evaluatorDiffer() {
+ return recordingDiffer;
+ }
+
+ @Override
+ protected Injectable injectable() {
+ return recordingDiffer;
+ }
+
+ @VisibleForTesting
+ public RecordingDifferencer getDifferencerForTesting() {
+ return recordingDiffer;
+ }
+
+ @Override
+ public void sync(PackageCacheOptions packageCacheOptions, Path workingDirectory,
+ String defaultsPackageContents, UUID commandId)
+ throws InterruptedException, AbruptExitException {
+ this.valueCacheEvictionLimit = packageCacheOptions.minLoadedPkgCountForCtNodeEviction;
+ super.sync(packageCacheOptions, workingDirectory, defaultsPackageContents, commandId);
+ handleDiffs();
+ }
+
+ /**
+ * The value types whose builders have direct access to the package locator, rather than accessing
+ * it via an explicit Skyframe dependency. They need to be invalidated if the package locator
+ * changes.
+ */
+ private static final Set<SkyFunctionName> PACKAGE_LOCATOR_DEPENDENT_VALUES = ImmutableSet.of(
+ SkyFunctions.FILE_STATE,
+ SkyFunctions.FILE,
+ SkyFunctions.DIRECTORY_LISTING_STATE,
+ SkyFunctions.TARGET_PATTERN,
+ SkyFunctions.WORKSPACE_FILE);
+
+ @Override
+ protected void onNewPackageLocator(PathPackageLocator oldLocator, PathPackageLocator pkgLocator) {
+ invalidate(SkyFunctionName.functionIsIn(PACKAGE_LOCATOR_DEPENDENT_VALUES));
+ }
+
+ @Override
+ protected void invalidate(Predicate<SkyKey> pred) {
+ recordingDiffer.invalidate(Iterables.filter(memoizingEvaluator.getValues().keySet(), pred));
+ }
+
+ private void invalidateDeletedPackages(Iterable<String> deletedPackages) {
+ ArrayList<SkyKey> packagesToInvalidate = Lists.newArrayList();
+ for (String deletedPackage : deletedPackages) {
+ PathFragment pathFragment = new PathFragment(deletedPackage);
+ packagesToInvalidate.add(PackageLookupValue.key(pathFragment));
+ }
+ recordingDiffer.invalidate(packagesToInvalidate);
+ }
+
+ /**
+ * Sets the packages that should be treated as deleted and ignored.
+ */
+ @Override
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public void setDeletedPackages(Iterable<String> pkgs) {
+ // Invalidate the old deletedPackages as they may exist now.
+ invalidateDeletedPackages(deletedPackages.get());
+ deletedPackages.set(ImmutableSet.copyOf(pkgs));
+ // Invalidate the new deletedPackages as we need to pretend that they don't exist now.
+ invalidateDeletedPackages(deletedPackages.get());
+ }
+
+ /**
+ * Uses diff awareness on all the package paths to invalidate changed files.
+ */
+ @VisibleForTesting
+ public void handleDiffs() throws InterruptedException {
+ if (lastAnalysisDiscarded) {
+ // Values were cleared last build, but they couldn't be deleted because they were needed for
+ // the execution phase. We can delete them now.
+ dropConfiguredTargetsNow();
+ lastAnalysisDiscarded = false;
+ }
+ modifiedFiles = 0;
+ Map<Path, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry =
+ Maps.newHashMap();
+ Set<Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet>>
+ pathEntriesWithoutDiffInformation = Sets.newHashSet();
+ for (Path pathEntry : pkgLocator.get().getPathEntries()) {
+ DiffAwarenessManager.ProcessableModifiedFileSet modifiedFileSet =
+ diffAwarenessManager.getDiff(pathEntry);
+ if (modifiedFileSet.getModifiedFileSet().treatEverythingAsModified()) {
+ pathEntriesWithoutDiffInformation.add(Pair.of(pathEntry, modifiedFileSet));
+ } else {
+ modifiedFilesByPathEntry.put(pathEntry, modifiedFileSet);
+ }
+ }
+ handleDiffsWithCompleteDiffInformation(modifiedFilesByPathEntry);
+ handleDiffsWithMissingDiffInformation(pathEntriesWithoutDiffInformation);
+ }
+
+ /**
+ * Invalidates files under path entries whose corresponding {@link DiffAwareness} gave an exact
+ * diff. Removes entries from the given map as they are processed. All of the files need to be
+ * invalidated, so the map should be empty upon completion of this function.
+ */
+ private void handleDiffsWithCompleteDiffInformation(
+ Map<Path, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry) {
+ // It's important that the below code be uninterruptible, since we already promised to
+ // invalidate these files.
+ for (Path pathEntry : ImmutableSet.copyOf(modifiedFilesByPathEntry.keySet())) {
+ DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet =
+ modifiedFilesByPathEntry.get(pathEntry);
+ ModifiedFileSet modifiedFileSet = processableModifiedFileSet.getModifiedFileSet();
+ Preconditions.checkState(!modifiedFileSet.treatEverythingAsModified(), pathEntry);
+ Iterable<SkyKey> dirtyValues = getSkyKeysPotentiallyAffected(
+ modifiedFileSet.modifiedSourceFiles(), pathEntry);
+ handleChangedFiles(new ImmutableDiff(dirtyValues, ImmutableMap.<SkyKey, SkyValue>of()));
+ processableModifiedFileSet.markProcessed();
+ }
+ }
+
+ /**
+ * Finds and invalidates changed files under path entries whose corresponding
+ * {@link DiffAwareness} said all files may have been modified.
+ */
+ private void handleDiffsWithMissingDiffInformation(
+ Set<Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet>>
+ pathEntriesWithoutDiffInformation) throws InterruptedException {
+ if (pathEntriesWithoutDiffInformation.isEmpty()) {
+ return;
+ }
+ // Before running the FilesystemValueChecker, ensure that all values marked for invalidation
+ // have actually been invalidated (recall that invalidation happens at the beginning of the
+ // next evaluate() call), because checking those is a waste of time.
+ buildDriver.evaluate(ImmutableList.<SkyKey>of(), false,
+ DEFAULT_THREAD_COUNT, reporter);
+ FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+ // We need to manually check for changes to known files. This entails finding all dirty file
+ // system values under package roots for which we don't have diff information. If at least
+ // one path entry doesn't have diff information, then we're going to have to iterate over
+ // the skyframe values at least once no matter what so we might as well do so now and avoid
+ // doing so more than once.
+ Iterable<SkyKey> filesystemSkyKeys = fsnc.getFilesystemSkyKeys();
+ // Partition by package path entry.
+ Multimap<Path, SkyKey> skyKeysByPathEntry = partitionSkyKeysByPackagePathEntry(
+ ImmutableSet.copyOf(pkgLocator.get().getPathEntries()), filesystemSkyKeys);
+ // Contains all file system values that we need to check for dirtiness.
+ List<Iterable<SkyKey>> valuesToCheckManually = Lists.newArrayList();
+ for (Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet> pair :
+ pathEntriesWithoutDiffInformation) {
+ Path pathEntry = pair.getFirst();
+ valuesToCheckManually.add(skyKeysByPathEntry.get(pathEntry));
+ }
+ Differencer.Diff diff = fsnc.getDirtyFilesystemValues(Iterables.concat(valuesToCheckManually));
+ handleChangedFiles(diff);
+ for (Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet> pair :
+ pathEntriesWithoutDiffInformation) {
+ DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet = pair.getSecond();
+ processableModifiedFileSet.markProcessed();
+ }
+ }
+
+ /**
+ * Partitions the given filesystem values based on which package path root they are under.
+ * Returns a {@link Multimap} {@code m} such that {@code m.containsEntry(k, pe)} is true for
+ * each filesystem valuekey {@code k} under a package path root {@code pe}. Note that values not
+ * under a package path root are not present in the returned {@link Multimap}; these values are
+ * unconditionally checked for changes on each incremental build.
+ */
+ private static Multimap<Path, SkyKey> partitionSkyKeysByPackagePathEntry(
+ Set<Path> pkgRoots, Iterable<SkyKey> filesystemSkyKeys) {
+ ImmutableSetMultimap.Builder<Path, SkyKey> multimapBuilder =
+ ImmutableSetMultimap.builder();
+ for (SkyKey key : filesystemSkyKeys) {
+ Preconditions.checkState(key.functionName() == SkyFunctions.FILE_STATE
+ || key.functionName() == SkyFunctions.DIRECTORY_LISTING_STATE, key);
+ Path root = ((RootedPath) key.argument()).getRoot();
+ if (pkgRoots.contains(root)) {
+ multimapBuilder.put(root, key);
+ }
+ // We don't need to worry about FileStateValues for external files because they have a
+ // dependency on the build_id and so they get invalidated each build.
+ }
+ return multimapBuilder.build();
+ }
+
+ private void handleChangedFiles(Differencer.Diff diff) {
+ recordingDiffer.invalidate(diff.changedKeysWithoutNewValues());
+ recordingDiffer.inject(diff.changedKeysWithNewValues());
+ modifiedFiles += getNumberOfModifiedFiles(diff.changedKeysWithoutNewValues());
+ modifiedFiles += getNumberOfModifiedFiles(diff.changedKeysWithNewValues().keySet());
+ incrementalBuildMonitor.accrue(diff.changedKeysWithoutNewValues());
+ incrementalBuildMonitor.accrue(diff.changedKeysWithNewValues().keySet());
+ }
+
+ private static int getNumberOfModifiedFiles(Iterable<SkyKey> modifiedValues) {
+ // We are searching only for changed files, DirectoryListingValues don't depend on
+ // child values, that's why they are invalidated separately
+ return Iterables.size(Iterables.filter(modifiedValues,
+ SkyFunctionName.functionIs(SkyFunctions.FILE_STATE)));
+ }
+
+ @Override
+ public void decideKeepIncrementalState(boolean batch, BuildView.Options viewOptions) {
+ Preconditions.checkState(!active);
+ if (viewOptions == null) {
+ // Some blaze commands don't include the view options. Don't bother with them.
+ return;
+ }
+ if (batch && viewOptions.keepGoing && viewOptions.discardAnalysisCache) {
+ Preconditions.checkState(keepGraphEdges, "May only be called once if successful");
+ keepGraphEdges = false;
+ // Graph will be recreated on next sync.
+ }
+ }
+
+ @Override
+ public boolean hasIncrementalState() {
+ // TODO(bazel-team): Combine this method with clearSkyframeRelevantCaches() once legacy
+ // execution is removed [skyframe-execution].
+ return keepGraphEdges;
+ }
+
+ @Override
+ public void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet, Path pathEntry)
+ throws InterruptedException {
+ if (lastAnalysisDiscarded) {
+ // Values were cleared last build, but they couldn't be deleted because they were needed for
+ // the execution phase. We can delete them now.
+ dropConfiguredTargetsNow();
+ lastAnalysisDiscarded = false;
+ }
+ Iterable<SkyKey> keys;
+ if (modifiedFileSet.treatEverythingAsModified()) {
+ Differencer.Diff diff =
+ new FilesystemValueChecker(memoizingEvaluator, tsgm).getDirtyFilesystemSkyKeys();
+ keys = diff.changedKeysWithoutNewValues();
+ recordingDiffer.inject(diff.changedKeysWithNewValues());
+ } else {
+ keys = getSkyKeysPotentiallyAffected(modifiedFileSet.modifiedSourceFiles(), pathEntry);
+ }
+ syscalls.set(new PerBuildSyscallCache());
+ recordingDiffer.invalidate(keys);
+ // Blaze invalidates transient errors on every build.
+ invalidateTransientErrors();
+ }
+
+ @Override
+ public void invalidateTransientErrors() {
+ checkActive();
+ recordingDiffer.invalidateTransientErrors();
+ }
+
+ @Override
+ protected void invalidateDirtyActions(Iterable<SkyKey> dirtyActionValues) {
+ recordingDiffer.invalidate(dirtyActionValues);
+ }
+
+ /**
+ * Save memory by removing references to configured targets and actions in Skyframe.
+ *
+ * <p>These values must be recreated on subsequent builds. We do not clear the top-level target
+ * values, since their configured targets are needed for the target completion middleman values.
+ *
+ * <p>The values are not deleted during this method call, because they are needed for the
+ * execution phase. Instead, their data is cleared. The next build will delete the values (and
+ * recreate them if necessary).
+ */
+ private void discardAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) {
+ lastAnalysisDiscarded = true;
+ for (Map.Entry<SkyKey, SkyValue> entry : memoizingEvaluator.getValues().entrySet()) {
+ if (!entry.getKey().functionName().equals(SkyFunctions.CONFIGURED_TARGET)) {
+ continue;
+ }
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue) entry.getValue();
+ // ctValue may be null if target was not successfully analyzed.
+ if (ctValue != null && !topLevelTargets.contains(ctValue.getConfiguredTarget())) {
+ ctValue.clear();
+ }
+ }
+ }
+
+ @Override
+ public void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) {
+ discardAnalysisCache(topLevelTargets);
+ }
+
+ @Override
+ public void dropConfiguredTargets() {
+ if (skyframeBuildView != null) {
+ skyframeBuildView.clearInvalidatedConfiguredTargets();
+ }
+ memoizingEvaluator.delete(
+ // We delete any value that can hold an action -- all subclasses of ActionLookupValue -- as
+ // well as ActionExecutionValues, since they do not depend on ActionLookupValues.
+ SkyFunctionName.functionIsIn(ImmutableSet.of(
+ SkyFunctions.CONFIGURED_TARGET,
+ SkyFunctions.ACTION_LOOKUP,
+ SkyFunctions.BUILD_INFO,
+ SkyFunctions.TARGET_COMPLETION,
+ SkyFunctions.BUILD_INFO_COLLECTION,
+ SkyFunctions.ACTION_EXECUTION))
+ );
+ }
+
+ /**
+ * Deletes all ConfiguredTarget values from the Skyframe cache.
+ *
+ * <p>After the execution of this method all invalidated and marked for deletion values
+ * (and the values depending on them) will be deleted from the cache.
+ *
+ * <p>WARNING: Note that a call to this method leaves legacy data inconsistent with Skyframe.
+ * The next build should clear the legacy caches.
+ */
+ private void dropConfiguredTargetsNow() {
+ dropConfiguredTargets();
+ // Run the invalidator to actually delete the values.
+ try {
+ progressReceiver.ignoreInvalidations = true;
+ callUninterruptibly(new Callable<Void>() {
+ @Override
+ public Void call() throws InterruptedException {
+ buildDriver.evaluate(ImmutableList.<SkyKey>of(), false,
+ ResourceUsage.getAvailableProcessors(), reporter);
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ } finally {
+ progressReceiver.ignoreInvalidations = false;
+ }
+ }
+
+ /**
+ * Returns true if the old set of Packages is a subset or superset of the new one.
+ *
+ * <p>Compares the names of packages instead of the Package objects themselves (Package doesn't
+ * yet override #equals). Since packages store their names as a String rather than a Label, it's
+ * easier to use strings here.
+ */
+ @VisibleForTesting
+ static boolean isBuildSubsetOrSupersetOfPreviousBuild(Set<PackageIdentifier> oldPackages,
+ Set<PackageIdentifier> newPackages) {
+ if (newPackages.size() <= oldPackages.size()) {
+ return Sets.difference(newPackages, oldPackages).isEmpty();
+ } else if (oldPackages.size() < newPackages.size()) {
+ // No need to check for <= here, since the first branch does that already.
+ // If size(A) = size(B), then then A\B = 0 iff B\A = 0
+ return Sets.difference(oldPackages, newPackages).isEmpty();
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void updateLoadedPackageSet(Set<PackageIdentifier> loadedPackages) {
+ Preconditions.checkState(valueCacheEvictionLimit >= 0,
+ "should have called setMinLoadedPkgCountForCtValueEviction earlier");
+
+ // Make a copy to avoid nesting SetView objects. It also computes size(), which we need below.
+ Set<PackageIdentifier> union = ImmutableSet.copyOf(
+ Sets.union(allLoadedPackages, loadedPackages));
+
+ if (union.size() < valueCacheEvictionLimit
+ || isBuildSubsetOrSupersetOfPreviousBuild(allLoadedPackages, loadedPackages)) {
+ allLoadedPackages = union;
+ } else {
+ dropConfiguredTargets();
+ allLoadedPackages = loadedPackages;
+ }
+ }
+
+ @Override
+ public void deleteOldNodes(long versionWindowForDirtyGc) {
+ // TODO(bazel-team): perhaps we should come up with a separate GC class dedicated to maintaining
+ // value garbage. If we ever do so, this logic should be moved there.
+ memoizingEvaluator.deleteDirty(versionWindowForDirtyGc);
+ }
+
+ @Override
+ public void dumpPackages(PrintStream out) {
+ Iterable<SkyKey> packageSkyKeys = Iterables.filter(memoizingEvaluator.getValues().keySet(),
+ SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE));
+ out.println(Iterables.size(packageSkyKeys) + " packages");
+ for (SkyKey packageSkyKey : packageSkyKeys) {
+ Package pkg = ((PackageValue) memoizingEvaluator.getValues().get(packageSkyKey)).getPackage();
+ pkg.dump(out);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
new file mode 100644
index 0000000..7e277a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Set;
+
+/**
+ * A factory of SkyframeExecutors that returns SequencedSkyframeExecutor.
+ */
+public class SequencedSkyframeExecutorFactory implements SkyframeExecutorFactory {
+
+ @Override
+ public SkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory, ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ return SequencedSkyframeExecutor.create(reporter, pkgFactory, tsgm, directories,
+ workspaceStatusActionFactory, buildInfoFactories, immutableDirectories,
+ diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier,
+ extraSkyFunctions, extraPrecomputedValues);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
new file mode 100644
index 0000000..316d27d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -0,0 +1,81 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Value types in Skyframe.
+ */
+public final class SkyFunctions {
+ public static final SkyFunctionName PRECOMPUTED = new SkyFunctionName("PRECOMPUTED", false);
+ public static final SkyFunctionName FILE_STATE = new SkyFunctionName("FILE_STATE", false);
+ public static final SkyFunctionName DIRECTORY_LISTING_STATE =
+ new SkyFunctionName("DIRECTORY_LISTING_STATE", false);
+ public static final SkyFunctionName FILE_SYMLINK_CYCLE_UNIQUENESS =
+ SkyFunctionName.computed("FILE_SYMLINK_CYCLE_UNIQUENESS_NODE");
+ public static final SkyFunctionName FILE = SkyFunctionName.computed("FILE");
+ public static final SkyFunctionName DIRECTORY_LISTING =
+ SkyFunctionName.computed("DIRECTORY_LISTING");
+ public static final SkyFunctionName PACKAGE_LOOKUP = SkyFunctionName.computed("PACKAGE_LOOKUP");
+ public static final SkyFunctionName CONTAINING_PACKAGE_LOOKUP =
+ SkyFunctionName.computed("CONTAINING_PACKAGE_LOOKUP");
+ public static final SkyFunctionName AST_FILE_LOOKUP = SkyFunctionName.computed("AST_FILE_LOOKUP");
+ public static final SkyFunctionName SKYLARK_IMPORTS_LOOKUP =
+ SkyFunctionName.computed("SKYLARK_IMPORTS_LOOKUP");
+ public static final SkyFunctionName GLOB = SkyFunctionName.computed("GLOB");
+ public static final SkyFunctionName PACKAGE = SkyFunctionName.computed("PACKAGE");
+ public static final SkyFunctionName TARGET_MARKER = SkyFunctionName.computed("TARGET_MARKER");
+ public static final SkyFunctionName TARGET_PATTERN = SkyFunctionName.computed("TARGET_PATTERN");
+ public static final SkyFunctionName RECURSIVE_PKG = SkyFunctionName.computed("RECURSIVE_PKG");
+ public static final SkyFunctionName TRANSITIVE_TARGET =
+ SkyFunctionName.computed("TRANSITIVE_TARGET");
+ public static final SkyFunctionName CONFIGURED_TARGET =
+ SkyFunctionName.computed("CONFIGURED_TARGET");
+ public static final SkyFunctionName ASPECT = SkyFunctionName.computed("ASPECT");
+ public static final SkyFunctionName POST_CONFIGURED_TARGET =
+ SkyFunctionName.computed("POST_CONFIGURED_TARGET");
+ public static final SkyFunctionName TARGET_COMPLETION =
+ SkyFunctionName.computed("TARGET_COMPLETION");
+ public static final SkyFunctionName TEST_COMPLETION =
+ SkyFunctionName.computed("TEST_COMPLETION");
+ public static final SkyFunctionName CONFIGURATION_FRAGMENT =
+ SkyFunctionName.computed("CONFIGURATION_FRAGMENT");
+ public static final SkyFunctionName CONFIGURATION_COLLECTION =
+ SkyFunctionName.computed("CONFIGURATION_COLLECTION");
+ public static final SkyFunctionName ARTIFACT = SkyFunctionName.computed("ARTIFACT");
+ public static final SkyFunctionName ACTION_EXECUTION =
+ SkyFunctionName.computed("ACTION_EXECUTION");
+ public static final SkyFunctionName ACTION_LOOKUP = SkyFunctionName.computed("ACTION_LOOKUP");
+ public static final SkyFunctionName RECURSIVE_FILESYSTEM_TRAVERSAL =
+ SkyFunctionName.computed("RECURSIVE_DIRECTORY_TRAVERSAL");
+ public static final SkyFunctionName FILESET_ENTRY = SkyFunctionName.computed("FILESET_ENTRY");
+ public static final SkyFunctionName BUILD_INFO_COLLECTION =
+ SkyFunctionName.computed("BUILD_INFO_COLLECTION");
+ public static final SkyFunctionName BUILD_INFO = SkyFunctionName.computed("BUILD_INFO");
+ public static final SkyFunctionName WORKSPACE_FILE = SkyFunctionName.computed("WORKSPACE_FILE");
+ public static final SkyFunctionName COVERAGE_REPORT = SkyFunctionName.computed("COVERAGE_REPORT");
+ public static final SkyFunctionName REPOSITORY = SkyFunctionName.computed("REPOSITORY");
+
+ public static Predicate<SkyKey> isSkyFunction(final SkyFunctionName functionName) {
+ return new Predicate<SkyKey>() {
+ @Override
+ public boolean apply(SkyKey key) {
+ return key.functionName() == functionName;
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
new file mode 100644
index 0000000..bd591e1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -0,0 +1,1152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.createDirectoryAndParents;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.eventbus.EventBus;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionCacheChecker.Token;
+import com.google.devtools.build.lib.actions.ActionCompletionEvent;
+import com.google.devtools.build.lib.actions.ActionExecutedEvent;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
+import com.google.devtools.build.lib.actions.ActionMiddlemanEvent;
+import com.google.devtools.build.lib.actions.ActionStartedEvent;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander;
+import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
+import com.google.devtools.build.lib.actions.CachedActionEvent;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.MapBasedActionGraph;
+import com.google.devtools.build.lib.actions.MutableActionGraph;
+import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
+import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.TargetOutOfDateException;
+import com.google.devtools.build.lib.actions.cache.Digest;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.actions.cache.Metadata;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil;
+import com.google.devtools.build.lib.concurrent.Sharder;
+import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Action executor: takes care of preparing an action for execution, executing it, validating that
+ * all output artifacts were created, error reporting, etc.
+ */
+public final class SkyframeActionExecutor {
+ private final Reporter reporter;
+ private final AtomicReference<EventBus> eventBus;
+ private final ResourceManager resourceManager;
+ private Executor executorEngine;
+ private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
+ private ActionCacheChecker actionCacheChecker;
+ private ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata = new ConcurrentHashMap<>();
+ private final Profiler profiler = Profiler.instance();
+ private boolean explain;
+
+ // We keep track of actions already executed this build in order to avoid executing a shared
+ // action twice. Note that we may still unnecessarily re-execute the action on a subsequent
+ // build: say actions A and B are shared. If A is requested on the first build and then B is
+ // requested on the second build, we will execute B even though its output files are up to date.
+ // However, we will not re-execute A on a subsequent build.
+ // We do not allow the shared action to re-execute in the same build, even after the first
+ // action has finished execution, because a downstream action might be reading the output file
+ // at the same time as the shared action was writing to it.
+ // This map is also used for Actions that try to execute twice because they have discovered
+ // headers -- the SkyFunction tries to declare a dep on the missing headers and has to restart.
+ // We don't want to execute the action again on the second entry to the SkyFunction.
+ // In both cases, we store the already-computed ActionExecutionValue to avoid having to compute it
+ // again.
+ private ConcurrentMap<Artifact, Pair<Action, FutureTask<ActionExecutionValue>>> buildActionMap;
+
+ // Errors found when examining all actions in the graph are stored here, so that they can be
+ // thrown when execution of the action is requested. This field is set during each call to
+ // findAndStoreArtifactConflicts, and is preserved across builds otherwise.
+ private ImmutableMap<Action, ConflictException> badActionMap = ImmutableMap.of();
+ private boolean keepGoing;
+ private boolean hadExecutionError;
+ private ActionInputFileCache perBuildFileCache;
+ private ProgressSupplier progressSupplier;
+ private ActionCompletedReceiver completionReceiver;
+ private final AtomicReference<ActionExecutionStatusReporter> statusReporterRef;
+
+ SkyframeActionExecutor(Reporter reporter, ResourceManager resourceManager,
+ AtomicReference<EventBus> eventBus,
+ AtomicReference<ActionExecutionStatusReporter> statusReporterRef) {
+ this.reporter = reporter;
+ this.resourceManager = resourceManager;
+ this.eventBus = eventBus;
+ this.statusReporterRef = statusReporterRef;
+ }
+
+ /**
+ * A typed union of {@link ActionConflictException}, which indicates two actions that generate
+ * the same {@link Artifact}, and {@link ArtifactPrefixConflictException}, which indicates that
+ * the path of one {@link Artifact} is a prefix of another.
+ */
+ public static class ConflictException extends Exception {
+ @Nullable private final ActionConflictException ace;
+ @Nullable private final ArtifactPrefixConflictException apce;
+
+ public ConflictException(ActionConflictException e) {
+ super(e);
+ this.ace = e;
+ this.apce = null;
+ }
+
+ public ConflictException(ArtifactPrefixConflictException e) {
+ super(e);
+ this.ace = null;
+ this.apce = e;
+ }
+
+ void rethrowTyped() throws ActionConflictException, ArtifactPrefixConflictException {
+ if (ace == null) {
+ throw Preconditions.checkNotNull(apce);
+ }
+ if (apce == null) {
+ throw Preconditions.checkNotNull(ace);
+ }
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Return the map of mostly recently executed bad actions to their corresponding exception.
+ * See {#findAndStoreArtifactConflicts()}.
+ */
+ public ImmutableMap<Action, ConflictException> badActions() {
+ // TODO(bazel-team): Move badActions() and findAndStoreArtifactConflicts() to SkyframeBuildView
+ // now that it's done in the analysis phase.
+ return badActionMap;
+ }
+
+ /**
+ * Basic implementation of {@link MetadataHandler} that delegates to Skyframe for metadata and
+ * caches missing source artifacts (which must be undeclared inputs: discovered headers) to avoid
+ * excessive filesystem access. The discovered-header cache is available across actions.
+ */
+ // TODO(bazel-team): remove when include scanning is skyframe-native.
+ private static class UndeclaredInputHandler implements MetadataHandler {
+ private final ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata;
+ private final MetadataHandler perActionHandler;
+
+ UndeclaredInputHandler(MetadataHandler perActionHandler,
+ ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata) {
+ // Shared across all UndeclaredInputHandlers in this build.
+ this.undeclaredInputsMetadata = undeclaredInputsMetadata;
+ this.perActionHandler = perActionHandler;
+ }
+
+ @Override
+ public Metadata getMetadataMaybe(Artifact artifact) {
+ try {
+ return getMetadata(artifact);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public Metadata getMetadata(Artifact artifact) throws IOException {
+ Metadata metadata = perActionHandler.getMetadata(artifact);
+ if (metadata != null) {
+ return metadata;
+ }
+ // Skyframe stats all generated artifacts, because either they are outputs of the action being
+ // executed or they are generated files already present in the graph.
+ Preconditions.checkState(artifact.isSourceArtifact(), artifact);
+ metadata = undeclaredInputsMetadata.get(artifact);
+ if (metadata != null) {
+ return metadata;
+ }
+ FileStatus stat = artifact.getPath().stat();
+ if (DigestUtils.useFileDigest(artifact, stat.isFile(), stat.getSize())) {
+ metadata = new Metadata(Preconditions.checkNotNull(
+ DigestUtils.getDigestOrFail(artifact.getPath(), stat.getSize()), artifact));
+ } else {
+ metadata = new Metadata(stat.getLastModifiedTime());
+ }
+ // Cache for other actions that may also include without declaring.
+ Metadata oldMetadata = undeclaredInputsMetadata.put(artifact, metadata);
+ FileAndMetadataCache.checkInconsistentData(artifact, oldMetadata, metadata);
+ return metadata;
+ }
+
+ @Override
+ public void setDigestForVirtualArtifact(Artifact artifact, Digest digest) {
+ perActionHandler.setDigestForVirtualArtifact(artifact, digest);
+ }
+
+ @Override
+ public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
+ perActionHandler.injectDigest(output, statNoFollow, digest);
+ }
+
+ @Override
+ public boolean artifactExists(Artifact artifact) {
+ return perActionHandler.artifactExists(artifact);
+ }
+
+ @Override
+ public boolean isRegularFile(Artifact artifact) {
+ return perActionHandler.isRegularFile(artifact);
+ }
+
+ @Override
+ public boolean isInjected(Artifact artifact) throws IOException {
+ return perActionHandler.isInjected(artifact);
+ }
+
+ @Override
+ public void discardMetadata(Collection<Artifact> artifactList) {
+ // This input handler only caches undeclared inputs, which never need to be discarded
+ // intra-build.
+ perActionHandler.discardMetadata(artifactList);
+ }
+ }
+
+ /**
+ * Find conflicts between generated artifacts. There are two ways to have conflicts. First, if
+ * two (unshareable) actions generate the same output artifact, this will result in an {@link
+ * ActionConflictException}. Second, if one action generates an artifact whose path is a prefix of
+ * another artifact's path, those two artifacts cannot exist simultaneously in the output tree.
+ * This causes an {@link ArtifactPrefixConflictException}. The relevant exceptions are stored in
+ * the executor in {@code badActionMap}, and will be thrown immediately when that action is
+ * executed. Those exceptions persist, so that even if the action is not executed this build, the
+ * first time it is executed, the correct exception will be thrown.
+ *
+ * <p>This method must be called if a new action was added to the graph this build, so
+ * whenever a new configured target was analyzed this build. It is somewhat expensive (~1s
+ * range for a medium build as of 2014), so it should only be called when necessary.
+ *
+ * <p>Conflicts found may not be requested this build, and so we may overzealously throw an error.
+ * For instance, if actions A and B generate the same artifact foo, and the user first requests
+ * A' depending on A, and then in a subsequent build B' depending on B, we will fail the second
+ * build, even though it would have succeeded if it had been the only build. However, since
+ * Skyframe does not know the transitive dependencies of the request, we err on the conservative
+ * side.
+ *
+ * <p>If the user first runs one action on the first build, and on the second build adds a
+ * conflicting action, only the second action's error may be reported (because the first action
+ * will be cached), whereas if both actions were requested for the first time, both errors would
+ * be reported. However, the first time an action is added to the build, we are guaranteed to find
+ * any conflicts it has, since this method will compare it against all other actions. So there is
+ * no sequence of builds that can evade the error.
+ */
+ void findAndStoreArtifactConflicts(Iterable<ActionLookupValue> actionLookupValues)
+ throws InterruptedException {
+ ConcurrentMap<Action, ConflictException> temporaryBadActionMap = new ConcurrentHashMap<>();
+ Pair<ActionGraph, SortedMap<PathFragment, Artifact>> result;
+ result = constructActionGraphAndPathMap(actionLookupValues, temporaryBadActionMap);
+ ActionGraph actionGraph = result.first;
+ SortedMap<PathFragment, Artifact> artifactPathMap = result.second;
+
+ // Report an error for every derived artifact which is a prefix of another.
+ // If x << y << z (where x << y means "y starts with x"), then we only report (x,y), (x,z), but
+ // not (y,z).
+ Iterator<PathFragment> iter = artifactPathMap.keySet().iterator();
+ if (!iter.hasNext()) {
+ // No actions in graph -- currently happens only in tests. Special-cased because .next() call
+ // below is unconditional.
+ this.badActionMap = ImmutableMap.of();
+ return;
+ }
+ for (PathFragment pathJ = iter.next(); iter.hasNext(); ) {
+ // For each comparison, we have a prefix candidate (pathI) and a suffix candidate (pathJ).
+ // At the beginning of the loop, we set pathI to the last suffix candidate, since it has not
+ // yet been tested as a prefix candidate, and then set pathJ to the paths coming after pathI,
+ // until we come to one that does not contain pathI as a prefix. pathI is then verified not to
+ // be the prefix of any path, so we start the next run of the loop.
+ PathFragment pathI = pathJ;
+ // Compare pathI to the paths coming after it.
+ while (iter.hasNext()) {
+ pathJ = iter.next();
+ if (pathJ.startsWith(pathI)) { // prefix conflict.
+ Artifact artifactI = Preconditions.checkNotNull(artifactPathMap.get(pathI), pathI);
+ Artifact artifactJ = Preconditions.checkNotNull(artifactPathMap.get(pathJ), pathJ);
+ Action actionI =
+ Preconditions.checkNotNull(actionGraph.getGeneratingAction(artifactI), artifactI);
+ Action actionJ =
+ Preconditions.checkNotNull(actionGraph.getGeneratingAction(artifactJ), artifactJ);
+ if (actionI.shouldReportPathPrefixConflict(actionJ)) {
+ ArtifactPrefixConflictException exception = new ArtifactPrefixConflictException(pathI,
+ pathJ, actionI.getOwner().getLabel(), actionJ.getOwner().getLabel());
+ temporaryBadActionMap.put(actionI, new ConflictException(exception));
+ temporaryBadActionMap.put(actionJ, new ConflictException(exception));
+ }
+ } else { // pathJ didn't have prefix pathI, so no conflict possible for pathI.
+ break;
+ }
+ }
+ }
+ this.badActionMap = ImmutableMap.copyOf(temporaryBadActionMap);
+ }
+
+ /**
+ * Simultaneously construct an action graph for all the actions in Skyframe and a map from
+ * {@link PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save
+ * around 1.5 seconds on a mid-sized build versus a single-threaded operation.
+ */
+ private static Pair<ActionGraph, SortedMap<PathFragment, Artifact>>
+ constructActionGraphAndPathMap(
+ Iterable<ActionLookupValue> values,
+ ConcurrentMap<Action, ConflictException> badActionMap) throws InterruptedException {
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ ConcurrentNavigableMap<PathFragment, Artifact> artifactPathMap = new ConcurrentSkipListMap<>();
+ // Action graph construction is CPU-bound.
+ int numJobs = Runtime.getRuntime().availableProcessors();
+ // No great reason for expecting 5000 action lookup values, but not worth counting size of
+ // values.
+ Sharder<ActionLookupValue> actionShards = new Sharder<>(numJobs, 5000);
+ for (ActionLookupValue value : values) {
+ actionShards.add(value);
+ }
+
+ ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper(
+ "SkyframeActionExecutor#constructActionGraphAndPathMap");
+
+ ExecutorService executor = Executors.newFixedThreadPool(
+ numJobs,
+ new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build());
+ for (List<ActionLookupValue> shard : actionShards) {
+ executor.execute(
+ wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap)));
+ }
+ boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ if (interrupted) {
+ throw new InterruptedException();
+ }
+ return Pair.<ActionGraph, SortedMap<PathFragment, Artifact>>of(actionGraph, artifactPathMap);
+ }
+
+ private static Runnable actionRegistration(
+ final List<ActionLookupValue> values,
+ final MutableActionGraph actionGraph,
+ final ConcurrentMap<PathFragment, Artifact> artifactPathMap,
+ final ConcurrentMap<Action, ConflictException> badActionMap) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ for (ActionLookupValue value : values) {
+ Set<Action> registeredActions = new HashSet<>();
+ for (Map.Entry<Artifact, Action> entry : value.getMapForConsistencyCheck().entrySet()) {
+ Action action = entry.getValue();
+ // We have an entry for each <action, artifact> pair. Only try to register each action
+ // once.
+ if (registeredActions.add(action)) {
+ try {
+ actionGraph.registerAction(action);
+ } catch (ActionConflictException e) {
+ Exception oldException = badActionMap.put(action, new ConflictException(e));
+ Preconditions.checkState(oldException == null,
+ "%s | %s | %s", action, e, oldException);
+ // We skip the rest of the loop, and do not add the path->artifact mapping for this
+ // artifact below -- we don't need to check it since this action is already in
+ // error.
+ continue;
+ }
+ }
+ artifactPathMap.put(entry.getKey().getExecPath(), entry.getKey());
+ }
+ }
+ }
+ };
+ }
+
+ void prepareForExecution(Executor executor, boolean keepGoing,
+ boolean explain, ActionCacheChecker actionCacheChecker) {
+ this.executorEngine = Preconditions.checkNotNull(executor);
+
+ // Start with a new map each build so there's no issue with internal resizing.
+ this.buildActionMap = Maps.newConcurrentMap();
+ this.keepGoing = keepGoing;
+ this.hadExecutionError = false;
+ this.actionCacheChecker = Preconditions.checkNotNull(actionCacheChecker);
+ // Don't cache possibly stale data from the last build.
+ undeclaredInputsMetadata = new ConcurrentHashMap<>();
+ this.explain = explain;
+ }
+
+ public void setActionLogBufferPathGenerator(
+ ActionLogBufferPathGenerator actionLogBufferPathGenerator) {
+ this.actionLogBufferPathGenerator = actionLogBufferPathGenerator;
+ }
+
+ void executionOver() {
+ // This transitively holds a bunch of heavy objects, so it's important to clear it at the
+ // end of a build.
+ this.executorEngine = null;
+ }
+
+ File getExecRoot() {
+ return executorEngine.getExecRoot().getPathFile();
+ }
+
+ boolean probeActionExecution(Action action) {
+ return buildActionMap.containsKey(action.getPrimaryOutput());
+ }
+
+ /**
+ * Executes the provided action on the current thread. Returns the ActionExecutionValue with the
+ * result, either computed here or already computed on another thread.
+ *
+ * <p>For use from {@link ArtifactFunction} only.
+ */
+ ActionExecutionValue executeAction(Action action, FileAndMetadataCache graphFileCache,
+ Token token, long actionStartTime,
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Exception exception = badActionMap.get(action);
+ if (exception != null) {
+ // If action had a conflict with some other action in the graph, report it now.
+ reportError(exception.getMessage(), exception, action, null);
+ }
+ Artifact primaryOutput = action.getPrimaryOutput();
+ FutureTask<ActionExecutionValue> actionTask =
+ new FutureTask<>(new ActionRunner(action, graphFileCache, token,
+ actionStartTime, actionExecutionContext));
+ // Check to see if another action is already executing/has executed this value.
+ Pair<Action, FutureTask<ActionExecutionValue>> oldAction =
+ buildActionMap.putIfAbsent(primaryOutput, Pair.of(action, actionTask));
+
+ if (oldAction == null) {
+ actionTask.run();
+ } else if (action == oldAction.first) {
+ // We only allow the same action to be executed twice if it discovers inputs. We allow that
+ // because we need to declare additional dependencies on those new inputs.
+ Preconditions.checkState(action.discoversInputs(),
+ "Same action shouldn't execute twice in build: %s", action);
+ actionTask = oldAction.second;
+ } else {
+ Preconditions.checkState(Actions.canBeShared(oldAction.first, action),
+ "Actions cannot be shared: %s %s", oldAction.first, action);
+ // Wait for other action to finish, so any actions that depend on its outputs can execute.
+ actionTask = oldAction.second;
+ }
+ try {
+ return actionTask.get();
+ } catch (ExecutionException e) {
+ Throwables.propagateIfPossible(e.getCause(),
+ ActionExecutionException.class, InterruptedException.class);
+ throw new IllegalStateException(e);
+ } finally {
+ String message = action.getProgressMessage();
+ if (message != null) {
+ // Tell the receiver that the action has completed *before* telling the reporter.
+ // This way the latter will correctly show the number of completed actions when task
+ // completion messages are enabled (--show_task_finish).
+ if (completionReceiver != null) {
+ completionReceiver.actionCompleted(action);
+ }
+ reporter.finishTask(null, prependExecPhaseStats(message));
+ }
+ }
+ }
+
+ /**
+ * Returns an ActionExecutionContext suitable for executing a particular action. The caller should
+ * pass the returned context to {@link #executeAction}, and any other method that needs to execute
+ * tasks related to that action.
+ */
+ ActionExecutionContext constructActionExecutionContext(final FileAndMetadataCache graphFileCache,
+ MetadataHandler metadataHandler) {
+ FileOutErr fileOutErr = actionLogBufferPathGenerator.generate();
+ return new ActionExecutionContext(
+ executorEngine,
+ new DelegatingPairFileCache(graphFileCache, perBuildFileCache),
+ metadataHandler,
+ fileOutErr,
+ new MiddlemanExpander() {
+ @Override
+ public void expand(Artifact middlemanArtifact,
+ Collection<? super Artifact> output) {
+ // Legacy code is more permissive regarding "mm" in that it expands any middleman,
+ // not just inputs of this action. Skyframe doesn't have access to a global action
+ // graph, therefore this implementation can't expand any middleman, only the
+ // inputs of this action.
+ // This is fine though: actions should only hold references to their input
+ // artifacts, otherwise hermeticity would be violated.
+ output.addAll(graphFileCache.expandInputMiddleman(middlemanArtifact));
+ }
+ });
+ }
+
+ /**
+ * Returns a MetadataHandler for use when executing a particular action. The caller can pass the
+ * returned handler in whenever a MetadataHandler is needed in the course of executing the action.
+ */
+ MetadataHandler constructMetadataHandler(MetadataHandler graphFileCache) {
+ return new UndeclaredInputHandler(graphFileCache, undeclaredInputsMetadata);
+ }
+
+ /**
+ * Checks the action cache to see if {@code action} needs to be executed, or is up to date.
+ * Returns a token with the semantics of {@link ActionCacheChecker#getTokenIfNeedToExecute}: null
+ * if the action is up to date, and non-null if it needs to be executed, in which case that token
+ * should be provided to the ActionCacheChecker after execution.
+ */
+ Token checkActionCache(Action action, MetadataHandler metadataHandler, long actionStartTime) {
+ profiler.startTask(ProfilerTask.ACTION_CHECK, action);
+ Token token = actionCacheChecker.getTokenIfNeedToExecute(
+ action, explain ? reporter : null, metadataHandler);
+ profiler.completeTask(ProfilerTask.ACTION_CHECK);
+ if (token == null) {
+ boolean eventPosted = false;
+ // Notify BlazeRuntimeStatistics about the action middleman 'execution'.
+ if (action.getActionType().isMiddleman()) {
+ postEvent(new ActionMiddlemanEvent(action, actionStartTime));
+ eventPosted = true;
+ }
+
+ if (action instanceof NotifyOnActionCacheHit) {
+ NotifyOnActionCacheHit notify = (NotifyOnActionCacheHit) action;
+ notify.actionCacheHit(executorEngine);
+ }
+
+ // We still need to check the outputs so that output file data is available to the value.
+ checkOutputs(action, metadataHandler);
+ if (!eventPosted) {
+ postEvent(new CachedActionEvent(action, actionStartTime));
+ }
+ }
+ return token;
+ }
+
+ /**
+ * Perform dependency discovery for action, which must discover its inputs.
+ *
+ * <p>This method is just a wrapper around {@link Action#discoverInputs} that properly processes
+ * any ActionExecutionException thrown before rethrowing it to the caller.
+ */
+ void discoverInputs(Action action, ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ try {
+ action.discoverInputs(actionExecutionContext);
+ } catch (ActionExecutionException e) {
+ processAndThrow(e, action, actionExecutionContext.getFileOutErr());
+ }
+ }
+
+ /**
+ * This method should be called if the builder encounters an error during
+ * execution. This allows the builder to record that it encountered at
+ * least one error, and may make it swallow its output to prevent
+ * spamming the user any further.
+ */
+ private void recordExecutionError() {
+ hadExecutionError = true;
+ }
+
+ /**
+ * Returns true if the Builder is winding down (i.e. cancelling outstanding
+ * actions and preparing to abort.)
+ * The builder is winding down iff:
+ * <ul>
+ * <li>we had an execution error
+ * <li>we are not running with --keep_going
+ * </ul>
+ */
+ private boolean isBuilderAborting() {
+ return hadExecutionError && !keepGoing;
+ }
+
+ void setFileCache(ActionInputFileCache fileCache) {
+ this.perBuildFileCache = fileCache;
+ }
+
+ private class ActionRunner implements Callable<ActionExecutionValue> {
+ private final Action action;
+ private final FileAndMetadataCache graphFileCache;
+ private Token token;
+ private long actionStartTime;
+ private ActionExecutionContext actionExecutionContext;
+
+ ActionRunner(Action action, FileAndMetadataCache graphFileCache, Token token,
+ long actionStartTime,
+ ActionExecutionContext actionExecutionContext) {
+ this.action = action;
+ this.graphFileCache = graphFileCache;
+ this.token = token;
+ this.actionStartTime = actionStartTime;
+ this.actionExecutionContext = actionExecutionContext;
+ }
+
+ @Override
+ public ActionExecutionValue call() throws ActionExecutionException, InterruptedException {
+ profiler.startTask(ProfilerTask.ACTION, action);
+ try {
+ if (actionCacheChecker.isActionExecutionProhibited(action)) {
+ // We can't execute an action (e.g. because --check_???_up_to_date option was used). Fail
+ // the build instead.
+ synchronized (reporter) {
+ TargetOutOfDateException e = new TargetOutOfDateException(action);
+ reporter.handle(Event.error(e.getMessage()));
+ recordExecutionError();
+ throw e;
+ }
+ }
+
+ String message = action.getProgressMessage();
+ if (message != null) {
+ reporter.startTask(null, prependExecPhaseStats(message));
+ }
+ statusReporterRef.get().setPreparing(action);
+
+ createOutputDirectories(action);
+
+ prepareScheduleExecuteAndCompleteAction(action, token,
+ actionExecutionContext, actionStartTime);
+ return new ActionExecutionValue(
+ graphFileCache.getOutputData(), graphFileCache.getAdditionalOutputData());
+ } finally {
+ profiler.completeTask(ProfilerTask.ACTION);
+ }
+ }
+ }
+
+ private void createOutputDirectories(Action action) throws ActionExecutionException {
+ try {
+ Set<Path> done = new HashSet<>(); // avoid redundant calls for the same directory.
+ for (Artifact outputFile : action.getOutputs()) {
+ Path outputDir = outputFile.getPath().getParentDirectory();
+ if (done.add(outputDir)) {
+ try {
+ createDirectoryAndParents(outputDir);
+ continue;
+ } catch (IOException e) {
+ /* Fall through to plan B. */
+ }
+
+ // Possibly some direct ancestors are not directories. In that case, we unlink all the
+ // ancestors until we reach a directory, then try again. This handles the case where a
+ // file becomes a directory, either from one build to another, or within a single build.
+ //
+ // Symlinks should not be followed so in order to clean up symlinks pointing to Fileset
+ // outputs from previous builds. See bug [incremental build of Fileset fails if
+ // Fileset.out was changed to be a subdirectory of the old value].
+ try {
+ for (Path p = outputDir; !p.isDirectory(Symlinks.NOFOLLOW);
+ p = p.getParentDirectory()) {
+ // p may be a file or dangling symlink, or a symlink to an old Fileset output
+ p.delete(); // throws IOException
+ }
+ createDirectoryAndParents(outputDir);
+ } catch (IOException e) {
+ throw new ActionExecutionException(
+ "failed to create output directory '" + outputDir + "'", e, action, false);
+ }
+ }
+ }
+ } catch (ActionExecutionException ex) {
+ printError(ex.getMessage(), action, null);
+ throw ex;
+ }
+ }
+
+ private String prependExecPhaseStats(String message) {
+ if (progressSupplier != null) {
+ // Prints a progress message like:
+ // [2608/6445] Compiling foo/bar.cc [host]
+ return progressSupplier.getProgressString() + " " + message;
+ } else {
+ // progressSupplier may be null in tests
+ return message;
+ }
+ }
+
+ /**
+ * Prepare, schedule, execute, and then complete the action.
+ * When this function is called, we know that this action needs to be executed.
+ * This function will prepare for the action's execution (i.e. delete the outputs);
+ * schedule its execution; execute the action;
+ * and then do some post-execution processing to complete the action:
+ * set the outputs readonly and executable, and insert the action results in the
+ * action cache.
+ *
+ * @param action The action to execute
+ * @param token The non-null token returned by dependencyChecker.getTokenIfNeedToExecute()
+ * @param context services in the scope of the action
+ * @param actionStartTime time when we started the first phase of the action execution.
+ * @throws ActionExecutionException if the execution of the specified action
+ * failed for any reason.
+ * @throws InterruptedException if the thread was interrupted.
+ */
+ private void prepareScheduleExecuteAndCompleteAction(Action action, Token token,
+ ActionExecutionContext context, long actionStartTime)
+ throws ActionExecutionException, InterruptedException {
+ Preconditions.checkNotNull(token, action);
+ // Delete the metadataHandler's cache of the action's outputs, since they are being deleted.
+ context.getMetadataHandler().discardMetadata(action.getOutputs());
+ // Delete the outputs before executing the action, just to ensure that
+ // the action really does produce the outputs.
+ try {
+ action.prepare(context.getExecutor().getExecRoot());
+ } catch (IOException e) {
+ reportError("failed to delete output files before executing action", e, action, null);
+ }
+
+ postEvent(new ActionStartedEvent(action, actionStartTime));
+ ResourceSet estimate = action.estimateResourceConsumption(executorEngine);
+ ActionExecutionStatusReporter statusReporter = statusReporterRef.get();
+ try {
+ if (estimate == null || estimate == ResourceSet.ZERO) {
+ statusReporter.setRunningFromBuildData(action);
+ } else {
+ // If estimated resource consumption is null, action will manually call
+ // resource manager when it knows what resources are needed.
+ resourceManager.acquireResources(action, estimate);
+ }
+ boolean outputDumped = executeActionTask(action, context);
+ completeAction(action, token, context.getMetadataHandler(),
+ context.getFileOutErr(), outputDumped);
+ } finally {
+ if (estimate != null) {
+ resourceManager.releaseResources(action, estimate);
+ }
+ statusReporter.remove(action);
+ postEvent(new ActionCompletionEvent(action));
+ }
+ }
+
+ private ActionExecutionException processAndThrow(
+ ActionExecutionException e, Action action, FileOutErr outErrBuffer)
+ throws ActionExecutionException {
+ reportActionExecution(action, e, outErrBuffer);
+ boolean reported = reportErrorIfNotAbortingMode(e, outErrBuffer);
+
+ ActionExecutionException toThrow = e;
+ if (reported){
+ // If we already printed the error for the exception we mark it as already reported
+ // so that we do not print it again in upper levels.
+ // Note that we need to report it here since we want immediate feedback of the errors
+ // and in some cases the upper-level printing mechanism only prints one of the errors.
+ toThrow = new AlreadyReportedActionExecutionException(e);
+ }
+
+ // Now, rethrow the exception.
+ // This can have two effects:
+ // If we're still building, the exception will get retrieved by the
+ // completor and rethrown.
+ // If we're aborting, the exception will never be retrieved from the
+ // completor, since the completor is waiting for all outstanding jobs
+ // to finish. After they have finished, it will only rethrow the
+ // exception that initially caused it to abort will and not check the
+ // exit status of any actions that had finished in the meantime.
+ throw toThrow;
+ }
+
+ /**
+ * Execute the specified action, in a profiler task.
+ * The caller is responsible for having already checked that we need to
+ * execute it and for acquiring/releasing any scheduling locks needed.
+ *
+ * <p>This is thread-safe so long as you don't try to execute the same action
+ * twice at the same time (or overlapping times).
+ * May execute in a worker thread.
+ *
+ * @throws ActionExecutionException if the execution of the specified action
+ * failed for any reason.
+ * @throws InterruptedException if the thread was interrupted.
+ * @return true if the action output was dumped, false otherwise.
+ */
+ private boolean executeActionTask(Action action, ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ profiler.startTask(ProfilerTask.ACTION_EXECUTE, action);
+ // ActionExecutionExceptions that occur as the thread is interrupted are
+ // assumed to be a result of that, so we throw InterruptedException
+ // instead.
+ FileOutErr outErrBuffer = actionExecutionContext.getFileOutErr();
+ try {
+ action.execute(actionExecutionContext);
+
+ // Action terminated fine, now report the output.
+ // The .showOutput() method is not necessarily a quick check: in its
+ // current implementation it uses regular expression matching.
+ if (outErrBuffer.hasRecordedOutput()
+ && (action.showsOutputUnconditionally()
+ || reporter.showOutput(Label.print(action.getOwner().getLabel())))) {
+ dumpRecordedOutErr(action, outErrBuffer);
+ return true;
+ }
+ // Defer reporting action success until outputs are checked
+ } catch (ActionExecutionException e) {
+ processAndThrow(e, action, outErrBuffer);
+ } finally {
+ profiler.completeTask(ProfilerTask.ACTION_EXECUTE);
+ }
+ return false;
+ }
+
+ private void completeAction(Action action, Token token, MetadataHandler metadataHandler,
+ FileOutErr fileOutErr, boolean outputAlreadyDumped) throws ActionExecutionException {
+ try {
+ Preconditions.checkState(action.inputsKnown(),
+ "Action %s successfully executed, but inputs still not known", action);
+
+ profiler.startTask(ProfilerTask.ACTION_COMPLETE, action);
+ try {
+ if (!checkOutputs(action, metadataHandler)) {
+ reportError("not all outputs were created", null, action,
+ outputAlreadyDumped ? null : fileOutErr);
+ }
+ // Prevent accidental stomping on files.
+ // This will also throw a FileNotFoundException
+ // if any of the output files doesn't exist.
+ try {
+ setOutputsReadOnlyAndExecutable(action, metadataHandler);
+ } catch (IOException e) {
+ reportError("failed to set outputs read-only", e, action, null);
+ }
+ try {
+ actionCacheChecker.afterExecution(action, token, metadataHandler);
+ } catch (IOException e) {
+ // Skyframe does all the filesystem access needed during the previous calls, and if those
+ // calls failed, we should already have thrown. So an IOException is impossible here.
+ throw new IllegalStateException(
+ "failed to update action cache for " + action.prettyPrint()
+ + ", but all outputs should already have been checked", e);
+ }
+ } finally {
+ profiler.completeTask(ProfilerTask.ACTION_COMPLETE);
+ }
+ reportActionExecution(action, null, fileOutErr);
+ } catch (ActionExecutionException actionException) {
+ // Success in execution but failure in completion.
+ reportActionExecution(action, actionException, fileOutErr);
+ throw actionException;
+ } catch (IllegalStateException exception) {
+ // More serious internal error, but failure still reported.
+ reportActionExecution(action,
+ new ActionExecutionException(exception, action, true), fileOutErr);
+ throw exception;
+ }
+ }
+
+ /**
+ * For each of the action's outputs that is a regular file (not a symbolic
+ * link or directory), make it read-only and executable.
+ *
+ * <p>Making the outputs read-only helps preventing accidental editing of
+ * them (e.g. in case of generated source code), while making them executable
+ * helps running generated files (such as generated shell scripts) on the
+ * command line.
+ *
+ * <p>May execute in a worker thread.
+ *
+ * <p>Note: setting these bits maintains transparency regarding the locality of the build;
+ * because the remote execution engine sets them, they should be set for local builds too.
+ *
+ * @throws IOException if an I/O error occurred.
+ */
+ private final void setOutputsReadOnlyAndExecutable(Action action, MetadataHandler metadataHandler)
+ throws IOException {
+ Preconditions.checkState(!action.getActionType().isMiddleman());
+
+ for (Artifact output : action.getOutputs()) {
+ Path path = output.getPath();
+ if (metadataHandler.isInjected(output)) {
+ // We trust the files created by the execution-engine to be non symlinks with expected
+ // chmod() settings already applied. The follow stanza implies a total of 6 system calls,
+ // since the UnixFileSystem implementation of setWritable() and setExecutable() both
+ // do a stat() internally.
+ continue;
+ }
+ if (path.isFile(Symlinks.NOFOLLOW)) { // i.e. regular files only.
+ path.setWritable(false);
+ path.setExecutable(true);
+ }
+ }
+ }
+
+ private void reportMissingOutputFile(Action action, Artifact output, Reporter reporter,
+ boolean isSymlink) {
+ boolean genrule = action.getMnemonic().equals("Genrule");
+ String prefix = (genrule ? "declared output '" : "output '") + output.prettyPrint() + "' ";
+ if (isSymlink) {
+ reporter.handle(Event.error(
+ action.getOwner().getLocation(), prefix + "is a dangling symbolic link"));
+ } else {
+ String suffix = genrule ? " by genrule. This is probably "
+ + "because the genrule actually didn't create this output, or because the output was a "
+ + "directory and the genrule was run remotely (note that only the contents of "
+ + "declared file outputs are copied from genrules run remotely)" : "";
+ reporter.handle(Event.error(
+ action.getOwner().getLocation(), prefix + "was not created" + suffix));
+ }
+ }
+
+ /**
+ * Validates that all action outputs were created.
+ *
+ * @return false if some outputs are missing, true - otherwise.
+ */
+ private boolean checkOutputs(Action action, MetadataHandler metadataHandler) {
+ boolean success = true;
+ for (Artifact output : action.getOutputs()) {
+ if (!metadataHandler.artifactExists(output)) {
+ reportMissingOutputFile(action, output, reporter, output.getPath().isSymbolicLink());
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ private void postEvent(Object event) {
+ EventBus bus = eventBus.get();
+ if (bus != null) {
+ bus.post(event);
+ }
+ }
+
+ /**
+ * Convenience function for reporting that the action failed due to a
+ * the exception cause, if there is an additional explanatory message that
+ * clarifies the message of the exception. Combines the user-provided message
+ * and the exceptions' message and reports the combination as error.
+ * Then, throws an ActionExecutionException with the reported error as
+ * message and the provided exception as the cause.
+ *
+ * @param message A small text that explains why the action failed
+ * @param cause The exception that caused the action to fail
+ * @param action The action that failed
+ * @param actionOutput The output of the failed Action.
+ * May be null, if there is no output to display
+ */
+ private void reportError(String message, Throwable cause, Action action, FileOutErr actionOutput)
+ throws ActionExecutionException {
+ ActionExecutionException ex;
+ if (cause == null) {
+ ex = new ActionExecutionException(message, action, false);
+ } else {
+ ex = new ActionExecutionException(message, cause, action, false);
+ }
+ printError(ex.getMessage(), action, actionOutput);
+ throw ex;
+ }
+
+ /**
+ * For the action 'action' that failed due to 'ex' with the output
+ * 'actionOutput', notify the user about the error. To notify the user, the
+ * method first displays the output of the action and then reports an error
+ * via the reporter. The method ensures that the two messages appear next to
+ * each other by locking the outErr object where the output is displayed.
+ *
+ * @param message The reason why the action failed
+ * @param action The action that failed, must not be null.
+ * @param actionOutput The output of the failed Action.
+ * May be null, if there is no output to display
+ */
+ private void printError(String message, Action action, FileOutErr actionOutput) {
+ synchronized (reporter) {
+ if (actionOutput != null && actionOutput.hasRecordedOutput()) {
+ dumpRecordedOutErr(action, actionOutput);
+ }
+ if (keepGoing) {
+ message = "Couldn't " + describeAction(action) + ": " + message;
+ }
+ reporter.handle(Event.error(action.getOwner().getLocation(), message));
+ recordExecutionError();
+ }
+ }
+
+ /** Describe an action, for use in error messages. */
+ private static String describeAction(Action action) {
+ if (action.getOutputs().isEmpty()) {
+ return "run " + action.prettyPrint();
+ } else if (action.getActionType().isMiddleman()) {
+ return "build " + action.prettyPrint();
+ } else {
+ return "build file " + action.getPrimaryOutput().prettyPrint();
+ }
+ }
+
+ /**
+ * Dump the output from the action.
+ *
+ * @param action The action whose output is being dumped
+ * @param outErrBuffer The OutErr that recorded the actions output
+ */
+ private void dumpRecordedOutErr(Action action, FileOutErr outErrBuffer) {
+ StringBuilder message = new StringBuilder("");
+ message.append("From ");
+ message.append(action.describe());
+ message.append(":");
+
+ // Synchronize this on the reporter, so that the output from multiple
+ // actions will not be interleaved.
+ synchronized (reporter) {
+ // Only print the output if we're not winding down.
+ if (isBuilderAborting()) {
+ return;
+ }
+ reporter.handle(Event.info(message.toString()));
+
+ OutErr outErr = this.reporter.getOutErr();
+ outErrBuffer.dumpOutAsLatin1(outErr.getOutputStream());
+ outErrBuffer.dumpErrAsLatin1(outErr.getErrorStream());
+ }
+ }
+
+ private void reportActionExecution(Action action,
+ ActionExecutionException exception, FileOutErr outErr) {
+ String stdout = null;
+ String stderr = null;
+
+ if (outErr.hasRecordedStdout()) {
+ stdout = outErr.getOutputFile().toString();
+ }
+ if (outErr.hasRecordedStderr()) {
+ stderr = outErr.getErrorFile().toString();
+ }
+ postEvent(new ActionExecutedEvent(action, exception, stdout, stderr));
+ }
+
+ /**
+ * Returns true if the exception was reported. False otherwise. Currently this is a copy of what
+ * we did in pre-Skyframe execution. The main implication is that we are printing the error to the
+ * top level reporter instead of the action reporter. Because of that Skyframe values do not know
+ * about the errors happening in the execution phase. Even if we change in the future to log to
+ * the action reporter (that would be done in ActionExecutionFunction.compute() when we get an
+ * ActionExecutionException), we probably do not want to also store the StdErr output, so
+ * dumpRecordedOutErr() should still be called here.
+ */
+ private boolean reportErrorIfNotAbortingMode(ActionExecutionException ex,
+ FileOutErr outErrBuffer) {
+ // For some actions (e.g. many local actions) the pollInterruptedStatus()
+ // won't notice that we had an interrupted job. It will continue.
+ // For that reason we must take care to NOT report errors if we're
+ // in the 'aborting' mode: Any cancelled action would show up here.
+ // For some actions (e.g. many local actions) the pollInterruptedStatus()
+ // won't notice that we had an interrupted job. It will continue.
+ // For that reason we must take care to NOT report errors if we're
+ // in the 'aborting' mode: Any cancelled action would show up here.
+ synchronized (this.reporter) {
+ if (!isBuilderAborting()) {
+ // Oops. The action aborted. Report the problem.
+ printError(ex.getMessage(), ex.getAction(), outErrBuffer);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** An object supplying data for action execution progress reporting. */
+ public interface ProgressSupplier {
+ /** Returns the progress string to prefix action execution messages with. */
+ String getProgressString();
+ }
+
+ /** An object that can be notified about action completion. */
+ public interface ActionCompletedReceiver {
+ /** Receives a completed action. */
+ void actionCompleted(Action action);
+ }
+
+ public void setActionExecutionProgressReportingObjects(
+ @Nullable ProgressSupplier progressSupplier,
+ @Nullable ActionCompletedReceiver completionReceiver) {
+ this.progressSupplier = progressSupplier;
+ this.completionReceiver = completionReceiver;
+ }
+
+ private static class DelegatingPairFileCache implements ActionInputFileCache {
+ private final ActionInputFileCache perActionCache;
+ private final ActionInputFileCache perBuildFileCache;
+
+ private DelegatingPairFileCache(ActionInputFileCache mainCache,
+ ActionInputFileCache perBuildFileCache) {
+ this.perActionCache = mainCache;
+ this.perBuildFileCache = perBuildFileCache;
+ }
+
+ @Override
+ public ByteString getDigest(ActionInput actionInput) throws IOException {
+ ByteString digest = perActionCache.getDigest(actionInput);
+ return digest != null ? digest : perBuildFileCache.getDigest(actionInput);
+ }
+
+ @Override
+ public long getSizeInBytes(ActionInput actionInput) throws IOException {
+ long size = perActionCache.getSizeInBytes(actionInput);
+ return size > -1 ? size : perBuildFileCache.getSizeInBytes(actionInput);
+ }
+
+ @Override
+ public boolean contentsAvailableLocally(ByteString digest) {
+ return perActionCache.contentsAvailableLocally(digest)
+ || perBuildFileCache.contentsAvailableLocally(digest);
+ }
+
+ @Nullable
+ @Override
+ public File getFileFromDigest(ByteString digest) throws IOException {
+ File file = perActionCache.getFileFromDigest(digest);
+ return file != null ? file : perBuildFileCache.getFileFromDigest(digest);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java
new file mode 100644
index 0000000..857c231
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java
@@ -0,0 +1,510 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
+import com.google.devtools.build.lib.actions.MutableActionGraph;
+import com.google.devtools.build.lib.analysis.AnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.AnalysisFailureEvent;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.ConfiguredTargetFactory;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey;
+import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig;
+import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.ConfiguredValueCreationException;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Skyframe-based driver of analysis.
+ *
+ * <p>Covers enough functionality to work as a substitute for {@code BuildView#configureTargets}.
+ */
+public final class SkyframeBuildView {
+
+ private final ConfiguredTargetFactory factory;
+ private final ArtifactFactory artifactFactory;
+ @Nullable private EventHandler warningListener;
+ private final SkyframeExecutor skyframeExecutor;
+ private final Runnable legacyDataCleaner;
+ private final BinTools binTools;
+ private boolean enableAnalysis = false;
+
+ // This hack allows us to see when a configured target has been invalidated, and thus when the set
+ // of artifact conflicts needs to be recomputed (whenever a configured target has been invalidated
+ // or newly evaluated).
+ private final EvaluationProgressReceiver invalidationReceiver =
+ new ConfiguredTargetValueInvalidationReceiver();
+ private final Set<SkyKey> evaluatedConfiguredTargets = Sets.newConcurrentHashSet();
+ // Used to see if checks of graph consistency need to be done after analysis.
+ private volatile boolean someConfiguredTargetEvaluated = false;
+
+ // We keep the set of invalidated configuration targets so that we can know if something
+ // has been invalidated after graph pruning has been executed.
+ private Set<ConfiguredTargetValue> dirtyConfiguredTargets = Sets.newConcurrentHashSet();
+ private volatile boolean anyConfiguredTargetDeleted = false;
+ private SkyKey configurationKey = null;
+
+ public SkyframeBuildView(ConfiguredTargetFactory factory,
+ ArtifactFactory artifactFactory,
+ SkyframeExecutor skyframeExecutor, Runnable legacyDataCleaner, BinTools binTools) {
+ this.factory = factory;
+ this.artifactFactory = artifactFactory;
+ this.skyframeExecutor = skyframeExecutor;
+ this.legacyDataCleaner = legacyDataCleaner;
+ this.binTools = binTools;
+ skyframeExecutor.setArtifactFactoryAndBinTools(artifactFactory, binTools);
+ }
+
+ public void setWarningListener(@Nullable EventHandler warningListener) {
+ this.warningListener = warningListener;
+ }
+
+ public void setConfigurationSkyKey(SkyKey skyKey) {
+ this.configurationKey = skyKey;
+ }
+
+ public void resetEvaluatedConfiguredTargetKeysSet() {
+ evaluatedConfiguredTargets.clear();
+ }
+
+ public Set<SkyKey> getEvaluatedTargetKeys() {
+ return ImmutableSet.copyOf(evaluatedConfiguredTargets);
+ }
+
+ private void setDeserializedArtifactOwners() throws ViewCreationFailedException {
+ Map<PathFragment, Artifact> deserializedArtifactMap =
+ artifactFactory.getDeserializedArtifacts();
+ Set<Artifact> deserializedArtifacts = new HashSet<>();
+ for (Artifact artifact : deserializedArtifactMap.values()) {
+ if (!artifact.getExecPath().getBaseName().endsWith(".gcda")) {
+ // gcda files are classified as generated artifacts, but are not actually generated. All
+ // others need owners.
+ deserializedArtifacts.add(artifact);
+ }
+ }
+ if (deserializedArtifacts.isEmpty()) {
+ // If there are no deserialized artifacts to process, don't pay the price of iterating over
+ // the graph.
+ return;
+ }
+ for (Map.Entry<SkyKey, ActionLookupValue> entry :
+ skyframeExecutor.getActionLookupValueMap().entrySet()) {
+ for (Action action : entry.getValue().getActionsForFindingArtifactOwners()) {
+ for (Artifact output : action.getOutputs()) {
+ Artifact deserializedArtifact = deserializedArtifactMap.get(output.getExecPath());
+ if (deserializedArtifact != null) {
+ deserializedArtifact.setArtifactOwner((ActionLookupKey) entry.getKey().argument());
+ deserializedArtifacts.remove(deserializedArtifact);
+ }
+ }
+ }
+ }
+ if (!deserializedArtifacts.isEmpty()) {
+ throw new ViewCreationFailedException("These artifacts were read in from the FDO profile but"
+ + " have no generating action that could be found. If you are confident that your profile was"
+ + " collected from the same source state at which you're building, please report this:\n"
+ + Artifact.asExecPaths(deserializedArtifacts));
+ }
+ artifactFactory.clearDeserializedArtifacts();
+ }
+
+ /**
+ * Analyzes the specified targets using Skyframe as the driving framework.
+ *
+ * @return the configured targets that should be built
+ */
+ public Collection<ConfiguredTarget> configureTargets(List<ConfiguredTargetKey> values,
+ EventBus eventBus, boolean keepGoing)
+ throws InterruptedException, ViewCreationFailedException {
+ enableAnalysis(true);
+ EvaluationResult<ConfiguredTargetValue> result;
+ try {
+ result = skyframeExecutor.configureTargets(values, keepGoing);
+ } finally {
+ enableAnalysis(false);
+ }
+ // For Skyframe m1, note that we already reported action conflicts during action registration
+ // in the legacy action graph.
+ ImmutableMap<Action, ConflictException> badActions = skyframeExecutor.findArtifactConflicts();
+
+ // Filter out all CTs that have a bad action and convert to a list of configured targets. This
+ // code ensures that the resulting list of configured targets has the same order as the incoming
+ // list of values, i.e., that the order is deterministic.
+ Collection<ConfiguredTarget> goodCts = Lists.newArrayListWithCapacity(values.size());
+ for (ConfiguredTargetKey value : values) {
+ ConfiguredTargetValue ctValue = result.get(ConfiguredTargetValue.key(value));
+ if (ctValue == null) {
+ continue;
+ }
+ goodCts.add(ctValue.getConfiguredTarget());
+ }
+
+ if (!result.hasError() && badActions.isEmpty()) {
+ setDeserializedArtifactOwners();
+ return goodCts;
+ }
+
+ // --nokeep_going so we fail with an exception for the first error.
+ // TODO(bazel-team): We might want to report the other errors through the event bus but
+ // for keeping this code in parity with legacy we just report the first error for now.
+ if (!keepGoing) {
+ for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
+ ConflictException ex = bad.getValue();
+ try {
+ ex.rethrowTyped();
+ } catch (MutableActionGraph.ActionConflictException ace) {
+ ace.reportTo(skyframeExecutor.getReporter());
+ String errorMsg = "Analysis of target '" + bad.getKey().getOwner().getLabel()
+ + "' failed; build aborted";
+ throw new ViewCreationFailedException(errorMsg);
+ } catch (ArtifactPrefixConflictException apce) {
+ skyframeExecutor.getReporter().handle(Event.error(apce.getMessage()));
+ }
+ throw new ViewCreationFailedException(ex.getMessage());
+ }
+
+ Map.Entry<SkyKey, ErrorInfo> error = result.errorMap().entrySet().iterator().next();
+ SkyKey topLevel = error.getKey();
+ ErrorInfo errorInfo = error.getValue();
+ assertSaneAnalysisError(errorInfo, topLevel);
+ skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), topLevel,
+ skyframeExecutor.getReporter());
+ Throwable cause = errorInfo.getException();
+ Preconditions.checkState(cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()),
+ errorInfo);
+ String errorMsg = "Analysis of target '" + ConfiguredTargetValue.extractLabel(topLevel)
+ + "' failed; build aborted";
+ throw new ViewCreationFailedException(errorMsg);
+ }
+
+ // --keep_going : We notify the error and return a ConfiguredTargetValue
+ for (Map.Entry<SkyKey, ErrorInfo> errorEntry : result.errorMap().entrySet()) {
+ if (values.contains(errorEntry.getKey().argument())) {
+ SkyKey errorKey = errorEntry.getKey();
+ ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument();
+ ErrorInfo errorInfo = errorEntry.getValue();
+ assertSaneAnalysisError(errorInfo, errorKey);
+
+ skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), errorKey,
+ skyframeExecutor.getReporter());
+ // We try to get the root cause key first from ErrorInfo rootCauses. If we don't have one
+ // we try to use the cycle culprit if the error is a cycle. Otherwise we use the top-level
+ // error key.
+ Label root;
+ if (!Iterables.isEmpty(errorEntry.getValue().getRootCauses())) {
+ SkyKey culprit = Preconditions.checkNotNull(Iterables.getFirst(
+ errorEntry.getValue().getRootCauses(), null));
+ root = ((ConfiguredTargetKey) culprit.argument()).getLabel();
+ } else {
+ root = maybeGetConfiguredTargetCycleCulprit(errorInfo.getCycleInfo());
+ }
+ if (warningListener != null) {
+ warningListener.handle(Event.warn("errors encountered while analyzing target '"
+ + label + "': it will not be built"));
+ }
+ eventBus.post(new AnalysisFailureEvent(
+ LabelAndConfiguration.of(label.getLabel(), label.getConfiguration()), root));
+ }
+ }
+
+ Collection<Exception> reportedExceptions = Sets.newHashSet();
+ for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
+ ConflictException ex = bad.getValue();
+ try {
+ ex.rethrowTyped();
+ } catch (MutableActionGraph.ActionConflictException ace) {
+ ace.reportTo(skyframeExecutor.getReporter());
+ if (warningListener != null) {
+ warningListener.handle(Event.warn("errors encountered while analyzing target '"
+ + bad.getKey().getOwner().getLabel() + "': it will not be built"));
+ }
+ } catch (ArtifactPrefixConflictException apce) {
+ if (reportedExceptions.add(apce)) {
+ skyframeExecutor.getReporter().handle(Event.error(apce.getMessage()));
+ }
+ }
+ }
+
+ if (!badActions.isEmpty()) {
+ // In order to determine the set of configured targets transitively error free from action
+ // conflict issues, we run a post-processing update() that uses the bad action map.
+ EvaluationResult<PostConfiguredTargetValue> actionConflictResult =
+ skyframeExecutor.postConfigureTargets(values, keepGoing, badActions);
+
+ goodCts = Lists.newArrayListWithCapacity(values.size());
+ for (ConfiguredTargetKey value : values) {
+ PostConfiguredTargetValue postCt =
+ actionConflictResult.get(PostConfiguredTargetValue.key(value));
+ if (postCt != null) {
+ goodCts.add(postCt.getCt());
+ }
+ }
+ }
+ setDeserializedArtifactOwners();
+ return goodCts;
+ }
+
+ @Nullable
+ Label maybeGetConfiguredTargetCycleCulprit(Iterable<CycleInfo> cycleInfos) {
+ for (CycleInfo cycleInfo : cycleInfos) {
+ SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null);
+ if (culprit == null) {
+ continue;
+ }
+ if (culprit.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) {
+ return ((LabelAndConfiguration) culprit.argument()).getLabel();
+ }
+ }
+ return null;
+ }
+
+ private static void assertSaneAnalysisError(ErrorInfo errorInfo, SkyKey key) {
+ Throwable cause = errorInfo.getException();
+ if (cause != null) {
+ // We should only be trying to configure targets when the loading phase succeeds, meaning
+ // that the only errors should be analysis errors.
+ Preconditions.checkState(cause instanceof ConfiguredValueCreationException,
+ "%s -> %s", key, errorInfo);
+ }
+ }
+
+ ArtifactFactory getArtifactFactory() {
+ return artifactFactory;
+ }
+
+ @Nullable
+ EventHandler getWarningListener() {
+ return warningListener;
+ }
+
+ /**
+ * Because we don't know what build-info artifacts this configured target may request, we
+ * conservatively register a dep on all of them.
+ */
+ // TODO(bazel-team): Allow analysis to return null so the value builder can exit and wait for a
+ // restart deps are not present.
+ private boolean getWorkspaceStatusValues(Environment env) {
+ env.getValue(WorkspaceStatusValue.SKY_KEY);
+ Map<BuildInfoKey, BuildInfoFactory> buildInfoFactories =
+ PrecomputedValue.BUILD_INFO_FACTORIES.get(env);
+ if (buildInfoFactories == null) {
+ return false;
+ }
+ BuildConfigurationCollection configurations = getBuildConfigurationCollection(env);
+ if (configurations == null) {
+ return false;
+ }
+ // These factories may each create their own build info artifacts, all depending on the basic
+ // build-info.txt and build-changelist.txt.
+ List<SkyKey> depKeys = Lists.newArrayList();
+ for (BuildInfoKey key : buildInfoFactories.keySet()) {
+ for (BuildConfiguration config : configurations.getAllConfigurations()) {
+ if (buildInfoFactories.get(key).isEnabled(config)) {
+ depKeys.add(BuildInfoCollectionValue.key(new BuildInfoKeyAndConfig(key, config)));
+ }
+ }
+ }
+ env.getValues(depKeys);
+ return !env.valuesMissing();
+ }
+
+ /** Returns null if any build-info values are not ready. */
+ @Nullable
+ CachingAnalysisEnvironment createAnalysisEnvironment(ArtifactOwner owner,
+ boolean isSystemEnv, boolean extendedSanityChecks, EventHandler eventHandler,
+ Environment env, boolean allowRegisteringActions) {
+ if (!getWorkspaceStatusValues(env)) {
+ return null;
+ }
+ return new CachingAnalysisEnvironment(
+ artifactFactory, owner, isSystemEnv, extendedSanityChecks, eventHandler, env,
+ allowRegisteringActions, binTools);
+ }
+
+ /**
+ * Invokes the appropriate constructor to create a {@link ConfiguredTarget} instance.
+ *
+ * <p>For use in {@code ConfiguredTargetFunction}.
+ *
+ * <p>Returns null if Skyframe deps are missing or upon certain errors.
+ */
+ @Nullable
+ ConfiguredTarget createConfiguredTarget(Target target, BuildConfiguration configuration,
+ CachingAnalysisEnvironment analysisEnvironment,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions)
+ throws InterruptedException {
+ Preconditions.checkState(enableAnalysis,
+ "Already in execution phase %s %s", target, configuration);
+ return factory.createConfiguredTarget(analysisEnvironment, artifactFactory, target,
+ configuration, prerequisiteMap, configConditions);
+ }
+
+ @Nullable
+ public Aspect createAspect(
+ AnalysisEnvironment env, RuleConfiguredTarget associatedTarget,
+ ConfiguredAspectFactory aspectFactory,
+ ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap,
+ Set<ConfigMatchingProvider> configConditions) {
+ return factory.createAspect(
+ env, associatedTarget, aspectFactory, prerequisiteMap, configConditions);
+ }
+
+ @Nullable
+ private BuildConfigurationCollection getBuildConfigurationCollection(Environment env) {
+ ConfigurationCollectionValue configurationsValue =
+ (ConfigurationCollectionValue) env.getValue(configurationKey);
+ return configurationsValue == null ? null : configurationsValue.getConfigurationCollection();
+ }
+
+ @Nullable
+ SkyframeDependencyResolver createDependencyResolver(Environment env) {
+ BuildConfigurationCollection configurations = getBuildConfigurationCollection(env);
+ return configurations == null ? null : new SkyframeDependencyResolver(env);
+ }
+
+ /**
+ * Workaround to clear all legacy data, like the action graph and the artifact factory. We need
+ * to clear them to avoid conflicts.
+ * TODO(bazel-team): Remove this workaround. [skyframe-execution]
+ */
+ void clearLegacyData() {
+ legacyDataCleaner.run();
+ }
+
+ /**
+ * Hack to invalidate actions in legacy action graph when their values are invalidated in
+ * skyframe.
+ */
+ EvaluationProgressReceiver getInvalidationReceiver() {
+ return invalidationReceiver;
+ }
+
+ /** Clear the invalidated configured targets detected during loading and analysis phases. */
+ public void clearInvalidatedConfiguredTargets() {
+ dirtyConfiguredTargets = Sets.newConcurrentHashSet();
+ anyConfiguredTargetDeleted = false;
+ }
+
+ public boolean isSomeConfiguredTargetInvalidated() {
+ return anyConfiguredTargetDeleted || !dirtyConfiguredTargets.isEmpty();
+ }
+
+ /**
+ * Called from SkyframeExecutor to see whether the graph needs to be checked for artifact
+ * conflicts. Returns true if some configured target has been evaluated since the last time the
+ * graph was checked for artifact conflicts (with that last time marked by a call to
+ * {@link #resetEvaluatedConfiguredTargetFlag()}).
+ */
+ boolean isSomeConfiguredTargetEvaluated() {
+ Preconditions.checkState(!enableAnalysis);
+ return someConfiguredTargetEvaluated;
+ }
+
+ /**
+ * Called from SkyframeExecutor after the graph is checked for artifact conflicts so that
+ * the next time {@link #isSomeConfiguredTargetEvaluated} is called, it will return true only if
+ * some configured target has been evaluated since the last check for artifact conflicts.
+ */
+ void resetEvaluatedConfiguredTargetFlag() {
+ someConfiguredTargetEvaluated = false;
+ }
+
+ /**
+ * {@link #createConfiguredTarget} will only create configured targets if this is set to true. It
+ * should be set to true before any Skyframe update call that might call into {@link
+ * #createConfiguredTarget}, and false immediately after the call. Use it to fail-fast in the case
+ * that a target is requested for analysis not during the analysis phase.
+ */
+ void enableAnalysis(boolean enable) {
+ this.enableAnalysis = enable;
+ }
+
+ private class ConfiguredTargetValueInvalidationReceiver implements EvaluationProgressReceiver {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ if (value instanceof ConfiguredTargetValue) {
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue) value;
+ // If the value was just dirtied and not deleted, then it may not be truly invalid, since
+ // it may later get re-validated.
+ if (state == InvalidationState.DELETED) {
+ anyConfiguredTargetDeleted = true;
+ } else {
+ dirtyConfiguredTargets.add(ctValue);
+ }
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {}
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ if (skyKey.functionName() == SkyFunctions.CONFIGURED_TARGET) {
+ if (state == EvaluationState.BUILT) {
+ evaluatedConfiguredTargets.add(skyKey);
+ // During multithreaded operation, this is only set to true, so no concurrency issues.
+ someConfiguredTargetEvaluated = true;
+ }
+ Preconditions.checkNotNull(value, "%s %s", skyKey, state);
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue) value;
+ dirtyConfiguredTargets.remove(ctValue);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java
new file mode 100644
index 0000000..1c7cbfa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.analysis.DependencyResolver;
+import com.google.devtools.build.lib.analysis.TargetAndConfiguration;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * A dependency resolver for use within Skyframe. Loads packages lazily when possible.
+ */
+public final class SkyframeDependencyResolver extends DependencyResolver {
+
+ private final Environment env;
+
+ public SkyframeDependencyResolver(Environment env) {
+ this.env = env;
+ }
+
+ @Override
+ protected void invalidVisibilityReferenceHook(TargetAndConfiguration value, Label label) {
+ env.getListener().handle(
+ Event.error(TargetUtils.getLocationMaybe(value.getTarget()), String.format(
+ "Label '%s' in visibility attribute does not refer to a package group", label)));
+ }
+
+ @Override
+ protected void invalidPackageGroupReferenceHook(TargetAndConfiguration value, Label label) {
+ env.getListener().handle(
+ Event.error(TargetUtils.getLocationMaybe(value.getTarget()), String.format(
+ "label '%s' does not refer to a package group", label)));
+ }
+
+ @Nullable
+ @Override
+ protected Target getTarget(Label label) throws NoSuchThingException {
+ if (env.getValue(TargetMarkerValue.key(label)) == null) {
+ return null;
+ }
+ SkyKey key = PackageValue.key(label.getPackageIdentifier());
+ SkyValue value = env.getValue(key);
+ if (value == null) {
+ return null;
+ }
+ PackageValue packageValue = (PackageValue) value;
+ return packageValue.getPackage().getTarget(label.getName());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
new file mode 100644
index 0000000..29b4c23
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -0,0 +1,1476 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactFactory;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.analysis.Aspect;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BuildView.Options;
+import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleVisibility;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.ResourceUsage;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.BuildDriver;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper object to support Skyframe-driven execution.
+ *
+ * <p>This object is mostly used to inject external state, such as the executor engine or
+ * some additional artifacts (workspace status and build info artifacts) into SkyFunctions
+ * for use during the build.
+ */
+public abstract class SkyframeExecutor {
+ private final EvaluatorSupplier evaluatorSupplier;
+ protected MemoizingEvaluator memoizingEvaluator;
+ private final MemoizingEvaluator.EmittedEventState emittedEventState =
+ new MemoizingEvaluator.EmittedEventState();
+ protected final Reporter reporter;
+ private final PackageFactory pkgFactory;
+ private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
+ private final BlazeDirectories directories;
+ @Nullable
+ private BatchStat batchStatter;
+
+ // TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
+ // operations may need to be handled in another (bigger?) thread pool. Also, we should detect
+ // the number of cores and use that as the thread-pool size for CPU-bound operations.
+ // I just bumped this to 200 to get reasonable execution phase performance; that may cause
+ // significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis]
+ @VisibleForTesting
+ public static final int DEFAULT_THREAD_COUNT = 200;
+
+ // Stores Packages between reruns of the PackageFunction (because of missing dependencies,
+ // within the same evaluate() run) to avoid loading the same package twice (first time loading
+ // to find subincludes and declare value dependencies).
+ // TODO(bazel-team): remove this cache once we have skyframe-native package loading
+ // [skyframe-loading]
+ private final ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache =
+ Maps.newConcurrentMap();
+ private final AtomicInteger numPackagesLoaded = new AtomicInteger(0);
+
+ protected SkyframeBuildView skyframeBuildView;
+ private EventHandler errorEventListener;
+ private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
+
+ protected BuildDriver buildDriver;
+
+ // AtomicReferences are used here as mutable boxes shared with value builders.
+ private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
+ protected final AtomicReference<UnixGlob.FilesystemCalls> syscalls =
+ new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
+ protected final AtomicReference<PathPackageLocator> pkgLocator =
+ new AtomicReference<>();
+ protected final AtomicReference<ImmutableSet<String>> deletedPackages =
+ new AtomicReference<>(ImmutableSet.<String>of());
+ private final AtomicReference<EventBus> eventBus = new AtomicReference<>();
+
+ private final ImmutableList<BuildInfoFactory> buildInfoFactories;
+ // Under normal circumstances, the artifact factory persists for the life of a Blaze server, but
+ // since it is not yet created when we create the value builders, we have to use a supplier,
+ // initialized when the build view is created.
+ private final MutableSupplier<ArtifactFactory> artifactFactory = new MutableSupplier<>();
+ // Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID
+ // would be preferable, but we have no way to have the Action depend on that value directly.
+ // Having the BuildInfoFunction own the supplier is currently not possible either, because then
+ // it would be invalidated on every build, since it would depend on the build id value.
+ private MutableSupplier<UUID> buildId = new MutableSupplier<>();
+
+ protected boolean active = true;
+ private final PackageManager packageManager;
+
+ private final Preprocessor.Factory.Supplier preprocessorFactorySupplier;
+ private Preprocessor.Factory preprocessorFactory;
+
+ protected final TimestampGranularityMonitor tsgm;
+
+ private final ResourceManager resourceManager;
+
+ /** Used to lock evaluator on legacy calls to get existing values. */
+ private final Object valueLookupLock = new Object();
+ private final AtomicReference<ActionExecutionStatusReporter> statusReporterRef =
+ new AtomicReference<>();
+ private final SkyframeActionExecutor skyframeActionExecutor;
+ protected SkyframeProgressReceiver progressReceiver;
+ private final AtomicReference<CyclesReporter> cyclesReporter = new AtomicReference<>();
+
+ private final Set<Path> immutableDirectories;
+
+ private BinTools binTools = null;
+ private boolean needToInjectEmbeddedArtifacts = true;
+ private boolean needToInjectPrecomputedValuesForAnalysis = true;
+ protected int modifiedFiles;
+ private final Predicate<PathFragment> allowedMissingInputs;
+
+ private final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions;
+ private final ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues;
+
+ protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor =
+ new SkyframeIncrementalBuildMonitor();
+
+ private MutableSupplier<ConfigurationFactory> configurationFactory = new MutableSupplier<>();
+ private MutableSupplier<Map<String, String>> clientEnv = new MutableSupplier<>();
+ private MutableSupplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments =
+ new MutableSupplier<>();
+ private MutableSupplier<Set<Package>> configurationPackages = new MutableSupplier<>();
+ private SkyKey configurationSkyKey = null;
+
+ private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName());
+
+ protected SkyframeExecutor(
+ Reporter reporter,
+ EvaluatorSupplier evaluatorSupplier,
+ PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm,
+ BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+ // Strictly speaking, these arguments are not required for initialization, but all current
+ // callsites have them at hand, so we might as well set them during construction.
+ this.reporter = Preconditions.checkNotNull(reporter);
+ this.evaluatorSupplier = evaluatorSupplier;
+ this.pkgFactory = pkgFactory;
+ this.pkgFactory.setSyscalls(syscalls);
+ this.tsgm = tsgm;
+ this.workspaceStatusActionFactory = workspaceStatusActionFactory;
+ this.packageManager = new SkyframePackageManager(
+ new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(),
+ new SkyframeTargetPatternEvaluator(this), syscalls, cyclesReporter, pkgLocator,
+ numPackagesLoaded, this);
+ this.errorEventListener = this.reporter;
+ this.resourceManager = ResourceManager.instance();
+ this.skyframeActionExecutor = new SkyframeActionExecutor(reporter, resourceManager, eventBus,
+ statusReporterRef);
+ this.directories = Preconditions.checkNotNull(directories);
+ this.buildInfoFactories = buildInfoFactories;
+ this.immutableDirectories = immutableDirectories;
+ this.allowedMissingInputs = allowedMissingInputs;
+ this.preprocessorFactorySupplier = preprocessorFactorySupplier;
+ this.extraSkyFunctions = extraSkyFunctions;
+ this.extraPrecomputedValues = extraPrecomputedValues;
+ }
+
+ private ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions(
+ Root buildDataDirectory,
+ PackageFactory pkgFactory,
+ Predicate<PathFragment> allowedMissingInputs) {
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator,
+ immutableDirectories);
+ // We use an immutable map builder for the nice side effect that it throws if a duplicate key
+ // is inserted.
+ ImmutableMap.Builder<SkyFunctionName, SkyFunction> map = ImmutableMap.builder();
+ map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
+ map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+ map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
+ new DirectoryListingStateFunction(externalFilesHelper));
+ map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS,
+ new FileSymlinkCycleUniquenessFunction());
+ map.put(SkyFunctions.FILE, new FileFunction(pkgLocator, externalFilesHelper));
+ map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
+ map.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
+ map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
+ map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(
+ pkgLocator, packageManager, pkgFactory.getRuleClassProvider()));
+ map.put(SkyFunctions.SKYLARK_IMPORTS_LOOKUP, new SkylarkImportLookupFunction(
+ pkgFactory.getRuleClassProvider(), pkgFactory));
+ map.put(SkyFunctions.GLOB, new GlobFunction());
+ map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction(pkgLocator));
+ map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction());
+ map.put(SkyFunctions.PACKAGE, new PackageFunction(
+ reporter, pkgFactory, packageManager, showLoadingProgress, packageFunctionCache,
+ eventBus, numPackagesLoaded));
+ map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction());
+ map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction());
+ map.put(SkyFunctions.CONFIGURED_TARGET,
+ new ConfiguredTargetFunction(new BuildViewProvider()));
+ map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider()));
+ map.put(SkyFunctions.POST_CONFIGURED_TARGET,
+ new PostConfiguredTargetFunction(new BuildViewProvider()));
+ map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction(
+ configurationFactory, clientEnv, configurationPackages));
+ map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction(
+ configurationFragments, configurationPackages));
+ map.put(SkyFunctions.WORKSPACE_FILE, new WorkspaceFileFunction(pkgFactory));
+ map.put(SkyFunctions.TARGET_COMPLETION, new TargetCompletionFunction(eventBus));
+ map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction());
+ map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs));
+ map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory,
+ buildDataDirectory));
+ map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction());
+ map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction());
+ map.put(SkyFunctions.ACTION_EXECUTION,
+ new ActionExecutionFunction(skyframeActionExecutor, tsgm));
+ map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL,
+ new RecursiveFilesystemTraversalFunction());
+ map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
+ map.putAll(extraSkyFunctions);
+ return map.build();
+ }
+
+ @ThreadCompatible
+ public void setActive(boolean active) {
+ this.active = active;
+ }
+
+ protected void checkActive() {
+ Preconditions.checkState(active);
+ }
+
+ public void setFileCache(ActionInputFileCache fileCache) {
+ this.skyframeActionExecutor.setFileCache(fileCache);
+ }
+
+ public void dump(boolean summarize, PrintStream out) {
+ memoizingEvaluator.dump(summarize, out);
+ }
+
+ public abstract void dumpPackages(PrintStream out);
+
+ public void setBatchStatter(@Nullable BatchStat batchStatter) {
+ this.batchStatter = batchStatter;
+ }
+
+ /**
+ * Notify listeners about changed files, and release any associated memory afterwards.
+ */
+ public void drainChangedFiles() {
+ incrementalBuildMonitor.alertListeners(getEventBus());
+ incrementalBuildMonitor = null;
+ }
+
+ @VisibleForTesting
+ public BuildDriver getDriverForTesting() {
+ return buildDriver;
+ }
+
+ /**
+ * This method exists only to allow a module to make a top-level Skyframe call during the
+ * transition to making it fully Skyframe-compatible. Do not add additional callers!
+ */
+ public <E extends Exception> SkyValue evaluateSkyKeyForCodeMigration(final SkyKey key,
+ final Class<E> clazz) throws E {
+ try {
+ return callUninterruptibly(new Callable<SkyValue>() {
+ @Override
+ public SkyValue call() throws E, InterruptedException {
+ synchronized (valueLookupLock) {
+ // We evaluate in keepGoing mode because in the case that the graph does not store its
+ // edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always
+ // permitted.
+ EvaluationResult<ActionLookupValue> result = buildDriver.evaluate(
+ ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(),
+ errorEventListener);
+ if (!result.hasError()) {
+ return Preconditions.checkNotNull(result.get(key), "%s %s", result, key);
+ }
+ ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key),
+ "%s %s", key, result);
+ Throwables.propagateIfPossible(errorInfo.getException(), clazz);
+ if (errorInfo.getException() != null) {
+ throw new IllegalStateException(errorInfo.getException());
+ }
+ throw new IllegalStateException(errorInfo.toString());
+ }
+ }
+ });
+ } catch (Exception e) {
+ Throwables.propagateIfPossible(e, clazz);
+ throw new IllegalStateException(e);
+ }
+ }
+
+ class BuildViewProvider {
+ /**
+ * Returns the current {@link SkyframeBuildView} instance.
+ */
+ SkyframeBuildView getSkyframeBuildView() {
+ return skyframeBuildView;
+ }
+ }
+
+ /**
+ * Must be called before the {@link SkyframeExecutor} can be used (should only be called in
+ * factory methods and as an implementation detail of {@link #resetEvaluator}).
+ */
+ protected void init() {
+ progressReceiver = new SkyframeProgressReceiver();
+ Map<SkyFunctionName, SkyFunction> skyFunctions = skyFunctions(
+ directories.getBuildDataDirectory(), pkgFactory, allowedMissingInputs);
+ memoizingEvaluator = evaluatorSupplier.create(
+ skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState,
+ hasIncrementalState());
+ buildDriver = newBuildDriver();
+ }
+
+ /**
+ * Reinitializes the Skyframe evaluator, dropping all previously computed values.
+ *
+ * <p>Be careful with this method as it also deletes all injected values. You need to make sure
+ * that any necessary precomputed values are reinjected before the next build. Constants can be
+ * put in {@link #reinjectConstantValuesLazily}.
+ */
+ public void resetEvaluator() {
+ init();
+ emittedEventState.clear();
+ if (skyframeBuildView != null) {
+ skyframeBuildView.clearLegacyData();
+ }
+ reinjectConstantValuesLazily();
+ }
+
+ protected abstract Differencer evaluatorDiffer();
+
+ protected abstract BuildDriver newBuildDriver();
+
+ /**
+ * Values whose values are known at startup and guaranteed constant are still wiped from the
+ * evaluator when we create a new one, so they must be re-injected each time we create a new
+ * evaluator.
+ */
+ private void reinjectConstantValuesLazily() {
+ needToInjectEmbeddedArtifacts = true;
+ needToInjectPrecomputedValuesForAnalysis = true;
+ }
+
+ /**
+ * Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g.
+ * on a configuration change); since the configuration is part of the key, these key/value pairs
+ * will be sitting around doing nothing until the configuration changes back to the previous
+ * value.
+ *
+ * <p>The next evaluation will delete all invalid values.
+ */
+ public abstract void dropConfiguredTargets();
+
+ /**
+ * Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache.
+ */
+ @VisibleForTesting
+ public void invalidateConfigurationCollection() {
+ invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT,
+ SkyFunctions.CONFIGURATION_COLLECTION)));
+ }
+
+ /**
+ * Decides if graph edges should be stored for this build. If not, re-creates the graph to not
+ * store graph edges. Necessary conditions to not store graph edges are:
+ * (1) batch (since incremental builds are not possible);
+ * (2) skyframe build (since otherwise the memory savings are too slight to bother);
+ * (3) keep-going (since otherwise bubbling errors up may require edges of done nodes);
+ * (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way).
+ */
+ public void decideKeepIncrementalState(boolean batch, Options viewOptions) {
+ // Assume incrementality.
+ }
+
+ public boolean hasIncrementalState() {
+ return true;
+ }
+
+ @VisibleForTesting
+ protected abstract Injectable injectable();
+
+ /**
+ * Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually
+ * deletes the relevant values. If using Skyframe execution, clears their data without deleting
+ * them (they will be deleted on the next build).
+ */
+ public abstract void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets);
+
+ /**
+ * Injects the contents of the computed tools/defaults package.
+ */
+ @VisibleForTesting
+ public void setupDefaultPackage(String defaultsPackageContents) {
+ PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents);
+ }
+
+ /**
+ * Injects the top-level artifact options.
+ */
+ public void injectTopLevelContext(TopLevelArtifactContext options) {
+ PrecomputedValue.TOP_LEVEL_CONTEXT.set(injectable(), options);
+ }
+
+ public void injectWorkspaceStatusData() {
+ PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(),
+ workspaceStatusActionFactory.createWorkspaceStatusAction(
+ artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId));
+ }
+
+ public void injectCoverageReportData(Action action) {
+ PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), action);
+ }
+
+ /**
+ * Sets the default visibility.
+ */
+ private void setDefaultVisibility(RuleVisibility defaultVisibility) {
+ PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility);
+ }
+
+ private void maybeInjectPrecomputedValuesForAnalysis() {
+ if (needToInjectPrecomputedValuesForAnalysis) {
+ injectBuildInfoFactories();
+ injectExtraPrecomputedValues();
+ needToInjectPrecomputedValuesForAnalysis = false;
+ }
+ }
+
+ private void injectExtraPrecomputedValues() {
+ for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
+ injected.inject(injectable());
+ }
+ }
+
+ /**
+ * Injects the build info factory map that will be used when constructing build info
+ * actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected
+ * each time the evaluator is created.
+ */
+ private void injectBuildInfoFactories() {
+ ImmutableMap.Builder<BuildInfoKey, BuildInfoFactory> factoryMapBuilder =
+ ImmutableMap.builder();
+ for (BuildInfoFactory factory : buildInfoFactories) {
+ factoryMapBuilder.put(factory.getKey(), factory);
+ }
+ PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build());
+ }
+
+ private void setShowLoadingProgress(boolean showLoadingProgressValue) {
+ showLoadingProgress.set(showLoadingProgressValue);
+ }
+
+ @VisibleForTesting
+ public void setCommandId(UUID commandId) {
+ PrecomputedValue.BUILD_ID.set(injectable(), commandId);
+ buildId.set(commandId);
+ }
+
+ /** Returns the build-info.txt and build-changelist.txt artifacts. */
+ public Collection<Artifact> getWorkspaceStatusArtifacts() throws InterruptedException {
+ // Should already be present, unless the user didn't request any targets for analysis.
+ EvaluationResult<WorkspaceStatusValue> result = buildDriver.evaluate(
+ ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1,
+ reporter);
+ WorkspaceStatusValue value =
+ Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY));
+ return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact());
+ }
+
+ // TODO(bazel-team): Make this take a PackageIdentifier.
+ public Map<PathFragment, Root> getArtifactRoots(Iterable<PathFragment> execPaths) {
+ final List<SkyKey> packageKeys = new ArrayList<>();
+ for (PathFragment execPath : execPaths) {
+ Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
+ packageKeys.add(ContainingPackageLookupValue.key(
+ PackageIdentifier.createInDefaultRepo(execPath)));
+ }
+
+ EvaluationResult<ContainingPackageLookupValue> result;
+ try {
+ result = callUninterruptibly(new Callable<EvaluationResult<ContainingPackageLookupValue>>() {
+ @Override
+ public EvaluationResult<ContainingPackageLookupValue> call() throws InterruptedException {
+ return buildDriver.evaluate(packageKeys, /*keepGoing=*/true, /*numThreads=*/1, reporter);
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e); // Should never happen.
+ }
+
+ Map<PathFragment, Root> roots = new HashMap<>();
+ for (PathFragment execPath : execPaths) {
+ ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key(
+ PackageIdentifier.createInDefaultRepo(execPath)));
+ if (value.hasContainingPackage()) {
+ roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot()));
+ } else {
+ roots.put(execPath, null);
+ }
+ }
+ return roots;
+ }
+
+ @VisibleForTesting
+ public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() {
+ PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting()
+ .getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting());
+ return (WorkspaceStatusAction) value.get();
+ }
+
+ /**
+ * Informs user about number of modified files (source and output files).
+ */
+ // Note, that number of modified files in some cases can be bigger than actual number of
+ // modified files for targets in current request. Skyframe may check for modification all files
+ // from previous requests.
+ protected void informAboutNumberOfModifiedFiles() {
+ LOG.info(String.format("Found %d modified files from last build", modifiedFiles));
+ }
+
+ public Reporter getReporter() {
+ return reporter;
+ }
+
+ public EventBus getEventBus() {
+ return eventBus.get();
+ }
+
+ /**
+ * The map from package names to the package root where each package was found; this is used to
+ * set up the symlink tree.
+ */
+ public ImmutableMap<PackageIdentifier, Path> getPackageRoots() {
+ // Make a map of the package names to their root paths.
+ ImmutableMap.Builder<PackageIdentifier, Path> packageRoots = ImmutableMap.builder();
+ for (Package pkg : configurationPackages.get()) {
+ packageRoots.put(pkg.getPackageIdentifier(), pkg.getSourceRoot());
+ }
+ return packageRoots.build();
+ }
+
+ @VisibleForTesting
+ ImmutableList<Path> getPathEntries() {
+ return pkgLocator.get().getPathEntries();
+ }
+
+ protected abstract void invalidate(Predicate<SkyKey> pred);
+
+ protected static Iterable<SkyKey> getSkyKeysPotentiallyAffected(
+ Iterable<PathFragment> modifiedSourceFiles, final Path pathEntry) {
+ // TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments.
+ Iterable<SkyKey> fileStateSkyKeys = Iterables.transform(modifiedSourceFiles,
+ new Function<PathFragment, SkyKey>() {
+ @Override
+ public SkyKey apply(PathFragment pathFragment) {
+ Preconditions.checkState(!pathFragment.isAbsolute(),
+ "found absolute PathFragment: %s", pathFragment);
+ return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment));
+ }
+ });
+ // TODO(bazel-team): Strictly speaking, we only need to invalidate directory values when a file
+ // has been created or deleted, not when it has been modified. Unfortunately we
+ // do not have that information here, although fancy filesystems could provide it with a
+ // hypothetically modified DiffAwareness interface.
+ // TODO(bazel-team): Even if we don't have that information, we could avoid invalidating
+ // directories when the state of a file does not change by statting them and comparing
+ // the new filetype (nonexistent/file/symlink/directory) with the old one.
+ Iterable<SkyKey> dirListingStateSkyKeys = Iterables.transform(
+ modifiedSourceFiles,
+ new Function<PathFragment, SkyKey>() {
+ @Override
+ public SkyKey apply(PathFragment pathFragment) {
+ Preconditions.checkState(!pathFragment.isAbsolute(),
+ "found absolute PathFragment: %s", pathFragment);
+ return DirectoryListingStateValue.key(RootedPath.toRootedPath(pathEntry,
+ pathFragment.getParentDirectory()));
+ }
+ });
+ return Iterables.concat(fileStateSkyKeys, dirListingStateSkyKeys);
+ }
+
+ protected static SkyKey createFileStateKey(RootedPath rootedPath) {
+ return FileStateValue.key(rootedPath);
+ }
+
+ protected static SkyKey createDirectoryListingStateKey(RootedPath rootedPath) {
+ return DirectoryListingStateValue.key(rootedPath);
+ }
+
+ /**
+ * Creates a FileValue pointing of type directory. No matter that the rootedPath points to a
+ * symlink.
+ *
+ * <p> Use it with caution as it would prevent invalidation when the destination file in the
+ * symlink changes.
+ */
+ protected static FileValue createFileDirValue(RootedPath rootedPath) {
+ return FileValue.value(rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE,
+ rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE);
+ }
+
+ /**
+ * Sets the packages that should be treated as deleted and ignored.
+ */
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public abstract void setDeletedPackages(Iterable<String> pkgs);
+
+ /**
+ * Prepares the evaluator for loading.
+ *
+ * <p>MUST be run before every incremental build.
+ */
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ public void preparePackageLoading(PathPackageLocator pkgLocator, RuleVisibility defaultVisibility,
+ boolean showLoadingProgress,
+ String defaultsPackageContents, UUID commandId) {
+ Preconditions.checkNotNull(pkgLocator);
+ setActive(true);
+
+ maybeInjectPrecomputedValuesForAnalysis();
+ setCommandId(commandId);
+ setShowLoadingProgress(showLoadingProgress);
+ setDefaultVisibility(defaultVisibility);
+ setupDefaultPackage(defaultsPackageContents);
+ setPackageLocator(pkgLocator);
+
+ syscalls.set(new PerBuildSyscallCache());
+ checkPreprocessorFactory();
+ emittedEventState.clear();
+
+ // If the PackageFunction was interrupted, there may be stale entries here.
+ packageFunctionCache.clear();
+ numPackagesLoaded.set(0);
+
+ // Reset the stateful SkyframeCycleReporter, which contains cycles from last run.
+ cyclesReporter.set(createCyclesReporter());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void setPackageLocator(PathPackageLocator pkgLocator) {
+ PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator);
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator);
+
+ if (!pkgLocator.equals(oldLocator)) {
+ // The package path is read not only by SkyFunctions but also by some other code paths.
+ // We need to take additional steps to keep the corresponding data structures in sync.
+ // (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener,
+ // and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.)
+ onNewPackageLocator(oldLocator, pkgLocator);
+ }
+ }
+
+ protected abstract void onNewPackageLocator(PathPackageLocator oldLocator,
+ PathPackageLocator pkgLocator);
+
+ private void checkPreprocessorFactory() {
+ if (preprocessorFactory == null) {
+ Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
+ packageManager);
+ pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
+ preprocessorFactory = newPreprocessorFactory;
+ } else if (!preprocessorFactory.isStillValid()) {
+ Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
+ packageManager);
+ invalidate(SkyFunctionName.functionIs(SkyFunctions.PACKAGE));
+ pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
+ preprocessorFactory = newPreprocessorFactory;
+ }
+ }
+
+ /**
+ * Specifies the current {@link SkyframeBuildView} instance. This should only be set once over the
+ * lifetime of the Blaze server, except in tests.
+ */
+ public void setSkyframeBuildView(SkyframeBuildView skyframeBuildView) {
+ this.skyframeBuildView = skyframeBuildView;
+ setConfigurationSkyKey(configurationSkyKey);
+ this.artifactFactory.set(skyframeBuildView.getArtifactFactory());
+ if (skyframeBuildView.getWarningListener() != null) {
+ setErrorEventListener(skyframeBuildView.getWarningListener());
+ }
+ }
+
+ /**
+ * Sets the eventBus to use for posting events.
+ */
+ public void setEventBus(EventBus eventBus) {
+ this.eventBus.set(eventBus);
+ }
+
+ /**
+ * Sets the eventHandler to use for reporting errors.
+ */
+ public void setErrorEventListener(EventHandler eventHandler) {
+ this.errorEventListener = eventHandler;
+ }
+
+ /**
+ * Sets the path for action log buffers.
+ */
+ public void setActionOutputRoot(Path actionOutputRoot) {
+ Preconditions.checkNotNull(actionOutputRoot);
+ this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot);
+ this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator);
+ }
+
+ private void setConfigurationSkyKey(SkyKey skyKey) {
+ this.configurationSkyKey = skyKey;
+ if (skyframeBuildView != null) {
+ skyframeBuildView.setConfigurationSkyKey(skyKey);
+ }
+ }
+
+ @VisibleForTesting
+ public void setConfigurationDataForTesting(BuildOptions options,
+ BlazeDirectories directories, ConfigurationFactory configurationFactory) {
+ SkyKey skyKey = ConfigurationCollectionValue.key(options, ImmutableSet.<String>of());
+ setConfigurationSkyKey(skyKey);
+ PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
+ this.configurationFactory.set(configurationFactory);
+ this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
+ this.configurationPackages.set(Sets.<Package>newConcurrentHashSet());
+ }
+
+ @VisibleForTesting
+ public BuildConfigurationCollection createConfigurations(
+ ConfigurationFactory configurationFactory, BuildConfigurationKey configurationKey)
+ throws InvalidConfigurationException, InterruptedException {
+ return createConfigurations(false, configurationFactory, configurationKey);
+ }
+
+ /**
+ * Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and
+ * returns result. Also invalidates {@link PrecomputedValue#TEST_ENVIRONMENT_VARIABLES} and
+ * {@link PrecomputedValue#BLAZE_DIRECTORIES} if they have changed.
+ */
+ public BuildConfigurationCollection createConfigurations(boolean keepGoing,
+ ConfigurationFactory configurationFactory, BuildConfigurationKey configurationKey)
+ throws InvalidConfigurationException, InterruptedException {
+
+ this.configurationPackages.set(Sets.<Package>newConcurrentHashSet());
+ this.clientEnv.set(configurationKey.getClientEnv());
+ this.configurationFactory.set(configurationFactory);
+ this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
+ BuildOptions buildOptions = configurationKey.getBuildOptions();
+ Map<String, String> testEnv = BuildConfiguration.getTestEnv(
+ buildOptions.get(BuildConfiguration.Options.class).testEnvironment,
+ configurationKey.getClientEnv());
+ // TODO(bazel-team): find a way to use only BuildConfigurationKey instead of
+ // TestEnvironmentVariables and BlazeDirectories. There is a problem only with
+ // TestEnvironmentVariables because BuildConfigurationKey stores client environment variables
+ // and we don't want to rebuild everything when any variable changes.
+ PrecomputedValue.TEST_ENVIRONMENT_VARIABLES.set(injectable(), testEnv);
+ PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), configurationKey.getDirectories());
+
+ SkyKey skyKey = ConfigurationCollectionValue.key(configurationKey.getBuildOptions(),
+ configurationKey.getMultiCpu());
+ setConfigurationSkyKey(skyKey);
+ EvaluationResult<ConfigurationCollectionValue> result = buildDriver.evaluate(
+ Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, errorEventListener);
+ if (result.hasError()) {
+ Throwable e = result.getError(skyKey).getException();
+ // Wrap loading failed exceptions
+ if (e instanceof NoSuchThingException) {
+ e = new InvalidConfigurationException(e);
+ }
+ Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class);
+ throw new IllegalStateException(
+ "Unknown error during ConfigurationCollectionValue evaluation", e);
+ }
+ Preconditions.checkState(result.values().size() == 1,
+ "Result of evaluate() must contain exactly one value %s", result);
+ ConfigurationCollectionValue configurationValue =
+ Iterables.getOnlyElement(result.values());
+ this.configurationPackages.set(
+ Sets.newConcurrentHashSet(configurationValue.getConfigurationPackages()));
+ return configurationValue.getConfigurationCollection();
+ }
+
+ private Iterable<ActionLookupValue> getActionLookupValues() {
+ // This filter keeps subclasses of ActionLookupValue.
+ return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ Map<SkyKey, ActionLookupValue> getActionLookupValueMap() {
+ return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(),
+ Predicates.instanceOf(ActionLookupValue.class));
+ }
+
+ /**
+ * Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to
+ * {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any
+ * conflicts found will only be reported during execution.
+ */
+ ImmutableMap<Action, SkyframeActionExecutor.ConflictException> findArtifactConflicts()
+ throws InterruptedException {
+ if (skyframeBuildView.isSomeConfiguredTargetEvaluated()
+ || skyframeBuildView.isSomeConfiguredTargetInvalidated()) {
+ // This operation is somewhat expensive, so we only do it if the graph might have changed in
+ // some way -- either we analyzed a new target or we invalidated an old one.
+ skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues());
+ skyframeBuildView.resetEvaluatedConfiguredTargetFlag();
+ // The invalidated configured targets flag will be reset later in the evaluate() call.
+ }
+ return skyframeActionExecutor.badActions();
+ }
+
+ /**
+ * Asks the Skyframe evaluator to build the given artifacts and targets, and to test the
+ * given test targets.
+ */
+ public EvaluationResult<?> buildArtifacts(
+ Executor executor,
+ Set<Artifact> artifactsToBuild,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Collection<ConfiguredTarget> targetsToTest,
+ boolean exclusiveTesting,
+ boolean keepGoing,
+ boolean explain,
+ int numJobs,
+ ActionCacheChecker actionCacheChecker,
+ @Nullable EvaluationProgressReceiver executionProgressReceiver) throws InterruptedException {
+ checkActive();
+ Preconditions.checkState(actionLogBufferPathGenerator != null);
+
+ skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, actionCacheChecker);
+
+ resourceManager.resetResourceUsage();
+ try {
+ progressReceiver.executionProgressReceiver = executionProgressReceiver;
+ Iterable<SkyKey> artifactKeys = ArtifactValue.mandatoryKeys(artifactsToBuild);
+ Iterable<SkyKey> targetKeys = TargetCompletionValue.keys(targetsToBuild);
+ Iterable<SkyKey> testKeys = TestCompletionValue.keys(targetsToTest, exclusiveTesting);
+ return buildDriver.evaluate(Iterables.concat(artifactKeys, targetKeys, testKeys), keepGoing,
+ numJobs, errorEventListener);
+ } finally {
+ progressReceiver.executionProgressReceiver = null;
+ // Also releases thread locks.
+ resourceManager.resetResourceUsage();
+ skyframeActionExecutor.executionOver();
+ }
+ }
+
+ @VisibleForTesting
+ public void prepareBuildingForTestingOnly(Executor executor, boolean keepGoing, boolean explain,
+ ActionCacheChecker checker) {
+ skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, checker);
+ }
+
+ EvaluationResult<TargetPatternValue> targetPatterns(Iterable<SkyKey> patternSkyKeys,
+ boolean keepGoing, EventHandler eventHandler) throws InterruptedException {
+ checkActive();
+ return buildDriver.evaluate(patternSkyKeys, keepGoing, DEFAULT_THREAD_COUNT,
+ eventHandler);
+ }
+
+ /**
+ * Returns the {@link ConfiguredTarget}s corresponding to the given keys.
+ *
+ * <p>For use for legacy support from {@code BuildView} only.
+ *
+ * <p>If a requested configured target is in error, the corresponding value is omitted from the
+ * returned list.
+ */
+ @ThreadSafety.ThreadSafe
+ public ImmutableList<ConfiguredTarget> getConfiguredTargets(Iterable<Dependency> keys) {
+ checkActive();
+ if (skyframeBuildView == null) {
+ // If build view has not yet been initialized, no configured targets can have been created.
+ // This is most likely to happen after a failed loading phase.
+ return ImmutableList.of();
+ }
+ final List<SkyKey> skyKeys = new ArrayList<>();
+ for (Dependency key : keys) {
+ skyKeys.add(ConfiguredTargetValue.key(key.getLabel(), key.getConfiguration()));
+ for (Class<? extends ConfiguredAspectFactory> aspect : key.getAspects()) {
+ skyKeys.add(AspectValue.key(key.getLabel(), key.getConfiguration(), aspect));
+ }
+ }
+
+ EvaluationResult<SkyValue> result;
+ try {
+ result = callUninterruptibly(new Callable<EvaluationResult<SkyValue>>() {
+ @Override
+ public EvaluationResult<SkyValue> call() throws Exception {
+ synchronized (valueLookupLock) {
+ try {
+ skyframeBuildView.enableAnalysis(true);
+ return buildDriver.evaluate(skyKeys, false, DEFAULT_THREAD_COUNT,
+ errorEventListener);
+ } finally {
+ skyframeBuildView.enableAnalysis(false);
+ }
+ }
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e); // Should never happen.
+ }
+
+ ImmutableList.Builder<ConfiguredTarget> cts = ImmutableList.builder();
+
+ DependentNodeLoop:
+ for (Dependency key : keys) {
+ SkyKey configuredTargetKey = ConfiguredTargetValue.key(
+ key.getLabel(), key.getConfiguration());
+ if (result.get(configuredTargetKey) == null) {
+ continue;
+ }
+
+ ConfiguredTarget configuredTarget =
+ ((ConfiguredTargetValue) result.get(configuredTargetKey)).getConfiguredTarget();
+ List<Aspect> aspects = new ArrayList<>();
+
+ for (Class<? extends ConfiguredAspectFactory> aspect : key.getAspects()) {
+ SkyKey aspectKey = AspectValue.key(key.getLabel(), key.getConfiguration(), aspect);
+ if (result.get(aspectKey) == null) {
+ continue DependentNodeLoop;
+ }
+
+ aspects.add(((AspectValue) result.get(aspectKey)).get());
+ }
+
+ cts.add(RuleConfiguredTarget.mergeAspects(configuredTarget, aspects));
+ }
+
+ return cts.build();
+ }
+
+ /**
+ * Returns a particular configured target.
+ *
+ * <p>Used only for testing.
+ */
+ @VisibleForTesting
+ @Nullable
+ public ConfiguredTarget getConfiguredTargetForTesting(
+ Label label, BuildConfiguration configuration) {
+ if (memoizingEvaluator.getExistingValueForTesting(
+ PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting()) == null) {
+ injectWorkspaceStatusData();
+ }
+ return Iterables.getFirst(getConfiguredTargets(ImmutableList.of(
+ new Dependency(label, configuration))), null);
+ }
+
+ /**
+ * Invalidates Skyframe values corresponding to the given set of modified files under the given
+ * path entry.
+ *
+ * <p>May throw an {@link InterruptedException}, which means that no values have been invalidated.
+ */
+ @VisibleForTesting
+ public abstract void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet,
+ Path pathEntry) throws InterruptedException;
+
+ /**
+ * Invalidates SkyFrame values that may have failed for transient reasons.
+ */
+ public abstract void invalidateTransientErrors();
+
+ @VisibleForTesting
+ public TimestampGranularityMonitor getTimestampGranularityMonitorForTesting() {
+ return tsgm;
+ }
+
+ /**
+ * Configures a given set of configured targets.
+ */
+ public EvaluationResult<ConfiguredTargetValue> configureTargets(
+ List<ConfiguredTargetKey> values, boolean keepGoing) throws InterruptedException {
+ checkActive();
+
+ // Make sure to not run too many analysis threads. This can cause memory thrashing.
+ return buildDriver.evaluate(ConfiguredTargetValue.keys(values), keepGoing,
+ ResourceUsage.getAvailableProcessors(), errorEventListener);
+ }
+
+ /**
+ * Post-process the targets. Values in the EvaluationResult are known to be transitively
+ * error-free from action conflicts.
+ */
+ public EvaluationResult<PostConfiguredTargetValue> postConfigureTargets(
+ List<ConfiguredTargetKey> values, boolean keepGoing,
+ ImmutableMap<Action, SkyframeActionExecutor.ConflictException> badActions)
+ throws InterruptedException {
+ checkActive();
+ PrecomputedValue.BAD_ACTIONS.set(injectable(), badActions);
+ // Make sure to not run too many analysis threads. This can cause memory thrashing.
+ EvaluationResult<PostConfiguredTargetValue> result =
+ buildDriver.evaluate(PostConfiguredTargetValue.keys(values), keepGoing,
+ ResourceUsage.getAvailableProcessors(), errorEventListener);
+
+ // Remove all post-configured target values immediately for memory efficiency. We are OK with
+ // this mini-phase being non-incremental as the failure mode of action conflict is rare.
+ memoizingEvaluator.delete(SkyFunctionName.functionIs(SkyFunctions.POST_CONFIGURED_TARGET));
+
+ return result;
+ }
+
+ /**
+ * Returns a Skyframe-based {@link SkyframeTransitivePackageLoader} implementation.
+ */
+ @VisibleForTesting
+ public TransitivePackageLoader pkgLoader() {
+ checkActive();
+ return new SkyframeLabelVisitor(new SkyframeTransitivePackageLoader(), cyclesReporter);
+ }
+
+ class SkyframeTransitivePackageLoader {
+ /**
+ * Loads the specified {@link TransitiveTargetValue}s.
+ */
+ EvaluationResult<TransitiveTargetValue> loadTransitiveTargets(
+ Iterable<Target> targetsToVisit, Iterable<Label> labelsToVisit, boolean keepGoing)
+ throws InterruptedException {
+ List<SkyKey> valueNames = new ArrayList<>();
+ for (Target target : targetsToVisit) {
+ valueNames.add(TransitiveTargetValue.key(target.getLabel()));
+ }
+ for (Label label : labelsToVisit) {
+ valueNames.add(TransitiveTargetValue.key(label));
+ }
+
+ return buildDriver.evaluate(valueNames, keepGoing, DEFAULT_THREAD_COUNT,
+ errorEventListener);
+ }
+
+ public Set<Package> retrievePackages(Set<PackageIdentifier> packageIds) {
+ final List<SkyKey> valueNames = new ArrayList<>();
+ for (PackageIdentifier pkgId : packageIds) {
+ valueNames.add(PackageValue.key(pkgId));
+ }
+
+ try {
+ return callUninterruptibly(new Callable<Set<Package>>() {
+ @Override
+ public Set<Package> call() throws Exception {
+ EvaluationResult<PackageValue> result = buildDriver.evaluate(
+ valueNames, false, ResourceUsage.getAvailableProcessors(), errorEventListener);
+ Preconditions.checkState(!result.hasError(),
+ "unexpected errors: %s", result.errorMap());
+ Set<Package> packages = Sets.newHashSet();
+ for (PackageValue value : result.values()) {
+ packages.add(value.getPackage());
+ }
+ return packages;
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+
+ }
+ }
+
+ /**
+ * Returns the generating {@link Action} of the given {@link Artifact}.
+ *
+ * <p>For use for legacy support from {@code BuildView} only.
+ */
+ @ThreadSafety.ThreadSafe
+ public Action getGeneratingAction(final Artifact artifact) {
+ if (artifact.isSourceArtifact()) {
+ return null;
+ }
+
+ try {
+ return callUninterruptibly(new Callable<Action>() {
+ @Override
+ public Action call() throws InterruptedException {
+ ArtifactOwner artifactOwner = artifact.getArtifactOwner();
+ Preconditions.checkState(artifactOwner instanceof ActionLookupValue.ActionLookupKey,
+ "%s %s", artifact, artifactOwner);
+ SkyKey actionLookupKey =
+ ActionLookupValue.key((ActionLookupValue.ActionLookupKey) artifactOwner);
+
+ synchronized (valueLookupLock) {
+ // Note that this will crash (attempting to run a configured target value builder after
+ // analysis) after a failed --nokeep_going analysis in which the configured target that
+ // failed was a (transitive) dependency of the configured target that should generate
+ // this action. We don't expect callers to query generating actions in such cases.
+ EvaluationResult<ActionLookupValue> result = buildDriver.evaluate(
+ ImmutableList.of(actionLookupKey), false, ResourceUsage.getAvailableProcessors(),
+ errorEventListener);
+ return result.hasError()
+ ? null
+ : result.get(actionLookupKey).getGeneratingAction(artifact);
+ }
+ }
+ });
+ } catch (Exception e) {
+ throw new IllegalStateException("Error getting generating action: " + artifact.prettyPrint(),
+ e);
+ }
+ }
+
+ public PackageManager getPackageManager() {
+ return packageManager;
+ }
+
+ class SkyframePackageLoader {
+ /**
+ * Looks up a particular package (mostly used after the loading phase, so packages should
+ * already be present, but occasionally used pre-loading phase). Use should be discouraged,
+ * since this cannot be used inside a Skyframe evaluation, and concurrent calls are
+ * synchronized.
+ *
+ * <p>Note that this method needs to be synchronized since InMemoryMemoizingEvaluator.evaluate()
+ * method does not support concurrent calls.
+ */
+ Package getPackage(EventHandler eventHandler, PackageIdentifier pkgName)
+ throws InterruptedException, NoSuchPackageException {
+ synchronized (valueLookupLock) {
+ SkyKey key = PackageValue.key(pkgName);
+ // Any call to this method post-loading phase should either be error-free or be in a
+ // keep_going build, since otherwise the build would have failed during loading. Thus
+ // we set keepGoing=true unconditionally.
+ EvaluationResult<PackageValue> result =
+ buildDriver.evaluate(ImmutableList.of(key), /*keepGoing=*/true,
+ DEFAULT_THREAD_COUNT, eventHandler);
+ if (result.hasError()) {
+ ErrorInfo error = result.getError();
+ if (!Iterables.isEmpty(error.getCycleInfo())) {
+ reportCycles(result.getError().getCycleInfo(), key);
+ // This can only happen if a package is freshly loaded outside of the target parsing
+ // or loading phase
+ throw new BuildFileContainsErrorsException(pkgName.toString(),
+ "Cycle encountered while loading package " + pkgName);
+ }
+ Throwable e = error.getException();
+ // PackageFunction should be catching, swallowing, and rethrowing all transitive
+ // errors as NoSuchPackageExceptions.
+ Throwables.propagateIfInstanceOf(e, NoSuchPackageException.class);
+ throw new IllegalStateException("Unexpected Exception type from PackageValue for '"
+ + pkgName + "'' with root causes: " + Iterables.toString(error.getRootCauses()), e);
+ }
+ return result.get(key).getPackage();
+ }
+ }
+
+ Package getLoadedPackage(final PackageIdentifier pkgName) throws NoSuchPackageException {
+ // Note that in Skyframe there is no way to tell if the package has been loaded before or not,
+ // so this will never throw for packages that are not loaded. However, no code currently
+ // relies on having the exception thrown.
+ try {
+ return callUninterruptibly(new Callable<Package>() {
+ @Override
+ public Package call() throws Exception {
+ return getPackage(errorEventListener, pkgName);
+ }
+ });
+ } catch (NoSuchPackageException e) {
+ if (e.getPackage() != null) {
+ return e.getPackage();
+ }
+ throw e;
+ } catch (Exception e) {
+ throw new IllegalStateException(e); // Should never happen.
+ }
+ }
+
+ /**
+ * Returns whether the given package should be consider deleted and thus should be ignored.
+ */
+ public boolean isPackageDeleted(String packageName) {
+ return deletedPackages.get().contains(packageName);
+ }
+
+ /** Same as {@link PackageManager#partiallyClear}. */
+ void partiallyClear() {
+ packageFunctionCache.clear();
+ }
+ }
+
+ /**
+ * Calls the given callable uninterruptibly.
+ *
+ * <p>If the callable throws {@link InterruptedException}, calls it again, until the callable
+ * returns a result. Sets the {@code currentThread().interrupted()} bit if the callable threw
+ * {@link InterruptedException} at least once.
+ *
+ * <p>This is almost identical to {@code Uninterruptibles#getUninterruptibly}.
+ */
+ protected static final <T> T callUninterruptibly(Callable<T> callable) throws Exception {
+ boolean interrupted = false;
+ try {
+ while (true) {
+ try {
+ return callable.call();
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+ } finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public MemoizingEvaluator getEvaluatorForTesting() {
+ return memoizingEvaluator;
+ }
+
+ /**
+ * Stores the set of loaded packages and, if needed, evicts ConfiguredTarget values.
+ *
+ * <p>The set represents all packages from the transitive closure of the top-level targets from
+ * the latest build.
+ */
+ @ThreadCompatible
+ public abstract void updateLoadedPackageSet(Set<PackageIdentifier> loadedPackages);
+
+ public void sync(PackageCacheOptions packageCacheOptions, Path workingDirectory,
+ String defaultsPackageContents, UUID commandId) throws InterruptedException,
+ AbruptExitException{
+
+ preparePackageLoading(
+ createPackageLocator(packageCacheOptions, directories.getWorkspace(), workingDirectory),
+ packageCacheOptions.defaultVisibility, packageCacheOptions.showLoadingProgress,
+ defaultsPackageContents, commandId);
+ setDeletedPackages(ImmutableSet.copyOf(packageCacheOptions.deletedPackages));
+
+ incrementalBuildMonitor = new SkyframeIncrementalBuildMonitor();
+ invalidateTransientErrors();
+ }
+
+ protected PathPackageLocator createPackageLocator(PackageCacheOptions packageCacheOptions,
+ Path workspace, Path workingDirectory) throws AbruptExitException{
+ return PathPackageLocator.create(
+ packageCacheOptions.packagePath, getReporter(), workspace, workingDirectory);
+ }
+
+ private CyclesReporter createCyclesReporter() {
+ return new CyclesReporter(
+ new TransitiveTargetCycleReporter(packageManager),
+ new ActionArtifactCycleReporter(packageManager),
+ new SkylarkModuleCycleReporter());
+ }
+
+ CyclesReporter getCyclesReporter() {
+ return cyclesReporter.get();
+ }
+
+ /** Convenience method with same semantics as {@link CyclesReporter#reportCycles}. */
+ public void reportCycles(Iterable<CycleInfo> cycles, SkyKey topLevelKey) {
+ getCyclesReporter().reportCycles(cycles, topLevelKey, errorEventListener);
+ }
+
+ public void setActionExecutionProgressReportingObjects(@Nullable ProgressSupplier supplier,
+ @Nullable ActionCompletedReceiver completionReceiver,
+ @Nullable ActionExecutionStatusReporter statusReporter) {
+ skyframeActionExecutor.setActionExecutionProgressReportingObjects(supplier, completionReceiver);
+ this.statusReporterRef.set(statusReporter);
+ }
+
+ /**
+ * This should be called at most once in the lifetime of the SkyframeExecutor (except for
+ * tests), and it should be called before the execution phase.
+ */
+ void setArtifactFactoryAndBinTools(ArtifactFactory artifactFactory, BinTools binTools) {
+ this.artifactFactory.set(artifactFactory);
+ this.binTools = binTools;
+ }
+
+ public void prepareExecution(boolean checkOutputFiles) throws AbruptExitException,
+ InterruptedException {
+ maybeInjectEmbeddedArtifacts();
+
+ if (checkOutputFiles) {
+ // Detect external modifications in the output tree.
+ FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+ invalidateDirtyActions(fsnc.getDirtyActionValues(batchStatter));
+ modifiedFiles += fsnc.getNumberOfModifiedOutputFiles();
+ }
+ informAboutNumberOfModifiedFiles();
+ }
+
+ protected abstract void invalidateDirtyActions(Iterable<SkyKey> dirtyActionValues);
+
+ @VisibleForTesting void maybeInjectEmbeddedArtifacts() throws AbruptExitException {
+ // The blaze client already ensures that the contents of the embedded binaries never change,
+ // so we just need to make sure that the appropriate artifacts are present in the skyframe
+ // graph.
+
+ if (!needToInjectEmbeddedArtifacts) {
+ return;
+ }
+
+ Preconditions.checkNotNull(artifactFactory.get());
+ Preconditions.checkNotNull(binTools);
+ Map<SkyKey, SkyValue> values = Maps.newHashMap();
+ // Blaze separately handles the symlinks that target these binaries. See BinTools#setupTool.
+ for (Artifact artifact : binTools.getAllEmbeddedArtifacts(artifactFactory.get())) {
+ FileArtifactValue fileArtifactValue;
+ try {
+ fileArtifactValue = FileArtifactValue.create(artifact);
+ } catch (IOException e) {
+ // See ExtractData in blaze.cc.
+ String message = "Error: corrupt installation: file " + artifact.getPath() + " missing. "
+ + "Please remove '" + directories.getInstallBase() + "' and try again.";
+ throw new AbruptExitException(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, e);
+ }
+ values.put(ArtifactValue.key(artifact, /*isMandatory=*/true), fileArtifactValue);
+ }
+ injectable().inject(values);
+ needToInjectEmbeddedArtifacts = false;
+ }
+
+ /**
+ * Mark dirty values for deletion if they've been dirty for longer than N versions.
+ *
+ * <p>Specifying a value N means, if the current version is V and a value was dirtied (and
+ * has remained so) in version U, and U + N <= V, then the value will be marked for deletion
+ * and purged in version V+1.
+ */
+ public abstract void deleteOldNodes(long versionWindowForDirtyGc);
+
+ /**
+ * A progress received to track analysis invalidation and update progress messages.
+ */
+ protected class SkyframeProgressReceiver implements EvaluationProgressReceiver {
+ /**
+ * This flag is needed in order to avoid invalidating legacy data when we clear the
+ * analysis cache because of --discard_analysis_cache flag. For that case we want to keep
+ * the legacy data but get rid of the Skyframe data.
+ */
+ protected boolean ignoreInvalidations = false;
+ /** This receiver is only needed for execution, so it is null otherwise. */
+ @Nullable EvaluationProgressReceiver executionProgressReceiver = null;
+
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ if (ignoreInvalidations) {
+ return;
+ }
+ if (skyframeBuildView != null) {
+ skyframeBuildView.getInvalidationReceiver().invalidated(value, state);
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ if (ignoreInvalidations) {
+ return;
+ }
+ if (skyframeBuildView != null) {
+ skyframeBuildView.getInvalidationReceiver().enqueueing(skyKey);
+ }
+ if (executionProgressReceiver != null) {
+ executionProgressReceiver.enqueueing(skyKey);
+ }
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ if (ignoreInvalidations) {
+ return;
+ }
+ if (skyframeBuildView != null) {
+ skyframeBuildView.getInvalidationReceiver().evaluated(skyKey, value, state);
+ }
+ if (executionProgressReceiver != null) {
+ executionProgressReceiver.evaluated(skyKey, value, state);
+ }
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
new file mode 100644
index 0000000..a1615cf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
+import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import java.util.Set;
+
+/**
+* A factory that creates instances of SkyframeExecutor.
+*/
+public interface SkyframeExecutorFactory {
+
+ /**
+ * Creates an instance of SkyframeExecutor
+ *
+ * @param reporter the reporter to be used by the executor
+ * @param pkgFactory the package factory
+ * @param skyframeBuild use Skyframe for the build phase. Should be always true after we are in
+ * the skyframe full mode.
+ * @param tsgm timestamp granularity monitor
+ * @param directories Blaze directories
+ * @param workspaceStatusActionFactory a factory for creating WorkspaceStatusAction objects
+ * @param buildInfoFactories list of BuildInfoFactories
+ * @param diffAwarenessFactories
+ * @param allowedMissingInputs
+ * @param preprocessorFactorySupplier
+ * @param extraSkyFunctions
+ * @param extraPrecomputedValues
+ * @return an instance of the SkyframeExecutor
+ * @throws AbruptExitException if the executor cannot be created
+ */
+ SkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory,
+ TimestampGranularityMonitor tsgm, BlazeDirectories directories,
+ Factory workspaceStatusActionFactory,
+ ImmutableList<BuildInfoFactory> buildInfoFactories,
+ Set<Path> immutableDirectories,
+ Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories,
+ Predicate<PathFragment> allowedMissingInputs,
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier,
+ ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions,
+ ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) throws AbruptExitException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java
new file mode 100644
index 0000000..c0fea26
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.ChangedFilesMessage;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A package-private class intended to track a small number of modified files during the build. This
+ * class should stop recording changed files if there are too many of them, instead of holding onto
+ * a large collection of files.
+ */
+@ThreadSafety.ThreadCompatible
+class SkyframeIncrementalBuildMonitor {
+ private Set<PathFragment> files = new HashSet<>();
+ private static final int MAX_FILES = 100;
+
+ public void accrue(Iterable<SkyKey> invalidatedValues) {
+ for (SkyKey skyKey : invalidatedValues) {
+ if (skyKey.functionName() == SkyFunctions.FILE_STATE) {
+ RootedPath file = (RootedPath) skyKey.argument();
+ maybeAddFile(file.getRelativePath());
+ }
+ }
+ }
+
+ private void maybeAddFile(PathFragment path) {
+ if (files != null) {
+ files.add(path);
+ if (files.size() >= MAX_FILES) {
+ files = null;
+ }
+ }
+ }
+
+ public void alertListeners(EventBus eventBus) {
+ if (files != null) {
+ eventBus.post(new ChangedFilesMessage(files));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java
new file mode 100644
index 0000000..2844cc0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java
@@ -0,0 +1,262 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframeTransitivePackageLoader;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Skyframe-based transitive package loader.
+ */
+final class SkyframeLabelVisitor implements TransitivePackageLoader {
+
+ private final SkyframeTransitivePackageLoader transitivePackageLoader;
+ private final AtomicReference<CyclesReporter> skyframeCyclesReporter;
+
+ private Set<PackageIdentifier> allVisitedPackages;
+ private Set<PackageIdentifier> errorFreeVisitedPackages;
+ private Set<Label> visitedTargets;
+ private Set<TransitiveTargetValue> previousBuildTargetValueSet = null;
+ private boolean lastBuildKeepGoing = false;
+ private final Multimap<Label, Label> rootCauses = HashMultimap.create();
+
+ SkyframeLabelVisitor(SkyframeTransitivePackageLoader transitivePackageLoader,
+ AtomicReference<CyclesReporter> skyframeCyclesReporter) {
+ this.transitivePackageLoader = transitivePackageLoader;
+ this.skyframeCyclesReporter = skyframeCyclesReporter;
+ }
+
+ @Override
+ public boolean sync(EventHandler eventHandler, Set<Target> targetsToVisit,
+ Set<Label> labelsToVisit, boolean keepGoing, int parallelThreads, int maxDepth)
+ throws InterruptedException {
+ rootCauses.clear();
+ lastBuildKeepGoing = false;
+ EvaluationResult<TransitiveTargetValue> result =
+ transitivePackageLoader.loadTransitiveTargets(targetsToVisit, labelsToVisit, keepGoing);
+ updateVisitedValues(result.values());
+ lastBuildKeepGoing = keepGoing;
+
+ if (!hasErrors(result)) {
+ return true;
+ }
+
+ Set<Entry<SkyKey, ErrorInfo>> errors = result.errorMap().entrySet();
+ if (!keepGoing) {
+ // We may have multiple errors, but in non keep_going builds, we're obligated to print only
+ // one of them.
+ Preconditions.checkState(!errors.isEmpty(), result);
+ Entry<SkyKey, ErrorInfo> error = errors.iterator().next();
+ ErrorInfo errorInfo = error.getValue();
+ SkyKey topLevel = error.getKey();
+ Label topLevelLabel = (Label) topLevel.argument();
+ if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
+ skyframeCyclesReporter.get().reportCycles(errorInfo.getCycleInfo(), topLevel, eventHandler);
+ errorAboutLoadingFailure(topLevelLabel, null, eventHandler);
+ } else if (isDirectErrorFromTopLevelLabel(topLevelLabel, labelsToVisit, errorInfo)) {
+ // An error caused by a non-top-level label has already been reported during error
+ // bubbling but an error caused by the top-level non-target label itself hasn't been
+ // reported yet. Note that errors from top-level targets have already been reported
+ // during target parsing.
+ errorAboutLoadingFailure(topLevelLabel, errorInfo.getException(), eventHandler);
+ }
+ return false;
+ }
+
+ for (Entry<SkyKey, ErrorInfo> errorEntry : errors) {
+ SkyKey key = errorEntry.getKey();
+ ErrorInfo errorInfo = errorEntry.getValue();
+ Preconditions.checkState(key.functionName().equals(SkyFunctions.TRANSITIVE_TARGET), errorEntry);
+ Label topLevelLabel = (Label) key.argument();
+ if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
+ skyframeCyclesReporter.get().reportCycles(errorInfo.getCycleInfo(), key, eventHandler);
+ for (Label rootCause : getRootCausesOfCycles(topLevelLabel, errorInfo.getCycleInfo())) {
+ rootCauses.put(topLevelLabel, rootCause);
+ }
+ }
+ if (isDirectErrorFromTopLevelLabel(topLevelLabel, labelsToVisit, errorInfo)) {
+ // Unlike top-level targets, which have already gone through target parsing,
+ // errors directly coming from top-level labels have not been reported yet.
+ //
+ // See the note in the --nokeep_going case above.
+ eventHandler.handle(Event.error(errorInfo.getException().getMessage()));
+ }
+ warnAboutLoadingFailure(topLevelLabel, eventHandler);
+ for (SkyKey badKey : errorInfo.getRootCauses()) {
+ Preconditions.checkState(badKey.argument() instanceof Label,
+ "%s %s %s", key, errorInfo, badKey);
+ rootCauses.put(topLevelLabel, (Label) badKey.argument());
+ }
+ }
+ for (Label topLevelLabel : result.<Label>keyNames()) {
+ SkyKey topLevelTransitiveTargetKey = TransitiveTargetValue.key(topLevelLabel);
+ TransitiveTargetValue topLevelTransitiveTargetValue = result.get(topLevelTransitiveTargetKey);
+ if (topLevelTransitiveTargetValue.getTransitiveRootCauses() != null) {
+ for (Label rootCause : topLevelTransitiveTargetValue.getTransitiveRootCauses()) {
+ rootCauses.put(topLevelLabel, rootCause);
+ }
+ warnAboutLoadingFailure(topLevelLabel, eventHandler);
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasErrors(EvaluationResult<TransitiveTargetValue> result) {
+ if (result.hasError()) {
+ return true;
+ }
+ for (TransitiveTargetValue transitiveTargetValue : result.values()) {
+ if (transitiveTargetValue.getTransitiveRootCauses() != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isDirectErrorFromTopLevelLabel(Label label, Set<Label> topLevelLabels,
+ ErrorInfo errorInfo) {
+ return errorInfo.getException() != null && topLevelLabels.contains(label)
+ && Iterables.contains(errorInfo.getRootCauses(), TransitiveTargetValue.key(label));
+ }
+
+ private static void errorAboutLoadingFailure(Label topLevelLabel, @Nullable Throwable throwable,
+ EventHandler eventHandler) {
+ eventHandler.handle(Event.error(
+ "Loading of target '" + topLevelLabel + "' failed; build aborted" +
+ (throwable == null ? "" : ": " + throwable.getMessage())));
+ }
+
+ private static void warnAboutLoadingFailure(Label label, EventHandler eventHandler) {
+ eventHandler.handle(Event.warn(
+ // TODO(bazel-team): We use 'analyzing' here so that we print the same message as legacy
+ // Blaze. Once we get rid of legacy we should be able to change to 'loading' or
+ // similar.
+ "errors encountered while analyzing target '" + label + "': it will not be built"));
+ }
+
+ private static Set<Label> getRootCausesOfCycles(Label labelToLoad, Iterable<CycleInfo> cycles) {
+ ImmutableSet.Builder<Label> builder = ImmutableSet.builder();
+ for (CycleInfo cycleInfo : cycles) {
+ // The root cause of a cycle depends on the type of a cycle.
+
+ SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null);
+ if (culprit == null) {
+ continue;
+ }
+ if (culprit.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) {
+ // For a cycle between build targets, the root cause is the first element of the cycle.
+ builder.add((Label) culprit.argument());
+ } else {
+ // For other types of cycles (e.g. file symlink cycles), the root cause is the furthest
+ // target dependency that itself depended on the cycle.
+ Label furthestTarget = labelToLoad;
+ for (SkyKey skyKey : cycleInfo.getPathToCycle()) {
+ if (skyKey.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) {
+ furthestTarget = (Label) skyKey.argument();
+ } else {
+ break;
+ }
+ }
+ builder.add(furthestTarget);
+ }
+ }
+ return builder.build();
+ }
+
+ // Unfortunately we have to do an effective O(TC) visitation after the eval() call above to
+ // determine all of the packages in the closure.
+ private void updateVisitedValues(Collection<TransitiveTargetValue> targetValues) {
+ Set<TransitiveTargetValue> currentBuildTargetValueSet = new HashSet<>(targetValues);
+ if (Objects.equals(previousBuildTargetValueSet, currentBuildTargetValueSet)) {
+ // The next stanza is slow (and scales with the edge count of the target graph), so avoid
+ // the computation if the previous build already did it.
+ return;
+ }
+ NestedSetBuilder<PackageIdentifier> nestedAllPkgsBuilder = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<PackageIdentifier> nestedErrorFreePkgsBuilder = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Label> nestedTargetBuilder = NestedSetBuilder.stableOrder();
+ for (TransitiveTargetValue value : targetValues) {
+ nestedAllPkgsBuilder.addTransitive(value.getTransitiveSuccessfulPackages());
+ nestedAllPkgsBuilder.addTransitive(value.getTransitiveUnsuccessfulPackages());
+ nestedErrorFreePkgsBuilder.addTransitive(value.getTransitiveSuccessfulPackages());
+ nestedTargetBuilder.addTransitive(value.getTransitiveTargets());
+ }
+ allVisitedPackages = nestedAllPkgsBuilder.build().toSet();
+ errorFreeVisitedPackages = nestedErrorFreePkgsBuilder.build().toSet();
+ visitedTargets = nestedTargetBuilder.build().toSet();
+ previousBuildTargetValueSet = currentBuildTargetValueSet;
+ }
+
+
+ @Override
+ public Set<PackageIdentifier> getVisitedPackageNames() {
+ return allVisitedPackages;
+ }
+
+ @Override
+ public Set<Package> getErrorFreeVisitedPackages() {
+ return transitivePackageLoader.retrievePackages(errorFreeVisitedPackages);
+ }
+
+ /**
+ * Doesn't necessarily include all top-level targets visited in error, because of issues with
+ * skyframe semantics (e.g. impossible to load a target if it transitively depends on a file
+ * symlink cycle). This is actually fine for the non-test usages of this method since such bad
+ * targets get filtered out.
+ */
+ @Override
+ public Set<Label> getVisitedTargets() {
+ return visitedTargets;
+ }
+
+ @Override
+ public Multimap<Label, Label> getRootCauses(final Collection<Label> targetsToLoad) {
+ Preconditions.checkState(lastBuildKeepGoing);
+ return Multimaps.filterKeys(rootCauses,
+ new Predicate<Label>() {
+ @Override
+ public boolean apply(Label label) {
+ return targetsToLoad.contains(label);
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java
new file mode 100644
index 0000000..e467ae0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java
@@ -0,0 +1,120 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Repeats functionality of {@link SkyframePackageLoader} but uses
+ * {@link SkyFunction.Environment#getValue} instead of {@link MemoizingEvaluator#evaluate}
+ * for node evaluation
+ */
+class SkyframePackageLoaderWithValueEnvironment implements
+ PackageProviderForConfigurations {
+ private final SkyFunction.Environment env;
+ private final Set<Package> packages;
+
+ public SkyframePackageLoaderWithValueEnvironment(SkyFunction.Environment env,
+ Set<Package> packages) {
+ this.env = env;
+ this.packages = packages;
+ }
+
+ private Package getPackage(PackageIdentifier pkgIdentifier) throws NoSuchPackageException{
+ SkyKey key = PackageValue.key(pkgIdentifier);
+ PackageValue value = (PackageValue) env.getValueOrThrow(key, NoSuchPackageException.class);
+ if (value != null) {
+ packages.add(value.getPackage());
+ return value.getPackage();
+ }
+ return null;
+ }
+
+ @Override
+ public Package getLoadedPackage(final PackageIdentifier pkgIdentifier)
+ throws NoSuchPackageException {
+ try {
+ return getPackage(pkgIdentifier);
+ } catch (NoSuchPackageException e) {
+ if (e.getPackage() != null) {
+ return e.getPackage();
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public Target getLoadedTarget(Label label) throws NoSuchPackageException,
+ NoSuchTargetException {
+ Package pkg = getLoadedPackage(label.getPackageIdentifier());
+ return pkg == null ? null : pkg.getTarget(label.getName());
+ }
+
+ @Override
+ public boolean isTargetCurrent(Target target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addDependency(Package pkg, String fileName) throws SyntaxException, IOException {
+ RootedPath fileRootedPath = RootedPath.toRootedPath(pkg.getSourceRoot(),
+ pkg.getNameFragment().getRelative(fileName));
+ FileValue result = (FileValue) env.getValue(FileValue.key(fileRootedPath));
+ if (result != null && !result.exists()) {
+ throw new IOException();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType)
+ throws InvalidConfigurationException {
+ ConfigurationFragmentValue fragmentNode = (ConfigurationFragmentValue) env.getValueOrThrow(
+ ConfigurationFragmentValue.key(buildOptions, fragmentType),
+ InvalidConfigurationException.class);
+ if (fragmentNode == null) {
+ return null;
+ }
+ return (T) fragmentNode.getFragment();
+ }
+
+ @Override
+ public BlazeDirectories getDirectories() {
+ return PrecomputedValue.BLAZE_DIRECTORIES.get(env);
+ }
+
+ @Override
+ public boolean valuesMissing() {
+ return env.valuesMissing();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
new file mode 100644
index 0000000..cc32bf8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
@@ -0,0 +1,177 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.CyclesReporter;
+
+import java.io.PrintStream;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Skyframe-based package manager.
+ *
+ * <p>This is essentially a compatibility shim between the native Skyframe and non-Skyframe
+ * parts of Blaze and should not be long-lived.
+ */
+class SkyframePackageManager implements PackageManager {
+
+ private final SkyframePackageLoader packageLoader;
+ private final SkyframeExecutor.SkyframeTransitivePackageLoader transitiveLoader;
+ private final TargetPatternEvaluator patternEvaluator;
+ private final AtomicReference<UnixGlob.FilesystemCalls> syscalls;
+ private final AtomicReference<CyclesReporter> skyframeCyclesReporter;
+ private final AtomicReference<PathPackageLocator> pkgLocator;
+ private final AtomicInteger numPackagesLoaded;
+ private final SkyframeExecutor skyframeExecutor;
+
+ public SkyframePackageManager(SkyframePackageLoader packageLoader,
+ SkyframeExecutor.SkyframeTransitivePackageLoader transitiveLoader,
+ TargetPatternEvaluator patternEvaluator,
+ AtomicReference<UnixGlob.FilesystemCalls> syscalls,
+ AtomicReference<CyclesReporter> skyframeCyclesReporter,
+ AtomicReference<PathPackageLocator> pkgLocator,
+ AtomicInteger numPackagesLoaded,
+ SkyframeExecutor skyframeExecutor) {
+ this.packageLoader = packageLoader;
+ this.transitiveLoader = transitiveLoader;
+ this.patternEvaluator = patternEvaluator;
+ this.skyframeCyclesReporter = skyframeCyclesReporter;
+ this.pkgLocator = pkgLocator;
+ this.syscalls = syscalls;
+ this.numPackagesLoaded = numPackagesLoaded;
+ this.skyframeExecutor = skyframeExecutor;
+ }
+
+ @Override
+ public Package getLoadedPackage(PackageIdentifier pkgIdentifier) throws NoSuchPackageException {
+ return packageLoader.getLoadedPackage(pkgIdentifier);
+ }
+
+ @ThreadSafe
+ @Override
+ public Package getPackage(EventHandler eventHandler, PackageIdentifier packageIdentifier)
+ throws NoSuchPackageException, InterruptedException {
+ try {
+ return packageLoader.getPackage(eventHandler, packageIdentifier);
+ } catch (NoSuchPackageException e) {
+ if (e.getPackage() != null) {
+ return e.getPackage();
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public Target getLoadedTarget(Label label) throws NoSuchPackageException, NoSuchTargetException {
+ return getLoadedPackage(label.getPackageIdentifier()).getTarget(label.getName());
+ }
+
+ @Override
+ public Target getTarget(EventHandler eventHandler, Label label)
+ throws NoSuchPackageException, NoSuchTargetException, InterruptedException {
+ return getPackage(eventHandler, label.getPackageIdentifier()).getTarget(label.getName());
+ }
+
+ @Override
+ public boolean isTargetCurrent(Target target) {
+ Package pkg = target.getPackage();
+ try {
+ return getLoadedPackage(target.getLabel().getPackageIdentifier()) == pkg;
+ } catch (NoSuchPackageException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void partiallyClear() {
+ packageLoader.partiallyClear();
+ }
+
+ @Override
+ public PackageManagerStatistics getStatistics() {
+ return new PackageManagerStatistics() {
+ @Override
+ public int getPackagesLoaded() {
+ return numPackagesLoaded.get();
+ }
+
+ @Override
+ public int getPackagesLookedUp() {
+ return -1;
+ }
+
+ @Override
+ public int getCacheSize() {
+ return -1;
+ }
+ };
+ }
+
+ @Override
+ public boolean isPackage(String packageName) {
+ return getBuildFileForPackage(packageName) != null;
+ }
+
+ @Override
+ public void dump(PrintStream printStream) {
+ skyframeExecutor.dumpPackages(printStream);
+ }
+
+ @ThreadSafe
+ @Override
+ public Path getBuildFileForPackage(String packageName) {
+ // Note that this method needs to be thread-safe, as it is currently used concurrently by
+ // legacy blaze code.
+ if (packageLoader.isPackageDeleted(packageName)
+ || LabelValidator.validatePackageName(packageName) != null) {
+ return null;
+ }
+ // TODO(bazel-team): Use a PackageLookupValue here [skyframe-loading]
+ // TODO(bazel-team): The implementation in PackageCache also checks for duplicate packages, see
+ // BuildFileCache#getBuildFile [skyframe-loading]
+ return pkgLocator.get().getPackageBuildFileNullable(packageName, syscalls);
+ }
+
+ @Override
+ public PathPackageLocator getPackagePath() {
+ return pkgLocator.get();
+ }
+
+ @Override
+ public TransitivePackageLoader newTransitiveLoader() {
+ return new SkyframeLabelVisitor(transitiveLoader, skyframeCyclesReporter);
+ }
+
+ @Override
+ public TargetPatternEvaluator getTargetPatternEvaluator() {
+ return patternEvaluator;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java
new file mode 100644
index 0000000..9e619e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java
@@ -0,0 +1,146 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.pkgcache.ParseFailureListener;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Skyframe-based target pattern parsing.
+ */
+final class SkyframeTargetPatternEvaluator implements TargetPatternEvaluator {
+ private final SkyframeExecutor skyframeExecutor;
+ private String offset = "";
+
+ SkyframeTargetPatternEvaluator(SkyframeExecutor skyframeExecutor) {
+ this.skyframeExecutor = skyframeExecutor;
+ }
+
+ @Override
+ public ResolvedTargets<Target> parseTargetPatternList(EventHandler eventHandler,
+ List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing)
+ throws TargetParsingException, InterruptedException {
+ return parseTargetPatternList(offset, eventHandler, targetPatterns, policy, keepGoing);
+ }
+
+ @Override
+ public ResolvedTargets<Target> parseTargetPattern(EventHandler eventHandler,
+ String pattern, boolean keepGoing) throws TargetParsingException, InterruptedException {
+ return parseTargetPatternList(eventHandler, ImmutableList.of(pattern),
+ FilteringPolicies.NO_FILTER, keepGoing);
+ }
+
+ @Override
+ public void updateOffset(PathFragment relativeWorkingDirectory) {
+ offset = relativeWorkingDirectory.getPathString();
+ }
+
+ @Override
+ public String getOffset() {
+ return offset;
+ }
+
+ @Override
+ public Map<String, ResolvedTargets<Target>> preloadTargetPatterns(EventHandler eventHandler,
+ Collection<String> patterns, boolean keepGoing)
+ throws TargetParsingException, InterruptedException {
+ // TODO(bazel-team): This is used only in "blaze query". There are plans to dramatically change
+ // how query works on Skyframe, in which case this method is likely to go away.
+ // We cannot use an ImmutableMap here because there may be null values.
+ Map<String, ResolvedTargets<Target>> result = Maps.newHashMapWithExpectedSize(patterns.size());
+ for (String pattern : patterns) {
+ // TODO(bazel-team): This could be parallelized to improve performance. [skyframe-loading]
+ result.put(pattern, parseTargetPattern(eventHandler, pattern, keepGoing));
+ }
+ return result;
+ }
+
+ /**
+ * Loads a list of target patterns (eg, "foo/...").
+ */
+ ResolvedTargets<Target> parseTargetPatternList(String offset, EventHandler eventHandler,
+ List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing)
+ throws InterruptedException, TargetParsingException {
+ Iterable<SkyKey> patternSkyKeys = TargetPatternValue.keys(targetPatterns, policy, offset);
+ EvaluationResult<TargetPatternValue> result =
+ skyframeExecutor.targetPatterns(patternSkyKeys, keepGoing, eventHandler);
+
+ String errorMessage = null;
+ ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder();
+ for (SkyKey key : patternSkyKeys) {
+ TargetPatternValue resultValue = result.get(key);
+ if (resultValue != null) {
+ ResolvedTargets<Target> results = resultValue.getTargets();
+ if (((TargetPatternValue.TargetPattern) key.argument()).isNegative()) {
+ builder.filter(Predicates.not(Predicates.in(results.getTargets())));
+ } else {
+ builder.merge(results);
+ }
+ } else {
+ TargetPatternValue.TargetPattern pattern =
+ (TargetPatternValue.TargetPattern) key.argument();
+ String rawPattern = pattern.getPattern();
+ ErrorInfo error = result.errorMap().get(key);
+ if (error == null) {
+ Preconditions.checkState(!keepGoing);
+ continue;
+ }
+ if (error.getException() != null) {
+ errorMessage = error.getException().getMessage();
+ } else if (!Iterables.isEmpty(error.getCycleInfo())) {
+ errorMessage = "cycles detected during target parsing";
+ skyframeExecutor.getCyclesReporter().reportCycles(
+ error.getCycleInfo(), key, eventHandler);
+ } else {
+ throw new IllegalStateException(error.toString());
+ }
+ if (keepGoing) {
+ eventHandler.handle(Event.error("Skipping '" + rawPattern + "': " + errorMessage));
+ }
+ builder.setError();
+
+ if (eventHandler instanceof ParseFailureListener) {
+ ParseFailureListener parseListener = (ParseFailureListener) eventHandler;
+ parseListener.parsingError(rawPattern, errorMessage);
+ }
+ }
+ }
+
+ if (!keepGoing && result.hasError()) {
+ Preconditions.checkState(errorMessage != null, "unexpected errors: %s", result.errorMap());
+ throw new TargetParsingException(errorMessage);
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java
new file mode 100644
index 0000000..02d2e91
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.syntax.Label;
+
+/**
+ * A simple value class to store the direct Skylark file dependencies of a Skylark
+ * extension file. It also contains a Label identifying the extension file.
+ */
+class SkylarkFileDependency {
+
+ private final Label label;
+ private final ImmutableList<SkylarkFileDependency> dependencies;
+
+ SkylarkFileDependency(Label label, ImmutableList<SkylarkFileDependency> dependencies) {
+ this.label = label;
+ this.dependencies = dependencies;
+ }
+
+ /**
+ * Returns the list of direct Skylark file dependencies of the Skylark extension file
+ * corresponding to this object.
+ */
+ ImmutableList<SkylarkFileDependency> getDependencies() {
+ return dependencies;
+ }
+
+ /**
+ * Returns the Label of the Skylark extension file corresponding to this object.
+ */
+ Label getLabel() {
+ return label;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java
new file mode 100644
index 0000000..02d41e6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java
@@ -0,0 +1,238 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A Skyframe function to look up and import a single Skylark extension.
+ */
+public class SkylarkImportLookupFunction implements SkyFunction {
+
+ private final RuleClassProvider ruleClassProvider;
+ private final ImmutableList<Function> nativeRuleFunctions;
+
+ public SkylarkImportLookupFunction(
+ RuleClassProvider ruleClassProvider, PackageFactory packageFactory) {
+ this.ruleClassProvider = ruleClassProvider;
+ this.nativeRuleFunctions = packageFactory.collectNativeRuleFunctions();
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ PackageIdentifier arg = (PackageIdentifier) skyKey.argument();
+ PathFragment file = arg.getPackageFragment();
+ ASTFileLookupValue astLookupValue = null;
+ try {
+ SkyKey astLookupKey = ASTFileLookupValue.key(file);
+ astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey,
+ ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class);
+ } catch (ErrorReadingSkylarkExtensionException e) {
+ throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.errorReadingFile(
+ file, e.getMessage()));
+ } catch (InconsistentFilesystemException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ } catch (ASTLookupInputException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ if (astLookupValue == null) {
+ return null;
+ }
+ if (astLookupValue == ASTFileLookupValue.NO_FILE) {
+ // Skylark import files have to exist.
+ throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.noFile(file));
+ }
+
+ Map<PathFragment, SkylarkEnvironment> importMap = new HashMap<>();
+ ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
+ BuildFileAST ast = astLookupValue.getAST();
+ // TODO(bazel-team): Refactor this code and PackageFunction to reduce code duplications.
+ for (PathFragment importFile : ast.getImports()) {
+ try {
+ SkyKey importsLookupKey = SkylarkImportLookupValue.key(arg.getRepository(), importFile);
+ SkylarkImportLookupValue importsLookupValue;
+ importsLookupValue = (SkylarkImportLookupValue) env.getValueOrThrow(
+ importsLookupKey, ASTLookupInputException.class);
+ if (importsLookupValue != null) {
+ importMap.put(importFile, importsLookupValue.getImportedEnvironment());
+ fileDependencies.add(importsLookupValue.getDependency());
+ }
+ } catch (ASTLookupInputException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ }
+ }
+ Label label = pathFragmentToLabel(arg.getRepository(), file, env);
+ if (env.valuesMissing()) {
+ // This means some imports are unavailable.
+ return null;
+ }
+
+ if (ast.containsErrors()) {
+ throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.skylarkErrors(
+ file));
+ }
+
+ SkylarkEnvironment extensionEnv = createEnv(ast, importMap, env);
+ // Skylark UserDefinedFunctions are sharing function definition Environments, so it's extremely
+ // important not to modify them from this point. Ideally they should be only used to import
+ // symbols and serve as global Environments of UserDefinedFunctions.
+ return new SkylarkImportLookupValue(
+ extensionEnv, new SkylarkFileDependency(label, fileDependencies.build()));
+ }
+
+ /**
+ * Converts the PathFragment of the Skylark file to a Label using the BUILD file closest to the
+ * Skylark file in its directory hierarchy - finds the package to which the Skylark file belongs.
+ * Throws an exception if no such BUILD file exists.
+ */
+ private Label pathFragmentToLabel(RepositoryName repo, PathFragment file, Environment env)
+ throws SkylarkImportLookupFunctionException {
+ ContainingPackageLookupValue containingPackageLookupValue = null;
+ try {
+ PackageIdentifier newPkgId = new PackageIdentifier(repo, file.getParentDirectory());
+ containingPackageLookupValue = (ContainingPackageLookupValue) env.getValueOrThrow(
+ ContainingPackageLookupValue.key(newPkgId),
+ BuildFileNotFoundException.class, InconsistentFilesystemException.class);
+ } catch (BuildFileNotFoundException e) {
+ // Thrown when there are IO errors looking for BUILD files.
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ } catch (InconsistentFilesystemException e) {
+ throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
+ }
+
+ if (containingPackageLookupValue == null) {
+ return null;
+ }
+
+ if (!containingPackageLookupValue.hasContainingPackage()) {
+ throw new SkylarkImportLookupFunctionException(
+ SkylarkImportFailedException.noBuildFile(file));
+ }
+
+ PathFragment pkgName =
+ containingPackageLookupValue.getContainingPackageName().getPackageFragment();
+ PathFragment fileInPkg = file.relativeTo(pkgName);
+
+ try {
+ // This code relies on PackageIdentifier.RepositoryName.toString()
+ return Label.parseRepositoryLabel(repo + "//" + pkgName.getPathString() + ":" + fileInPkg);
+ } catch (SyntaxException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Creates the SkylarkEnvironment to be imported. After it's returned, the Environment
+ * must not be modified.
+ */
+ private SkylarkEnvironment createEnv(BuildFileAST ast,
+ Map<PathFragment, SkylarkEnvironment> importMap, Environment env)
+ throws InterruptedException {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ // TODO(bazel-team): this method overestimates the changes which can affect the
+ // Skylark RuleClass. For example changes to comments or unused functions can modify the hash.
+ // A more accurate - however much more complicated - way would be to calculate a hash based on
+ // the transitive closure of the accessible AST nodes.
+ SkylarkEnvironment extensionEnv = ruleClassProvider
+ .createSkylarkRuleClassEnvironment(eventHandler, ast.getContentHashCode());
+ // Adding native rules module for build extensions.
+ // TODO(bazel-team): this might not be the best place to do this.
+ extensionEnv.update("native", ruleClassProvider.getNativeModule());
+ for (Function function : nativeRuleFunctions) {
+ extensionEnv.registerFunction(
+ ruleClassProvider.getNativeModule().getClass(), function.getName(), function);
+ }
+ extensionEnv.setImportedExtensions(importMap);
+ ast.exec(extensionEnv, eventHandler);
+ // Don't fail just replay the events so the original package lookup can fail.
+ Event.replayEventsOn(env.getListener(), eventHandler.getEvents());
+ return extensionEnv;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ static final class SkylarkImportFailedException extends Exception {
+ private SkylarkImportFailedException(String errorMessage) {
+ super(errorMessage);
+ }
+
+ static SkylarkImportFailedException errorReadingFile(PathFragment file, String error) {
+ return new SkylarkImportFailedException(
+ String.format("Encountered error while reading extension file '%s': %s", file, error));
+ }
+
+ static SkylarkImportFailedException noFile(PathFragment file) {
+ return new SkylarkImportFailedException(
+ String.format("Extension file not found: '%s'", file));
+ }
+
+ static SkylarkImportFailedException noBuildFile(PathFragment file) {
+ return new SkylarkImportFailedException(
+ String.format("Every .bzl file must have a corresponding package, but '%s' "
+ + "does not have one. Please create a BUILD file in the same or any parent directory."
+ + " Note that this BUILD file does not need to do anything except exist.", file));
+ }
+
+ static SkylarkImportFailedException skylarkErrors(PathFragment file) {
+ return new SkylarkImportFailedException(String.format("Extension '%s' has errors", file));
+ }
+ }
+
+ private static final class SkylarkImportLookupFunctionException extends SkyFunctionException {
+ private SkylarkImportLookupFunctionException(SkylarkImportFailedException cause) {
+ super(cause, Transience.PERSISTENT);
+ }
+
+ private SkylarkImportLookupFunctionException(InconsistentFilesystemException e,
+ Transience transience) {
+ super(e, transience);
+ }
+
+ private SkylarkImportLookupFunctionException(ASTLookupInputException e,
+ Transience transience) {
+ super(e, transience);
+ }
+
+ private SkylarkImportLookupFunctionException(BuildFileNotFoundException e,
+ Transience transience) {
+ super(e, transience);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java
new file mode 100644
index 0000000..3c87431
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException;
+import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A value that represents a Skylark import lookup result. The lookup value corresponds to
+ * exactly one Skylark file, identified by the PathFragment SkyKey argument.
+ */
+public class SkylarkImportLookupValue implements SkyValue {
+
+ private final SkylarkEnvironment importedEnvironment;
+ /**
+ * The immediate Skylark file dependency descriptor class corresponding to this value.
+ * Using this reference it's possible to reach the transitive closure of Skylark files
+ * on which this Skylark file depends.
+ */
+ private final SkylarkFileDependency dependency;
+
+ public SkylarkImportLookupValue(
+ SkylarkEnvironment importedEnvironment, SkylarkFileDependency dependency) {
+ this.importedEnvironment = Preconditions.checkNotNull(importedEnvironment);
+ this.dependency = Preconditions.checkNotNull(dependency);
+ }
+
+ /**
+ * Returns the imported SkylarkEnvironment.
+ */
+ public SkylarkEnvironment getImportedEnvironment() {
+ return importedEnvironment;
+ }
+
+ /**
+ * Returns the immediate Skylark file dependency corresponding to this import lookup value.
+ */
+ public SkylarkFileDependency getDependency() {
+ return dependency;
+ }
+
+ static SkyKey key(PackageIdentifier pkgIdentifier) throws ASTLookupInputException {
+ return key(pkgIdentifier.getRepository(), pkgIdentifier.getPackageFragment());
+ }
+
+ static SkyKey key(RepositoryName repo, PathFragment fileToImport) throws ASTLookupInputException {
+ // Skylark import lookup keys need to be valid AST file lookup keys.
+ ASTFileLookupValue.checkInputArgument(fileToImport);
+ return new SkyKey(
+ SkyFunctions.SKYLARK_IMPORTS_LOOKUP,
+ new PackageIdentifier(repo, fileToImport));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java
new file mode 100644
index 0000000..a0f37a9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.CyclesReporter;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Reports cycles of recursive import of Skylark files.
+ */
+public class SkylarkModuleCycleReporter implements CyclesReporter.SingleCycleReporter {
+
+ private static final Predicate<SkyKey> IS_SKYLARK_MODULE_SKY_KEY =
+ SkyFunctions.isSkyFunction(SkyFunctions.SKYLARK_IMPORTS_LOOKUP);
+
+ private static final Predicate<SkyKey> IS_PACKAGE_SKY_KEY =
+ SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE);
+
+ @Override
+ public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, boolean alreadyReported,
+ EventHandler eventHandler) {
+ ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle();
+ if (pathToCycle.size() == 0) {
+ return false;
+ }
+ SkyKey lastPathElement = cycleInfo.getPathToCycle().get(pathToCycle.size() - 1);
+ if (alreadyReported) {
+ return true;
+ } else if (Iterables.all(cycleInfo.getCycle(), IS_SKYLARK_MODULE_SKY_KEY)
+ // The last element of the path to the cycle has to be a PackageFunction.
+ && IS_PACKAGE_SKY_KEY.apply(lastPathElement)) {
+ StringBuilder cycleMessage = new StringBuilder()
+ .append(((PackageIdentifier) lastPathElement.argument()).toString() + "/BUILD: ")
+ .append("cycle in referenced extension files: ");
+
+ AbstractLabelCycleReporter.printCycle(cycleInfo.getCycle(), cycleMessage,
+ new Function<SkyKey, String>() {
+ @Override
+ public String apply(SkyKey input) {
+ return ((PackageIdentifier) input.argument()).toString();
+ }
+ });
+
+ // TODO(bazel-team): it would be nice to pass the Location of the load Statement in the
+ // BUILD file.
+ eventHandler.handle(Event.error(null, cycleMessage.toString()));
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java
new file mode 100644
index 0000000..2e4aea9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java
@@ -0,0 +1,138 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException2;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * TargetCompletionFunction builds the artifactsToBuild collection of a {@link ConfiguredTarget}.
+ */
+public final class TargetCompletionFunction implements SkyFunction {
+
+ private final AtomicReference<EventBus> eventBusRef;
+
+ public TargetCompletionFunction(AtomicReference<EventBus> eventBusRef) {
+ this.eventBusRef = eventBusRef;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws TargetCompletionFunctionException {
+ LabelAndConfiguration lac = (LabelAndConfiguration) skyKey.argument();
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue)
+ env.getValue(ConfiguredTargetValue.key(lac.getLabel(), lac.getConfiguration()));
+ TopLevelArtifactContext topLevelContext = PrecomputedValue.TOP_LEVEL_CONTEXT.get(env);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps =
+ env.getValuesOrThrow(ArtifactValue.mandatoryKeys(
+ TopLevelArtifactHelper.getAllArtifactsToBuild(
+ ctValue.getConfiguredTarget(), topLevelContext)),
+ MissingInputFileException.class, ActionExecutionException.class);
+
+ int missingCount = 0;
+ ActionExecutionException firstActionExecutionException = null;
+ MissingInputFileException missingInputException = null;
+ NestedSetBuilder<Label> rootCausesBuilder = NestedSetBuilder.stableOrder();
+ for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException,
+ ActionExecutionException>> depsEntry : inputDeps.entrySet()) {
+ Artifact input = ArtifactValue.artifact(depsEntry.getKey());
+ try {
+ depsEntry.getValue().get();
+ } catch (MissingInputFileException e) {
+ missingCount++;
+ if (input.getOwner() != null) {
+ rootCausesBuilder.add(input.getOwner());
+ env.getListener().handle(Event.error(
+ ctValue.getConfiguredTarget().getTarget().getLocation(),
+ String.format("%s: missing input file '%s'",
+ lac.getLabel(), input.getOwner())));
+ }
+ } catch (ActionExecutionException e) {
+ rootCausesBuilder.addTransitive(e.getRootCauses());
+ if (firstActionExecutionException == null) {
+ firstActionExecutionException = e;
+ }
+ }
+ }
+
+ if (missingCount > 0) {
+ missingInputException = new MissingInputFileException(
+ ctValue.getConfiguredTarget().getTarget().getLocation() + " " + missingCount
+ + " input file(s) do not exist", ctValue.getConfiguredTarget().getTarget().getLocation());
+ }
+
+ NestedSet<Label> rootCauses = rootCausesBuilder.build();
+ if (!rootCauses.isEmpty()) {
+ eventBusRef.get().post(
+ TargetCompleteEvent.createFailed(ctValue.getConfiguredTarget(), rootCauses));
+ if (firstActionExecutionException != null) {
+ throw new TargetCompletionFunctionException(firstActionExecutionException);
+ } else {
+ throw new TargetCompletionFunctionException(missingInputException);
+ }
+ }
+
+ return env.valuesMissing() ? null : new TargetCompletionValue(ctValue.getConfiguredTarget());
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel());
+ }
+
+ private static final class TargetCompletionFunctionException extends SkyFunctionException {
+
+ private final ActionExecutionException actionException;
+
+ public TargetCompletionFunctionException(ActionExecutionException e) {
+ super(e, Transience.PERSISTENT);
+ this.actionException = e;
+ }
+
+ public TargetCompletionFunctionException(MissingInputFileException e) {
+ super(e, Transience.TRANSIENT);
+ this.actionException = null;
+ }
+
+ @Override
+ public boolean isCatastrophic() {
+ return actionException != null && actionException.isCatastrophe();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java
new file mode 100644
index 0000000..5d7153d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+
+/**
+ * The value of a TargetCompletion. Currently this just stores a ConfiguredTarget.
+ */
+public class TargetCompletionValue implements SkyValue {
+ private final ConfiguredTarget ct;
+
+ TargetCompletionValue(ConfiguredTarget ct) {
+ this.ct = ct;
+ }
+
+ public ConfiguredTarget getConfiguredTarget() {
+ return ct;
+ }
+
+ public static SkyKey key(LabelAndConfiguration labelAndConfiguration) {
+ return new SkyKey(SkyFunctions.TARGET_COMPLETION, labelAndConfiguration);
+ }
+
+ public static Iterable<SkyKey> keys(Collection<ConfiguredTarget> targets) {
+ return Iterables.transform(targets, new Function<ConfiguredTarget, SkyKey>() {
+ @Override
+ public SkyKey apply(ConfiguredTarget ct) {
+ return new SkyKey(SkyFunctions.TARGET_COMPLETION, new LabelAndConfiguration(ct));
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java
new file mode 100644
index 0000000..3f9f22f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java
@@ -0,0 +1,134 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * A SkyFunction for {@link TargetMarkerValue}s.
+ */
+public final class TargetMarkerFunction implements SkyFunction {
+
+ public TargetMarkerFunction() {
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws TargetMarkerFunctionException {
+ Label label = (Label) key.argument();
+ PathFragment pkgForLabel = label.getPackageFragment();
+
+ if (label.getName().contains("/")) {
+ // This target is in a subdirectory, therefore it could potentially be invalidated by
+ // a new BUILD file appearing in the hierarchy.
+ PathFragment containingDirectory = label.toPathFragment().getParentDirectory();
+ ContainingPackageLookupValue containingPackageLookupValue = null;
+ try {
+ PackageIdentifier newPkgId = new PackageIdentifier(
+ label.getPackageIdentifier().getRepository(), containingDirectory);
+ containingPackageLookupValue = (ContainingPackageLookupValue) env.getValueOrThrow(
+ ContainingPackageLookupValue.key(newPkgId),
+ BuildFileNotFoundException.class, InconsistentFilesystemException.class);
+ } catch (BuildFileNotFoundException e) {
+ // Thrown when there are IO errors looking for BUILD files.
+ throw new TargetMarkerFunctionException(e);
+ } catch (InconsistentFilesystemException e) {
+ throw new TargetMarkerFunctionException(new NoSuchTargetException(label,
+ e.getMessage()));
+ }
+ if (containingPackageLookupValue == null) {
+ return null;
+ }
+ if (!containingPackageLookupValue.hasContainingPackage()) {
+ // This means the label's package doesn't exist. E.g. there is no package 'a' and we are
+ // trying to build the target for label 'a:b/foo'.
+ throw new TargetMarkerFunctionException(new BuildFileNotFoundException(
+ pkgForLabel.getPathString(), "BUILD file not found on package path for '"
+ + pkgForLabel.getPathString() + "'"));
+ }
+ if (!containingPackageLookupValue.getContainingPackageName().equals(
+ label.getPackageIdentifier())) {
+ throw new TargetMarkerFunctionException(new NoSuchTargetException(label,
+ String.format("Label '%s' crosses boundary of subpackage '%s'", label,
+ containingPackageLookupValue.getContainingPackageName())));
+ }
+ }
+
+ SkyKey pkgSkyKey = PackageValue.key(label.getPackageIdentifier());
+ NoSuchPackageException nspe = null;
+ Package pkg;
+ try {
+ PackageValue value = (PackageValue)
+ env.getValueOrThrow(pkgSkyKey, NoSuchPackageException.class);
+ if (value == null) {
+ return null;
+ }
+ pkg = value.getPackage();
+ } catch (NoSuchPackageException e) {
+ // For consistency with pre-Skyframe Blaze, we can return a valid Target from a Package
+ // containing errors.
+ pkg = e.getPackage();
+ if (pkg == null) {
+ // Re-throw this exception with our key because root causes should be targets, not packages.
+ throw new TargetMarkerFunctionException(e);
+ }
+ nspe = e;
+ }
+
+ Target target;
+ try {
+ target = pkg.getTarget(label.getName());
+ } catch (NoSuchTargetException e) {
+ throw new TargetMarkerFunctionException(e);
+ }
+
+ if (nspe != null) {
+ // There is a target, but its package is in error. We rethrow so that the root cause is the
+ // target, not the package. Note that targets are only in error when their package is
+ // "in error" (because a package is in error if there was an error evaluating the package, or
+ // if one of its targets was in error).
+ throw new TargetMarkerFunctionException(new NoSuchTargetException(target, nspe));
+ }
+ return TargetMarkerValue.TARGET_MARKER_INSTANCE;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print((Label) skyKey.argument());
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link TargetMarkerFunction#compute}.
+ */
+ private static final class TargetMarkerFunctionException extends SkyFunctionException {
+ public TargetMarkerFunctionException(NoSuchTargetException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ public TargetMarkerFunctionException(NoSuchPackageException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java
new file mode 100644
index 0000000..eef7084
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * Value represents visited target in the Skyframe graph after error checking.
+ */
+@Immutable
+@ThreadSafe
+public final class TargetMarkerValue implements SkyValue {
+
+ static final TargetMarkerValue TARGET_MARKER_INSTANCE = new TargetMarkerValue();
+
+ private TargetMarkerValue() {
+ }
+
+ @ThreadSafe
+ public static SkyKey key(Label label) {
+ return new SkyKey(SkyFunctions.TARGET_MARKER, label);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java
new file mode 100644
index 0000000..6e631ed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java
@@ -0,0 +1,278 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.TargetParsingException;
+import com.google.devtools.build.lib.cmdline.TargetPattern;
+import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.pkgcache.TargetPatternResolverUtil;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * TargetPatternFunction translates a target pattern (eg, "foo/...") into a set of resolved
+ * Targets.
+ */
+public class TargetPatternFunction implements SkyFunction {
+
+ private final AtomicReference<PathPackageLocator> pkgPath;
+
+ public TargetPatternFunction(AtomicReference<PathPackageLocator> pkgPath) {
+ this.pkgPath = pkgPath;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws TargetPatternFunctionException,
+ InterruptedException {
+ TargetPatternValue.TargetPattern patternKey =
+ ((TargetPatternValue.TargetPattern) key.argument());
+
+ TargetPattern.Parser parser = new TargetPattern.Parser(patternKey.getOffset());
+ try {
+ Resolver resolver = new Resolver(env, patternKey.getPolicy(), pkgPath);
+ TargetPattern resolvedPattern = parser.parse(patternKey.getPattern());
+ return new TargetPatternValue(resolvedPattern.eval(resolver));
+ } catch (TargetParsingException e) {
+ throw new TargetPatternFunctionException(e);
+ } catch (TargetPatternResolver.MissingDepException e) {
+ return null;
+ }
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static class Resolver implements TargetPatternResolver<Target> {
+ private final Environment env;
+ private final FilteringPolicy policy;
+ private final AtomicReference<PathPackageLocator> pkgPath;
+
+ public Resolver(Environment env, FilteringPolicy policy,
+ AtomicReference<PathPackageLocator> pkgPath) {
+ this.policy = policy;
+ this.env = env;
+ this.pkgPath = pkgPath;
+ }
+
+ @Override
+ public void warn(String msg) {
+ env.getListener().handle(Event.warn(msg));
+ }
+
+ /**
+ * Gets a Package via the Skyframe env. May return a Package that has errors.
+ */
+ private Package getPackage(PackageIdentifier pkgIdentifier)
+ throws MissingDepException, NoSuchThingException {
+ SkyKey pkgKey = PackageValue.key(pkgIdentifier);
+ Package pkg;
+ try {
+ PackageValue pkgValue =
+ (PackageValue) env.getValueOrThrow(pkgKey, NoSuchThingException.class);
+ if (pkgValue == null) {
+ throw new MissingDepException();
+ }
+ pkg = pkgValue.getPackage();
+ } catch (NoSuchPackageException e) {
+ pkg = e.getPackage();
+ if (pkg == null) {
+ throw e;
+ }
+ }
+ return pkg;
+ }
+
+ @Override
+ public Target getTargetOrNull(String targetName) throws InterruptedException,
+ MissingDepException {
+ try {
+ Label label = Label.parseAbsolute(targetName);
+ if (!isPackage(label.getPackageName())) {
+ return null;
+ }
+ Package pkg = getPackage(label.getPackageIdentifier());
+ return pkg.getTarget(label.getName());
+ } catch (Label.SyntaxException | NoSuchThingException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Target> getExplicitTarget(String targetName)
+ throws TargetParsingException, InterruptedException, MissingDepException {
+ Label label = TargetPatternResolverUtil.label(targetName);
+ try {
+ Package pkg = getPackage(label.getPackageIdentifier());
+ Target target = pkg.getTarget(label.getName());
+ return policy.shouldRetain(target, true)
+ ? ResolvedTargets.of(target)
+ : ResolvedTargets.<Target>empty();
+ } catch (NoSuchThingException e) {
+ throw new TargetParsingException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName,
+ boolean rulesOnly)
+ throws TargetParsingException, InterruptedException, MissingDepException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+ return getTargetsInPackage(originalPattern, packageName, actualPolicy);
+ }
+
+ private ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName,
+ FilteringPolicy policy)
+ throws TargetParsingException, MissingDepException {
+ // Normalise, e.g "foo//bar" -> "foo/bar"; "foo/" -> "foo":
+ PathFragment packageNameFragment = new PathFragment(packageName);
+ packageName = packageNameFragment.toString();
+
+ // It's possible for this check to pass, but for
+ // Label.validatePackageNameFull to report an error because the
+ // package name is illegal. That's a little weird, but we can live with
+ // that for now--see test case: testBadPackageNameButGoodEnoughForALabel.
+ // (BTW I tried duplicating that validation logic in Label but it was
+ // extremely tricky.)
+ if (LabelValidator.validatePackageName(packageName) != null) {
+ throw new TargetParsingException("'" + packageName + "' is not a valid package name");
+ }
+ if (!isPackage(packageName)) {
+ throw new TargetParsingException(
+ TargetPatternResolverUtil.getParsingErrorMessage(
+ "no such package '" + packageName + "': BUILD file not found on package path",
+ originalPattern));
+ }
+
+ try {
+ Package pkg = getPackage(
+ PackageIdentifier.createInDefaultRepo(packageNameFragment.toString()));
+ return TargetPatternResolverUtil.resolvePackageTargets(pkg, policy);
+ } catch (NoSuchThingException e) {
+ String message = TargetPatternResolverUtil.getParsingErrorMessage(
+ "package contains errors", originalPattern);
+ throw new TargetParsingException(message, e);
+ }
+ }
+
+ @Override
+ public boolean isPackage(String packageName) throws MissingDepException {
+ SkyKey packageLookupKey;
+ packageLookupKey = PackageLookupValue.key(new PathFragment(packageName));
+ PackageLookupValue packageLookupValue = (PackageLookupValue) env.getValue(packageLookupKey);
+ if (packageLookupValue == null) {
+ throw new MissingDepException();
+ }
+ return packageLookupValue.packageExists();
+ }
+
+ @Override
+ public String getTargetKind(Target target) {
+ return target.getTargetKind();
+ }
+
+ @Override
+ public ResolvedTargets<Target> findTargetsBeneathDirectory(
+ String originalPattern, String pathPrefix, boolean rulesOnly)
+ throws TargetParsingException, MissingDepException {
+ FilteringPolicy actualPolicy = rulesOnly
+ ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy)
+ : policy;
+
+ PathFragment directory = new PathFragment(pathPrefix);
+ if (directory.containsUplevelReferences()) {
+ throw new TargetParsingException("up-level references are not permitted: '"
+ + directory.getPathString() + "'");
+ }
+ if (!pathPrefix.isEmpty() && (LabelValidator.validatePackageName(pathPrefix) != null)) {
+ throw new TargetParsingException("'" + pathPrefix + "' is not a valid package name");
+ }
+
+ ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder();
+
+ List<RecursivePkgValue> lookupValues = new ArrayList<>();
+ for (Path root : pkgPath.get().getPathEntries()) {
+ SkyKey key = RecursivePkgValue.key(RootedPath.toRootedPath(root, directory));
+ RecursivePkgValue lookup = (RecursivePkgValue) env.getValue(key);
+ if (lookup != null) {
+ lookupValues.add(lookup);
+ }
+ }
+ if (env.valuesMissing()) {
+ throw new MissingDepException();
+ }
+
+ for (RecursivePkgValue value : lookupValues) {
+ for (String pkg : value.getPackages()) {
+ builder.merge(getTargetsInPackage(originalPattern, pkg, FilteringPolicies.NO_FILTER));
+ }
+ }
+
+ if (builder.isEmpty()) {
+ throw new TargetParsingException("no targets found beneath '" + directory + "'");
+ }
+
+ // Apply the transform after the check so we only return the
+ // error if the tree really contains no targets.
+ ResolvedTargets<Target> intermediateResult = builder.build();
+ ResolvedTargets.Builder<Target> filteredBuilder = ResolvedTargets.builder();
+ if (intermediateResult.hasError()) {
+ filteredBuilder.setError();
+ }
+ for (Target target : intermediateResult.getTargets()) {
+ if (actualPolicy.shouldRetain(target, false)) {
+ filteredBuilder.add(target);
+ }
+ }
+ return filteredBuilder.build();
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link TargetPatternFunction#compute}.
+ */
+ private static final class TargetPatternFunctionException extends SkyFunctionException {
+ public TargetPatternFunctionException(TargetParsingException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java
new file mode 100644
index 0000000..c97194f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java
@@ -0,0 +1,212 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets;
+import com.google.devtools.build.lib.cmdline.ResolvedTargets.Builder;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
+import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A value referring to a computed set of resolved targets. This is used for the results of target
+ * pattern parsing.
+ */
+@Immutable
+@ThreadSafe
+public final class TargetPatternValue implements SkyValue {
+
+ private ResolvedTargets<Target> targets;
+
+ TargetPatternValue(ResolvedTargets<Target> targets) {
+ this.targets = Preconditions.checkNotNull(targets);
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ Set<Package> packages = new LinkedHashSet<>();
+ List<String> ts = new ArrayList<>();
+ List<String> filteredTs = new ArrayList<>();
+ for (Target target : targets.getTargets()) {
+ packages.add(target.getPackage());
+ ts.add(target.getLabel().toString());
+ }
+ for (Target target : targets.getFilteredTargets()) {
+ packages.add(target.getPackage());
+ filteredTs.add(target.getLabel().toString());
+ }
+
+ out.writeObject(packages);
+ out.writeObject(ts);
+ out.writeObject(filteredTs);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ Set<Package> packages = (Set<Package>) in.readObject();
+ List<String> ts = (List<String>) in.readObject();
+ List<String> filteredTs = (List<String>) in.readObject();
+
+ Map<String, Package> packageMap = new HashMap<>();
+ for (Package p : packages) {
+ packageMap.put(p.getName(), p);
+ }
+
+ Builder<Target> builder = ResolvedTargets.<Target>builder();
+ for (String labelString : ts) {
+ builder.add(lookupTarget(packageMap, labelString));
+ }
+
+ for (String labelString : filteredTs) {
+ builder.remove(lookupTarget(packageMap, labelString));
+ }
+ this.targets = builder.build();
+ }
+
+ private static Target lookupTarget(Map<String, Package> packageMap, String labelString) {
+ Label label = Label.parseAbsoluteUnchecked(labelString);
+ Package p = packageMap.get(label.getPackageName());
+ try {
+ return p.getTarget(label.getName());
+ } catch (NoSuchTargetException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private void readObjectNoData() {
+ throw new IllegalStateException();
+ }
+
+ /**
+ * Create a target pattern value key.
+ *
+ * @param pattern The pattern, eg "-foo/biz...". If the first character is "-", the pattern
+ * is treated as a negative pattern.
+ * @param policy The filtering policy, eg "only return test targets"
+ * @param offset The offset to apply to relative target patterns.
+ */
+ @ThreadSafe
+ public static SkyKey key(String pattern,
+ FilteringPolicy policy,
+ String offset) {
+ return new SkyKey(SkyFunctions.TARGET_PATTERN,
+ pattern.startsWith("-")
+ // Don't apply filters to negative patterns.
+ ? new TargetPattern(pattern.substring(1), FilteringPolicies.NO_FILTER, true, offset)
+ : new TargetPattern(pattern, policy, false, offset));
+ }
+
+ /**
+ * Like above, but accepts a collection of target patterns for the same filtering policy.
+ *
+ * @param patterns The collection of patterns, eg "-foo/biz...". If the first character is "-",
+ * the pattern is treated as a negative pattern.
+ * @param policy The filtering policy, eg "only return test targets"
+ * @param offset The offset to apply to relative target patterns.
+ */
+ @ThreadSafe
+ public static Iterable<SkyKey> keys(Collection<String> patterns,
+ FilteringPolicy policy,
+ String offset) {
+ List<SkyKey> keys = Lists.newArrayListWithCapacity(patterns.size());
+ for (String pattern : patterns) {
+ keys.add(key(pattern, policy, offset));
+ }
+ return keys;
+ }
+
+ public ResolvedTargets<Target> getTargets() {
+ return targets;
+ }
+
+ /**
+ * A TargetPattern is a tuple of pattern (eg, "foo/..."), filtering policy, a relative pattern
+ * offset, and whether it is a positive or negative match.
+ */
+ @ThreadSafe
+ public static class TargetPattern implements Serializable {
+ private final String pattern;
+ private final FilteringPolicy policy;
+ private final boolean isNegative;
+
+ private final String offset;
+
+ public TargetPattern(String pattern, FilteringPolicy policy,
+ boolean isNegative, String offset) {
+ this.pattern = Preconditions.checkNotNull(pattern);
+ this.policy = Preconditions.checkNotNull(policy);
+ this.isNegative = isNegative;
+ this.offset = offset;
+ }
+
+ public String getPattern() {
+ return pattern;
+ }
+
+ public boolean isNegative() {
+ return isNegative;
+ }
+
+ public FilteringPolicy getPolicy() {
+ return policy;
+ }
+
+ public String getOffset() {
+ return offset;
+ }
+
+ @Override
+ public String toString() {
+ return (isNegative ? "-" : "") + pattern;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(pattern, isNegative, policy, offset);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof TargetPattern)) {
+ return false;
+ }
+ TargetPattern other = (TargetPattern) obj;
+
+ return other.isNegative == this.isNegative && other.pattern.equals(this.pattern) &&
+ other.offset.equals(this.offset) && other.policy.equals(this.policy);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java
new file mode 100644
index 0000000..b6f9606
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/**
+ * TargetCompletionFunction builds all relevant test artifacts of a {@link
+ * com.google.devtools.build.lib.analysis.ConfiguredTarget}. This includes test shards and repeated
+ * runs.
+ */
+public final class TestCompletionFunction implements SkyFunction {
+
+ public TestCompletionFunction() {
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ TestCompletionValue.TestCompletionKey key =
+ (TestCompletionValue.TestCompletionKey) skyKey.argument();
+ LabelAndConfiguration lac = key.getLabelAndConfiguration();
+ if (env.getValue(TargetCompletionValue.key(lac)) == null) {
+ return null;
+ }
+
+ ConfiguredTargetValue ctValue = (ConfiguredTargetValue)
+ env.getValue(ConfiguredTargetValue.key(lac.getLabel(), lac.getConfiguration()));
+ if (ctValue == null) {
+ return null;
+ }
+
+ ConfiguredTarget ct = ctValue.getConfiguredTarget();
+ if (key.isExclusiveTesting()) {
+ // Request test artifacts iteratively if testing exclusively.
+ for (Artifact testArtifact : TestProvider.getTestStatusArtifacts(ct)) {
+ if (env.getValue(ArtifactValue.key(testArtifact, /*isMandatory=*/true)) == null) {
+ return null;
+ }
+ }
+ } else {
+ env.getValues(ArtifactValue.mandatoryKeys(TestProvider.getTestStatusArtifacts(ct)));
+ if (env.valuesMissing()) {
+ return null;
+ }
+ }
+ return TestCompletionValue.TEST_COMPLETION_MARKER;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java
new file mode 100644
index 0000000..da944ed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Collection;
+
+/**
+ * A test completion value represents the completion of a test target. This includes the execution
+ * of all test shards and repeated runs, if applicable.
+ */
+public class TestCompletionValue implements SkyValue {
+ static final TestCompletionValue TEST_COMPLETION_MARKER = new TestCompletionValue();
+
+ private TestCompletionValue() { }
+
+ public static SkyKey key(LabelAndConfiguration lac, boolean exclusive) {
+ return new SkyKey(SkyFunctions.TEST_COMPLETION, new TestCompletionKey(lac, exclusive));
+ }
+
+ public static Iterable<SkyKey> keys(Collection<ConfiguredTarget> targets,
+ final boolean exclusive) {
+ return Iterables.transform(targets, new Function<ConfiguredTarget, SkyKey>() {
+ @Override
+ public SkyKey apply(ConfiguredTarget ct) {
+ return new SkyKey(SkyFunctions.TEST_COMPLETION,
+ new TestCompletionKey(new LabelAndConfiguration(ct), exclusive));
+ }
+ });
+ }
+
+ static class TestCompletionKey {
+ private final LabelAndConfiguration lac;
+ private final boolean exclusiveTesting;
+
+ TestCompletionKey(LabelAndConfiguration lac, boolean exclusive) {
+ this.lac = lac;
+ this.exclusiveTesting = exclusive;
+ }
+
+ public LabelAndConfiguration getLabelAndConfiguration() {
+ return lac;
+ }
+
+ public boolean isExclusiveTesting() {
+ return exclusiveTesting;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java
new file mode 100644
index 0000000..03bbd25
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.List;
+
+/**
+ * Reports cycles between {@link TransitiveTargetValue}s. These indicates cycles between targets
+ * (e.g. '//a:foo' depends on '//b:bar' and '//b:bar' depends on '//a:foo').
+ */
+class TransitiveTargetCycleReporter extends AbstractLabelCycleReporter {
+
+ private static final Predicate<SkyKey> IS_TRANSITIVE_TARGET_SKY_KEY =
+ SkyFunctions.isSkyFunction(SkyFunctions.TRANSITIVE_TARGET);
+
+ TransitiveTargetCycleReporter(LoadedPackageProvider loadedPackageProvider) {
+ super(loadedPackageProvider);
+ }
+
+ @Override
+ protected boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ return Iterables.all(Iterables.concat(ImmutableList.of(topLevelKey),
+ cycleInfo.getPathToCycle(), cycleInfo.getCycle()),
+ IS_TRANSITIVE_TARGET_SKY_KEY);
+ }
+
+ @Override
+ public String prettyPrint(SkyKey key) {
+ return getLabel(key).toString();
+ }
+
+ @Override
+ protected Label getLabel(SkyKey key) {
+ return (Label) key.argument();
+ }
+
+ @Override
+ protected String getAdditionalMessageAboutCycle(SkyKey topLevelKey, CycleInfo cycleInfo) {
+ Target currentTarget = getTargetForLabel(getLabel(topLevelKey));
+ List<SkyKey> keys = Lists.newArrayList();
+ if (!cycleInfo.getPathToCycle().isEmpty()) {
+ keys.add(topLevelKey);
+ keys.addAll(cycleInfo.getPathToCycle());
+ }
+ keys.addAll(cycleInfo.getCycle());
+ // Make sure we check the edge from the last element of the cycle to the first element of the
+ // cycle.
+ keys.add(cycleInfo.getCycle().get(0));
+ for (SkyKey nextKey : keys) {
+ Label nextLabel = getLabel(nextKey);
+ Target nextTarget = getTargetForLabel(nextLabel);
+ // This is inefficient but it's no big deal since we only do this when there's a cycle.
+ if (currentTarget.getVisibility().getDependencyLabels().contains(nextLabel)
+ && !nextTarget.getTargetKind().equals(PackageGroup.targetKind())) {
+ return "\nThe cycle is caused by a visibility edge from " + currentTarget.getLabel()
+ + " to the non-package-group target " + nextTarget.getLabel() + " . Note that "
+ + "visibility labels are supposed to be package group targets (which prevents cycles "
+ + "of this form)";
+ }
+ currentTarget = nextTarget;
+ }
+ return "";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java
new file mode 100644
index 0000000..417cfca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java
@@ -0,0 +1,234 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.packages.InputFile;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.PackageGroup;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.ValueOrException;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class builds transitive Target values such that evaluating a Target value is similar to
+ * running it through the LabelVisitor.
+ */
+public class TransitiveTargetFunction implements SkyFunction {
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws TransitiveTargetFunctionException {
+ Label label = (Label) key.argument();
+ SkyKey packageKey = PackageValue.key(label.getPackageIdentifier());
+ SkyKey targetKey = TargetMarkerValue.key(label);
+ Target target;
+ boolean packageLoadedSuccessfully;
+ boolean successfulTransitiveLoading = true;
+ NestedSetBuilder<Label> transitiveRootCauses = NestedSetBuilder.stableOrder();
+ NoSuchTargetException errorLoadingTarget = null;
+ try {
+ TargetMarkerValue targetValue = (TargetMarkerValue) env.getValueOrThrow(targetKey,
+ NoSuchThingException.class);
+ if (targetValue == null) {
+ return null;
+ }
+ PackageValue packageValue = (PackageValue) env.getValueOrThrow(packageKey,
+ NoSuchThingException.class);
+ if (packageValue == null) {
+ return null;
+ }
+
+ packageLoadedSuccessfully = true;
+ target = packageValue.getPackage().getTarget(label.getName());
+ } catch (NoSuchTargetException e) {
+ target = e.getTarget();
+ if (target == null) {
+ throw new TransitiveTargetFunctionException(e);
+ }
+ successfulTransitiveLoading = false;
+ transitiveRootCauses.add(label);
+ errorLoadingTarget = e;
+ packageLoadedSuccessfully = e.getPackageLoadedSuccessfully();
+ } catch (NoSuchPackageException e) {
+ throw new TransitiveTargetFunctionException(e);
+ } catch (NoSuchThingException e) {
+ throw new IllegalStateException(e
+ + " not NoSuchTargetException or NoSuchPackageException");
+ }
+
+ NestedSetBuilder<PackageIdentifier> transitiveSuccessfulPkgs = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<PackageIdentifier> transitiveUnsuccessfulPkgs = NestedSetBuilder.stableOrder();
+ NestedSetBuilder<Label> transitiveTargets = NestedSetBuilder.stableOrder();
+
+ PackageIdentifier packageId = target.getPackage().getPackageIdentifier();
+ if (packageLoadedSuccessfully) {
+ transitiveSuccessfulPkgs.add(packageId);
+ } else {
+ transitiveUnsuccessfulPkgs.add(packageId);
+ }
+ transitiveTargets.add(target.getLabel());
+ for (Map.Entry<SkyKey, ValueOrException<NoSuchThingException>> entry :
+ env.getValuesOrThrow(getLabelDepKeys(target), NoSuchThingException.class).entrySet()) {
+ Label depLabel = (Label) entry.getKey().argument();
+ TransitiveTargetValue transitiveTargetValue;
+ try {
+ transitiveTargetValue = (TransitiveTargetValue) entry.getValue().get();
+ if (transitiveTargetValue == null) {
+ continue;
+ }
+ } catch (NoSuchPackageException | NoSuchTargetException e) {
+ successfulTransitiveLoading = false;
+ transitiveRootCauses.add(depLabel);
+ maybeReportErrorAboutMissingEdge(target, depLabel, e, env.getListener());
+ continue;
+ } catch (NoSuchThingException e) {
+ throw new IllegalStateException("Unexpected Exception type from TransitiveTargetValue.", e);
+ }
+ transitiveSuccessfulPkgs.addTransitive(
+ transitiveTargetValue.getTransitiveSuccessfulPackages());
+ transitiveUnsuccessfulPkgs.addTransitive(
+ transitiveTargetValue.getTransitiveUnsuccessfulPackages());
+ transitiveTargets.addTransitive(transitiveTargetValue.getTransitiveTargets());
+ NestedSet<Label> rootCauses = transitiveTargetValue.getTransitiveRootCauses();
+ if (rootCauses != null) {
+ successfulTransitiveLoading = false;
+ transitiveRootCauses.addTransitive(rootCauses);
+ if (transitiveTargetValue.getErrorLoadingTarget() != null) {
+ maybeReportErrorAboutMissingEdge(target, depLabel,
+ transitiveTargetValue.getErrorLoadingTarget(), env.getListener());
+ }
+ }
+ }
+
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ NestedSet<PackageIdentifier> successfullyLoadedPackages = transitiveSuccessfulPkgs.build();
+ NestedSet<PackageIdentifier> unsuccessfullyLoadedPackages = transitiveUnsuccessfulPkgs.build();
+ NestedSet<Label> loadedTargets = transitiveTargets.build();
+ if (successfulTransitiveLoading) {
+ return TransitiveTargetValue.successfulTransitiveLoading(successfullyLoadedPackages,
+ unsuccessfullyLoadedPackages, loadedTargets);
+ } else {
+ NestedSet<Label> rootCauses = transitiveRootCauses.build();
+ return TransitiveTargetValue.unsuccessfulTransitiveLoading(successfullyLoadedPackages,
+ unsuccessfullyLoadedPackages, loadedTargets, rootCauses, errorLoadingTarget);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return Label.print(((Label) skyKey.argument()));
+ }
+
+ private static void maybeReportErrorAboutMissingEdge(Target target, Label depLabel,
+ NoSuchThingException e, EventHandler eventHandler) {
+ if (e instanceof NoSuchTargetException) {
+ NoSuchTargetException nste = (NoSuchTargetException) e;
+ if (depLabel.equals(nste.getLabel())) {
+ eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target),
+ TargetUtils.formatMissingEdge(target, depLabel, e)));
+ }
+ } else if (e instanceof NoSuchPackageException) {
+ NoSuchPackageException nspe = (NoSuchPackageException) e;
+ if (nspe.getPackageName().equals(depLabel.getPackageName())) {
+ eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target),
+ TargetUtils.formatMissingEdge(target, depLabel, e)));
+ }
+ }
+ }
+
+ private static Iterable<SkyKey> getLabelDepKeys(Target target) {
+ List<SkyKey> depKeys = Lists.newArrayList();
+ for (Label depLabel : getLabelDeps(target)) {
+ depKeys.add(TransitiveTargetValue.key(depLabel));
+ }
+ return depKeys;
+ }
+
+ // TODO(bazel-team): Unify this logic with that in LabelVisitor, and possibly DependencyResolver.
+ private static Iterable<Label> getLabelDeps(Target target) {
+ final Set<Label> labels = new HashSet<>();
+ if (target instanceof OutputFile) {
+ Rule rule = ((OutputFile) target).getGeneratingRule();
+ labels.add(rule.getLabel());
+ visitTargetVisibility(target, labels);
+ } else if (target instanceof InputFile) {
+ visitTargetVisibility(target, labels);
+ } else if (target instanceof Rule) {
+ visitTargetVisibility(target, labels);
+ labels.addAll(((Rule) target).getLabels(Rule.NO_NODEP_ATTRIBUTES));
+ } else if (target instanceof PackageGroup) {
+ visitPackageGroup((PackageGroup) target, labels);
+ }
+ return labels;
+ }
+
+ private static void visitTargetVisibility(Target target, Set<Label> labels) {
+ for (Label label : target.getVisibility().getDependencyLabels()) {
+ labels.add(label);
+ }
+ }
+
+ private static void visitPackageGroup(PackageGroup packageGroup, Set<Label> labels) {
+ for (final Label include : packageGroup.getIncludes()) {
+ labels.add(include);
+ }
+ }
+
+ /**
+ * Used to declare all the exception types that can be wrapped in the exception thrown by
+ * {@link TransitiveTargetFunction#compute}.
+ */
+ private static class TransitiveTargetFunctionException extends SkyFunctionException {
+ /**
+ * Used to propagate an error from a direct target dependency to the
+ * target that depended on it.
+ */
+ public TransitiveTargetFunctionException(NoSuchPackageException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ /**
+ * In nokeep_going mode, used to propagate an error from a direct target dependency to the
+ * target that depended on it.
+ *
+ * In keep_going mode, used the same way, but only for targets that could not be loaded at all
+ * (we proceed with transitive loading on targets that contain errors).
+ */
+ public TransitiveTargetFunctionException(NoSuchTargetException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java
new file mode 100644
index 0000000..69b9638
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java
@@ -0,0 +1,142 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A <i>transitive</i> target reference that, when built in skyframe, loads the entire
+ * transitive closure of a target.
+ *
+ * This will probably be unnecessary once other refactorings occur throughout the codebase
+ * which make loading/analysis interleaving more feasible, or we will migrate "blaze query" to
+ * use this to evaluate its Target graph.
+ */
+@Immutable
+@ThreadSafe
+public class TransitiveTargetValue implements SkyValue {
+
+ // Non-final for serialization purposes.
+ private NestedSet<PackageIdentifier> transitiveSuccessfulPkgs;
+ private NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs;
+ private NestedSet<Label> transitiveTargets;
+ @Nullable private NestedSet<Label> transitiveRootCauses;
+ @Nullable private NoSuchTargetException errorLoadingTarget;
+
+ private TransitiveTargetValue(NestedSet<PackageIdentifier> transitiveSuccessfulPkgs,
+ NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, NestedSet<Label> transitiveTargets,
+ @Nullable NestedSet<Label> transitiveRootCauses,
+ @Nullable NoSuchTargetException errorLoadingTarget) {
+ this.transitiveSuccessfulPkgs = transitiveSuccessfulPkgs;
+ this.transitiveUnsuccessfulPkgs = transitiveUnsuccessfulPkgs;
+ this.transitiveTargets = transitiveTargets;
+ this.transitiveRootCauses = transitiveRootCauses;
+ this.errorLoadingTarget = errorLoadingTarget;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ // It helps to flatten the transitiveSuccessfulPkgs nested set as it has lots of duplicates.
+ Set<PackageIdentifier> successfulPkgs = transitiveSuccessfulPkgs.toSet();
+ out.writeInt(successfulPkgs.size());
+ for (PackageIdentifier pkg : successfulPkgs) {
+ out.writeObject(pkg);
+ }
+
+ out.writeObject(transitiveUnsuccessfulPkgs);
+ // Deliberately do not write out transitiveTargets. There is a lot of those and they drive
+ // serialization costs through the roof, both in terms of space and of time.
+ // TODO(bazel-team): Deal with this properly once we have efficient serialization of NestedSets.
+ out.writeObject(transitiveRootCauses);
+ out.writeObject(errorLoadingTarget);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ int successfulPkgCount = in.readInt();
+ NestedSetBuilder<PackageIdentifier> pkgs = NestedSetBuilder.stableOrder();
+ for (int i = 0; i < successfulPkgCount; i++) {
+ pkgs.add((PackageIdentifier) in.readObject());
+ }
+ transitiveSuccessfulPkgs = pkgs.build();
+ transitiveUnsuccessfulPkgs = (NestedSet<PackageIdentifier>) in.readObject();
+ // TODO(bazel-team): Deal with transitiveTargets properly.
+ transitiveTargets = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ transitiveRootCauses = (NestedSet<Label>) in.readObject();
+ errorLoadingTarget = (NoSuchTargetException) in.readObject();
+ }
+
+ static TransitiveTargetValue unsuccessfulTransitiveLoading(
+ NestedSet<PackageIdentifier> transitiveSuccessfulPkgs,
+ NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, NestedSet<Label> transitiveTargets,
+ NestedSet<Label> rootCauses, @Nullable NoSuchTargetException errorLoadingTarget) {
+ return new TransitiveTargetValue(transitiveSuccessfulPkgs, transitiveUnsuccessfulPkgs,
+ transitiveTargets, rootCauses, errorLoadingTarget);
+ }
+
+ static TransitiveTargetValue successfulTransitiveLoading(
+ NestedSet<PackageIdentifier> transitiveSuccessfulPkgs,
+ NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs,
+ NestedSet<Label> transitiveTargets) {
+ return new TransitiveTargetValue(transitiveSuccessfulPkgs, transitiveUnsuccessfulPkgs,
+ transitiveTargets, null, null);
+ }
+
+ /** Returns the error, if any, from loading the target. */
+ @Nullable
+ public NoSuchTargetException getErrorLoadingTarget() {
+ return errorLoadingTarget;
+ }
+
+ /** Returns the packages that were transitively successfully loaded. */
+ public NestedSet<PackageIdentifier> getTransitiveSuccessfulPackages() {
+ return transitiveSuccessfulPkgs;
+ }
+
+ /** Returns the packages that were transitively successfully loaded. */
+ public NestedSet<PackageIdentifier> getTransitiveUnsuccessfulPackages() {
+ return transitiveUnsuccessfulPkgs;
+ }
+
+ /** Returns the targets that were transitively loaded. */
+ public NestedSet<Label> getTransitiveTargets() {
+ return transitiveTargets;
+ }
+
+ /** Returns the root causes, if any, of why targets weren't loaded. */
+ @Nullable
+ public NestedSet<Label> getTransitiveRootCauses() {
+ return transitiveRootCauses;
+ }
+
+ @ThreadSafe
+ public static SkyKey key(Label label) {
+ return new SkyKey(SkyFunctions.TRANSITIVE_TARGET, label);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
new file mode 100644
index 0000000..f518b8a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
@@ -0,0 +1,224 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.devtools.build.lib.syntax.Environment.NONE;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.packages.ExternalPackage.Binding;
+import com.google.devtools.build.lib.packages.ExternalPackage.ExternalPackageBuilder;
+import com.google.devtools.build.lib.packages.ExternalPackage.ExternalPackageBuilder.NoSuchBindingException;
+import com.google.devtools.build.lib.packages.Package.NameConflictException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleFactory;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.AbstractFunction;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.FuncallExpression;
+import com.google.devtools.build.lib.syntax.Function;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.syntax.MixedModeFunction;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A SkyFunction to parse WORKSPACE files.
+ */
+public class WorkspaceFileFunction implements SkyFunction {
+
+ private static final String BIND = "bind";
+
+ private final PackageFactory packageFactory;
+
+ WorkspaceFileFunction(PackageFactory packageFactory) {
+ this.packageFactory = packageFactory;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws WorkspaceFileFunctionException,
+ InterruptedException {
+ RootedPath workspaceRoot = (RootedPath) skyKey.argument();
+ // Explicitly make skyframe load this file.
+ if (env.getValue(FileValue.key(workspaceRoot)) == null) {
+ return null;
+ }
+ Path workspaceFilePath = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath());
+ WorkspaceNameHolder holder = new WorkspaceNameHolder();
+ ExternalPackageBuilder builder = new ExternalPackageBuilder(workspaceFilePath);
+ StoredEventHandler localReporter = new StoredEventHandler();
+ BuildFileAST buildFileAST;
+ ParserInputSource inputSource = null;
+
+ try {
+ inputSource = ParserInputSource.create(workspaceFilePath);
+ } catch (IOException e) {
+ throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
+ }
+ buildFileAST = BuildFileAST.parseBuildFile(inputSource, localReporter, null, false);
+ if (buildFileAST.containsErrors()) {
+ localReporter.handle(Event.error("WORKSPACE file could not be parsed"));
+ } else {
+ try {
+ if (!evaluateWorkspaceFile(buildFileAST, holder, builder)) {
+ localReporter.handle(
+ Event.error("Error evaluating WORKSPACE file " + workspaceFilePath));
+ }
+ } catch (EvalException e) {
+ throw new WorkspaceFileFunctionException(e);
+ }
+ }
+
+ builder.addEvents(localReporter.getEvents());
+ if (localReporter.hasErrors()) {
+ builder.setContainsErrors();
+ }
+ return new WorkspaceFileValue(holder.workspaceName, builder.build());
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static Function newWorkspaceNameFunction(final WorkspaceNameHolder holder) {
+ List<String> params = ImmutableList.of("name");
+ return new MixedModeFunction("workspace", params, 1, true) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast) throws EvalException,
+ ConversionException, InterruptedException {
+ String name = Type.STRING.convert(namedArgs[0], "'name' argument");
+ String errorMessage = LabelValidator.validateTargetName(name);
+ if (errorMessage != null) {
+ throw new EvalException(ast.getLocation(), errorMessage);
+ }
+ holder.workspaceName = name;
+ return NONE;
+ }
+ };
+ }
+
+ private static Function newBindFunction(final ExternalPackageBuilder builder) {
+ List<String> params = ImmutableList.of("name", "actual");
+ return new MixedModeFunction(BIND, params, 2, true) {
+ @Override
+ public Object call(Object[] namedArgs, FuncallExpression ast)
+ throws EvalException, ConversionException {
+ String name = Type.STRING.convert(namedArgs[0], "'name' argument");
+ String actual = Type.STRING.convert(namedArgs[1], "'actual' argument");
+
+ Label nameLabel = null;
+ try {
+ nameLabel = Label.parseAbsolute("//external:" + name);
+ builder.addBinding(
+ nameLabel, new Binding(Label.parseRepositoryLabel(actual), ast.getLocation()));
+ } catch (SyntaxException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+
+ return NONE;
+ }
+ };
+ }
+
+ /**
+ * Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the
+ * specified package context.
+ */
+ private static Function newRuleFunction(final RuleFactory ruleFactory,
+ final ExternalPackageBuilder builder, final String ruleClassName) {
+ return new AbstractFunction(ruleClassName) {
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ com.google.devtools.build.lib.syntax.Environment env)
+ throws EvalException {
+ if (!args.isEmpty()) {
+ throw new EvalException(ast.getLocation(),
+ "build rules do not accept positional parameters");
+ }
+
+ try {
+ RuleClass ruleClass = ruleFactory.getRuleClass(ruleClassName);
+ builder.createAndAddRepositoryRule(ruleClass, kwargs, ast);
+ } catch (RuleFactory.InvalidRuleException | NameConflictException | SyntaxException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ return NONE;
+ }
+ };
+ }
+
+ public boolean evaluateWorkspaceFile(BuildFileAST buildFileAST, WorkspaceNameHolder holder,
+ ExternalPackageBuilder builder)
+ throws InterruptedException, EvalException, WorkspaceFileFunctionException {
+ // Environment is defined in SkyFunction and the syntax package.
+ com.google.devtools.build.lib.syntax.Environment workspaceEnv =
+ new com.google.devtools.build.lib.syntax.Environment();
+
+ RuleFactory ruleFactory = new RuleFactory(packageFactory.getRuleClassProvider());
+ for (String ruleClass : ruleFactory.getRuleClassNames()) {
+ Function ruleFunction = newRuleFunction(ruleFactory, builder, ruleClass);
+ workspaceEnv.update(ruleClass, ruleFunction);
+ }
+
+ workspaceEnv.update(BIND, newBindFunction(builder));
+ workspaceEnv.update("workspace", newWorkspaceNameFunction(holder));
+
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ if (!buildFileAST.exec(workspaceEnv, eventHandler)) {
+ return false;
+ }
+ try {
+ builder.resolveBindTargets(packageFactory.getRuleClass(BIND));
+ } catch (NoSuchBindingException e) {
+ throw new WorkspaceFileFunctionException(e);
+ }
+ return true;
+ }
+
+ private static final class WorkspaceNameHolder {
+ String workspaceName;
+ }
+
+ private static final class WorkspaceFileFunctionException extends SkyFunctionException {
+ public WorkspaceFileFunctionException(IOException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public WorkspaceFileFunctionException(NoSuchBindingException e) {
+ super(e, Transience.PERSISTENT);
+ }
+
+ public WorkspaceFileFunctionException(EvalException e) {
+ super(e, Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java
new file mode 100644
index 0000000..200f23f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.packages.ExternalPackage;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import javax.annotation.Nullable;
+
+/**
+ * Holds the contents of a WORKSPACE file as the //external package.
+ */
+public class WorkspaceFileValue implements SkyValue {
+
+ private final String workspace;
+ private final ExternalPackage pkg;
+
+ public WorkspaceFileValue(String workspace, ExternalPackage pkg) {
+ this.workspace = workspace;
+ this.pkg = pkg;
+ }
+
+ /**
+ * Returns the name of this workspace (or null for the default workspace).
+ */
+ @Nullable
+ public String getWorkspace() {
+ return workspace;
+ }
+
+ /**
+ * Returns the //external package.
+ */
+ public ExternalPackage getPackage() {
+ return pkg;
+ }
+
+ /**
+ * Generates a SkyKey based on the path to the WORKSPACE file.
+ */
+ public static SkyKey key(RootedPath workspacePath) {
+ return new SkyKey(SkyFunctions.WORKSPACE_FILE, workspacePath);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java
new file mode 100644
index 0000000..b1c5ef3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+/** Creates the workspace status artifacts and action. */
+public class WorkspaceStatusFunction implements SkyFunction {
+ WorkspaceStatusFunction() {
+ }
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ Preconditions.checkState(
+ WorkspaceStatusValue.SKY_KEY.equals(skyKey), WorkspaceStatusValue.SKY_KEY);
+
+ WorkspaceStatusAction action = PrecomputedValue.WORKSPACE_STATUS_KEY.get(env);
+ if (action == null) {
+ return null;
+ }
+
+ return new WorkspaceStatusValue(
+ action.getStableStatus(),
+ action.getVolatileStatus(),
+ action);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java
new file mode 100644
index 0000000..21b9215
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/**
+ * Value that stores the workspace status artifacts and their generating action. There should be
+ * only one of these values in the graph at any time.
+ */
+// TODO(bazel-team): This seems to be superfluous now, but it cannot be removed without making
+// PrecomputedValue public instead of package-private
+public class WorkspaceStatusValue extends ActionLookupValue {
+ private final Artifact stableArtifact;
+ private final Artifact volatileArtifact;
+
+ // There should only ever be one BuildInfo value in the graph.
+ public static final SkyKey SKY_KEY = new SkyKey(SkyFunctions.BUILD_INFO, "BUILD_INFO");
+ static final ArtifactOwner ARTIFACT_OWNER = new BuildInfoKey();
+
+ public WorkspaceStatusValue(Artifact stableArtifact, Artifact volatileArtifact,
+ WorkspaceStatusAction action) {
+ super(action);
+ this.stableArtifact = stableArtifact;
+ this.volatileArtifact = volatileArtifact;
+ }
+
+ public Artifact getStableArtifact() {
+ return stableArtifact;
+ }
+
+ public Artifact getVolatileArtifact() {
+ return volatileArtifact;
+ }
+
+ private static class BuildInfoKey extends ActionLookupKey {
+ @Override
+ SkyFunctionName getType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ SkyKey getSkyKey() {
+ return SKY_KEY;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java
new file mode 100644
index 0000000..dc32ddc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java
@@ -0,0 +1,227 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.standalone;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.rules.cpp.CppCompileAction;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.unix.FilesystemUtils;
+import com.google.devtools.build.lib.util.CommandFailureUtils;
+import com.google.devtools.build.lib.util.DependencySet;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeSet;
+
+/**
+ * Strategy that uses sandboxing to execute a process.
+ */
+@ExecutionStrategy(name = {"sandboxed"},
+ contextType = SpawnActionContext.class)
+public class LinuxSandboxedStrategy implements SpawnActionContext {
+ private final boolean verboseFailures;
+ private final BlazeDirectories directories;
+
+ public LinuxSandboxedStrategy(BlazeDirectories blazeDirectories, boolean verboseFailures) {
+ this.directories = blazeDirectories;
+ this.verboseFailures = verboseFailures;
+ }
+
+ /**
+ * Executes the given {@code spawn}.
+ */
+ @Override
+ public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
+ throws ExecException {
+ Executor executor = actionExecutionContext.getExecutor();
+ if (executor.reportsSubcommands()) {
+ executor.reportSubcommand(Label.print(spawn.getOwner().getLabel()),
+ spawn.asShellCommand(executor.getExecRoot()));
+ }
+ boolean processHeaders = spawn.getResourceOwner() instanceof CppCompileAction;
+
+ Path execPath = this.directories.getExecRoot();
+ List<String> spawnArguments = new ArrayList<>();
+
+ for (String arg : spawn.getArguments()) {
+ if (arg.startsWith(execPath.getPathString())) {
+ // make all paths relative for the sandbox
+ spawnArguments.add(arg.substring(execPath.getPathString().length()));
+ } else {
+ spawnArguments.add(arg);
+ }
+ }
+
+ List<? extends ActionInput> expandedInputs =
+ ActionInputHelper.expandMiddlemen(spawn.getInputFiles(),
+ actionExecutionContext.getMiddlemanExpander());
+
+ String cwd = executor.getExecRoot().getPathString();
+
+ FileOutErr outErr = actionExecutionContext.getFileOutErr();
+ try {
+ PathFragment includePrefix = null; // null when there's no include mangling to do
+ List<PathFragment> includeDirectories = ImmutableList.of();
+ if (processHeaders) {
+ CppCompileAction cppAction = (CppCompileAction) spawn.getResourceOwner();
+ // headers are mounted in the sandbox in a separate include dir, so their names are mangled
+ // when running the compilation and will have to be unmangled after it's done in the *.pic.d
+ includeDirectories = extractIncludeDirs(execPath, cppAction, spawnArguments);
+ includePrefix = getSandboxIncludeDir(cppAction);
+ }
+
+ NamespaceSandboxRunner runner = new NamespaceSandboxRunner(directories, spawn, includePrefix,
+ includeDirectories, spawn.getRunfilesManifests());
+ runner.setupSandbox(expandedInputs, spawn.getOutputFiles());
+ runner.run(spawnArguments, spawn.getEnvironment(), new File(cwd), outErr);
+ runner.copyOutputs(spawn.getOutputFiles(), outErr);
+ if (processHeaders) {
+ CppCompileAction cppAction = (CppCompileAction) spawn.getResourceOwner();
+ unmangleHeaderFiles(cppAction);
+ }
+ runner.cleanup();
+ } catch (CommandException e) {
+ String message = CommandFailureUtils.describeCommandFailure(verboseFailures,
+ spawn.getArguments(), spawn.getEnvironment(), cwd);
+ throw new UserExecException(String.format("%s: %s", message, e));
+ } catch (IOException e) {
+ throw new UserExecException(e.getMessage());
+ }
+ }
+
+ private void unmangleHeaderFiles(CppCompileAction cppCompileAction) throws IOException {
+ Path execPath = this.directories.getExecRoot();
+ CppCompileAction.DotdFile dotdfile = cppCompileAction.getDotdFile();
+ DependencySet depset = new DependencySet(execPath).read(dotdfile.getPath());
+ DependencySet unmangled = new DependencySet(execPath);
+ PathFragment sandboxIncludeDir = getSandboxIncludeDir(cppCompileAction);
+ PathFragment prefix = sandboxIncludeDir.getRelative(execPath.asFragment().relativeTo("/"));
+ for (PathFragment dep : depset.getDependencies()) {
+ if (dep.startsWith(prefix)) {
+ dep = dep.relativeTo(prefix);
+ }
+ unmangled.addDependency(dep);
+ }
+ unmangled.write(execPath.getRelative(depset.getOutputFileName()), ".d");
+ }
+
+ private PathFragment getSandboxIncludeDir(CppCompileAction cppCompileAction) {
+ return new PathFragment(
+ "include-" + Actions.escapedPath(cppCompileAction.getPrimaryOutput().toString()));
+ }
+
+ private ImmutableList<PathFragment> extractIncludeDirs(Path execPath,
+ CppCompileAction cppCompileAction, List<String> spawnArguments) throws IOException {
+ List<PathFragment> includes = new ArrayList<>();
+ includes.addAll(cppCompileAction.getQuoteIncludeDirs());
+ includes.addAll(cppCompileAction.getIncludeDirs());
+ includes.addAll(cppCompileAction.getSystemIncludeDirs());
+
+ // gcc implicitly includes headers in the same dir as .cc file
+ PathFragment sourceDirectory =
+ cppCompileAction.getSourceFile().getPath().getParentDirectory().asFragment();
+ includes.add(sourceDirectory);
+ spawnArguments.add("-iquote");
+ spawnArguments.add(sourceDirectory.toString());
+
+ TreeSet<PathFragment> processedIncludes = new TreeSet<>();
+ for (int i = 0; i < includes.size(); i++) {
+ PathFragment absolutePath;
+ if (!includes.get(i).isAbsolute()) {
+ absolutePath = execPath.getRelative(includes.get(i)).asFragment();
+ } else {
+ absolutePath = includes.get(i);
+ }
+ // CppCompileAction may provide execPath as one of the include directories. This is a big
+ // overestimation of what is actually needed and doesn't make for very hermetic sandbox
+ // (since everything from the workspace will be somehow accessed in the sandbox). To have
+ // some more hermeticity in this situation we mount all the include dirs in:
+ // sandbox-directory/include-prefix/actual-include-dir
+ // (where include-prefix is obtained from this.getSandboxIncludeDir(cppCompileAction))
+ // and make so gcc looks there for includes. This should prevent the user from accessing
+ // files that technically should not be in the sandbox.
+ // TODO(bazel-team): change CppCompileAction so that include dirs contain only subsets of the
+ // execPath
+ if (absolutePath.equals(execPath.asFragment())) {
+ // we can't mount execPath because it will lead to a circular mount; instead mount its
+ // subdirs inside (other than the ones containing sandbox)
+ String[] subdirs = FilesystemUtils.readdir(absolutePath.toString());
+ for (String dirName : subdirs) {
+ if (dirName.equals("_bin") || dirName.equals("bazel-out")) {
+ continue;
+ }
+ PathFragment child = absolutePath.getChild(dirName);
+ processedIncludes.add(child);
+ }
+ } else {
+ processedIncludes.add(absolutePath);
+ }
+ }
+
+ // pseudo random name for include directory inside sandbox, so it won't be accessed by accident
+ String prefix = getSandboxIncludeDir(cppCompileAction).toString();
+
+ // change names in the invocation
+ for (int i = 0; i < spawnArguments.size(); i++) {
+ if (spawnArguments.get(i).startsWith("-I")) {
+ String argument = spawnArguments.get(i).substring(2);
+ spawnArguments.set(i, setIncludeDirSandboxPath(execPath, argument, "-I" + prefix));
+ }
+ if (spawnArguments.get(i).equals("-iquote") || spawnArguments.get(i).equals("-isystem")) {
+ spawnArguments.set(i + 1, setIncludeDirSandboxPath(execPath,
+ spawnArguments.get(i + 1), prefix));
+ }
+ }
+ return ImmutableList.copyOf(processedIncludes);
+ }
+
+ private String setIncludeDirSandboxPath(Path execPath, String argument, String prefix) {
+ StringBuilder builder = new StringBuilder(prefix);
+ if (argument.charAt(0) != '/') {
+ // relative path
+ builder.append(execPath);
+ builder.append('/');
+ }
+ builder.append(argument);
+
+ return builder.toString();
+ }
+
+ @Override
+ public String strategyLocality(String mnemonic, boolean remotable) {
+ return "linux-sandboxing";
+ }
+
+ @Override
+ public boolean isRemotable(String mnemonic, boolean remotable) {
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java
new file mode 100644
index 0000000..fc2387c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.standalone;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.CommandFailureUtils;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Strategy that uses subprocessing to execute a process.
+ */
+@ExecutionStrategy(name = { "standalone" }, contextType = SpawnActionContext.class)
+public class LocalSpawnStrategy implements SpawnActionContext {
+ private final boolean verboseFailures;
+
+ private final Path processWrapper;
+
+ public LocalSpawnStrategy(Path execRoot, boolean verboseFailures) {
+ this.verboseFailures = verboseFailures;
+ this.processWrapper = execRoot.getRelative(
+ "_bin/process-wrapper" + OsUtils.executableExtension());
+ }
+
+ /**
+ * Executes the given {@code spawn}.
+ */
+ @Override
+ public void exec(Spawn spawn,
+ ActionExecutionContext actionExecutionContext)
+ throws ExecException {
+ Executor executor = actionExecutionContext.getExecutor();
+ if (executor.reportsSubcommands()) {
+ executor.reportSubcommand(Label.print(spawn.getOwner().getLabel()),
+ spawn.asShellCommand(executor.getExecRoot()));
+ }
+
+ // We must wrap the subprocess with process-wrapper to kill the process tree.
+ // All actions therefore depend on the process-wrapper file. Since it's embedded,
+ // we don't bother with declaring it as an input.
+ List<String> args = new ArrayList<>();
+ if (OS.getCurrent() != OS.WINDOWS) {
+ // TODO(bazel-team): process-wrapper seems to work on Windows, but requires
+ // additional setup as it is an msys2 binary, so it needs msys2 DLLs on %PATH%.
+ // Disable it for now to make the setup easier and to avoid further PATH hacks.
+ // Ideally we should have a native implementation of process-wrapper for Windows.
+ args.add(processWrapper.getPathString());
+ args.add("-1"); /* timeout */
+ args.add("0"); /* kill delay. */
+
+ // TODO(bazel-team): use process-wrapper redirection so we don't have to
+ // pass test logs through the Java heap.
+ args.add("-"); /* stdout. */
+ args.add("-"); /* stderr. */
+ }
+ args.addAll(spawn.getArguments());
+
+ String cwd = executor.getExecRoot().getPathString();
+ Command cmd = new Command(args.toArray(new String[]{}), spawn.getEnvironment(), new File(cwd));
+
+ FileOutErr outErr = actionExecutionContext.getFileOutErr();
+ try {
+ cmd.execute(
+ /* stdin */ new byte[]{},
+ Command.NO_OBSERVER,
+ outErr.getOutputStream(),
+ outErr.getErrorStream(),
+ /*killSubprocessOnInterrupt*/ true);
+ } catch (CommandException e) {
+ String message = CommandFailureUtils.describeCommandFailure(
+ verboseFailures, spawn.getArguments(), spawn.getEnvironment(), cwd);
+ throw new UserExecException(String.format("%s: %s", message, e));
+ }
+ }
+
+ @Override
+ public String strategyLocality(String mnemonic, boolean remotable) {
+ return "standalone";
+ }
+
+ @Override
+ public boolean isRemotable(String mnemonic, boolean remotable) {
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java b/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java
new file mode 100644
index 0000000..3c7a8e0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java
@@ -0,0 +1,267 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.standalone;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.unix.FilesystemUtils;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map.Entry;
+
+/**
+ * Helper class for running the namespace sandbox. This runner prepares environment inside the
+ * sandbox (copies inputs, creates file structure), handles sandbox output, performs cleanup and
+ * changes invocation if necessary.
+ */
+public class NamespaceSandboxRunner {
+ private final boolean debug = true;
+ private final PathFragment sandboxDirectory;
+ private final Path sandboxPath;
+ private final List<String> mounts;
+ private final Path embeddedBinaries;
+ private final Path tools;
+ private final ImmutableList<PathFragment> includeDirectories;
+ private final PathFragment includePrefix;
+ private final ImmutableMap<PathFragment, Artifact> manifests;
+ private final Path execRoot;
+
+ public NamespaceSandboxRunner(BlazeDirectories directories, Spawn spawn,
+ PathFragment includePrefix, List<PathFragment> includeDirectories,
+ ImmutableMap<PathFragment, Artifact> manifests) {
+ String md5sum = Fingerprint.md5Digest(spawn.getResourceOwner().getPrimaryOutput().toString());
+ this.sandboxDirectory = new PathFragment("sandbox-root-" + md5sum);
+ this.sandboxPath =
+ directories.getExecRoot().getRelative("sandboxes").getRelative(sandboxDirectory);
+ this.mounts = new ArrayList<>();
+ this.tools = directories.getExecRoot().getChild("tools");
+ this.embeddedBinaries = directories.getEmbeddedBinariesRoot();
+ this.includePrefix = includePrefix;
+ this.includeDirectories = ImmutableList.copyOf(includeDirectories);
+ this.manifests = manifests;
+ this.execRoot = directories.getExecRoot();
+ }
+
+ private void createFileSystem(Collection<? extends ActionInput> outputs) throws IOException {
+ // create the sandboxes' parent directory if needed
+ // TODO(bazel-team): create this with rest of the workspace dirs
+ if (!sandboxPath.getParentDirectory().isDirectory()) {
+ FilesystemUtils.mkdir(sandboxPath.getParentDirectory().getPathString(), 0755);
+ }
+
+ FilesystemUtils.mkdir(sandboxPath.getPathString(), 0755);
+ String[] dirs = { "bin", "etc" };
+ for (String dir : dirs) {
+ FilesystemUtils.mkdir(sandboxPath.getChild(dir).getPathString(), 0755);
+ mounts.add("/" + dir);
+ }
+
+ // usr
+ String[] dirsUsr = { "bin", "include" };
+ FilesystemUtils.mkdir(sandboxPath.getChild("usr").getPathString(), 0755);
+ Path usr = sandboxPath.getChild("usr");
+ for (String dir : dirsUsr) {
+ FilesystemUtils.mkdir(usr.getChild(dir).getPathString(), 0755);
+ mounts.add("/usr/" + dir);
+ }
+ FileSystemUtils.createDirectoryAndParents(usr.getChild("local").getChild("include"));
+ mounts.add("/usr/local/include");
+
+ // shared libs
+ String[] rootDirs = FilesystemUtils.readdir("/");
+ for (String entry : rootDirs) {
+ if (entry.startsWith("lib")) {
+ FilesystemUtils.mkdir(sandboxPath.getChild(entry).getPathString(), 0755);
+ mounts.add("/" + entry);
+ }
+ }
+
+ String[] usrDirs = FilesystemUtils.readdir("/usr/");
+ for (String entry : usrDirs) {
+ if (entry.startsWith("lib")) {
+ String lib = usr.getChild(entry).getPathString();
+ FilesystemUtils.mkdir(lib, 0755);
+ mounts.add("/usr/" + entry);
+ }
+ }
+
+ if (this.includePrefix != null) {
+ FilesystemUtils.mkdir(sandboxPath.getRelative(includePrefix).getPathString(), 0755);
+
+ for (PathFragment fullPath : includeDirectories) {
+ // includeDirectories should be absolute paths like /usr/include/foo.h. we want to combine
+ // them into something like sandbox/include-prefix/usr/include/foo.h - for that we remove
+ // the leading '/' from the path string and concatenate with sandbox/include/prefix
+ FileSystemUtils.createDirectoryAndParents(sandboxPath.getRelative(includePrefix)
+ .getRelative(fullPath.getPathString().substring(1)));
+ }
+ }
+
+ // output directories
+ for (ActionInput output : outputs) {
+ PathFragment parentDirectory =
+ new PathFragment(output.getExecPathString()).getParentDirectory();
+ FileSystemUtils.createDirectoryAndParents(sandboxPath.getRelative(parentDirectory));
+ }
+ }
+
+ public void setupSandbox(List<? extends ActionInput> inputs,
+ Collection<? extends ActionInput> outputs) throws IOException {
+ createFileSystem(outputs);
+ setupBlazeUtils();
+ includeManifests();
+ copyInputs(inputs);
+ }
+
+ private void copyInputs(List<? extends ActionInput> inputs) throws IOException {
+ for (ActionInput input : inputs) {
+ if (input.getExecPathString().contains("internal/_middlemen/")) {
+ continue;
+ }
+ // entire tools will be mounted in the sandbox, so don't copy parts of it
+ if (input.getExecPathString().startsWith("tools/")) {
+ continue;
+ }
+ Path target = sandboxPath.getRelative(input.getExecPathString());
+ Path source = execRoot.getRelative(input.getExecPathString());
+ FileSystemUtils.createDirectoryAndParents(target.getParentDirectory());
+ File targetFile = new File(target.getPathString());
+ // TODO(bazel-team): mount inputs inside sandbox instead of copying
+ Files.copy(new File(source.getPathString()), targetFile);
+ FilesystemUtils.chmod(targetFile, 0755);
+ }
+ }
+
+ private void includeManifests() throws IOException {
+ for (Entry<PathFragment, Artifact> manifest : this.manifests.entrySet()) {
+ String path = manifest.getValue().getPath().getPathString();
+ for (String line : Files.readLines(new File(path), Charset.defaultCharset())) {
+ String[] fields = line.split(" ");
+ String targetPath = sandboxPath.getPathString() + PathFragment.SEPARATOR_CHAR + fields[0];
+ String sourcePath = fields[1];
+ File source = new File(sourcePath);
+ File target = new File(targetPath);
+ Files.createParentDirs(target);
+ Files.copy(source, target);
+ }
+ }
+ }
+
+ private void setupBlazeUtils() throws IOException {
+ Path bin = this.sandboxPath.getChild("_bin");
+ if (!bin.isDirectory()) {
+ FilesystemUtils.mkdir(bin.getPathString(), 0755);
+ }
+ Files.copy(new File(this.embeddedBinaries.getChild("build-runfiles").getPathString()),
+ new File(bin.getChild("build-runfiles").getPathString()));
+ FilesystemUtils.chmod(bin.getChild("build-runfiles").getPathString(), 0755);
+ // TODO(bazel-team) filter tools out of input files instead
+ // some of the tools could be in inputs; we will mount entire tools anyway so it's just
+ // easier to remove them and remount inside sandbox
+ FilesystemUtils.rmTree(sandboxPath.getChild("tools").getPathString());
+ }
+
+
+ /**
+ * Runs given
+ *
+ * @param spawnArguments - arguments of spawn to run inside the sandbox
+ * @param env - environment to run sandbox in
+ * @param cwd - current working directory
+ * @param outErr - error output to capture sandbox's and command's stderr
+ * @throws CommandException
+ */
+ public void run(List<String> spawnArguments, ImmutableMap<String, String> env, File cwd,
+ FileOutErr outErr) throws CommandException {
+ List<String> args = new ArrayList<>();
+ args.add(execRoot.getRelative("_bin/namespace-sandbox").getPathString());
+
+ // Only for c++ compilation
+ if (includePrefix != null) {
+ for (PathFragment include : includeDirectories) {
+ args.add("-n");
+ args.add(include.getPathString());
+ }
+
+ args.add("-N");
+ args.add(includePrefix.getPathString());
+ }
+
+ if (debug) {
+ args.add("-D");
+ }
+ args.add("-t");
+ args.add(tools.getPathString());
+
+ args.add("-S");
+ args.add(sandboxPath.getPathString());
+ for (String mount : mounts) {
+ args.add("-m");
+ args.add(mount);
+ }
+
+ args.add("-C");
+ args.addAll(spawnArguments);
+ Command cmd = new Command(args.toArray(new String[] {}), env, cwd);
+
+ cmd.execute(
+ /* stdin */new byte[] {},
+ Command.NO_OBSERVER,
+ outErr.getOutputStream(),
+ outErr.getErrorStream(),
+ /* killSubprocessOnInterrupt */true);
+ }
+
+
+ public void cleanup() throws IOException {
+ FilesystemUtils.rmTree(sandboxPath.getPathString());
+ }
+
+
+ public void copyOutputs(Collection<? extends ActionInput> outputs, FileOutErr outErr)
+ throws IOException {
+ for (ActionInput output : outputs) {
+ Path source = this.sandboxPath.getRelative(output.getExecPathString());
+ Path target = this.execRoot.getRelative(output.getExecPathString());
+ FileSystemUtils.createDirectoryAndParents(target.getParentDirectory());
+ // TODO(bazel-team): eliminate cases when there are excessive outputs in spawns
+ // (java compilation expects "srclist" file in its outputs which is sometimes not produced)
+ if (source.isFile()) {
+ Files.move(new File(source.getPathString()), new File(target.getPathString()));
+ } else {
+ outErr.getErrorStream().write(("Output wasn't created by action: " + output + "\n")
+ .getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java
new file mode 100644
index 0000000..2011327
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.standalone;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.devtools.build.lib.actions.ActionContextConsumer;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext;
+import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext;
+import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext;
+import com.google.devtools.build.lib.rules.cpp.LinkStrategy;
+import com.google.devtools.build.lib.rules.test.TestStrategy;
+
+import java.util.Map;
+
+/**
+ * {@link ActionContextConsumer} that requests the action contexts necessary for standalone
+ * execution.
+ */
+public class StandaloneContextConsumer implements ActionContextConsumer {
+
+ @Override
+ public Map<String, String> getSpawnActionContexts() {
+ return ImmutableMap.of();
+ }
+
+ @Override
+ public Map<Class<? extends ActionContext>, String> getActionContexts() {
+ Builder<Class<? extends ActionContext>, String> actionContexts =
+ new ImmutableMap.Builder<Class<? extends ActionContext>, String>();
+
+ actionContexts.put(SpawnActionContext.class, "standalone");
+
+ // C++.
+ actionContexts.put(LinkStrategy.class, "");
+ actionContexts.put(IncludeScanningContext.class, "");
+ actionContexts.put(CppCompileActionContext.class, "");
+ actionContexts.put(TestStrategy.class, "");
+ actionContexts.put(FileWriteActionContext.class, "");
+
+ return actionContexts.build();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java
new file mode 100644
index 0000000..dbfac2e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java
@@ -0,0 +1,126 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.standalone;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionMetadata;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactResolver;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor.ActionContext;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.exec.FileWriteStrategy;
+import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext;
+import com.google.devtools.build.lib.rules.cpp.LocalGccStrategy;
+import com.google.devtools.build.lib.rules.cpp.LocalLinkStrategy;
+import com.google.devtools.build.lib.rules.test.ExclusiveTestStrategy;
+import com.google.devtools.build.lib.rules.test.StandaloneTestStrategy;
+import com.google.devtools.build.lib.rules.test.TestActionContext;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+
+import java.io.IOException;
+
+/**
+ * Provide a standalone, local execution context.
+ */
+public class StandaloneContextProvider implements ActionContextProvider {
+
+ /**
+ * a IncludeScanningContext that does nothing. Since local execution does not need to
+ * discover inclusion in advance, we do not need include scanning.
+ */
+ @ExecutionStrategy(contextType = IncludeScanningContext.class)
+ class DummyIncludeScanningContext implements IncludeScanningContext {
+ @Override
+ public void extractIncludes(ActionExecutionContext actionExecutionContext,
+ ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput)
+ throws IOException, InterruptedException {
+ FileSystemUtils.writeContent(primaryOutput.getPath(), new byte[]{});
+ }
+
+ @Override
+ public ArtifactResolver getArtifactResolver() {
+ return runtime.getView().getArtifactFactory();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private final ActionContext localSpawnStrategy;
+ private final ImmutableList<ActionContext> strategies;
+ private final BlazeRuntime runtime;
+
+ public StandaloneContextProvider(
+ BlazeRuntime runtime, BuildRequest buildRequest) {
+ boolean verboseFailures = buildRequest.getOptions(ExecutionOptions.class).verboseFailures;
+
+ localSpawnStrategy = new LocalSpawnStrategy(
+ runtime.getDirectories().getExecRoot(), verboseFailures);
+ this.runtime = runtime;
+
+ TestActionContext testStrategy = new StandaloneTestStrategy(buildRequest,
+ runtime.getStartupOptionsProvider(), runtime.getBinTools(), runtime.getRunfilesPrefix());
+ Builder<ActionContext> strategiesBuilder = ImmutableList.builder();
+ // order of strategies passed to builder is significant - when there are many strategies that
+ // could potentially be used and a spawnActionContext doesn't specify which one it wants, the
+ // last one from strategies list will be used
+
+ // put sandboxed strategy first, as we don't want it by default
+ if (OS.getCurrent() == OS.LINUX) {
+ LinuxSandboxedStrategy sandboxedLinuxStrategy =
+ new LinuxSandboxedStrategy(runtime.getDirectories(), verboseFailures);
+ strategiesBuilder.add(sandboxedLinuxStrategy);
+ }
+ strategiesBuilder.add(
+ localSpawnStrategy,
+ new DummyIncludeScanningContext(),
+ new LocalLinkStrategy(),
+ testStrategy,
+ new ExclusiveTestStrategy(testStrategy),
+ new LocalGccStrategy(buildRequest),
+ new FileWriteStrategy());
+
+
+ this.strategies = strategiesBuilder.build();
+ }
+
+ @Override
+ public Iterable<ActionContext> getActionContexts() {
+ return strategies;
+ }
+
+ @Override
+ public void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException {
+ }
+
+ @Override
+ public void executionPhaseStarting(
+ ActionInputFileCache actionInputFileCache,
+ ActionGraph actionGraph,
+ Iterable<Artifact> topLevelArtifacts) throws ExecutorInitException {
+ }
+
+ @Override
+ public void executionPhaseEnding() {}
+}
+
+
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java
new file mode 100644
index 0000000..06bffd0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.standalone;
+
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.actions.ActionContextConsumer;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+
+/**
+ * StandaloneModule provides pluggable functionality for blaze.
+ */
+public class StandaloneModule extends BlazeModule {
+ private final ActionContextConsumer actionContextConsumer = new StandaloneContextConsumer();
+ private BuildRequest buildRequest;
+ private BlazeRuntime runtime;
+
+ /**
+ * Returns the action context provider the module contributes to Blaze, if any.
+ */
+ @Override
+ public ActionContextProvider getActionContextProvider() {
+ return new StandaloneContextProvider(runtime, buildRequest);
+ }
+
+ /**
+ * Returns the action context consumer the module contributes to Blaze, if any.
+ */
+ @Override
+ public ActionContextConsumer getActionContextConsumer() {
+ return actionContextConsumer;
+ }
+
+ @Override
+ public void beforeCommand(BlazeRuntime runtime, Command command) {
+ this.runtime = runtime;
+ runtime.getEventBus().register(this);
+ }
+
+ @Subscribe
+ public void buildStarting(BuildStartingEvent event) {
+ buildRequest = event.getRequest();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java b/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java
new file mode 100644
index 0000000..81ca584
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java
@@ -0,0 +1,65 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.events.Location;
+
+import java.io.Serializable;
+
+/**
+ * Root class for nodes in the Abstract Syntax Tree of the Build language.
+ */
+public abstract class ASTNode implements Serializable {
+
+ private Location location;
+
+ protected ASTNode() {}
+
+ @VisibleForTesting // productionVisibility = Visibility.PACKAGE_PRIVATE
+ public void setLocation(Location location) {
+ this.location = location;
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+
+ /**
+ * Print the syntax node in a form useful for debugging. The output is not
+ * precisely specified, and should not be used by pretty-printing routines.
+ */
+ @Override
+ public abstract String toString();
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException(); // avoid nondeterminism
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Implements the double dispatch by calling into the node specific
+ * <code>visit</code> method of the {@link SyntaxTreeVisitor}
+ *
+ * @param visitor the {@link SyntaxTreeVisitor} instance to dispatch to.
+ */
+ public abstract void accept(SyntaxTreeVisitor visitor);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java
new file mode 100644
index 0000000..f444c23
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Partial implementation of Function interface.
+ */
+public abstract class AbstractFunction implements Function {
+
+ private final String name;
+
+ protected AbstractFunction(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Returns the name of this function.
+ */
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return null;
+ }
+
+ /**
+ * Abstract implementation of Function that accepts no parameters.
+ */
+ public abstract static class NoArgFunction extends AbstractFunction {
+
+ public NoArgFunction(String name) {
+ super(name);
+ }
+
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ Environment env) throws EvalException, InterruptedException {
+ if (args.size() != 1 || kwargs.size() != 0) {
+ throw new EvalException(ast.getLocation(), "Invalid number of arguments (expected 0)");
+ }
+ return call(args.get(0), ast, env);
+ }
+
+ public abstract Object call(Object self, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Argument.java b/src/main/java/com/google/devtools/build/lib/syntax/Argument.java
new file mode 100644
index 0000000..0706dee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Argument.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for a function argument. This can be a key/value pair such as
+ * appears as a keyword argument to a function call or just an expression that
+ * is used as a positional argument. It also can be used for function definitions
+ * to identify the name (and optionally the default value) of the argument.
+ */
+public final class Argument extends ASTNode {
+
+ private final Ident name;
+
+ private final Expression value;
+
+ private final boolean kwargs;
+
+ /**
+ * Create a new argument.
+ * At call site: name is optional, value is mandatory. kwargs is true for ** arguments.
+ * At definition site: name is mandatory, (default) value is optional.
+ */
+ public Argument(Ident name, Expression value, boolean kwargs) {
+ this.name = name;
+ this.value = value;
+ this.kwargs = kwargs;
+ }
+
+ public Argument(Ident name, Expression value) {
+ this.name = name;
+ this.value = value;
+ this.kwargs = false;
+ }
+
+ /**
+ * Creates an Argument with null as name. It can be used as positional arguments
+ * of function calls.
+ */
+ public Argument(Expression value) {
+ this(null, value);
+ }
+
+ /**
+ * Creates an Argument with null as value. It can be used as a mandatory keyword argument
+ * of a function definition.
+ */
+ public Argument(Ident name) {
+ this(name, null);
+ }
+
+ /**
+ * Returns the name of this keyword argument or null if this argument is
+ * positional.
+ */
+ public Ident getName() {
+ return name;
+ }
+
+ /**
+ * Returns the String value of the Ident of this argument. Shortcut for arg.getName().getName().
+ */
+ public String getArgName() {
+ return name.getName();
+ }
+
+ /**
+ * Returns the syntax of this argument expression.
+ */
+ public Expression getValue() {
+ return value;
+ }
+
+ /**
+ * Returns true if this argument is positional.
+ */
+ public boolean isPositional() {
+ return name == null && !kwargs;
+ }
+
+ /**
+ * Returns true if this argument is a keyword argument.
+ */
+ public boolean isNamed() {
+ return name != null;
+ }
+
+ /**
+ * Returns true if this argument is a **kwargs argument.
+ */
+ public boolean isKwargs() {
+ return kwargs;
+ }
+
+ /**
+ * Returns true if this argument has value.
+ */
+ public boolean hasValue() {
+ return value != null;
+ }
+
+ @Override
+ public String toString() {
+ return isNamed() ? name + "=" + value : String.valueOf(value);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java
new file mode 100644
index 0000000..619e841
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Syntax node for an assignment statement.
+ */
+public final class AssignmentStatement extends Statement {
+
+ private final Expression lvalue;
+
+ private final Expression expression;
+
+ /**
+ * Constructs an assignment: "lvalue := value".
+ */
+ AssignmentStatement(Expression lvalue, Expression expression) {
+ this.lvalue = lvalue;
+ this.expression = expression;
+ }
+
+ /**
+ * Returns the LHS of the assignment.
+ */
+ public Expression getLValue() {
+ return lvalue;
+ }
+
+ /**
+ * Returns the RHS of the assignment.
+ */
+ public Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ public String toString() {
+ return lvalue + " = " + expression + '\n';
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ if (!(lvalue instanceof Ident)) {
+ throw new EvalException(getLocation(),
+ "can only assign to variables, not to '" + lvalue + "'");
+ }
+
+ Ident ident = (Ident) lvalue;
+ Object result = expression.eval(env);
+ Preconditions.checkNotNull(result, "result of " + expression + " is null");
+
+ if (env.isSkylarkEnabled()) {
+ // The variable may have been referenced successfully if a global variable
+ // with the same name exists. In this case an Exception needs to be thrown.
+ SkylarkEnvironment skylarkEnv = (SkylarkEnvironment) env;
+ if (skylarkEnv.hasBeenReadGlobalVariable(ident.getName())) {
+ throw new EvalException(getLocation(), "Variable '" + ident.getName()
+ + "' is referenced before assignment."
+ + "The variable is defined in the global scope.");
+ }
+ Class<?> variableType = skylarkEnv.getVariableType(ident.getName());
+ Class<?> resultType = EvalUtils.getSkylarkType(result.getClass());
+ if (variableType != null && !variableType.equals(resultType)
+ && !resultType.equals(Environment.NoneType.class)
+ && !variableType.equals(Environment.NoneType.class)) {
+ throw new EvalException(getLocation(), String.format("Incompatible variable types, "
+ + "trying to assign %s (type of %s) to variable %s which is already %s",
+ EvalUtils.prettyPrintValue(result),
+ EvalUtils.getDatatypeName(result),
+ ident.getName(),
+ EvalUtils.getDataTypeNameFromClass(variableType)));
+ }
+ }
+ env.update(ident.getName(), result);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): Implement other validations.
+ if (lvalue instanceof Ident) {
+ Ident ident = (Ident) lvalue;
+ SkylarkType resultType = expression.validate(env);
+ env.update(ident.getName(), resultType, getLocation());
+ } else {
+ throw new EvalException(getLocation(),
+ "can only assign to variables, not to '" + lvalue + "'");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
new file mode 100644
index 0000000..f53538f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
@@ -0,0 +1,412 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.IllegalFormatException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Syntax node for a binary operator expression.
+ */
+public final class BinaryOperatorExpression extends Expression {
+
+ private final Expression lhs;
+
+ private final Expression rhs;
+
+ private final Operator operator;
+
+ public BinaryOperatorExpression(Operator operator,
+ Expression lhs,
+ Expression rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ this.operator = operator;
+ }
+
+ public Expression getLhs() {
+ return lhs;
+ }
+
+ public Expression getRhs() {
+ return rhs;
+ }
+
+ /**
+ * Returns the operator kind for this binary operation.
+ */
+ public Operator getOperator() {
+ return operator;
+ }
+
+ @Override
+ public String toString() {
+ return lhs + " " + operator + " " + rhs;
+ }
+
+ private int compare(Object lval, Object rval) throws EvalException {
+ if (!(lval instanceof Comparable)) {
+ throw new EvalException(getLocation(), lval + " is not comparable");
+ }
+ try {
+ return ((Comparable) lval).compareTo(rval);
+ } catch (ClassCastException e) {
+ throw new EvalException(getLocation(), "Cannot compare " + EvalUtils.getDatatypeName(lval)
+ + " with " + EvalUtils.getDatatypeName(rval));
+ }
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ Object lval = lhs.eval(env);
+
+ // Short-circuit operators
+ if (operator == Operator.AND) {
+ if (EvalUtils.toBoolean(lval)) {
+ return rhs.eval(env);
+ } else {
+ return lval;
+ }
+ }
+
+ if (operator == Operator.OR) {
+ if (EvalUtils.toBoolean(lval)) {
+ return lval;
+ } else {
+ return rhs.eval(env);
+ }
+ }
+
+ Object rval = rhs.eval(env);
+
+ switch (operator) {
+ case PLUS: {
+ // int + int
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() + ((Integer) rval).intValue();
+ }
+
+ // string + string
+ if (lval instanceof String && rval instanceof String) {
+ return (String) lval + (String) rval;
+ }
+
+ // list + list, tuple + tuple (list + tuple, tuple + list => error)
+ if (lval instanceof List<?> && rval instanceof List<?>) {
+ List<?> llist = (List<?>) lval;
+ List<?> rlist = (List<?>) rval;
+ if (EvalUtils.isImmutable(llist) != EvalUtils.isImmutable(rlist)) {
+ throw new EvalException(getLocation(), "can only concatenate "
+ + EvalUtils.getDatatypeName(rlist) + " (not \""
+ + EvalUtils.getDatatypeName(llist) + "\") to "
+ + EvalUtils.getDatatypeName(rlist));
+ }
+ if (llist instanceof GlobList<?> || rlist instanceof GlobList<?>) {
+ return GlobList.concat(llist, rlist);
+ } else {
+ List<Object> result = Lists.newArrayListWithCapacity(llist.size() + rlist.size());
+ result.addAll(llist);
+ result.addAll(rlist);
+ return EvalUtils.makeSequence(result, EvalUtils.isImmutable(llist));
+ }
+ }
+
+ if (lval instanceof SkylarkList && rval instanceof SkylarkList) {
+ return SkylarkList.concat((SkylarkList) lval, (SkylarkList) rval, getLocation());
+ }
+
+ if (env.isSkylarkEnabled() && lval instanceof Map<?, ?> && rval instanceof Map<?, ?>) {
+ Map<?, ?> ldict = (Map<?, ?>) lval;
+ Map<?, ?> rdict = (Map<?, ?>) rval;
+ Map<Object, Object> result = Maps.newHashMapWithExpectedSize(ldict.size() + rdict.size());
+ result.putAll(ldict);
+ result.putAll(rdict);
+ return result;
+ }
+
+ if (env.isSkylarkEnabled()
+ && lval instanceof SkylarkClassObject && rval instanceof SkylarkClassObject) {
+ return SkylarkClassObject.concat(
+ (SkylarkClassObject) lval, (SkylarkClassObject) rval, getLocation());
+ }
+
+ if (env.isSkylarkEnabled() && lval instanceof SkylarkNestedSet) {
+ return new SkylarkNestedSet((SkylarkNestedSet) lval, rval, getLocation());
+ }
+ break;
+ }
+
+ case MINUS: {
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() - ((Integer) rval).intValue();
+ }
+ break;
+ }
+
+ case MULT: {
+ // int * int
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() * ((Integer) rval).intValue();
+ }
+
+ // string * int
+ if (lval instanceof String && rval instanceof Integer) {
+ return Strings.repeat((String) lval, ((Integer) rval).intValue());
+ }
+
+ // int * string
+ if (lval instanceof Integer && rval instanceof String) {
+ return Strings.repeat((String) rval, ((Integer) lval).intValue());
+ }
+ break;
+ }
+
+ case PERCENT: {
+ // int % int
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() % ((Integer) rval).intValue();
+ }
+
+ // string % tuple, string % dict, string % anything-else
+ if (lval instanceof String) {
+ try {
+ String pattern = (String) lval;
+ if (rval instanceof List<?>) {
+ List<?> rlist = (List<?>) rval;
+ if (EvalUtils.isTuple(rlist)) {
+ return EvalUtils.formatString(pattern, rlist);
+ }
+ /* string % list: fall thru */
+ }
+ if (rval instanceof SkylarkList) {
+ SkylarkList rlist = (SkylarkList) rval;
+ if (rlist.isTuple()) {
+ return EvalUtils.formatString(pattern, rlist.toList());
+ }
+ }
+
+ return EvalUtils.formatString(pattern,
+ Collections.singletonList(rval));
+ } catch (IllegalFormatException e) {
+ throw new EvalException(getLocation(), e.getMessage());
+ }
+ }
+ break;
+ }
+
+ case EQUALS_EQUALS: {
+ return lval.equals(rval);
+ }
+
+ case NOT_EQUALS: {
+ return !lval.equals(rval);
+ }
+
+ case LESS: {
+ return compare(lval, rval) < 0;
+ }
+
+ case LESS_EQUALS: {
+ return compare(lval, rval) <= 0;
+ }
+
+ case GREATER: {
+ return compare(lval, rval) > 0;
+ }
+
+ case GREATER_EQUALS: {
+ return compare(lval, rval) >= 0;
+ }
+
+ case IN: {
+ if (rval instanceof SkylarkList) {
+ for (Object obj : (SkylarkList) rval) {
+ if (obj.equals(lval)) {
+ return true;
+ }
+ }
+ return false;
+ } else if (rval instanceof Collection<?>) {
+ return ((Collection<?>) rval).contains(lval);
+ } else if (rval instanceof Map<?, ?>) {
+ return ((Map<?, ?>) rval).containsKey(lval);
+ } else if (rval instanceof String) {
+ if (lval instanceof String) {
+ return ((String) rval).contains((String) lval);
+ } else {
+ throw new EvalException(getLocation(),
+ "in operator only works on strings if the left operand is also a string");
+ }
+ } else {
+ throw new EvalException(getLocation(),
+ "in operator only works on lists, tuples, dictionaries and strings");
+ }
+ }
+
+ default: {
+ throw new AssertionError("Unsupported binary operator: " + operator);
+ }
+ } // endswitch
+
+ throw new EvalException(getLocation(),
+ "unsupported operand types for '" + operator + "': '"
+ + EvalUtils.getDatatypeName(lval) + "' and '"
+ + EvalUtils.getDatatypeName(rval) + "'");
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType ltype = lhs.validate(env);
+ SkylarkType rtype = rhs.validate(env);
+ String lname = EvalUtils.getDataTypeNameFromClass(ltype.getType());
+ String rname = EvalUtils.getDataTypeNameFromClass(rtype.getType());
+
+ switch (operator) {
+ case AND: {
+ return ltype.infer(rtype, "and operator", rhs.getLocation(), lhs.getLocation());
+ }
+
+ case OR: {
+ return ltype.infer(rtype, "or operator", rhs.getLocation(), lhs.getLocation());
+ }
+
+ case PLUS: {
+ // int + int
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+
+ // string + string
+ if (ltype == SkylarkType.STRING && rtype == SkylarkType.STRING) {
+ return SkylarkType.STRING;
+ }
+
+ // list + list
+ if (ltype.isList() && rtype.isList()) {
+ return ltype.infer(rtype, "list concatenation", rhs.getLocation(), lhs.getLocation());
+ }
+
+ // dict + dict
+ if (ltype.isDict() && rtype.isDict()) {
+ return ltype.infer(rtype, "dict concatenation", rhs.getLocation(), lhs.getLocation());
+ }
+
+ // struct + struct
+ if (ltype.isStruct() && rtype.isStruct()) {
+ return SkylarkType.of(ClassObject.class);
+ }
+
+ if (ltype.isNset()) {
+ if (rtype.isNset()) {
+ return ltype.infer(rtype, "nested set", rhs.getLocation(), lhs.getLocation());
+ } else if (rtype.isList()) {
+ return ltype.infer(SkylarkType.of(SkylarkNestedSet.class, rtype.getGenericType1()),
+ "nested set", rhs.getLocation(), lhs.getLocation());
+ }
+ if (rtype != SkylarkType.UNKNOWN) {
+ throw new EvalException(getLocation(), String.format("can only concatenate nested sets "
+ + "with other nested sets or list of items, not '" + rname + "'"));
+ }
+ }
+
+ break;
+ }
+
+ case MULT: {
+ // int * int
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+
+ // string * int
+ if (ltype == SkylarkType.STRING && rtype == SkylarkType.INT) {
+ return SkylarkType.STRING;
+ }
+
+ // int * string
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.STRING) {
+ return SkylarkType.STRING;
+ }
+ break;
+ }
+
+ case MINUS: {
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+ break;
+ }
+
+ case PERCENT: {
+ // int % int
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+
+ // string % tuple, string % dict, string % anything-else
+ if (ltype == SkylarkType.STRING) {
+ return SkylarkType.STRING;
+ }
+ break;
+ }
+
+ case EQUALS_EQUALS:
+ case NOT_EQUALS:
+ case LESS:
+ case LESS_EQUALS:
+ case GREATER:
+ case GREATER_EQUALS: {
+ if (ltype != SkylarkType.UNKNOWN && !(Comparable.class.isAssignableFrom(ltype.getType()))) {
+ throw new EvalException(getLocation(), lname + " is not comparable");
+ }
+ ltype.infer(rtype, "comparison", lhs.getLocation(), rhs.getLocation());
+ return SkylarkType.BOOL;
+ }
+
+ case IN: {
+ if (rtype.isList()
+ || rtype.isSet()
+ || rtype.isDict()
+ || rtype == SkylarkType.STRING) {
+ return SkylarkType.BOOL;
+ } else {
+ if (rtype != SkylarkType.UNKNOWN) {
+ throw new EvalException(getLocation(), String.format("operand 'in' only works on "
+ + "strings, dictionaries, lists, sets or tuples, not on a(n) %s",
+ EvalUtils.getDataTypeNameFromClass(rtype.getType())));
+ }
+ }
+ }
+ } // endswitch
+
+ if (ltype != SkylarkType.UNKNOWN && rtype != SkylarkType.UNKNOWN) {
+ throw new EvalException(getLocation(),
+ "unsupported operand types for '" + operator + "': '" + lname + "' and '" + rname + "'");
+ }
+ return SkylarkType.UNKNOWN;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
new file mode 100644
index 0000000..6c85ab1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
@@ -0,0 +1,244 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Abstract syntax node for an entire BUILD file.
+ */
+public class BuildFileAST extends ASTNode {
+
+ private final ImmutableList<Statement> stmts;
+
+ private final ImmutableList<Comment> comments;
+
+ private final ImmutableSet<PathFragment> imports;
+
+ /**
+ * Whether any errors were encountered during scanning or parsing.
+ */
+ private final boolean containsErrors;
+
+ private final String contentHashCode;
+
+ private BuildFileAST(Lexer lexer, List<Statement> preludeStatements, Parser.ParseResult result) {
+ this(lexer, preludeStatements, result, null);
+ }
+
+ private BuildFileAST(Lexer lexer, List<Statement> preludeStatements,
+ Parser.ParseResult result, String contentHashCode) {
+ this.stmts = ImmutableList.<Statement>builder()
+ .addAll(preludeStatements)
+ .addAll(result.statements)
+ .build();
+ this.comments = ImmutableList.copyOf(result.comments);
+ this.containsErrors = result.containsErrors;
+ this.contentHashCode = contentHashCode;
+ this.imports = fetchImports(this.stmts);
+ if (result.statements.size() > 0) {
+ setLocation(lexer.createLocation(
+ result.statements.get(0).getLocation().getStartOffset(),
+ result.statements.get(result.statements.size() - 1).getLocation().getEndOffset()));
+ } else {
+ setLocation(Location.fromFile(lexer.getFilename()));
+ }
+ }
+
+ private ImmutableSet<PathFragment> fetchImports(List<Statement> stmts) {
+ Set<PathFragment> imports = new HashSet<>();
+ for (Statement stmt : stmts) {
+ if (stmt instanceof LoadStatement) {
+ LoadStatement imp = (LoadStatement) stmt;
+ imports.add(imp.getImportPath());
+ }
+ }
+ return ImmutableSet.copyOf(imports);
+ }
+
+ /**
+ * Returns true if any errors were encountered during scanning or parsing. If
+ * set, clients should not rely on the correctness of the AST for builds or
+ * BUILD-file editing.
+ */
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ /**
+ * Returns an (immutable, ordered) list of statements in this BUILD file.
+ */
+ public ImmutableList<Statement> getStatements() {
+ return stmts;
+ }
+
+ /**
+ * Returns an (immutable, ordered) list of comments in this BUILD file.
+ */
+ public ImmutableList<Comment> getComments() {
+ return comments;
+ }
+
+ /**
+ * Returns an (immutable) set of imports in this BUILD file.
+ */
+ public ImmutableCollection<PathFragment> getImports() {
+ return imports;
+ }
+
+ /**
+ * Executes this build file in a given Environment.
+ *
+ * <p>If, for any reason, execution of a statement cannot be completed, an
+ * {@link EvalException} is thrown by {@link Statement#exec(Environment)}.
+ * This exception is caught here and reported through reporter and execution
+ * continues on the next statement. In effect, there is a "try/except" block
+ * around every top level statement. Such exceptions are not ignored, though:
+ * they are visible via the return value. Rules declared in a package
+ * containing any error (including loading-phase semantical errors that
+ * cannot be checked here) must also be considered "in error".
+ *
+ * <p>Note that this method will not affect the value of {@link
+ * #containsErrors()}; that refers only to lexer/parser errors.
+ *
+ * @return true if no error occurred during execution.
+ */
+ public boolean exec(Environment env, EventHandler eventHandler) throws InterruptedException {
+ boolean ok = true;
+ for (Statement stmt : stmts) {
+ try {
+ stmt.exec(env);
+ } catch (EvalException e) {
+ ok = false;
+ // Do not report errors caused by a previous parsing error, as it has already been
+ // reported.
+ if (e.isDueToIncompleteAST()) {
+ continue;
+ }
+ // When the exception is raised from another file, report first the location in the
+ // BUILD file (as it is the most probable cause for the error).
+ Location exnLoc = e.getLocation();
+ Location nodeLoc = stmt.getLocation();
+ if (exnLoc == null || !nodeLoc.getPath().equals(exnLoc.getPath())) {
+ eventHandler.handle(Event.error(nodeLoc,
+ e.getMessage() + " (raised from " + exnLoc + ")"));
+ } else {
+ eventHandler.handle(Event.error(exnLoc, e.getMessage()));
+ }
+ }
+ }
+ return ok;
+ }
+
+ @Override
+ public String toString() {
+ return "BuildFileAST" + getStatements();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ /**
+ * Parse the specified build file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ *
+ * @throws IOException if the file cannot not be read.
+ */
+ public static BuildFileAST parseBuildFile(Path buildFile, EventHandler eventHandler,
+ CachingPackageLocator locator, boolean parsePython)
+ throws IOException {
+ ParserInputSource inputSource = ParserInputSource.create(buildFile);
+ return parseBuildFile(inputSource, eventHandler, locator, parsePython);
+ }
+
+ /**
+ * Parse the specified build file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ */
+ public static BuildFileAST parseBuildFile(ParserInputSource input,
+ List<Statement> preludeStatements,
+ EventHandler eventHandler,
+ CachingPackageLocator locator,
+ boolean parsePython) {
+ Lexer lexer = new Lexer(input, eventHandler, parsePython);
+ Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, locator, parsePython);
+ return new BuildFileAST(lexer, preludeStatements, result);
+ }
+
+ public static BuildFileAST parseBuildFile(ParserInputSource input, EventHandler eventHandler,
+ CachingPackageLocator locator, boolean parsePython) {
+ Lexer lexer = new Lexer(input, eventHandler, parsePython);
+ Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, locator, parsePython);
+ return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result);
+ }
+
+ /**
+ * Parse the specified build file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ */
+ public static BuildFileAST parseBuildFile(Lexer lexer, EventHandler eventHandler) {
+ Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, null, false);
+ return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result);
+ }
+
+ /**
+ * Parse the specified Skylark file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ *
+ * @throws IOException if the file cannot not be read.
+ */
+ public static BuildFileAST parseSkylarkFile(Path file, EventHandler eventHandler,
+ CachingPackageLocator locator, ValidationEnvironment validationEnvironment)
+ throws IOException {
+ ParserInputSource input = ParserInputSource.create(file);
+ Lexer lexer = new Lexer(input, eventHandler, false);
+ Parser.ParseResult result =
+ Parser.parseFileForSkylark(lexer, eventHandler, locator, validationEnvironment);
+ return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result, input.contentHashCode());
+ }
+
+ /**
+ * Parse the specified build file, without building the AST.
+ *
+ * @return true if the input file is syntactically valid
+ */
+ public static boolean checkSyntax(ParserInputSource input,
+ EventHandler eventHandler, boolean parsePython) {
+ return !parseBuildFile(input, eventHandler, null, parsePython).containsErrors();
+ }
+
+ /**
+ * Returns a hash code calculated from the string content of the source file of this AST.
+ */
+ @Nullable public String getContentHashCode() {
+ return contentHashCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java b/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java
new file mode 100644
index 0000000..3b1cccf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Sets.SetView;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * An interface for objects behaving like Skylark structs.
+ */
+// TODO(bazel-team): type checks
+public interface ClassObject {
+
+ /**
+ * Returns the value associated with the name field in this struct,
+ * or null if the field does not exist.
+ */
+ @Nullable
+ Object getValue(String name);
+
+ /**
+ * Returns the fields of this struct.
+ */
+ ImmutableCollection<String> getKeys();
+
+ /**
+ * Returns a customized error message to print if the name is not a valid struct field
+ * of this struct, or returns null to use the default error message.
+ */
+ @Nullable String errorMessage(String name);
+
+ /**
+ * An implementation class of ClassObject for structs created in Skylark code.
+ */
+ @Immutable
+ @SkylarkModule(name = "struct",
+ doc = "A special language element to support structs (i.e. simple value objects). "
+ + "See the global <code>struct</code> method for more details.")
+ public class SkylarkClassObject implements ClassObject {
+
+ private final ImmutableMap<String, Object> values;
+ private final Location creationLoc;
+ private final String errorMessage;
+
+ /**
+ * Creates a built-in struct (i.e. without creation loc). The errorMessage has to have
+ * exactly one '%s' parameter to substitute the struct field name.
+ */
+ public SkylarkClassObject(Map<String, Object> values, String errorMessage) {
+ this.values = ImmutableMap.copyOf(values);
+ this.creationLoc = null;
+ this.errorMessage = errorMessage;
+ }
+
+ public SkylarkClassObject(Map<String, Object> values, Location creationLoc) {
+ this.values = ImmutableMap.copyOf(values);
+ this.creationLoc = Preconditions.checkNotNull(creationLoc);
+ this.errorMessage = null;
+ }
+
+ @Override
+ public Object getValue(String name) {
+ return values.get(name);
+ }
+
+ @Override
+ public ImmutableCollection<String> getKeys() {
+ return values.keySet();
+ }
+
+ public Location getCreationLoc() {
+ return Preconditions.checkNotNull(creationLoc,
+ "This struct was not created in a Skylark code");
+ }
+
+ static SkylarkClassObject concat(
+ SkylarkClassObject lval, SkylarkClassObject rval, Location loc) throws EvalException {
+ SetView<String> commonFields = Sets.intersection(lval.values.keySet(), rval.values.keySet());
+ if (!commonFields.isEmpty()) {
+ throw new EvalException(loc, "Cannot concat structs with common field(s): "
+ + Joiner.on(",").join(commonFields));
+ }
+ return new SkylarkClassObject(ImmutableMap.<String, Object>builder()
+ .putAll(lval.values).putAll(rval.values).build(), loc);
+ }
+
+ @Override
+ public String errorMessage(String name) {
+ return errorMessage != null ? String.format(errorMessage, name) : null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java b/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java
new file mode 100644
index 0000000..070e928
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.List;
+
+/**
+ * A converter from strings containing comma-separated names of packages to lists of strings.
+ */
+public class CommaSeparatedPackageNameListConverter
+ implements Converter<List<String>> {
+
+ private static final Splitter SPACE_SPLITTER = Splitter.on(',');
+
+ @Override
+ public List<String> convert(String input) throws OptionsParsingException {
+ if (Strings.isNullOrEmpty(input)) {
+ return ImmutableList.of();
+ }
+ ImmutableList.Builder<String> list = ImmutableList.builder();
+ for (String s : SPACE_SPLITTER.split(input)) {
+ String errorMessage = LabelValidator.validatePackageName(s);
+ if (errorMessage != null) {
+ throw new OptionsParsingException(errorMessage);
+ }
+ list.add(s);
+ }
+ return list.build();
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "comma-separated list of package names";
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Comment.java b/src/main/java/com/google/devtools/build/lib/syntax/Comment.java
new file mode 100644
index 0000000..29d9474
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Comment.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for comments.
+ */
+public final class Comment extends ASTNode {
+
+ protected final String value;
+
+ public Comment(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java b/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java
new file mode 100644
index 0000000..a69605e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java
@@ -0,0 +1,102 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Syntax node for dictionary comprehension expressions.
+ */
+public class DictComprehension extends Expression {
+
+ private final Expression keyExpression;
+ private final Expression valueExpression;
+ private final Ident loopVar;
+ private final Expression listExpression;
+
+ public DictComprehension(Expression keyExpression, Expression valueExpression, Ident loopVar,
+ Expression listExpression) {
+ this.keyExpression = keyExpression;
+ this.valueExpression = valueExpression;
+ this.loopVar = loopVar;
+ this.listExpression = listExpression;
+ }
+
+ Expression getKeyExpression() {
+ return keyExpression;
+ }
+
+ Expression getValueExpression() {
+ return valueExpression;
+ }
+
+ Ident getLoopVar() {
+ return loopVar;
+ }
+
+ Expression getListExpression() {
+ return listExpression;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ // We want to keep the iteration order
+ LinkedHashMap<Object, Object> map = new LinkedHashMap<>();
+ Iterable<?> elements = EvalUtils.toIterable(listExpression.eval(env), getLocation());
+ for (Object element : elements) {
+ env.update(loopVar.getName(), element);
+ Object key = keyExpression.eval(env);
+ map.put(key, valueExpression.eval(env));
+ }
+ return ImmutableMap.copyOf(map);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType elementsType = listExpression.validate(env);
+ // TODO(bazel-team): GenericType1 should be a SkylarkType.
+ Class<?> listElementType = elementsType.getGenericType1();
+ SkylarkType listElementSkylarkType = listElementType.equals(Object.class)
+ ? SkylarkType.UNKNOWN : SkylarkType.of(listElementType);
+ env.update(loopVar.getName(), listElementSkylarkType, getLocation());
+ SkylarkType keyType = keyExpression.validate(env);
+ if (!keyType.isSimple()) {
+ // TODO(bazel-team): this is most probably dead code but it's better to have it here
+ // in case we enable e.g. list of lists or we validate function calls on Java objects
+ throw new EvalException(getLocation(), "Dict comprehension key must be of a simple type");
+ }
+ valueExpression.validate(env);
+ if (elementsType != SkylarkType.UNKNOWN && !elementsType.isList()) {
+ throw new EvalException(getLocation(), "Dict comprehension elements must be a list");
+ }
+ return SkylarkType.of(Map.class, keyType.getType());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('{').append(keyExpression).append(": ").append(valueExpression);
+ sb.append(" for ").append(loopVar).append(" in ").append(listExpression);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.accept(this);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java
new file mode 100644
index 0000000..8f79739
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java
@@ -0,0 +1,117 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Syntax node for dictionary literals.
+ */
+public class DictionaryLiteral extends Expression {
+
+ static final class DictionaryEntryLiteral extends ASTNode {
+
+ private final Expression key;
+ private final Expression value;
+
+ public DictionaryEntryLiteral(Expression key, Expression value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ Expression getKey() {
+ return key;
+ }
+
+ Expression getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(key);
+ sb.append(": ");
+ sb.append(value);
+ return sb.toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+ }
+
+ private final ImmutableList<DictionaryEntryLiteral> entries;
+
+ public DictionaryLiteral(List<DictionaryEntryLiteral> exprs) {
+ this.entries = ImmutableList.copyOf(exprs);
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ // We need LinkedHashMap to maintain the order during iteration (e.g. for loops)
+ Map<Object, Object> map = new LinkedHashMap<>();
+ for (DictionaryEntryLiteral entry : entries) {
+ if (entry == null) {
+ throw new EvalException(getLocation(), "null expression in " + this);
+ }
+ map.put(entry.key.eval(env), entry.value.eval(env));
+
+ }
+ return map;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("{");
+ String sep = "";
+ for (DictionaryEntryLiteral e : entries) {
+ sb.append(sep);
+ sb.append(e);
+ sep = ", ";
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ public ImmutableList<DictionaryEntryLiteral> getEntries() {
+ return entries;
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType type = SkylarkType.UNKNOWN;
+ for (DictionaryEntryLiteral entry : entries) {
+ SkylarkType nextType = entry.key.validate(env);
+ entry.value.validate(env);
+ if (!nextType.isSimple()) {
+ throw new EvalException(getLocation(),
+ String.format("Dict cannot contain composite type '%s' as key", nextType));
+ }
+ type = type.infer(nextType, "dict literal", entry.getLocation(), getLocation());
+ }
+ return SkylarkType.of(Map.class, type.getType());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java
new file mode 100644
index 0000000..b0ae5a9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java
@@ -0,0 +1,110 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.FuncallExpression.MethodDescriptor;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Syntax node for a dot expression.
+ * e.g. obj.field, but not obj.method()
+ */
+public final class DotExpression extends Expression {
+
+ private final Expression obj;
+
+ private final Ident field;
+
+ public DotExpression(Expression obj, Ident field) {
+ this.obj = obj;
+ this.field = field;
+ }
+
+ public Expression getObj() {
+ return obj;
+ }
+
+ public Ident getField() {
+ return field;
+ }
+
+ @Override
+ public String toString() {
+ return obj + "." + field;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ Object objValue = obj.eval(env);
+ String name = field.getName();
+ Object result = eval(objValue, name, getLocation());
+ if (result == null) {
+ if (objValue instanceof ClassObject) {
+ String customErrorMessage = ((ClassObject) objValue).errorMessage(name);
+ if (customErrorMessage != null) {
+ throw new EvalException(getLocation(), customErrorMessage);
+ }
+ }
+ throw new EvalException(getLocation(), "Object of type '"
+ + EvalUtils.getDatatypeName(objValue) + "' has no field '" + name + "'");
+ }
+ return result;
+ }
+
+ /**
+ * Returns the field of the given name of the struct objValue, or null if no such field exists.
+ */
+ public static Object eval(Object objValue, String name, Location loc) throws EvalException {
+ Object result = null;
+ if (objValue instanceof ClassObject) {
+ result = ((ClassObject) objValue).getValue(name);
+ result = SkylarkType.convertToSkylark(result, loc);
+ // If we access NestedSets using ClassObject.getValue() we won't know the generic type,
+ // so we have to disable it. This should not happen.
+ SkylarkType.checkTypeAllowedInSkylark(result, loc);
+ } else {
+ try {
+ List<MethodDescriptor> methods = FuncallExpression.getMethods(objValue.getClass(), name, 0);
+ if (methods != null && methods.size() > 0) {
+ MethodDescriptor method = Iterables.getOnlyElement(methods);
+ if (method.getAnnotation().structField()) {
+ result = FuncallExpression.callMethod(
+ method, name, objValue, new Object[] {}, loc);
+ }
+ }
+ } catch (ExecutionException | IllegalAccessException | InvocationTargetException e) {
+ throw new EvalException(loc, "Method invocation failed: " + e);
+ }
+ }
+
+ return result;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ obj.validate(env);
+ // TODO(bazel-team): check existance of field
+ return SkylarkType.UNKNOWN;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Environment.java b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
new file mode 100644
index 0000000..a148a70
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
@@ -0,0 +1,345 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * The BUILD environment.
+ */
+public class Environment {
+
+ @SkylarkBuiltin(name = "True", returnType = Boolean.class, doc = "Literal for the boolean true.")
+ private static final Boolean TRUE = true;
+
+ @SkylarkBuiltin(name = "False", returnType = Boolean.class,
+ doc = "Literal for the boolean false.")
+ private static final Boolean FALSE = false;
+
+ @SkylarkBuiltin(name = "PACKAGE_NAME", returnType = String.class,
+ doc = "The name of the package the rule or build extension is called from. "
+ + "This variable is special, because its value comes from outside of the extension "
+ + "module (it comes from the BUILD file), so it can only be accessed in functions "
+ + "(transitively) called from BUILD files. For example:<br>"
+ + "<pre class=language-python>def extension():\n"
+ + " return PACKAGE_NAME</pre>"
+ + "In this case calling <code>extension()</code> works from the BUILD file (if the "
+ + "function is loaded), but not as a top level function call in the extension module.")
+ public static final String PKG_NAME = "PACKAGE_NAME";
+
+ /**
+ * There should be only one instance of this type to allow "== None" tests.
+ */
+ @Immutable
+ public static final class NoneType {
+ @Override
+ public String toString() { return "None"; }
+ private NoneType() {}
+ }
+
+ @SkylarkBuiltin(name = "None", returnType = NoneType.class, doc = "Literal for the None value.")
+ public static final NoneType NONE = new NoneType();
+
+ protected final Map<String, Object> env = new HashMap<>();
+
+ // Functions with namespaces. Works only in the global environment.
+ protected final Map<Class<?>, Map<String, Function>> functions = new HashMap<>();
+
+ /**
+ * The parent environment. For Skylark it's the global environment,
+ * used for global read only variable lookup.
+ */
+ protected final Environment parent;
+
+ /**
+ * Map from a Skylark extension to an environment, which contains all symbols defined in the
+ * extension.
+ */
+ protected Map<PathFragment, SkylarkEnvironment> importedExtensions;
+
+ /**
+ * A set of disable variables propagating through function calling. This is needed because
+ * UserDefinedFunctions lock the definition Environment which should be immutable.
+ */
+ protected Set<String> disabledVariables = new HashSet<>();
+
+ /**
+ * A set of disable namespaces propagating through function calling. See disabledVariables.
+ */
+ protected Set<Class<?>> disabledNameSpaces = new HashSet<>();
+
+ /**
+ * A set of variables propagating through function calling. It's only used to call
+ * native rules from Skylark build extensions.
+ */
+ protected Set<String> propagatingVariables = new HashSet<>();
+
+ /**
+ * An EventHandler for errors and warnings. This is not used in the BUILD language,
+ * however it might be used in Skylark code called from the BUILD language.
+ */
+ @Nullable protected EventHandler eventHandler;
+
+ /**
+ * Constructs an empty root non-Skylark environment.
+ * The root environment is also the global environment.
+ */
+ public Environment() {
+ this.parent = null;
+ this.importedExtensions = new HashMap<>();
+ setupGlobal();
+ }
+
+ /**
+ * Constructs an empty child environment.
+ */
+ public Environment(Environment parent) {
+ Preconditions.checkNotNull(parent);
+ this.parent = parent;
+ this.importedExtensions = new HashMap<>();
+ }
+
+ /**
+ * Constructs an empty child environment with an EventHandler.
+ */
+ public Environment(Environment parent, EventHandler eventHandler) {
+ this(parent);
+ this.eventHandler = Preconditions.checkNotNull(eventHandler);
+ }
+
+ // Sets up the global environment
+ private void setupGlobal() {
+ // In Python 2.x, True and False are global values and can be redefined by the user.
+ // In Python 3.x, they are keywords. We implement them as values, for the sake of
+ // simplicity. We define them as Boolean objects.
+ env.put("False", FALSE);
+ env.put("True", TRUE);
+ env.put("None", NONE);
+ }
+
+ public boolean isSkylarkEnabled() {
+ return false;
+ }
+
+ protected boolean hasVariable(String varname) {
+ return env.containsKey(varname);
+ }
+
+ /**
+ * @return the value from the environment whose name is "varname".
+ * @throws NoSuchVariableException if the variable is not defined in the Environment.
+ *
+ */
+ public Object lookup(String varname) throws NoSuchVariableException {
+ if (disabledVariables.contains(varname)) {
+ throw new NoSuchVariableException(varname);
+ }
+ Object value = env.get(varname);
+ if (value == null) {
+ if (parent != null) {
+ return parent.lookup(varname);
+ }
+ throw new NoSuchVariableException(varname);
+ }
+ return value;
+ }
+
+ /**
+ * Like <code>lookup(String)</code>, but instead of throwing an exception in
+ * the case where "varname" is not defined, "defaultValue" is returned instead.
+ *
+ */
+ public Object lookup(String varname, Object defaultValue) {
+ Object value = env.get(varname);
+ if (value == null) {
+ if (parent != null) {
+ return parent.lookup(varname, defaultValue);
+ }
+ return defaultValue;
+ }
+ return value;
+ }
+
+ /**
+ * Updates the value of variable "varname" in the environment, corresponding
+ * to an {@link AssignmentStatement}.
+ */
+ public void update(String varname, Object value) {
+ Preconditions.checkNotNull(value, "update(value == null)");
+ env.put(varname, value);
+ }
+
+ /**
+ * Same as {@link #update}, but also marks the variable propagating, meaning it will
+ * be present in the execution environment of a UserDefinedFunction called from this
+ * Environment. Using this method is discouraged.
+ */
+ public void updateAndPropagate(String varname, Object value) {
+ update(varname, value);
+ propagatingVariables.add(varname);
+ }
+
+ /**
+ * Remove the variable from the environment, returning
+ * any previous mapping (null if there was none).
+ */
+ public Object remove(String varname) {
+ return env.remove(varname);
+ }
+
+ /**
+ * Returns the (immutable) set of names of all variables defined in this
+ * environment. Exposed for testing; not very efficient!
+ */
+ @VisibleForTesting
+ public Set<String> getVariableNames() {
+ if (parent == null) {
+ return env.keySet();
+ } else {
+ Set<String> vars = new HashSet<>();
+ vars.addAll(env.keySet());
+ vars.addAll(parent.getVariableNames());
+ return vars;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException(); // avoid nondeterminism
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder out = new StringBuilder();
+ out.append("Environment{");
+ List<String> keys = new ArrayList<>(env.keySet());
+ Collections.sort(keys);
+ for (String key: keys) {
+ out.append(key).append(" -> ").append(env.get(key)).append(", ");
+ }
+ out.append("}");
+ if (parent != null) {
+ out.append("=>");
+ out.append(parent.toString());
+ }
+ return out.toString();
+ }
+
+ /**
+ * An exception thrown when an attempt is made to lookup a non-existent
+ * variable in the environment.
+ */
+ public static class NoSuchVariableException extends Exception {
+ NoSuchVariableException(String variable) {
+ super("no such variable: " + variable);
+ }
+ }
+
+ /**
+ * An exception thrown when an attempt is made to import a symbol from a file
+ * that was not properly loaded.
+ */
+ public static class LoadFailedException extends Exception {
+ LoadFailedException(String file) {
+ super("file '" + file + "' was not correctly loaded. Make sure the 'load' statement appears "
+ + "in the global scope, in the BUILD file");
+ }
+ }
+
+ public void setImportedExtensions(Map<PathFragment, SkylarkEnvironment> importedExtensions) {
+ this.importedExtensions = importedExtensions;
+ }
+
+ public void importSymbol(PathFragment extension, String symbol)
+ throws NoSuchVariableException, LoadFailedException {
+ if (!importedExtensions.containsKey(extension)) {
+ throw new LoadFailedException(extension.toString());
+ }
+ Object value = importedExtensions.get(extension).lookup(symbol);
+ if (!isSkylarkEnabled()) {
+ value = SkylarkType.convertFromSkylark(value);
+ }
+ update(symbol, value);
+ }
+
+ /**
+ * Registers a function with namespace to this global environment.
+ */
+ public void registerFunction(Class<?> nameSpace, String name, Function function) {
+ Preconditions.checkArgument(parent == null);
+ if (!functions.containsKey(nameSpace)) {
+ functions.put(nameSpace, new HashMap<String, Function>());
+ }
+ functions.get(nameSpace).put(name, function);
+ }
+
+ private Map<String, Function> getNamespaceFunctions(Class<?> nameSpace) {
+ if (disabledNameSpaces.contains(nameSpace)
+ || (parent != null && parent.disabledNameSpaces.contains(nameSpace))) {
+ return null;
+ }
+ Environment topLevel = this;
+ while (topLevel.parent != null) {
+ topLevel = topLevel.parent;
+ }
+ return topLevel.functions.get(nameSpace);
+ }
+
+ /**
+ * Returns the function of the namespace of the given name or null of it does not exists.
+ */
+ public Function getFunction(Class<?> nameSpace, String name) {
+ Map<String, Function> nameSpaceFunctions = getNamespaceFunctions(nameSpace);
+ return nameSpaceFunctions != null ? nameSpaceFunctions.get(name) : null;
+ }
+
+ /**
+ * Returns the function names registered with the namespace.
+ */
+ public Set<String> getFunctionNames(Class<?> nameSpace) {
+ Map<String, Function> nameSpaceFunctions = getNamespaceFunctions(nameSpace);
+ return nameSpaceFunctions != null ? nameSpaceFunctions.keySet() : ImmutableSet.<String>of();
+ }
+
+ /**
+ * Return the current stack trace (list of function names).
+ */
+ public ImmutableList<String> getStackTrace() {
+ // Empty list, since this environment does not allow function definition
+ // (see SkylarkEnvironment)
+ return ImmutableList.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java
new file mode 100644
index 0000000..27aba0f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java
@@ -0,0 +1,105 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.events.Location;
+
+/**
+ * Exceptions thrown during evaluation of BUILD ASTs or Skylark extensions.
+ *
+ * <p>This exception must always correspond to a repeatable, permanent error, i.e. evaluating the
+ * same package again must yield the same exception. Notably, do not use this for reporting I/O
+ * errors.
+ *
+ * <p>This requirement is in place so that we can cache packages where an error is reported by way
+ * of {@link EvalException}.
+ */
+public class EvalException extends Exception {
+
+ private final Location location;
+ private final String message;
+ private final boolean dueToIncompleteAST;
+
+ /**
+ * @param location the location where evaluation/execution failed.
+ * @param message the error message.
+ */
+ public EvalException(Location location, String message) {
+ this.location = location;
+ this.message = Preconditions.checkNotNull(message);
+ this.dueToIncompleteAST = false;
+ }
+
+ /**
+ * @param location the location where evaluation/execution failed.
+ * @param message the error message.
+ * @param dueToIncompleteAST if the error is caused by a previous error, such as parsing.
+ */
+ public EvalException(Location location, String message, boolean dueToIncompleteAST) {
+ this.location = location;
+ this.message = Preconditions.checkNotNull(message);
+ this.dueToIncompleteAST = dueToIncompleteAST;
+ }
+
+ private EvalException(Location location, Throwable cause) {
+ super(cause);
+ this.location = location;
+ // This is only used from Skylark, it's useful for debugging. Note that this only happens
+ // when the Precondition below kills the execution anyway.
+ if (cause.getMessage() == null) {
+ cause.printStackTrace();
+ }
+ this.message = Preconditions.checkNotNull(cause.getMessage());
+ this.dueToIncompleteAST = false;
+ }
+
+ /**
+ * Returns the error message.
+ */
+ @Override
+ public String getMessage() {
+ return message;
+ }
+
+ /**
+ * Returns the location of the evaluation error.
+ */
+ public Location getLocation() {
+ return location;
+ }
+
+ public boolean isDueToIncompleteAST() {
+ return dueToIncompleteAST;
+ }
+
+ /**
+ * A class to support a special case of EvalException when the cause of the error is an
+ * Exception during a direct Java call.
+ */
+ public static final class EvalExceptionWithJavaCause extends EvalException {
+
+ public EvalExceptionWithJavaCause(Location location, Throwable cause) {
+ super(location, cause);
+ }
+ }
+
+ /**
+ * Returns the error message with location info if exists.
+ */
+ public String print() {
+ return getLocation() == null ? getMessage() : getLocation().print() + ": " + getMessage();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
new file mode 100644
index 0000000..70d89bc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
@@ -0,0 +1,590 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Formattable;
+import java.util.Formatter;
+import java.util.IllegalFormatException;
+import java.util.List;
+import java.util.Map;
+import java.util.MissingFormatWidthException;
+import java.util.Set;
+
+/**
+ * Utilities used by the evaluator.
+ */
+public abstract class EvalUtils {
+
+ // TODO(bazel-team): Yet an other hack committed in the name of Skylark. One problem is that the
+ // syntax package cannot depend on actions so we have to have this until Actions are immutable.
+ // The other is that BuildConfigurations are technically not immutable but they cannot be modified
+ // from Skylark.
+ private static final ImmutableSet<Class<?>> quasiImmutableClasses;
+ static {
+ try {
+ ImmutableSet.Builder<Class<?>> builder = ImmutableSet.builder();
+ builder.add(Class.forName("com.google.devtools.build.lib.actions.Action"));
+ builder.add(Class.forName("com.google.devtools.build.lib.analysis.config.BuildConfiguration"));
+ builder.add(Class.forName("com.google.devtools.build.lib.actions.Root"));
+ quasiImmutableClasses = builder.build();
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private EvalUtils() {
+ }
+
+ /**
+ * @return true if the specified sequence is a tuple; false if it's a modifiable list.
+ */
+ public static boolean isTuple(List<?> l) {
+ return isTuple(l.getClass());
+ }
+
+ public static boolean isTuple(Class<?> c) {
+ Preconditions.checkState(List.class.isAssignableFrom(c));
+ if (ImmutableList.class.isAssignableFrom(c)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * @return true if the specified value is immutable (suitable for use as a
+ * dictionary key) according to the rules of the Build language.
+ */
+ public static boolean isImmutable(Object o) {
+ if (o instanceof Map<?, ?> || o instanceof Function
+ || o instanceof FilesetEntry || o instanceof GlobList<?>) {
+ return false;
+ } else if (o instanceof List<?>) {
+ return isTuple((List<?>) o); // tuples are immutable, lists are not.
+ } else {
+ return true; // string/int
+ }
+ }
+
+ /**
+ * Returns true if the type is immutable in the skylark language.
+ */
+ public static boolean isSkylarkImmutable(Class<?> c) {
+ if (c.isAnnotationPresent(Immutable.class)) {
+ return true;
+ } else if (c.equals(String.class) || c.equals(Integer.class) || c.equals(Boolean.class)
+ || SkylarkList.class.isAssignableFrom(c) || ImmutableMap.class.isAssignableFrom(c)
+ || NestedSet.class.isAssignableFrom(c)) {
+ return true;
+ } else {
+ for (Class<?> classObject : quasiImmutableClasses) {
+ if (classObject.isAssignableFrom(c)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns a transitive superclass or interface implemented by c which is annotated
+ * with SkylarkModule. Returns null if no such class or interface exists.
+ */
+ @VisibleForTesting
+ static Class<?> getParentWithSkylarkModule(Class<?> c) {
+ if (c == null) {
+ return null;
+ }
+ if (c.isAnnotationPresent(SkylarkModule.class)) {
+ return c;
+ }
+ Class<?> parent = getParentWithSkylarkModule(c.getSuperclass());
+ if (parent != null) {
+ return parent;
+ }
+ for (Class<?> ifparent : c.getInterfaces()) {
+ ifparent = getParentWithSkylarkModule(ifparent);
+ if (ifparent != null) {
+ return ifparent;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the Skylark equivalent type of the parameter. Note that the Skylark
+ * language doesn't have inheritance.
+ */
+ public static Class<?> getSkylarkType(Class<?> c) {
+ if (ImmutableList.class.isAssignableFrom(c)) {
+ return ImmutableList.class;
+ } else if (List.class.isAssignableFrom(c)) {
+ return List.class;
+ } else if (SkylarkList.class.isAssignableFrom(c)) {
+ return SkylarkList.class;
+ } else if (Map.class.isAssignableFrom(c)) {
+ return Map.class;
+ } else if (NestedSet.class.isAssignableFrom(c)) {
+ // This could be removed probably
+ return NestedSet.class;
+ } else if (Set.class.isAssignableFrom(c)) {
+ return Set.class;
+ } else {
+ // Check if one of the superclasses or implemented interfaces has the SkylarkModule
+ // annotation. If yes return that class.
+ Class<?> parent = getParentWithSkylarkModule(c);
+ if (parent != null) {
+ return parent;
+ }
+ }
+ return c;
+ }
+
+ /**
+ * Returns a pretty name for the datatype of object 'o' in the Build language.
+ */
+ public static String getDatatypeName(Object o) {
+ Preconditions.checkNotNull(o);
+ if (o instanceof SkylarkList) {
+ return ((SkylarkList) o).isTuple() ? "tuple" : "list";
+ }
+ return getDataTypeNameFromClass(o.getClass());
+ }
+
+ /**
+ * Returns a pretty name for the datatype equivalent of class 'c' in the Build language.
+ */
+ public static String getDataTypeNameFromClass(Class<?> c) {
+ if (c.equals(Object.class)) {
+ return "unknown";
+ } else if (c.equals(String.class)) {
+ return "string";
+ } else if (c.equals(Integer.class)) {
+ return "int";
+ } else if (c.equals(Boolean.class)) {
+ return "bool";
+ } else if (c.equals(Void.TYPE) || c.equals(Environment.NoneType.class)) {
+ return "None";
+ } else if (List.class.isAssignableFrom(c)) {
+ return isTuple(c) ? "tuple" : "list";
+ } else if (GlobList.class.isAssignableFrom(c)) {
+ return "list";
+ } else if (Map.class.isAssignableFrom(c)) {
+ return "dict";
+ } else if (Function.class.isAssignableFrom(c)) {
+ return "function";
+ } else if (c.equals(FilesetEntry.class)) {
+ return "FilesetEntry";
+ } else if (NestedSet.class.isAssignableFrom(c) || SkylarkNestedSet.class.isAssignableFrom(c)) {
+ return "set";
+ } else if (SkylarkClassObject.class.isAssignableFrom(c)) {
+ return "struct";
+ } else if (SkylarkList.class.isAssignableFrom(c)) {
+ // TODO(bazel-team): this is not entirely correct, it can also be a tuple.
+ return "list";
+ } else if (c.isAnnotationPresent(SkylarkModule.class)) {
+ SkylarkModule module = c.getAnnotation(SkylarkModule.class);
+ return c.getAnnotation(SkylarkModule.class).name()
+ + (module.namespace() ? " (a language module)" : "");
+ } else {
+ if (c.getSimpleName().isEmpty()) {
+ return c.getName();
+ } else {
+ return c.getSimpleName();
+ }
+ }
+ }
+
+ /**
+ * Returns a sequence of the appropriate list/tuple datatype for 'seq', based on 'isTuple'.
+ */
+ public static List<?> makeSequence(List<?> seq, boolean isTuple) {
+ return isTuple ? ImmutableList.copyOf(seq) : seq;
+ }
+
+ /**
+ * Print build-language value 'o' in display format into the specified buffer.
+ */
+ public static void printValue(Object o, Appendable buffer) {
+ // Exception-swallowing wrapper due to annoying Appendable interface.
+ try {
+ printValueX(o, buffer);
+ } catch (IOException e) {
+ throw new AssertionError(e); // can't happen
+ }
+ }
+
+ private static void printValueX(Object o, Appendable buffer)
+ throws IOException {
+ if (o == null) {
+ throw new NullPointerException(); // None is not a build language value.
+ } else if (o instanceof String ||
+ o instanceof Integer ||
+ o instanceof Double) {
+ buffer.append(o.toString());
+
+ } else if (o == Environment.NONE) {
+ buffer.append("None");
+
+ } else if (o == Boolean.TRUE) {
+ buffer.append("True");
+
+ } else if (o == Boolean.FALSE) {
+ buffer.append("False");
+
+ } else if (o instanceof List<?>) {
+ List<?> seq = (List<?>) o;
+ boolean isTuple = isImmutable(seq);
+ String sep = "";
+ buffer.append(isTuple ? '(' : '[');
+ for (int ii = 0, len = seq.size(); ii < len; ++ii) {
+ buffer.append(sep);
+ prettyPrintValue(seq.get(ii), buffer);
+ sep = ", ";
+ }
+ buffer.append(isTuple ? ')' : ']');
+
+ } else if (o instanceof Map<?, ?>) {
+ Map<?, ?> dict = (Map<?, ?>) o;
+ buffer.append('{');
+ String sep = "";
+ for (Map.Entry<?, ?> entry : dict.entrySet()) {
+ buffer.append(sep);
+ prettyPrintValue(entry.getKey(), buffer);
+ buffer.append(": ");
+ prettyPrintValue(entry.getValue(), buffer);
+ sep = ", ";
+ }
+ buffer.append('}');
+
+ } else if (o instanceof Function) {
+ Function func = (Function) o;
+ buffer.append("<function " + func.getName() + ">");
+
+ } else if (o instanceof FilesetEntry) {
+ FilesetEntry entry = (FilesetEntry) o;
+ buffer.append("FilesetEntry(srcdir = ");
+ prettyPrintValue(entry.getSrcLabel().toString(), buffer);
+ buffer.append(", files = ");
+ prettyPrintValue(makeStringList(entry.getFiles()), buffer);
+ buffer.append(", excludes = ");
+ prettyPrintValue(makeList(entry.getExcludes()), buffer);
+ buffer.append(", destdir = ");
+ prettyPrintValue(entry.getDestDir().getPathString(), buffer);
+ buffer.append(", strip_prefix = ");
+ prettyPrintValue(entry.getStripPrefix(), buffer);
+ buffer.append(", symlinks = \"");
+ buffer.append(entry.getSymlinkBehavior().toString());
+ buffer.append("\")");
+ } else if (o instanceof PathFragment) {
+ buffer.append(((PathFragment) o).getPathString());
+ } else {
+ buffer.append(o.toString());
+ }
+ }
+
+ private static List<?> makeList(Collection<?> list) {
+ return list == null ? Lists.newArrayList() : Lists.newArrayList(list);
+ }
+
+ private static List<String> makeStringList(List<Label> labels) {
+ if (labels == null) { return Collections.emptyList(); }
+ List<String> strings = Lists.newArrayListWithCapacity(labels.size());
+ for (Label label : labels) {
+ strings.add(label.toString());
+ }
+ return strings;
+ }
+
+ /**
+ * Print build-language value 'o' in parseable format into the specified
+ * buffer. (Only differs from printValueX in treatment of strings at toplevel,
+ * i.e. not within a sequence or dict)
+ */
+ public static void prettyPrintValue(Object o, Appendable buffer) {
+ // Exception-swallowing wrapper due to annoying Appendable interface.
+ try {
+ prettyPrintValueX(o, buffer);
+ } catch (IOException e) {
+ throw new AssertionError(e); // can't happen
+ }
+ }
+
+ private static void prettyPrintValueX(Object o, Appendable buffer)
+ throws IOException {
+ if (o instanceof Label) {
+ o = o.toString(); // Pretty-print a label like a string
+ }
+ if (o instanceof String) {
+ String s = (String) o;
+ buffer.append('"');
+ for (int ii = 0, len = s.length(); ii < len; ++ii) {
+ char c = s.charAt(ii);
+ switch (c) {
+ case '\r':
+ buffer.append('\\').append('r');
+ break;
+ case '\n':
+ buffer.append('\\').append('n');
+ break;
+ case '\t':
+ buffer.append('\\').append('t');
+ break;
+ case '\"':
+ buffer.append('\\').append('"');
+ break;
+ default:
+ if (c < 32) {
+ buffer.append(String.format("\\x%02x", (int) c));
+ } else {
+ buffer.append(c); // no need to support UTF-8
+ }
+ } // endswitch
+ }
+ buffer.append('\"');
+ } else {
+ printValueX(o, buffer);
+ }
+ }
+
+ /**
+ * Pretty-print value 'o' to a string. Convenience overloading of
+ * prettyPrintValue(Object, Appendable).
+ */
+ public static String prettyPrintValue(Object o) {
+ StringBuffer buffer = new StringBuffer();
+ prettyPrintValue(o, buffer);
+ return buffer.toString();
+ }
+
+ /**
+ * Pretty-print values of 'o' separated by the separator.
+ */
+ public static String prettyPrintValues(String separator, Iterable<Object> o) {
+ return Joiner.on(separator).join(Iterables.transform(o,
+ new com.google.common.base.Function<Object, String>() {
+ @Override
+ public String apply(Object input) {
+ return prettyPrintValue(input);
+ }
+ }));
+ }
+
+ /**
+ * Print value 'o' to a string. Convenience overloading of printValue(Object, Appendable).
+ */
+ public static String printValue(Object o) {
+ StringBuffer buffer = new StringBuffer();
+ printValue(o, buffer);
+ return buffer.toString();
+ }
+
+ public static Object checkNotNull(Expression expr, Object obj) throws EvalException {
+ if (obj == null) {
+ throw new EvalException(expr.getLocation(),
+ "Unexpected null value, please send a bug report. "
+ + "This was generated by '" + expr + "'");
+ }
+ return obj;
+ }
+
+ /**
+ * Convert BUILD language objects to Formattable so JDK can render them correctly.
+ * Don't do this for numeric or string types because we want %d, %x, %s to work.
+ */
+ private static Object makeFormattable(final Object o) {
+ if (o instanceof Integer || o instanceof Double || o instanceof String) {
+ return o;
+ } else {
+ return new Formattable() {
+ @Override
+ public String toString() {
+ return "Formattable[" + o + "]";
+ }
+
+ @Override
+ public void formatTo(Formatter formatter, int flags, int width,
+ int precision) {
+ printValue(o, formatter.out());
+ }
+ };
+ }
+ }
+
+ private static final Object[] EMPTY = new Object[0];
+
+ /*
+ * N.B. MissingFormatWidthException is the only kind of IllegalFormatException
+ * whose constructor can take and display arbitrary error message, hence its use below.
+ */
+
+ /**
+ * Perform Python-style string formatting. Implemented by delegation to Java's
+ * own string formatting routine to avoid reinventing the wheel. In more
+ * obscure cases, semantics follow JDK (not Python) rules.
+ *
+ * @param pattern a format string.
+ * @param tuple a tuple containing positional arguments
+ */
+ public static String formatString(String pattern, List<?> tuple)
+ throws IllegalFormatException {
+ int count = countPlaceholders(pattern);
+ if (count < tuple.size()) {
+ throw new MissingFormatWidthException(
+ "not all arguments converted during string formatting");
+ }
+
+ List<Object> args = new ArrayList<>();
+
+ for (Object o : tuple) {
+ args.add(makeFormattable(o));
+ }
+
+ try {
+ return String.format(pattern, args.toArray(EMPTY));
+ } catch (IllegalFormatException e) {
+ throw new MissingFormatWidthException(
+ "invalid arguments for format string");
+ }
+ }
+
+ private static int countPlaceholders(String pattern) {
+ int length = pattern.length();
+ boolean afterPercent = false;
+ int i = 0;
+ int count = 0;
+ while (i < length) {
+ switch (pattern.charAt(i)) {
+ case 's':
+ case 'd':
+ if (afterPercent) {
+ count++;
+ afterPercent = false;
+ }
+ break;
+
+ case '%':
+ afterPercent = !afterPercent;
+ break;
+
+ default:
+ if (afterPercent) {
+ throw new MissingFormatWidthException("invalid arguments for format string");
+ }
+ afterPercent = false;
+ break;
+ }
+ i++;
+ }
+
+ return count;
+ }
+
+ /**
+ * @return the truth value of an object, according to Python rules.
+ * http://docs.python.org/2/library/stdtypes.html#truth-value-testing
+ */
+ public static boolean toBoolean(Object o) {
+ if (o == null || o == Environment.NONE) {
+ return false;
+ } else if (o instanceof Boolean) {
+ return (Boolean) o;
+ } else if (o instanceof String) {
+ return !((String) o).isEmpty();
+ } else if (o instanceof Integer) {
+ return (Integer) o != 0;
+ } else if (o instanceof Collection<?>) {
+ return !((Collection<?>) o).isEmpty();
+ } else if (o instanceof Map<?, ?>) {
+ return !((Map<?, ?>) o).isEmpty();
+ } else if (o instanceof NestedSet<?>) {
+ return !((NestedSet<?>) o).isEmpty();
+ } else if (o instanceof SkylarkNestedSet) {
+ return !((SkylarkNestedSet) o).isEmpty();
+ } else if (o instanceof Iterable<?>) {
+ return !(Iterables.isEmpty((Iterable<?>) o));
+ } else {
+ return true;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Collection<Object> toCollection(Object o, Location loc) throws EvalException {
+ if (o instanceof Collection) {
+ return (Collection<Object>) o;
+ } else if (o instanceof Map<?, ?>) {
+ // For dictionaries we iterate through the keys only
+ return ((Map<Object, Object>) o).keySet();
+ } else if (o instanceof SkylarkNestedSet) {
+ return ((SkylarkNestedSet) o).toCollection();
+ } else {
+ throw new EvalException(loc,
+ "type '" + EvalUtils.getDatatypeName(o) + "' is not a collection");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Iterable<Object> toIterable(Object o, Location loc) throws EvalException {
+ if (o instanceof String) {
+ // This is not as efficient as special casing String in for and dict and list comprehension
+ // statements. However this is a more unified way.
+ // The regex matches every character in the string until the end of the string,
+ // so "abc" will be split into ["a", "b", "c"].
+ return ImmutableList.<Object>copyOf(((String) o).split("(?!^)"));
+ } else if (o instanceof Iterable) {
+ return (Iterable<Object>) o;
+ } else if (o instanceof Map<?, ?>) {
+ // For dictionaries we iterate through the keys only
+ return ((Map<Object, Object>) o).keySet();
+ } else {
+ throw new EvalException(loc,
+ "type '" + EvalUtils.getDatatypeName(o) + "' is not an iterable");
+ }
+ }
+
+ /**
+ * Returns the size of the Skylark object or -1 in case the object doesn't have a size.
+ */
+ public static int size(Object arg) {
+ if (arg instanceof String) {
+ return ((String) arg).length();
+ } else if (arg instanceof Map) {
+ return ((Map<?, ?>) arg).size();
+ } else if (arg instanceof SkylarkList) {
+ return ((SkylarkList) arg).size();
+ } else if (arg instanceof Iterable) {
+ // Iterables.size() checks if arg is a Collection so it's efficient in that sense.
+ return Iterables.size((Iterable<?>) arg);
+ }
+ return -1;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Expression.java b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
new file mode 100644
index 0000000..1659eb0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Base class for all expression nodes in the AST.
+ */
+public abstract class Expression extends ASTNode {
+
+ /**
+ * Returns the result of evaluating this build-language expression in the
+ * specified environment. All BUILD language datatypes are mapped onto the
+ * corresponding Java types as follows:
+ *
+ * <pre>
+ * int -> Integer
+ * float -> Double (currently not generated by the grammar)
+ * str -> String
+ * [...] -> List<Object> (mutable)
+ * (...) -> List<Object> (immutable)
+ * {...} -> Map<Object, Object>
+ * func -> Function
+ * </pre>
+ *
+ * @return the result of evaluting the expression: a Java object corresponding
+ * to a datatype in the BUILD language.
+ * @throws EvalException if the expression could not be evaluated.
+ */
+ abstract Object eval(Environment env) throws EvalException, InterruptedException;
+
+ /**
+ * Returns the inferred type of the result of the Expression.
+ *
+ * <p>Checks the semantics of the Expression using the SkylarkEnvironment according to
+ * the rules of the Skylark language, throws EvalException in case of a semantical error.
+ *
+ * @see Statement
+ */
+ abstract SkylarkType validate(ValidationEnvironment env) throws EvalException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java
new file mode 100644
index 0000000..f742d40
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for a function call statement. Used for build rules.
+ */
+public final class ExpressionStatement extends Statement {
+
+ private final Expression expr;
+
+ public ExpressionStatement(Expression expr) {
+ this.expr = expr;
+ }
+
+ public Expression getExpression() {
+ return expr;
+ }
+
+ @Override
+ public String toString() {
+ return expr.toString() + '\n';
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ expr.eval(env);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ expr.validate(env);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java b/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java
new file mode 100644
index 0000000..4586b64
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java
@@ -0,0 +1,175 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * FilesetEntry is a value object used to represent a "FilesetEntry" inside a "Fileset" BUILD rule.
+ */
+public final class FilesetEntry {
+ /** SymlinkBehavior decides what to do when a source file of a FilesetEntry is a symlink. */
+ public enum SymlinkBehavior {
+ /** Just copies the symlink as-is. May result in dangling links. */
+ COPY,
+ /** Follow the link and make the destination point to the absolute path of the final target. */
+ DEREFERENCE;
+
+ public static SymlinkBehavior parse(String value) throws IllegalArgumentException {
+ return valueOf(value.toUpperCase());
+ }
+
+ @Override
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+ }
+
+ private final Label srcLabel;
+ @Nullable private final ImmutableList<Label> files;
+ @Nullable private final ImmutableSet<String> excludes;
+ private final PathFragment destDir;
+ private final SymlinkBehavior symlinkBehavior;
+ private final String stripPrefix;
+
+ /**
+ * Constructs a FilesetEntry with the given values.
+ *
+ * @param srcLabel the label of the source directory. Must be non-null.
+ * @param files The explicit files to include. May be null.
+ * @param excludes The files to exclude. Man be null. May only be non-null if files is null.
+ * @param destDir The target-relative output directory.
+ * @param symlinkBehavior how to treat symlinks on the input. See
+ * {@link FilesetEntry.SymlinkBehavior}.
+ * @param stripPrefix the prefix to strip from the package-relative path. If ".", keep only the
+ * basename.
+ */
+ public FilesetEntry(Label srcLabel,
+ @Nullable List<Label> files,
+ @Nullable List<String> excludes,
+ String destDir,
+ SymlinkBehavior symlinkBehavior,
+ String stripPrefix) {
+ this.srcLabel = checkNotNull(srcLabel);
+ this.destDir = new PathFragment((destDir == null) ? "" : destDir);
+ this.files = files == null ? null : ImmutableList.copyOf(files);
+ this.excludes = (excludes == null || excludes.isEmpty()) ? null : ImmutableSet.copyOf(excludes);
+ this.symlinkBehavior = symlinkBehavior;
+ this.stripPrefix = stripPrefix;
+ }
+
+ /**
+ * @return the source label.
+ */
+ public Label getSrcLabel() {
+ return srcLabel;
+ }
+
+ /**
+ * @return the destDir. Non null.
+ */
+ public PathFragment getDestDir() {
+ return destDir;
+ }
+
+ /**
+ * @return how symlinks should be handled.
+ */
+ public SymlinkBehavior getSymlinkBehavior() {
+ return symlinkBehavior;
+ }
+
+ /**
+ * @return an immutable list of excludes. Null if none specified.
+ */
+ @Nullable
+ public ImmutableSet<String> getExcludes() {
+ return excludes;
+ }
+
+ /**
+ * @return an immutable list of file labels. Null if none specified.
+ */
+ @Nullable
+ public ImmutableList<Label> getFiles() {
+ return files;
+ }
+
+ /**
+ * @return true if this Fileset should get files from the source directory.
+ */
+ public boolean isSourceFileset() {
+ return "BUILD".equals(srcLabel.getName());
+ }
+
+ /**
+ * @return all prerequisite labels in the FilesetEntry.
+ */
+ public Collection<Label> getLabels() {
+ Set<Label> labels = new LinkedHashSet<>();
+ if (files != null) {
+ labels.addAll(files);
+ } else {
+ labels.add(srcLabel);
+ }
+ return labels;
+ }
+
+ /**
+ * @return the prefix that should be stripped from package-relative path names.
+ */
+ public String getStripPrefix() {
+ return stripPrefix;
+ }
+
+ /**
+ * @return null if the entry is valid, and a human-readable error message otherwise.
+ */
+ @Nullable
+ public String validate() {
+ if (excludes != null && files != null) {
+ return "Cannot specify both 'files' and 'excludes' in a FilesetEntry";
+ } else if (files != null && !isSourceFileset()) {
+ return "Cannot specify files with Fileset label '" + srcLabel + "'";
+ } else if (destDir.isAbsolute()) {
+ return "Cannot specify absolute destdir '" + destDir + "'";
+ } else if (!stripPrefix.equals(".") && files == null) {
+ return "If the strip prefix is not '.', files must be specified";
+ } else if (new PathFragment(stripPrefix).containsUplevelReferences()) {
+ return "Strip prefix must not contain uplevel references";
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("FilesetEntry(srcdir=%s, destdir=%s, strip_prefix=%s, symlinks=%s, "
+ + "%d file(s) and %d excluded)", srcLabel, destDir, stripPrefix, symlinkBehavior,
+ files != null ? files.size() : 0,
+ excludes != null ? excludes.size() : 0);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java
new file mode 100644
index 0000000..34a4eea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Syntax node for a for loop statement.
+ */
+public final class ForStatement extends Statement {
+
+ private final Ident variable;
+ private final Expression collection;
+ private final ImmutableList<Statement> block;
+
+ /**
+ * Constructs a for loop statement.
+ */
+ ForStatement(Ident variable, Expression collection, List<Statement> block) {
+ this.variable = Preconditions.checkNotNull(variable);
+ this.collection = Preconditions.checkNotNull(collection);
+ this.block = ImmutableList.copyOf(block);
+ }
+
+ public Ident getVariable() {
+ return variable;
+ }
+
+ public Expression getCollection() {
+ return collection;
+ }
+
+ public ImmutableList<Statement> block() {
+ return block;
+ }
+
+ @Override
+ public String toString() {
+ // TODO(bazel-team): if we want to print the complete statement, the function
+ // needs an extra argument to specify indentation level.
+ return "for " + variable + " in " + collection + ": ...\n";
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ Object o = collection.eval(env);
+ Iterable<?> col = EvalUtils.toIterable(o, getLocation());
+
+ int i = 0;
+ for (Object it : ImmutableList.copyOf(col)) {
+ env.update(variable.getName(), it);
+ for (Statement stmt : block) {
+ stmt.exec(env);
+ }
+ i++;
+ }
+ // TODO(bazel-team): This should not happen if every collection is immutable.
+ if (i != EvalUtils.size(col)) {
+ throw new EvalException(getLocation(),
+ String.format("Cannot modify '%s' during during iteration.", collection.toString()));
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ if (env.isTopLevel()) {
+ throw new EvalException(getLocation(),
+ "'For' is not allowed as a the top level statement");
+ }
+ // TODO(bazel-team): validate variable. Maybe make it temporarily readonly.
+ SkylarkType type = collection.validate(env);
+ env.checkIterable(type, getLocation());
+ env.update(variable.getName(), SkylarkType.UNKNOWN, getLocation());
+ for (Statement stmt : block) {
+ stmt.validate(env);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java
new file mode 100644
index 0000000..e24d97f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java
@@ -0,0 +1,550 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.EvalException.EvalExceptionWithJavaCause;
+import com.google.devtools.build.lib.util.StringUtilities;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Syntax node for a function call expression.
+ */
+public final class FuncallExpression extends Expression {
+
+ private static enum ArgConversion {
+ FROM_SKYLARK,
+ TO_SKYLARK,
+ NO_CONVERSION
+ }
+
+ /**
+ * A value class to store Methods with their corresponding SkylarkCallable annotations.
+ * This is needed because the annotation is sometimes in a superclass.
+ */
+ public static final class MethodDescriptor {
+ private final Method method;
+ private final SkylarkCallable annotation;
+
+ private MethodDescriptor(Method method, SkylarkCallable annotation) {
+ this.method = method;
+ this.annotation = annotation;
+ }
+
+ Method getMethod() {
+ return method;
+ }
+
+ /**
+ * Returns the SkylarkCallable annotation corresponding to this method.
+ */
+ public SkylarkCallable getAnnotation() {
+ return annotation;
+ }
+ }
+
+ private static final LoadingCache<Class<?>, Map<String, List<MethodDescriptor>>> methodCache =
+ CacheBuilder.newBuilder()
+ .initialCapacity(10)
+ .maximumSize(100)
+ .build(new CacheLoader<Class<?>, Map<String, List<MethodDescriptor>>>() {
+
+ @Override
+ public Map<String, List<MethodDescriptor>> load(Class<?> key) throws Exception {
+ Map<String, List<MethodDescriptor>> methodMap = new HashMap<>();
+ for (Method method : key.getMethods()) {
+ // Synthetic methods lead to false multiple matches
+ if (method.isSynthetic()) {
+ continue;
+ }
+ SkylarkCallable callable = getAnnotationFromParentClass(
+ method.getDeclaringClass(), method);
+ if (callable == null) {
+ continue;
+ }
+ String name = callable.name();
+ if (name.isEmpty()) {
+ name = StringUtilities.toPythonStyleFunctionName(method.getName());
+ }
+ String signature = name + "#" + method.getParameterTypes().length;
+ if (methodMap.containsKey(signature)) {
+ methodMap.get(signature).add(new MethodDescriptor(method, callable));
+ } else {
+ methodMap.put(signature, Lists.newArrayList(new MethodDescriptor(method, callable)));
+ }
+ }
+ return ImmutableMap.copyOf(methodMap);
+ }
+ });
+
+ /**
+ * Returns a map of methods and corresponding SkylarkCallable annotations
+ * of the methods of the classObj class reachable from Skylark.
+ */
+ public static ImmutableMap<Method, SkylarkCallable> collectSkylarkMethodsWithAnnotation(
+ Class<?> classObj) {
+ ImmutableMap.Builder<Method, SkylarkCallable> methodMap = ImmutableMap.builder();
+ for (Method method : classObj.getMethods()) {
+ // Synthetic methods lead to false multiple matches
+ if (!method.isSynthetic()) {
+ SkylarkCallable annotation = getAnnotationFromParentClass(classObj, method);
+ if (annotation != null) {
+ methodMap.put(method, annotation);
+ }
+ }
+ }
+ return methodMap.build();
+ }
+
+ private static SkylarkCallable getAnnotationFromParentClass(Class<?> classObj, Method method) {
+ boolean keepLooking = false;
+ try {
+ Method superMethod = classObj.getMethod(method.getName(), method.getParameterTypes());
+ if (classObj.isAnnotationPresent(SkylarkModule.class)
+ && superMethod.isAnnotationPresent(SkylarkCallable.class)) {
+ return superMethod.getAnnotation(SkylarkCallable.class);
+ } else {
+ keepLooking = true;
+ }
+ } catch (NoSuchMethodException e) {
+ // The class might not have the specified method, so an exceptions is OK.
+ keepLooking = true;
+ }
+ if (keepLooking) {
+ if (classObj.getSuperclass() != null) {
+ SkylarkCallable annotation = getAnnotationFromParentClass(classObj.getSuperclass(), method);
+ if (annotation != null) {
+ return annotation;
+ }
+ }
+ for (Class<?> interfaceObj : classObj.getInterfaces()) {
+ SkylarkCallable annotation = getAnnotationFromParentClass(interfaceObj, method);
+ if (annotation != null) {
+ return annotation;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * An exception class to handle exceptions in direct Java API calls.
+ */
+ public static final class FuncallException extends Exception {
+
+ public FuncallException(String msg) {
+ super(msg);
+ }
+ }
+
+ private final Expression obj;
+
+ private final Ident func;
+
+ private final List<Argument> args;
+
+ private final int numPositionalArgs;
+
+ /**
+ * Note: the grammar definition restricts the function value in a function
+ * call expression to be a global identifier; however, the representation of
+ * values in the interpreter is flexible enough to allow functions to be
+ * arbitrary expressions. In any case, the "func" expression is always
+ * evaluated, so functions and variables share a common namespace.
+ */
+ public FuncallExpression(Expression obj, Ident func,
+ List<Argument> args) {
+ for (Argument arg : args) {
+ Preconditions.checkArgument(arg.hasValue());
+ }
+ this.obj = obj;
+ this.func = func;
+ this.args = args;
+ this.numPositionalArgs = countPositionalArguments();
+ }
+
+ /**
+ * Note: the grammar definition restricts the function value in a function
+ * call expression to be a global identifier; however, the representation of
+ * values in the interpreter is flexible enough to allow functions to be
+ * arbitrary expressions. In any case, the "func" expression is always
+ * evaluated, so functions and variables share a common namespace.
+ */
+ public FuncallExpression(Ident func, List<Argument> args) {
+ this(null, func, args);
+ }
+
+ /**
+ * Returns the number of positional arguments.
+ */
+ private int countPositionalArguments() {
+ int num = 0;
+ for (Argument arg : args) {
+ if (arg.isPositional()) {
+ num++;
+ }
+ }
+ return num;
+ }
+
+ /**
+ * Returns the function expression.
+ */
+ public Ident getFunction() {
+ return func;
+ }
+
+ /**
+ * Returns the object the function called on.
+ * It's null if the function is not called on an object.
+ */
+ public Expression getObject() {
+ return obj;
+ }
+
+ /**
+ * Returns an (immutable, ordered) list of function arguments. The first n are
+ * positional and the remaining ones are keyword args, where n =
+ * getNumPositionalArguments().
+ */
+ public List<Argument> getArguments() {
+ return Collections.unmodifiableList(args);
+ }
+
+ /**
+ * Returns the number of arguments which are positional; the remainder are
+ * keyword arguments.
+ */
+ public int getNumPositionalArguments() {
+ return numPositionalArgs;
+ }
+
+ @Override
+ public String toString() {
+ if (func.getName().equals("$substring")) {
+ return obj + "[" + args.get(0) + ":" + args.get(1) + "]";
+ }
+ if (func.getName().equals("$index")) {
+ return obj + "[" + args.get(0) + "]";
+ }
+ if (obj != null) {
+ return obj + "." + func + "(" + args + ")";
+ }
+ return func + "(" + args + ")";
+ }
+
+ /**
+ * Returns the list of Skylark callable Methods of objClass with the given name
+ * and argument number.
+ */
+ public static List<MethodDescriptor> getMethods(Class<?> objClass, String methodName, int argNum)
+ throws ExecutionException {
+ return methodCache.get(objClass).get(methodName + "#" + argNum);
+ }
+
+ /**
+ * Returns the list of the Skylark name of all Skylark callable methods.
+ */
+ public static List<String> getMethodNames(Class<?> objClass)
+ throws ExecutionException {
+ List<String> names = new ArrayList<>();
+ for (List<MethodDescriptor> methods : methodCache.get(objClass).values()) {
+ for (MethodDescriptor method : methods) {
+ // TODO(bazel-team): store the Skylark name in the MethodDescriptor.
+ String name = method.annotation.name();
+ if (name.isEmpty()) {
+ name = StringUtilities.toPythonStyleFunctionName(method.method.getName());
+ }
+ names.add(name);
+ }
+ }
+ return names;
+ }
+
+ static Object callMethod(MethodDescriptor methodDescriptor, String methodName, Object obj,
+ Object[] args, Location loc) throws EvalException, IllegalAccessException,
+ IllegalArgumentException, InvocationTargetException {
+ Method method = methodDescriptor.getMethod();
+ if (obj == null && !Modifier.isStatic(method.getModifiers())) {
+ throw new EvalException(loc, "Method '" + methodName + "' is not static");
+ }
+ // This happens when the interface is public but the implementation classes
+ // have reduced visibility.
+ method.setAccessible(true);
+ Object result = method.invoke(obj, args);
+ if (method.getReturnType().equals(Void.TYPE)) {
+ return Environment.NONE;
+ }
+ if (result == null) {
+ if (methodDescriptor.getAnnotation().allowReturnNones()) {
+ return Environment.NONE;
+ } else {
+ throw new EvalException(loc,
+ "Method invocation returned None, please contact Skylark developers: " + methodName
+ + "(" + EvalUtils.prettyPrintValues(", ", ImmutableList.copyOf(args)) + ")");
+ }
+ }
+ result = SkylarkType.convertToSkylark(result, method);
+ if (result != null && !EvalUtils.isSkylarkImmutable(result.getClass())) {
+ throw new EvalException(loc, "Method '" + methodName
+ + "' returns a mutable object (type of " + EvalUtils.getDatatypeName(result) + ")");
+ }
+ return result;
+ }
+
+ // TODO(bazel-team): If there's exactly one usable method, this works. If there are multiple
+ // matching methods, it still can be a problem. Figure out how the Java compiler does it
+ // exactly and copy that behaviour.
+ // TODO(bazel-team): check if this and SkylarkBuiltInFunctions.createObject can be merged.
+ private Object invokeJavaMethod(
+ Object obj, Class<?> objClass, String methodName, List<Object> args) throws EvalException {
+ try {
+ MethodDescriptor matchingMethod = null;
+ List<MethodDescriptor> methods = getMethods(objClass, methodName, args.size());
+ if (methods != null) {
+ for (MethodDescriptor method : methods) {
+ Class<?>[] params = method.getMethod().getParameterTypes();
+ int i = 0;
+ boolean matching = true;
+ for (Class<?> param : params) {
+ if (!param.isAssignableFrom(args.get(i).getClass())) {
+ matching = false;
+ break;
+ }
+ i++;
+ }
+ if (matching) {
+ if (matchingMethod == null) {
+ matchingMethod = method;
+ } else {
+ throw new EvalException(func.getLocation(),
+ "Multiple matching methods for " + formatMethod(methodName, args)
+ + " in " + EvalUtils.getDataTypeNameFromClass(objClass));
+ }
+ }
+ }
+ }
+ if (matchingMethod != null && !matchingMethod.getAnnotation().structField()) {
+ return callMethod(matchingMethod, methodName, obj, args.toArray(), getLocation());
+ } else {
+ throw new EvalException(getLocation(), "No matching method found for "
+ + formatMethod(methodName, args) + " in "
+ + EvalUtils.getDataTypeNameFromClass(objClass));
+ }
+ } catch (IllegalAccessException e) {
+ // TODO(bazel-team): Print a nice error message. Maybe the method exists
+ // and an argument is missing or has the wrong type.
+ throw new EvalException(getLocation(), "Method invocation failed: " + e);
+ } catch (InvocationTargetException e) {
+ if (e.getCause() instanceof FuncallException) {
+ throw new EvalException(getLocation(), e.getCause().getMessage());
+ } else if (e.getCause() != null) {
+ throw new EvalExceptionWithJavaCause(getLocation(), e.getCause());
+ } else {
+ // This is unlikely to happen
+ throw new EvalException(getLocation(), "Method invocation failed: " + e);
+ }
+ } catch (ExecutionException e) {
+ throw new EvalException(getLocation(), "Method invocation failed: " + e);
+ }
+ }
+
+ private String formatMethod(String methodName, List<Object> args) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(methodName).append("(");
+ boolean first = true;
+ for (Object obj : args) {
+ if (!first) {
+ sb.append(", ");
+ }
+ sb.append(EvalUtils.getDatatypeName(obj));
+ first = false;
+ }
+ return sb.append(")").toString();
+ }
+
+ /**
+ * Add one argument to the keyword map, raising an exception when names conflict.
+ */
+ private void addKeywordArg(Map<String, Object> kwargs, String name, Object value)
+ throws EvalException {
+ if (kwargs.put(name, value) != null) {
+ throw new EvalException(getLocation(),
+ "duplicate keyword '" + name + "' in call to '" + func + "'");
+ }
+ }
+
+ /**
+ * Add multiple arguments to the keyword map (**kwargs).
+ */
+ private void addKeywordArgs(Map<String, Object> kwargs, Object items)
+ throws EvalException {
+ if (!(items instanceof Map<?, ?>)) {
+ throw new EvalException(getLocation(),
+ "Argument after ** must be a dictionary, not " + EvalUtils.getDatatypeName(items));
+ }
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>) items).entrySet()) {
+ if (!(entry.getKey() instanceof String)) {
+ throw new EvalException(getLocation(),
+ "Keywords must be strings, not " + EvalUtils.getDatatypeName(entry.getKey()));
+ }
+ addKeywordArg(kwargs, (String) entry.getKey(), entry.getValue());
+ }
+ }
+
+ private void evalArguments(List<Object> posargs, Map<String, Object> kwargs,
+ Environment env, Function function)
+ throws EvalException, InterruptedException {
+ ArgConversion conversion = getArgConversion(function);
+ for (Argument arg : args) {
+ Object value = arg.getValue().eval(env);
+ if (conversion == ArgConversion.FROM_SKYLARK) {
+ value = SkylarkType.convertFromSkylark(value);
+ } else if (conversion == ArgConversion.TO_SKYLARK) {
+ // We try to auto convert the type if we can.
+ value = SkylarkType.convertToSkylark(value, getLocation());
+ // We call into Skylark so we need to be sure that the caller uses the appropriate types.
+ SkylarkType.checkTypeAllowedInSkylark(value, getLocation());
+ }
+ if (arg.isPositional()) {
+ posargs.add(value);
+ } else if (arg.isKwargs()) { // expand the kwargs
+ addKeywordArgs(kwargs, value);
+ } else {
+ addKeywordArg(kwargs, arg.getArgName(), value);
+ }
+ }
+ if (function instanceof UserDefinedFunction) {
+ // Adding the default values for a UserDefinedFunction if needed.
+ UserDefinedFunction func = (UserDefinedFunction) function;
+ if (args.size() < func.getArgs().size()) {
+ for (Map.Entry<String, Object> entry : func.getDefaultValues().entrySet()) {
+ String key = entry.getKey();
+ if (func.getArgIndex(key) >= numPositionalArgs && !kwargs.containsKey(key)) {
+ kwargs.put(key, entry.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ static boolean isNamespace(Class<?> classObject) {
+ return classObject.isAnnotationPresent(SkylarkModule.class)
+ && classObject.getAnnotation(SkylarkModule.class).namespace();
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ List<Object> posargs = new ArrayList<>();
+ Map<String, Object> kwargs = new LinkedHashMap<>();
+
+ if (obj != null) {
+ Object objValue = obj.eval(env);
+ // Strings, lists and dictionaries (maps) have functions that we want to use in MethodLibrary.
+ // For other classes, we can call the Java methods.
+ Function function =
+ env.getFunction(EvalUtils.getSkylarkType(objValue.getClass()), func.getName());
+ if (function != null) {
+ if (!isNamespace(objValue.getClass())) {
+ posargs.add(objValue);
+ }
+ evalArguments(posargs, kwargs, env, function);
+ return EvalUtils.checkNotNull(this, function.call(posargs, kwargs, this, env));
+ } else if (env.isSkylarkEnabled()) {
+
+ // When calling a Java method, the name is not in the Environment, so
+ // evaluating 'func' would fail. For arguments we don't need to consider the default
+ // arguments since the Java function doesn't have any.
+
+ evalArguments(posargs, kwargs, env, null);
+ if (!kwargs.isEmpty()) {
+ throw new EvalException(func.getLocation(),
+ "Keyword arguments are not allowed when calling a java method");
+ }
+ if (objValue instanceof Class<?>) {
+ // Static Java method call
+ return invokeJavaMethod(null, (Class<?>) objValue, func.getName(), posargs);
+ } else {
+ return invokeJavaMethod(objValue, objValue.getClass(), func.getName(), posargs);
+ }
+ } else {
+ throw new EvalException(getLocation(), String.format(
+ "function '%s' is not defined on '%s'", func.getName(),
+ EvalUtils.getDatatypeName(objValue)));
+ }
+ }
+
+ Object funcValue = func.eval(env);
+ if (!(funcValue instanceof Function)) {
+ throw new EvalException(getLocation(),
+ "'" + EvalUtils.getDatatypeName(funcValue)
+ + "' object is not callable");
+ }
+ Function function = (Function) funcValue;
+ evalArguments(posargs, kwargs, env, function);
+ return EvalUtils.checkNotNull(this, function.call(posargs, kwargs, this, env));
+ }
+
+ private ArgConversion getArgConversion(Function function) {
+ if (function == null) {
+ // It means we try to call a Java function.
+ return ArgConversion.FROM_SKYLARK;
+ }
+ // If we call a UserDefinedFunction we call into Skylark. If we call from Skylark
+ // the argument conversion is invariant, but if we call from the BUILD language
+ // we might need an auto conversion.
+ return function instanceof UserDefinedFunction
+ ? ArgConversion.TO_SKYLARK : ArgConversion.NO_CONVERSION;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): implement semantical check.
+
+ if (obj != null) {
+ // TODO(bazel-team): validate function calls on objects too.
+ return env.getReturnType(obj.validate(env), func.getName(), getLocation());
+ } else {
+ // TODO(bazel-team): Imported functions are not validated properly.
+ if (!env.hasSymbolInEnvironment(func.getName())) {
+ throw new EvalException(getLocation(),
+ String.format("function '%s' does not exist", func.getName()));
+ }
+ return env.getReturnType(func.getName(), getLocation());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Function.java b/src/main/java/com/google/devtools/build/lib/syntax/Function.java
new file mode 100644
index 0000000..5636a95
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Function.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Function values in the BUILD language.
+ *
+ * <p>Each implementation of this interface defines a function in the BUILD language.
+ *
+ */
+public interface Function {
+
+ /**
+ * Implements the behavior of a call to a function with positional arguments
+ * "args" and keyword arguments "kwargs". The "ast" argument is provided to
+ * allow construction of EvalExceptions containing source information.
+ */
+ Object call(List<Object> args,
+ Map<String, Object> kwargs,
+ FuncallExpression ast,
+ Environment env)
+ throws EvalException, InterruptedException;
+
+ /**
+ * Returns the name of the function.
+ */
+ String getName();
+
+ // TODO(bazel-team): implement this for MethodLibrary functions as well.
+ /**
+ * Returns the type of the object on which this function is defined or null
+ * if this is a global function.
+ */
+ Class<?> getObjectType();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java
new file mode 100644
index 0000000..29ed579
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+import java.util.Collection;
+
+/**
+ * Syntax node for a function definition.
+ */
+public class FunctionDefStatement extends Statement {
+
+ private final Ident ident;
+ private final ImmutableList<Argument> args;
+ private final ImmutableList<Statement> statements;
+
+ public FunctionDefStatement(Ident ident, Collection<Argument> args,
+ Collection<Statement> statements) {
+ for (Argument arg : args) {
+ Preconditions.checkArgument(arg.isNamed());
+ }
+ this.ident = ident;
+ this.args = ImmutableList.copyOf(args);
+ this.statements = ImmutableList.copyOf(statements);
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ ImmutableMap.Builder<String, Object> defaultValues = ImmutableMap.builder();
+ for (Argument arg : args) {
+ if (arg.hasValue()) {
+ defaultValues.put(arg.getArgName(), arg.getValue().eval(env));
+ }
+ }
+ env.update(ident.getName(), new UserDefinedFunction(
+ ident, args, defaultValues.build(), statements, (SkylarkEnvironment) env));
+ }
+
+ @Override
+ public String toString() {
+ return "def " + ident + "(" + args + "):\n";
+ }
+
+ public Ident getIdent() {
+ return ident;
+ }
+
+ public ImmutableList<Statement> getStatements() {
+ return statements;
+ }
+
+ public ImmutableList<Argument> getArgs() {
+ return args;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ SkylarkFunctionType type = SkylarkFunctionType.of(ident.getName());
+ ValidationEnvironment localEnv = new ValidationEnvironment(env, type);
+ for (Argument i : args) {
+ SkylarkType argType = SkylarkType.UNKNOWN;
+ if (i.hasValue()) {
+ argType = i.getValue().validate(env);
+ if (argType.equals(SkylarkType.NONE)) {
+ argType = SkylarkType.UNKNOWN;
+ }
+ }
+ localEnv.update(i.getArgName(), argType, getLocation());
+ }
+ for (Statement stmts : statements) {
+ stmts.validate(localEnv);
+ }
+ env.updateFunction(ident.getName(), type, getLocation());
+ // Register a dummy return value with an incompatible type if there was no return statement.
+ type.setReturnType(SkylarkType.NONE, getLocation());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java b/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java
new file mode 100644
index 0000000..577bd4a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java
@@ -0,0 +1,214 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Iterables;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Either the arguments to a glob call (the include and exclude lists) or the
+ * contents of a fixed list that was appended to a list of glob results.
+ * (The latter need to be stored by {@link GlobList} in order to fully
+ * reproduce the inputs that created the output list.)
+ *
+ * <p>For example, the expression
+ * <code>glob(['*.java']) + ['x.properties']</code>
+ * will result in two GlobCriteria: one has include = ['*.java'], glob = true
+ * and the other, include = ['x.properties'], glob = false.
+ */
+public class GlobCriteria {
+
+ /**
+ * A list of names or patterns that are included by this glob. They should
+ * consist of characters that are valid in labels in the BUILD language.
+ */
+ private final ImmutableList<String> include;
+
+ /**
+ * A list of names or patterns that are excluded by this glob. They should
+ * consist of characters that are valid in labels in the BUILD language.
+ */
+ private final ImmutableList<String> exclude;
+
+ /** True if the includes list was passed to glob(), false if not. */
+ private final boolean glob;
+
+ /**
+ * Parses criteria from its {@link #toExpression} form.
+ * Package-private for use by tests and GlobList.
+ * @throws IllegalArgumentException if the expression cannot be parsed
+ */
+ public static GlobCriteria parse(String text) {
+ if (text.startsWith("glob([") && text.endsWith("])")) {
+ int excludeIndex = text.indexOf("], exclude=[");
+ if (excludeIndex == -1) {
+ String listText = text.substring(6, text.length() - 2);
+ return new GlobCriteria(parseList(listText), ImmutableList.<String>of(), true);
+ } else {
+ String listText = text.substring(6, excludeIndex);
+ String excludeText = text.substring(excludeIndex + 12, text.length() - 2);
+ return new GlobCriteria(parseList(listText), parseList(excludeText), true);
+ }
+ } else if (text.startsWith("[") && text.endsWith("]")) {
+ String listText = text.substring(1, text.length() - 1);
+ return new GlobCriteria(parseList(listText), ImmutableList.<String>of(), false);
+ } else {
+ throw new IllegalArgumentException(
+ "unrecognized format (not from toExpression?): " + text);
+ }
+ }
+
+ /**
+ * Constructs a copy of a given glob critera object, with additional exclude patterns added.
+ *
+ * @param base a glob criteria object to copy. Must be an actual glob
+ * @param excludes a list of pattern strings indicating new excludes to provide
+ * @return a new glob criteria object which contains the same parameters as {@code base}, with
+ * the additional patterns in {@code excludes} added.
+ * @throws IllegalArgumentException if {@code base} is not a glob
+ */
+ public static GlobCriteria createWithAdditionalExcludes(GlobCriteria base,
+ List<String> excludes) {
+ Preconditions.checkArgument(base.isGlob());
+ return fromGlobCall(base.include,
+ ImmutableList.copyOf(Iterables.concat(base.exclude, excludes)));
+ }
+
+ /**
+ * Constructs a copy of a fixed list, converted to Strings.
+ */
+ public static GlobCriteria fromList(Iterable<?> list) {
+ Iterable<String> strings = Iterables.transform(list, Functions.toStringFunction());
+ return new GlobCriteria(ImmutableList.copyOf(strings), ImmutableList.<String>of(), false);
+ }
+
+ /**
+ * Constructs a glob call with include and exclude list.
+ *
+ * @param include list of included patterns
+ * @param exclude list of excluded patterns
+ */
+ public static GlobCriteria fromGlobCall(
+ ImmutableList<String> include, ImmutableList<String> exclude) {
+ return new GlobCriteria(include, exclude, true);
+ }
+
+ /**
+ * Constructs a glob call with include and exclude list.
+ */
+ private GlobCriteria(ImmutableList<String> include, ImmutableList<String> exclude, boolean glob) {
+ this.include = include;
+ this.exclude = exclude;
+ this.glob = glob;
+ }
+
+ /**
+ * Returns the patterns that were included in this {@code glob()} call.
+ */
+ public ImmutableList<String> getIncludePatterns() {
+ return include;
+ }
+
+ /**
+ * Returns the patterns that were excluded in this {@code glob()} call.
+ */
+ public ImmutableList<String> getExcludePatterns() {
+ return exclude;
+ }
+
+ /**
+ * Returns true if the include list was passed to {@code glob()}, false
+ * if it was a fixed list. If this returns false, the exclude list will
+ * always be empty.
+ */
+ public boolean isGlob() {
+ return glob;
+ }
+
+ /**
+ * Returns a String that represents this glob as a BUILD expression.
+ * For example, <code>glob(['abc', 'def'], exclude=['uvw', 'xyz'])</code>
+ * or <code>['foo', 'bar', 'baz']</code>.
+ */
+ public String toExpression() {
+ StringBuilder sb = new StringBuilder();
+ if (glob) {
+ sb.append("glob(");
+ }
+ sb.append('[');
+ appendList(sb, include);
+ if (!exclude.isEmpty()) {
+ sb.append("], exclude=[");
+ appendList(sb, exclude);
+ }
+ sb.append(']');
+ if (glob) {
+ sb.append(')');
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toExpression();
+ }
+
+ /**
+ * Takes a list of Strings, quotes them in single quotes, and appends them to
+ * a StringBuilder separated by a comma and space. This can be parsed back
+ * out by {@link #parseList}.
+ */
+ private static void appendList(StringBuilder sb, List<String> list) {
+ boolean first = true;
+ for (String content : list) {
+ if (!first) {
+ sb.append(", ");
+ }
+ sb.append('\'').append(content).append('\'');
+ first = false;
+ }
+ }
+
+ /**
+ * Takes a String in the format created by {@link #appendList} and returns
+ * the original Strings. A null String (which may be returned when Pattern
+ * does not find a match) or the String "" (which will be captured in "[]")
+ * will result in an empty list.
+ */
+ private static ImmutableList<String> parseList(@Nullable String text) {
+ if (text == null) {
+ return ImmutableList.of();
+ }
+ Iterable<String> split = Splitter.on(", ").split(text);
+ Builder<String> listBuilder = ImmutableList.builder();
+ for (String element : split) {
+ if (!element.isEmpty()) {
+ if ((element.length() < 2) || !element.startsWith("'") || !element.endsWith("'")) {
+ throw new IllegalArgumentException("expected a filename or pattern in quotes: " + text);
+ }
+ listBuilder.add(element.substring(1, element.length() - 1));
+ }
+ }
+ return listBuilder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java b/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java
new file mode 100644
index 0000000..82afd01
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ForwardingList;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Glob matches and information about glob patterns, which are useful to
+ * ide_build_info. Its implementation of the List interface is as an immutable
+ * list of the matching files. Glob criteria can be retrieved through
+ * {@link #getCriteria}.
+ *
+ * @param <E> the element this List contains (generally either String or Label)
+ */
+public class GlobList<E> extends ForwardingList<E> {
+
+ /** Include/exclude criteria. */
+ private final ImmutableList<GlobCriteria> criteria;
+
+ /** Matching files (usually either String or Label). */
+ private final ImmutableList<E> matches;
+
+ /**
+ * Constructs a list with {@code glob()} call results.
+ *
+ * @param includes the patterns that the glob includes
+ * @param excludes the patterns that the glob excludes
+ * @param matches the filenames that matched the includes/excludes criteria
+ */
+ public static <T> GlobList<T> captureResults(List<String> includes,
+ List<String> excludes, List<T> matches) {
+ GlobCriteria criteria = GlobCriteria.fromGlobCall(
+ ImmutableList.copyOf(includes), ImmutableList.copyOf(excludes));
+ return new GlobList<>(ImmutableList.of(criteria), matches);
+ }
+
+ /**
+ * Parses a GlobInfo from its {@link #toExpression} representation.
+ */
+ public static GlobList<String> parse(String text) {
+ List<GlobCriteria> criteria = new ArrayList<>();
+ Iterable<String> globs = Splitter.on(" + ").split(text);
+ for (String glob : globs) {
+ criteria.add(GlobCriteria.parse(glob));
+ }
+ return new GlobList<>(criteria, ImmutableList.<String>of());
+ }
+
+ /**
+ * Concatenates two lists into a new GlobList. If either of the lists is a
+ * GlobList, its GlobCriteria are preserved. Otherwise a simple GlobCriteria
+ * is created to represent the fixed list.
+ */
+ public static <T> GlobList<T> concat(
+ List<? extends T> list1, List<? extends T> list2) {
+ // we add the list to both includes and matches, preserving order
+ Builder<GlobCriteria> criteriaBuilder = ImmutableList.<GlobCriteria>builder();
+ if (list1 instanceof GlobList<?>) {
+ criteriaBuilder.addAll(((GlobList<?>) list1).criteria);
+ } else {
+ criteriaBuilder.add(GlobCriteria.fromList(list1));
+ }
+ if (list2 instanceof GlobList<?>) {
+ criteriaBuilder.addAll(((GlobList<?>) list2).criteria);
+ } else {
+ criteriaBuilder.add(GlobCriteria.fromList(list2));
+ }
+ List<T> matches = ImmutableList.copyOf(Iterables.concat(list1, list2));
+ return new GlobList<>(criteriaBuilder.build(), matches);
+ }
+
+ /**
+ * Constructs a list with given criteria and matches.
+ */
+ public GlobList(List<GlobCriteria> criteria, List<E> matches) {
+ Preconditions.checkNotNull(criteria);
+ Preconditions.checkNotNull(matches);
+ this.criteria = ImmutableList.copyOf(criteria);
+ this.matches = ImmutableList.copyOf(matches);
+ }
+
+ /**
+ * Returns the criteria used to create this list, from which the
+ * includes/excludes can be retrieved.
+ */
+ public ImmutableList<GlobCriteria> getCriteria() {
+ return criteria;
+ }
+
+ /**
+ * Returns a String that represents this glob list as a BUILD expression.
+ */
+ public String toExpression() {
+ return Joiner.on(" + ").join(criteria);
+ }
+
+ @Override
+ protected ImmutableList<E> delegate() {
+ return matches;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Ident.java b/src/main/java/com/google/devtools/build/lib/syntax/Ident.java
new file mode 100644
index 0000000..86bd458
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Ident.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for an identifier.
+ */
+public final class Ident extends Expression {
+
+ private final String name;
+
+ public Ident(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Returns the name of the Ident.
+ */
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException {
+ try {
+ return env.lookup(name);
+ } catch (Environment.NoSuchVariableException e) {
+ if (name.equals("$error$")) {
+ throw new EvalException(getLocation(), "contains syntax error(s)", true);
+ } else {
+ throw new EvalException(getLocation(), "name '" + name + "' is not defined");
+ }
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ if (env.hasSymbolInEnvironment(name)) {
+ return env.getVartype(name);
+ } else {
+ throw new EvalException(getLocation(), "name '" + name + "' is not defined");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java
new file mode 100644
index 0000000..3877a9c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java
@@ -0,0 +1,138 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+// TODO(bazel-team): maybe we should get rid of the ConditionalStatements and
+// create a chain of if-else statements for elif-s.
+/**
+ * Syntax node for an if/else statement.
+ */
+public final class IfStatement extends Statement {
+
+ /**
+ * Syntax node for an [el]if statement.
+ */
+ static final class ConditionalStatements extends Statement {
+
+ private final Expression condition;
+ private final ImmutableList<Statement> stmts;
+
+ public ConditionalStatements(Expression condition, List<Statement> stmts) {
+ this.condition = Preconditions.checkNotNull(condition);
+ this.stmts = ImmutableList.copyOf(stmts);
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ for (Statement stmt : stmts) {
+ stmt.exec(env);
+ }
+ }
+
+ @Override
+ public String toString() {
+ // TODO(bazel-team): see TODO in the outer class
+ return "[el]if " + condition + ": ...\n";
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ Expression getCondition() {
+ return condition;
+ }
+
+ ImmutableList<Statement> getStmts() {
+ return stmts;
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // EvalUtils.toBoolean() evaluates everything so we don't need type check here.
+ condition.validate(env);
+ validateStmts(env, stmts);
+ }
+ }
+
+ private final ImmutableList<ConditionalStatements> thenBlocks;
+ private final ImmutableList<Statement> elseBlock;
+
+ /**
+ * Constructs a if-elif-else statement. The else part is mandatory, but the list may be empty.
+ * ThenBlocks has to have at least one element.
+ */
+ IfStatement(List<ConditionalStatements> thenBlocks, List<Statement> elseBlock) {
+ Preconditions.checkArgument(thenBlocks.size() > 0);
+ this.thenBlocks = ImmutableList.copyOf(thenBlocks);
+ this.elseBlock = ImmutableList.copyOf(elseBlock);
+ }
+
+ public ImmutableList<ConditionalStatements> getThenBlocks() {
+ return thenBlocks;
+ }
+
+ public ImmutableList<Statement> getElseBlock() {
+ return elseBlock;
+ }
+
+ @Override
+ public String toString() {
+ // TODO(bazel-team): if we want to print the complete statement, the function
+ // needs an extra argument to specify indentation level.
+ return "if : ...\n";
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ for (ConditionalStatements stmt : thenBlocks) {
+ if (EvalUtils.toBoolean(stmt.getCondition().eval(env))) {
+ stmt.exec(env);
+ return;
+ }
+ }
+ for (Statement stmt : elseBlock) {
+ stmt.exec(env);
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ env.startTemporarilyDisableReadonlyCheckSession();
+ for (ConditionalStatements stmts : thenBlocks) {
+ stmts.validate(env);
+ }
+ validateStmts(env, elseBlock);
+ env.finishTemporarilyDisableReadonlyCheckSession();
+ }
+
+ private static void validateStmts(ValidationEnvironment env, List<Statement> stmts)
+ throws EvalException {
+ for (Statement stmt : stmts) {
+ stmt.validate(env);
+ }
+ env.finishTemporarilyDisableReadonlyCheckBranch();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java
new file mode 100644
index 0000000..e6852e6b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for an integer literal.
+ */
+public final class IntegerLiteral extends Literal<Integer> {
+
+ public IntegerLiteral(Integer value) {
+ super(value);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ return SkylarkType.INT;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Label.java b/src/main/java/com/google/devtools/build/lib/syntax/Label.java
new file mode 100644
index 0000000..89db4de
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Label.java
@@ -0,0 +1,412 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.cmdline.LabelValidator.BadLabelException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+
+/**
+ * A class to identify a BUILD target. All targets belong to exactly one package.
+ * The name of a target is called its label. A typical label looks like this:
+ * //dir1/dir2:target_name where 'dir1/dir2' identifies the package containing a BUILD file,
+ * and 'target_name' identifies the target within the package.
+ *
+ * <p>Parsing is robust against bad input, for example, from the command line.
+ */
+@SkylarkModule(name = "Label", doc = "A BUILD target identifier.")
+@Immutable @ThreadSafe
+public final class Label implements Comparable<Label>, Serializable {
+
+ /**
+ * Thrown by the parsing methods to indicate a bad label.
+ */
+ public static class SyntaxException extends Exception {
+ public SyntaxException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Factory for Labels from absolute string form, possibly including a repository name prefix. For
+ * example:
+ * <pre>
+ * //foo/bar
+ * {@literal @}foo//bar
+ * {@literal @}foo//bar:baz
+ * </pre>
+ */
+ public static Label parseRepositoryLabel(String absName) throws SyntaxException {
+ String repo = PackageIdentifier.DEFAULT_REPOSITORY;
+ int packageStartPos = absName.indexOf("//");
+ if (packageStartPos > 0) {
+ repo = absName.substring(0, packageStartPos);
+ absName = absName.substring(packageStartPos);
+ }
+ try {
+ LabelValidator.PackageAndTarget labelParts = LabelValidator.parseAbsoluteLabel(absName);
+ return new Label(new PackageIdentifier(repo, new PathFragment(labelParts.getPackageName())),
+ labelParts.getTargetName());
+ } catch (BadLabelException e) {
+ throw new SyntaxException(e.getMessage());
+ }
+ }
+
+ /**
+ * Factory for Labels from absolute string form. e.g.
+ * <pre>
+ * //foo/bar
+ * //foo/bar:quux
+ * </pre>
+ */
+ public static Label parseAbsolute(String absName) throws SyntaxException {
+ try {
+ LabelValidator.PackageAndTarget labelParts = LabelValidator.parseAbsoluteLabel(absName);
+ return create(labelParts.getPackageName(), labelParts.getTargetName());
+ } catch (BadLabelException e) {
+ throw new SyntaxException(e.getMessage());
+ }
+ }
+
+ /**
+ * Alternate factory method for Labels from absolute strings. This is a convenience method for
+ * cases when a Label needs to be initialized statically, so the declared exception is
+ * inconvenient.
+ *
+ * <p>Do not use this when the argument is not hard-wired.
+ */
+ public static Label parseAbsoluteUnchecked(String absName) {
+ try {
+ return parseAbsolute(absName);
+ } catch (SyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Factory for Labels from separate components.
+ *
+ * @param packageName The name of the package. The package name does
+ * <b>not</b> include {@code //}. Must be valid according to
+ * {@link LabelValidator#validatePackageName}.
+ * @param targetName The name of the target within the package. Must be
+ * valid according to {@link LabelValidator#validateTargetName}.
+ * @throws SyntaxException if either of the arguments was invalid.
+ */
+ public static Label create(String packageName, String targetName) throws SyntaxException {
+ return new Label(packageName, targetName);
+ }
+
+ /**
+ * Similar factory to above, but takes a package identifier to allow external repository labels
+ * to be created.
+ */
+ public static Label create(PackageIdentifier packageId, String targetName)
+ throws SyntaxException {
+ return new Label(packageId, targetName);
+ }
+
+ /**
+ * Resolves a relative label using a workspace-relative path to the current working directory. The
+ * method handles these cases:
+ * <ul>
+ * <li>The label is absolute.
+ * <li>The label starts with a colon.
+ * <li>The label consists of a relative path, a colon, and a local part.
+ * <li>The label consists only of a local part.
+ * </ul>
+ *
+ * <p>Note that this method does not support any of the special syntactic constructs otherwise
+ * supported on the command line, like ":all", "/...", and so on.
+ *
+ * <p>It would be cleaner to use the TargetPatternEvaluator for this resolution, but that is not
+ * possible, because it is sometimes necessary to resolve a relative label before the package path
+ * is setup; in particular, before the tools/defaults package is created.
+ *
+ * @throws SyntaxException if the resulting label is not valid
+ */
+ public static Label parseCommandLineLabel(String label, PathFragment workspaceRelativePath)
+ throws SyntaxException {
+ Preconditions.checkArgument(!workspaceRelativePath.isAbsolute());
+ if (label.startsWith("//")) {
+ return parseAbsolute(label);
+ }
+ int index = label.indexOf(':');
+ if (index < 0) {
+ index = 0;
+ label = ":" + label;
+ }
+ PathFragment path = workspaceRelativePath.getRelative(label.substring(0, index));
+ // Use the String, String constructor, to make sure that the package name goes through the
+ // validity check.
+ return new Label(path.getPathString(), label.substring(index + 1));
+ }
+
+ /**
+ * Validates the given target name and returns a canonical String instance if it is valid.
+ * Otherwise it throws a SyntaxException.
+ */
+ private static String canonicalizeTargetName(String name) throws SyntaxException {
+ String error = LabelValidator.validateTargetName(name);
+ if (error != null) {
+ error = "invalid target name '" + StringUtilities.sanitizeControlChars(name) + "': " + error;
+ throw new SyntaxException(error);
+ }
+
+ // TODO(bazel-team): This should be an error, but we can't make it one for legacy reasons.
+ if (name.endsWith("/.")) {
+ name = name.substring(0, name.length() - 2);
+ }
+
+ return StringCanonicalizer.intern(name);
+ }
+
+ /**
+ * Validates the given package name and returns a canonical PathFragment instance if it is valid.
+ * Otherwise it throws a SyntaxException.
+ */
+ private static PathFragment validate(String packageName, String name) throws SyntaxException {
+ String error = LabelValidator.validatePackageName(packageName);
+ if (error != null) {
+ error = "invalid package name '" + packageName + "': " + error;
+ // This check is just for a more helpful error message
+ // i.e. valid target name, invalid package name, colon-free label form
+ // used => probably they meant "//foo:bar.c" not "//foo/bar.c".
+ if (packageName.endsWith("/" + name)) {
+ error += " (perhaps you meant \":" + name + "\"?)";
+ }
+ throw new SyntaxException(error);
+ }
+ return new PathFragment(packageName);
+ }
+
+ /** The name and repository of the package. */
+ private final PackageIdentifier packageIdentifier;
+
+ /** The name of the target within the package. Canonical. */
+ private final String name;
+
+ /**
+ * Constructor from a package name, target name. Both are checked for validity
+ * and a SyntaxException is thrown if either is invalid.
+ * TODO(bazel-team): move the validation to {@link PackageIdentifier}. Unfortunately, there are a
+ * bazillion tests that use invalid package names (taking advantage of the fact that calling
+ * Label(PathFragment, String) doesn't validate the package name).
+ */
+ private Label(String packageName, String name) throws SyntaxException {
+ this(validate(packageName, name), name);
+ }
+
+ /**
+ * Constructor from canonical valid package name and a target name. The target
+ * name is checked for validity and a SyntaxException is throw if it isn't.
+ */
+ private Label(PathFragment packageName, String name) throws SyntaxException {
+ this(PackageIdentifier.createInDefaultRepo(packageName), name);
+ }
+
+ private Label(PackageIdentifier packageIdentifier, String name)
+ throws SyntaxException {
+ Preconditions.checkNotNull(packageIdentifier);
+ Preconditions.checkNotNull(name);
+
+ try {
+ this.packageIdentifier = packageIdentifier;
+ this.name = canonicalizeTargetName(name);
+ } catch (SyntaxException e) {
+ // This check is just for a more helpful error message
+ // i.e. valid target name, invalid package name, colon-free label form
+ // used => probably they meant "//foo:bar.c" not "//foo/bar.c".
+ if (packageIdentifier.getPackageFragment().getPathString().endsWith("/" + name)) {
+ throw new SyntaxException(e.getMessage() + " (perhaps you meant \":" + name + "\"?)");
+ }
+ throw e;
+ }
+ }
+
+ private Object writeReplace() {
+ return new LabelSerializationProxy(toString());
+ }
+
+ private void readObject(ObjectInputStream stream) throws InvalidObjectException {
+ throw new InvalidObjectException("Serialization is allowed only by proxy");
+ }
+
+ public PackageIdentifier getPackageIdentifier() {
+ return packageIdentifier;
+ }
+
+ /**
+ * Returns the name of the package in which this rule was declared (e.g. {@code
+ * //file/base:fileutils_test} returns {@code file/base}).
+ */
+ @SkylarkCallable(name = "package", structField = true,
+ doc = "The package part of this label. "
+ + "For instance:<br>"
+ + "<pre class=language-python>label(\"//pkg/foo:abc\").package == \"pkg/foo\"</pre>")
+ public String getPackageName() {
+ return packageIdentifier.getPackageFragment().getPathString();
+ }
+
+ /**
+ * Returns the path fragment of the package in which this rule was declared (e.g. {@code
+ * //file/base:fileutils_test} returns {@code file/base}).
+ */
+ public PathFragment getPackageFragment() {
+ return packageIdentifier.getPackageFragment();
+ }
+
+ public static final com.google.common.base.Function<Label, PathFragment> PACKAGE_FRAGMENT =
+ new com.google.common.base.Function<Label, PathFragment>() {
+ @Override
+ public PathFragment apply(Label label) {
+ return label.getPackageFragment();
+ }
+ };
+
+ /**
+ * Returns the label as a path fragment, using the package and the label name.
+ */
+ public PathFragment toPathFragment() {
+ return packageIdentifier.getPackageFragment().getRelative(name);
+ }
+
+ /**
+ * Returns the name by which this rule was declared (e.g. {@code //foo/bar:baz}
+ * returns {@code baz}).
+ */
+ @SkylarkCallable(name = "name", structField = true,
+ doc = "The name of this label within the package. "
+ + "For instance:<br>"
+ + "<pre class=language-python>label(\"//pkg/foo:abc\").name == \"abc\"</pre>")
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Renders this label in canonical form.
+ *
+ * <p>invariant: {@code parseAbsolute(x.toString()).equals(x)}
+ */
+ @Override
+ public String toString() {
+ return packageIdentifier.getRepository() + "//" + packageIdentifier.getPackageFragment()
+ + ":" + name;
+ }
+
+ /**
+ * Renders this label in shorthand form.
+ *
+ * <p>Labels with canonical form {@code //foo/bar:bar} have the shorthand form {@code //foo/bar}.
+ * All other labels have identical shorthand and canonical forms.
+ */
+ public String toShorthandString() {
+ return packageIdentifier.getRepository() + (getPackageFragment().getBaseName().equals(name)
+ ? "//" + getPackageFragment()
+ : toString());
+ }
+
+ /**
+ * Returns a label in the same package as this label with the given target name.
+ *
+ * @throws SyntaxException if {@code targetName} is not a valid target name
+ */
+ public Label getLocalTargetLabel(String targetName) throws SyntaxException {
+ return new Label(packageIdentifier, targetName);
+ }
+
+ /**
+ * Resolves a relative or absolute label name. If given name is absolute, then this method calls
+ * {@link #parseAbsolute}. Otherwise, it calls {@link #getLocalTargetLabel}.
+ *
+ * <p>For example:
+ * {@code :quux} relative to {@code //foo/bar:baz} is {@code //foo/bar:quux};
+ * {@code //wiz:quux} relative to {@code //foo/bar:baz} is {@code //wiz:quux}.
+ *
+ * @param relName the relative label name; must be non-empty.
+ */
+ @SkylarkCallable(name = "relative", doc =
+ "Resolves a relative or absolute label name.<br>"
+ + "For example:<br><ul>"
+ + "<li><code>:quux</code> relative to <code>//foo/bar:baz</code> is "
+ + "<code>//foo/bar:quux</code></li>"
+ + "<li><code>//wiz:quux</code> relative to <code>//foo/bar:baz</code> is "
+ + "<code>//wiz:quux</code></li></ul>")
+ public Label getRelative(String relName) throws SyntaxException {
+ if (relName.length() == 0) {
+ throw new SyntaxException("empty package-relative label");
+ }
+ if (relName.startsWith("//")) {
+ return parseAbsolute(relName);
+ } else if (relName.equals(":")) {
+ throw new SyntaxException("':' is not a valid package-relative label");
+ } else if (relName.charAt(0) == ':') {
+ return getLocalTargetLabel(relName.substring(1));
+ } else {
+ return getLocalTargetLabel(relName);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode() ^ packageIdentifier.hashCode();
+ }
+
+ /**
+ * Two labels are equal iff both their name and their package name are equal.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Label)) {
+ return false;
+ }
+ Label otherLabel = (Label) other;
+ return name.equals(otherLabel.name) // least likely one first
+ && packageIdentifier.equals(otherLabel.packageIdentifier);
+ }
+
+ /**
+ * Defines the order between labels.
+ *
+ * <p>Labels are ordered primarily by package name and secondarily by target name. Both components
+ * are ordered lexicographically. Thus {@code //a:b/c} comes before {@code //a/b:a}, i.e. the
+ * position of the colon is significant to the order.
+ */
+ @Override
+ public int compareTo(Label other) {
+ return ComparisonChain.start()
+ .compare(packageIdentifier, other.packageIdentifier)
+ .compare(name, other.name)
+ .result();
+ }
+
+ /**
+ * Returns a suitable string for the user-friendly representation of the Label. Works even if the
+ * argument is null.
+ */
+ public static String print(Label label) {
+ return label == null ? "(unknown)" : label.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java b/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java
new file mode 100644
index 0000000..5b4556a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectOutput;
+
+/**
+ * A serialization proxy for {@code Label}.
+ */
+public class LabelSerializationProxy implements Externalizable {
+
+ private String labelString;
+
+ public LabelSerializationProxy(String labelString) {
+ this.labelString = labelString;
+ }
+
+ // For deserialization machinery.
+ public LabelSerializationProxy() {
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ // Manual serialization gives us about a 30% reduction in size.
+ out.writeUTF(labelString);
+ }
+
+ @Override
+ public void readExternal(java.io.ObjectInput in) throws IOException {
+ this.labelString = in.readUTF();
+ }
+
+ private Object readResolve() {
+ return Label.parseAbsoluteUnchecked(labelString);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
new file mode 100644
index 0000000..fc12c66
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -0,0 +1,803 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+/**
+ * A tokenizer for the BUILD language.
+ * <p>
+ * See: <a href="https://docs.python.org/2/reference/lexical_analysis.html"/>
+ * for some details.
+ * <p>
+ * Since BUILD files are small, we just tokenize the entire file a-priori
+ * instead of interleaving scanning with parsing.
+ */
+public final class Lexer {
+
+ private final EventHandler eventHandler;
+
+ // Input buffer and position
+ private char[] buffer;
+ private int pos;
+
+ /**
+ * The part of the location information that is common to all LexerLocation
+ * instances created by this Lexer. Factored into a separate object so that
+ * many Locations instances can share the same information as compactly as
+ * possible, without closing over a Lexer instance.
+ */
+ private static class LocationInfo {
+ final LineNumberTable lineNumberTable;
+ final Path filename;
+ LocationInfo(Path filename, LineNumberTable lineNumberTable) {
+ this.filename = filename;
+ this.lineNumberTable = lineNumberTable;
+ }
+ }
+
+ private final LocationInfo locationInfo;
+
+ // The stack of enclosing indentation levels; always contains '0' at the
+ // bottom.
+ private final Stack<Integer> indentStack = new Stack<>();
+
+ private final List<Token> tokens = new ArrayList<>();
+
+ // The number of unclosed open-parens ("(", '{', '[') at the current point in
+ // the stream. Whitespace is handled differently when this is nonzero.
+ private int openParenStackDepth = 0;
+
+ private boolean containsErrors;
+
+ private boolean parsePython;
+
+ /**
+ * Constructs a lexer which tokenizes the contents of the specified
+ * InputBuffer. Any errors during lexing are reported on "handler".
+ */
+ public Lexer(ParserInputSource input, EventHandler eventHandler, boolean parsePython) {
+ this.buffer = input.getContent();
+ this.pos = 0;
+ this.parsePython = parsePython;
+ this.eventHandler = eventHandler;
+ this.locationInfo = new LocationInfo(input.getPath(),
+ LineNumberTable.create(buffer, input.getPath()));
+
+ indentStack.push(0);
+ tokenize();
+ }
+
+ public Lexer(ParserInputSource input, EventHandler eventHandler) {
+ this(input, eventHandler, false);
+ }
+
+ /**
+ * Returns the filename from which the lexer's input came. Returns a dummy
+ * value if the input came from a string.
+ */
+ public Path getFilename() {
+ return locationInfo.filename;
+ }
+
+ /**
+ * Returns true if there were errors during scanning of this input file or
+ * string. The Lexer may attempt to recover from errors, but clients should
+ * not rely on the results of scanning if this flag is set.
+ */
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ /**
+ * Returns the (mutable) list of tokens generated by the Lexer.
+ */
+ public List<Token> getTokens() {
+ return tokens;
+ }
+
+ private void popParen() {
+ if (openParenStackDepth == 0) {
+ error("indentation error");
+ } else {
+ openParenStackDepth--;
+ }
+ }
+
+ private void error(String message) {
+ error(message, pos - 1, pos - 1);
+ }
+
+ private void error(String message, int start, int end) {
+ this.containsErrors = true;
+ eventHandler.handle(Event.error(createLocation(start, end), message));
+ }
+
+ Location createLocation(int start, int end) {
+ return new LexerLocation(locationInfo, start, end);
+ }
+
+ // Don't use an inner class as we don't want to close over the Lexer, only
+ // the LocationInfo.
+ @Immutable
+ private static final class LexerLocation extends Location {
+
+ private final LineNumberTable lineNumberTable;
+
+ LexerLocation(LocationInfo locationInfo, int start, int end) {
+ super(start, end);
+ this.lineNumberTable = locationInfo.lineNumberTable;
+ }
+
+ @Override
+ public PathFragment getPath() {
+ return lineNumberTable.getPath(getStartOffset()).asFragment();
+ }
+
+ @Override
+ public LineAndColumn getStartLineAndColumn() {
+ return lineNumberTable.getLineAndColumn(getStartOffset());
+ }
+
+ @Override
+ public LineAndColumn getEndLineAndColumn() {
+ return lineNumberTable.getLineAndColumn(getEndOffset());
+ }
+ }
+
+ /** invariant: symbol positions are half-open intervals. */
+ private void addToken(Token s) {
+ tokens.add(s);
+ }
+
+ /**
+ * Parses an end-of-line sequence, handling statement indentation correctly.
+ *
+ * UNIX newlines are assumed (LF). Carriage returns are always ignored.
+ *
+ * ON ENTRY: 'pos' is the index of the char after '\n'.
+ * ON EXIT: 'pos' is the index of the next non-space char after '\n'.
+ */
+ private void newline() {
+ if (openParenStackDepth > 0) {
+ newlineInsideExpression(); // in an expression: ignore space
+ } else {
+ newlineOutsideExpression(); // generate NEWLINE/INDENT/OUTDENT tokens
+ }
+ }
+
+ private void newlineInsideExpression() {
+ while (pos < buffer.length) {
+ switch (buffer[pos]) {
+ case ' ': case '\t': case '\r':
+ pos++;
+ break;
+ default:
+ return;
+ }
+ }
+ }
+
+ private void newlineOutsideExpression() {
+ if (pos > 1) { // skip over newline at start of file
+ addToken(new Token(TokenKind.NEWLINE, pos - 1, pos));
+ }
+
+ // we're in a stmt: suck up space at beginning of next line
+ int indentLen = 0;
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ if (c == ' ') {
+ indentLen++;
+ pos++;
+ } else if (c == '\t') {
+ indentLen += 8 - indentLen % 8;
+ pos++;
+ } else if (c == '\n') { // entirely blank line: discard
+ indentLen = 0;
+ pos++;
+ } else if (c == '#') { // line containing only indented comment
+ int oldPos = pos;
+ while (pos < buffer.length && c != '\n') {
+ c = buffer[pos++];
+ }
+ addToken(new Token(TokenKind.COMMENT, oldPos, pos - 1, bufferSlice(oldPos, pos - 1)));
+ indentLen = 0;
+ } else { // printing character
+ break;
+ }
+ }
+
+ if (pos == buffer.length) {
+ indentLen = 0;
+ } // trailing space on last line
+
+ int peekedIndent = indentStack.peek();
+ if (peekedIndent < indentLen) { // push a level
+ indentStack.push(indentLen);
+ addToken(new Token(TokenKind.INDENT, pos - 1, pos));
+
+ } else if (peekedIndent > indentLen) { // pop one or more levels
+ while (peekedIndent > indentLen) {
+ indentStack.pop();
+ addToken(new Token(TokenKind.OUTDENT, pos - 1, pos));
+ peekedIndent = indentStack.peek();
+ }
+
+ if (peekedIndent < indentLen) {
+ error("indentation error");
+ }
+ }
+ }
+
+ /**
+ * Returns true if current position is in the middle of a triple quote
+ * delimiter (3 x quot), and advances 'pos' by two if so.
+ */
+ private boolean skipTripleQuote(char quot) {
+ if (pos + 1 < buffer.length && buffer[pos] == quot && buffer[pos + 1] == quot) {
+ pos += 2;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Scans a string literal delimited by 'quot', containing escape sequences.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first delimiter
+ * ON EXIT: 'pos' is 1 + the index of the last delimiter.
+ *
+ * @return the string-literal token.
+ */
+ private Token escapedStringLiteral(char quot) {
+ boolean inTriplequote = skipTripleQuote(quot);
+
+ int oldPos = pos - 1;
+ // more expensive second choice that expands escaped into a buffer
+ StringBuilder literal = new StringBuilder();
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '\n':
+ if (inTriplequote) {
+ literal.append(c);
+ break;
+ } else {
+ error("unterminated string literal at eol", oldPos, pos);
+ newline();
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+ case '\\':
+ if (pos == buffer.length) {
+ error("unterminated string literal at eof", oldPos, pos);
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+ c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '\n':
+ // ignore end of line character
+ break;
+ case 'n':
+ literal.append('\n');
+ break;
+ case 'r':
+ literal.append('\r');
+ break;
+ case 't':
+ literal.append('\t');
+ break;
+ case '\\':
+ literal.append('\\');
+ break;
+ case '\'':
+ literal.append('\'');
+ break;
+ case '"':
+ literal.append('"');
+ break;
+ case '0': case '1': case '2': case '3':
+ case '4': case '5': case '6': case '7': { // octal escape
+ int octal = c - '0';
+ if (pos < buffer.length) {
+ c = buffer[pos];
+ if (c >= '0' && c <= '7') {
+ pos++;
+ octal = (octal << 3) | (c - '0');
+ if (pos < buffer.length) {
+ c = buffer[pos];
+ if (c >= '0' && c <= '7') {
+ pos++;
+ octal = (octal << 3) | (c - '0');
+ }
+ }
+ }
+ }
+ literal.append((char) (octal & 0xff));
+ break;
+ }
+ case 'a': case 'b': case 'f': case 'N': case 'u': case 'U': case 'v': case 'x':
+ // exists in Python but not implemented in Blaze => error
+ error("escape sequence not implemented: \\" + c, oldPos, pos);
+ break;
+ default:
+ // unknown char escape => "\literal"
+ literal.append('\\');
+ literal.append(c);
+ break;
+ }
+ break;
+ case '\'':
+ case '"':
+ if (c != quot
+ || (inTriplequote && !skipTripleQuote(quot))) {
+ // Non-matching quote, treat it like a regular char.
+ literal.append(c);
+ } else {
+ // Matching close-delimiter, all done.
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+ break;
+ default:
+ literal.append(c);
+ break;
+ }
+ }
+ error("unterminated string literal at eof", oldPos, pos);
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+
+ /**
+ * Scans a string literal delimited by 'quot'.
+ *
+ * <ul>
+ * <li> ON ENTRY: 'pos' is 1 + the index of the first delimiter
+ * <li> ON EXIT: 'pos' is 1 + the index of the last delimiter.
+ * </ul>
+ *
+ * @param isRaw if true, do not escape the string.
+ * @return the string-literal token.
+ */
+ private Token stringLiteral(char quot, boolean isRaw) {
+ int oldPos = pos - 1;
+
+ // Don't even attempt to parse triple-quotes here.
+ if (skipTripleQuote(quot)) {
+ pos -= 2;
+ return escapedStringLiteral(quot);
+ }
+
+ // first quick optimistic scan for a simple non-escaped string
+ while (pos < buffer.length) {
+ char c = buffer[pos++];
+ switch (c) {
+ case '\n':
+ error("unterminated string literal at eol", oldPos, pos);
+ Token t = new Token(TokenKind.STRING, oldPos, pos,
+ bufferSlice(oldPos + 1, pos - 1));
+ newline();
+ return t;
+ case '\\':
+ if (isRaw) {
+ // skip the next character
+ pos++;
+ break;
+ } else {
+ // oops, hit an escape, need to start over & build a new string buffer
+ pos = oldPos + 1;
+ return escapedStringLiteral(quot);
+ }
+ case '\'':
+ case '"':
+ if (c == quot) {
+ // close-quote, all done.
+ return new Token(TokenKind.STRING, oldPos, pos,
+ bufferSlice(oldPos + 1, pos - 1));
+ }
+ }
+ }
+
+ error("unterminated string literal at eof", oldPos, pos);
+ return new Token(TokenKind.STRING, oldPos, pos,
+ bufferSlice(oldPos + 1, pos));
+ }
+
+ private static final Map<String, TokenKind> keywordMap = new HashMap<>();
+
+ static {
+ keywordMap.put("and", TokenKind.AND);
+ keywordMap.put("as", TokenKind.AS);
+ keywordMap.put("class", TokenKind.CLASS); // reserved for future expansion
+ keywordMap.put("def", TokenKind.DEF);
+ keywordMap.put("elif", TokenKind.ELIF);
+ keywordMap.put("else", TokenKind.ELSE);
+ keywordMap.put("except", TokenKind.EXCEPT);
+ keywordMap.put("finally", TokenKind.FINALLY);
+ keywordMap.put("for", TokenKind.FOR);
+ keywordMap.put("from", TokenKind.FROM);
+ keywordMap.put("if", TokenKind.IF);
+ keywordMap.put("import", TokenKind.IMPORT);
+ keywordMap.put("in", TokenKind.IN);
+ keywordMap.put("not", TokenKind.NOT);
+ keywordMap.put("or", TokenKind.OR);
+ keywordMap.put("return", TokenKind.RETURN);
+ keywordMap.put("try", TokenKind.TRY);
+ }
+
+ private TokenKind getTokenKindForIdentfier(String id) {
+ TokenKind kind = keywordMap.get(id);
+ return kind == null ? TokenKind.IDENTIFIER : kind;
+ }
+
+ private String scanIdentifier() {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ switch (buffer[pos]) {
+ case '_':
+ case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
+ case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
+ case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
+ case 's': case 't': case 'u': case 'v': case 'w': case 'x':
+ case 'y': case 'z':
+ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
+ case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
+ case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
+ case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
+ case 'Y': case 'Z':
+ case '0': case '1': case '2': case '3': case '4': case '5':
+ case '6': case '7': case '8': case '9':
+ pos++;
+ break;
+ default:
+ return bufferSlice(oldPos, pos);
+ }
+ }
+ return bufferSlice(oldPos, pos);
+ }
+
+ /**
+ * Scans an identifier or keyword.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first char in the identifier.
+ * ON EXIT: 'pos' is 1 + the index of the last char in the identifier.
+ *
+ * @return the identifier or keyword token.
+ */
+ private Token identifierOrKeyword() {
+ int oldPos = pos - 1;
+ String id = scanIdentifier();
+ TokenKind kind = getTokenKindForIdentfier(id);
+ return new Token(kind, oldPos, pos,
+ (kind == TokenKind.IDENTIFIER) ? id : null);
+ }
+
+ private String scanInteger() {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ switch (c) {
+ case 'X': case 'x':
+ case 'a': case 'A':
+ case 'b': case 'B':
+ case 'c': case 'C':
+ case 'd': case 'D':
+ case 'e': case 'E':
+ case 'f': case 'F':
+ case '0': case '1':
+ case '2': case '3':
+ case '4': case '5':
+ case '6': case '7':
+ case '8': case '9':
+ pos++;
+ break;
+ default:
+ return bufferSlice(oldPos, pos);
+ }
+ }
+ // TODO(bazel-team): (2009) to do roundtripping when we evaluate the integer
+ // constants, we must save the actual text of the tokens, not just their
+ // integer value.
+
+ return bufferSlice(oldPos, pos);
+ }
+
+ /**
+ * Scans an integer literal.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first char in the literal.
+ * ON EXIT: 'pos' is 1 + the index of the last char in the literal.
+ *
+ * @return the integer token.
+ */
+ private Token integer() {
+ int oldPos = pos - 1;
+ String literal = scanInteger();
+
+ final String substring;
+ final int radix;
+ if (literal.startsWith("0x") || literal.startsWith("0X")) {
+ radix = 16;
+ substring = literal.substring(2);
+ } else if (literal.startsWith("0") && literal.length() > 1) {
+ radix = 8;
+ substring = literal.substring(1);
+ } else {
+ radix = 10;
+ substring = literal;
+ }
+
+ int value = 0;
+ try {
+ value = Integer.parseInt(substring, radix);
+ } catch (NumberFormatException e) {
+ error("invalid base-" + radix + " integer constant: " + literal);
+ }
+
+ return new Token(TokenKind.INT, oldPos, pos, value);
+ }
+
+ /**
+ * Tokenizes a two-char operator.
+ * @return true if it tokenized an operator
+ */
+ private boolean tokenizeTwoChars() {
+ if (pos + 2 >= buffer.length) {
+ return false;
+ }
+ char c1 = buffer[pos];
+ char c2 = buffer[pos + 1];
+ if (c2 == '=') {
+ switch (c1) {
+ case '=': {
+ addToken(new Token(TokenKind.EQUALS_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '!': {
+ addToken(new Token(TokenKind.NOT_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '>': {
+ addToken(new Token(TokenKind.GREATER_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '<': {
+ addToken(new Token(TokenKind.LESS_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '+': {
+ addToken(new Token(TokenKind.PLUS_EQUALS, pos, pos + 2));
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Performs tokenization of the character buffer of file contents provided to
+ * the constructor.
+ */
+ private void tokenize() {
+ while (pos < buffer.length) {
+ if (tokenizeTwoChars()) {
+ pos += 2;
+ continue;
+ }
+ char c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '{': {
+ addToken(new Token(TokenKind.LBRACE, pos - 1, pos));
+ openParenStackDepth++;
+ break;
+ }
+ case '}': {
+ addToken(new Token(TokenKind.RBRACE, pos - 1, pos));
+ popParen();
+ break;
+ }
+ case '(': {
+ addToken(new Token(TokenKind.LPAREN, pos - 1, pos));
+ openParenStackDepth++;
+ break;
+ }
+ case ')': {
+ addToken(new Token(TokenKind.RPAREN, pos - 1, pos));
+ popParen();
+ break;
+ }
+ case '[': {
+ addToken(new Token(TokenKind.LBRACKET, pos - 1, pos));
+ openParenStackDepth++;
+ break;
+ }
+ case ']': {
+ addToken(new Token(TokenKind.RBRACKET, pos - 1, pos));
+ popParen();
+ break;
+ }
+ case '>': {
+ addToken(new Token(TokenKind.GREATER, pos - 1, pos));
+ break;
+ }
+ case '<': {
+ addToken(new Token(TokenKind.LESS, pos - 1, pos));
+ break;
+ }
+ case ':': {
+ addToken(new Token(TokenKind.COLON, pos - 1, pos));
+ break;
+ }
+ case ',': {
+ addToken(new Token(TokenKind.COMMA, pos - 1, pos));
+ break;
+ }
+ case '+': {
+ addToken(new Token(TokenKind.PLUS, pos - 1, pos));
+ break;
+ }
+ case '-': {
+ addToken(new Token(TokenKind.MINUS, pos - 1, pos));
+ break;
+ }
+ case '=': {
+ addToken(new Token(TokenKind.EQUALS, pos - 1, pos));
+ break;
+ }
+ case '%': {
+ addToken(new Token(TokenKind.PERCENT, pos - 1, pos));
+ break;
+ }
+ case ';': {
+ addToken(new Token(TokenKind.SEMI, pos - 1, pos));
+ break;
+ }
+ case '.': {
+ addToken(new Token(TokenKind.DOT, pos - 1, pos));
+ break;
+ }
+ case '*': {
+ addToken(new Token(TokenKind.STAR, pos - 1, pos));
+ break;
+ }
+ case ' ':
+ case '\t':
+ case '\r': {
+ /* ignore */
+ break;
+ }
+ case '\\': {
+ // Backslash character is valid only at the end of a line (or in a string)
+ if (pos + 1 < buffer.length && buffer[pos] == '\n') {
+ pos++; // skip the end of line character
+ } else {
+ addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c)));
+ }
+ break;
+ }
+ case '\n': {
+ newline();
+ break;
+ }
+ case '#': {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ c = buffer[pos];
+ if (c == '\n') {
+ break;
+ } else {
+ pos++;
+ }
+ }
+ addToken(new Token(TokenKind.COMMENT, oldPos, pos, bufferSlice(oldPos, pos)));
+ break;
+ }
+ case '\'':
+ case '\"': {
+ addToken(stringLiteral(c, false));
+ break;
+ }
+ default: {
+ // detect raw strings, e.g. r"str"
+ if (c == 'r' && pos < buffer.length
+ && (buffer[pos] == '\'' || buffer[pos] == '\"')) {
+ c = buffer[pos];
+ pos++;
+ addToken(stringLiteral(c, true));
+ break;
+ }
+
+ if (Character.isDigit(c)) {
+ addToken(integer());
+ } else if (Character.isJavaIdentifierStart(c) && c != '$') {
+ addToken(identifierOrKeyword());
+ } else {
+ // Some characters in Python are not recognized in Blaze syntax (e.g. '!')
+ if (parsePython) {
+ addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c)));
+ } else {
+ error("invalid character: '" + c + "'");
+ }
+ }
+ break;
+ } // default
+ } // switch
+ } // while
+
+ if (indentStack.size() > 1) { // top of stack is always zero
+ addToken(new Token(TokenKind.NEWLINE, pos - 1, pos));
+ while (indentStack.size() > 1) {
+ indentStack.pop();
+ addToken(new Token(TokenKind.OUTDENT, pos - 1, pos));
+ }
+ }
+
+ // Like Python, always end with a NEWLINE token, even if no '\n' in input:
+ if (tokens.size() == 0
+ || tokens.get(tokens.size() - 1).kind != TokenKind.NEWLINE) {
+ addToken(new Token(TokenKind.NEWLINE, pos - 1, pos));
+ }
+
+ addToken(new Token(TokenKind.EOF, pos, pos));
+ }
+
+ /**
+ * Returns the character in the input buffer at the given position.
+ *
+ * @param at the position to get the character at.
+ * @return the character at the given position.
+ */
+ public char charAt(int at) {
+ return buffer[at];
+ }
+
+ /**
+ * Returns the string at the current line, minus the new line.
+ *
+ * @param line the line from which to retrieve the String, 1-based
+ * @return the text of the line
+ */
+ public String stringAtLine(int line) {
+ Pair<Integer, Integer> offsets = locationInfo.lineNumberTable.getOffsetsForLine(line);
+ return bufferSlice(offsets.first, offsets.second);
+ }
+
+ /**
+ * Returns parts of the source buffer based on offsets
+ *
+ * @param start the beginning offset for the slice
+ * @param end the offset immediately following the slice
+ * @return the text at offset start with length end - start
+ */
+ private String bufferSlice(int start, int end) {
+ return new String(this.buffer, start, end - start);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java b/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java
new file mode 100644
index 0000000..4842a16
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java
@@ -0,0 +1,235 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location.LineAndColumn;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A table to keep track of line numbers in source files. The client creates a LineNumberTable for
+ * their buffer using {@link #create}. The client can then ask for the line and column given a
+ * position using ({@link #getLineAndColumn(int)}).
+ */
+abstract class LineNumberTable implements Serializable {
+
+ /**
+ * Returns the (line, column) pair for the specified offset.
+ */
+ abstract LineAndColumn getLineAndColumn(int offset);
+
+ /**
+ * Returns the (start, end) offset pair for a specified line, not including
+ * newline chars.
+ */
+ abstract Pair<Integer, Integer> getOffsetsForLine(int line);
+
+ /**
+ * Returns the path corresponding to the given offset.
+ */
+ abstract Path getPath(int offset);
+
+ static LineNumberTable create(char[] buffer, Path path) {
+ // If #line appears within a BUILD file, we assume it has been preprocessed
+ // by gconfig2blaze. We ignore all actual newlines and compute the logical
+ // LNT based only on the presence of #line markers.
+ return StringUtilities.containsSubarray(buffer, "\n#line ".toCharArray())
+ ? new HashLine(buffer, path)
+ : new Regular(buffer, path);
+ }
+
+ /**
+ * Line number table implementation for regular source files. Records
+ * offsets of newlines.
+ */
+ @Immutable
+ private static class Regular extends LineNumberTable {
+
+ /**
+ * A mapping from line number (line >= 1) to character offset into the file.
+ */
+ private final int[] linestart;
+ private final Path path;
+ private final int bufferLength;
+
+ private Regular(char[] buffer, Path path) {
+ // Compute the size.
+ int size = 2;
+ for (int i = 0; i < buffer.length; i++) {
+ if (buffer[i] == '\n') {
+ size++;
+ }
+ }
+ linestart = new int[size];
+
+ int index = 0;
+ linestart[index++] = 0; // The 0th line does not exist - so we fill something in
+ // to make sure the start pos for the 1st line ends up at
+ // linestart[1]. Using 0 is useful for tables that are
+ // completely empty.
+ linestart[index++] = 0; // The first line ("line 1") starts at offset 0.
+
+ // Scan the buffer and record the offset of each line start. Doing this
+ // once upfront is faster than checking each char as it is pulled from
+ // the buffer.
+ for (int i = 0; i < buffer.length; i++) {
+ if (buffer[i] == '\n') {
+ linestart[index++] = i + 1;
+ }
+ }
+ this.bufferLength = buffer.length;
+ this.path = path;
+ }
+
+ private int getLineAt(int pos) {
+ if (pos < 0) {
+ throw new IllegalArgumentException("Illegal position: " + pos);
+ }
+ int lowBoundary = 1, highBoundary = linestart.length - 1;
+ while (true) {
+ if ((highBoundary - lowBoundary) <= 1) {
+ if (linestart[highBoundary] > pos) {
+ return lowBoundary;
+ } else {
+ return highBoundary;
+ }
+ }
+ int medium = lowBoundary + ((highBoundary - lowBoundary) >> 1);
+ if (linestart[medium] > pos) {
+ highBoundary = medium;
+ } else {
+ lowBoundary = medium;
+ }
+ }
+ }
+
+ @Override
+ LineAndColumn getLineAndColumn(int offset) {
+ int line = getLineAt(offset);
+ int column = offset - linestart[line] + 1;
+ return new LineAndColumn(line, column);
+ }
+
+ @Override
+ Path getPath(int offset) {
+ return path;
+ }
+
+ @Override
+ Pair<Integer, Integer> getOffsetsForLine(int line) {
+ if (line <= 0 || line >= linestart.length) {
+ throw new IllegalArgumentException("Illegal line: " + line);
+ }
+ return Pair.of(linestart[line], line < linestart.length - 1
+ ? linestart[line + 1]
+ : bufferLength);
+ }
+ }
+
+ /**
+ * Line number table implementation for source files that have been
+ * preprocessed. Ignores newlines and uses only #line directives.
+ */
+ // TODO(bazel-team): Use binary search instead of linear search.
+ @Immutable
+ private static class HashLine extends LineNumberTable {
+
+ /**
+ * Represents a "#line" directive
+ */
+ private static class SingleHashLine implements Serializable {
+ final private int offset;
+ final private int line;
+ final private Path path;
+
+ SingleHashLine(int offset, int line, Path path) {
+ this.offset = offset;
+ this.line = line;
+ this.path = path;
+ }
+ }
+
+ private static final Pattern pattern = Pattern.compile("\n#line ([0-9]+) \"([^\"\\n]+)\"");
+
+ private final List<SingleHashLine> table;
+ private final Path defaultPath;
+ private final int bufferLength;
+
+ private HashLine(char[] buffer, Path defaultPath) {
+ // Not especially efficient, but that's fine: we just exec'd Python.
+ String bufString = new String(buffer);
+ Matcher m = pattern.matcher(bufString);
+ ImmutableList.Builder<SingleHashLine> tableBuilder = ImmutableList.builder();
+ while (m.find()) {
+ tableBuilder.add(new SingleHashLine(
+ m.start(0) + 1, //offset (+1 to skip \n in pattern)
+ Integer.valueOf(m.group(1)), // line number
+ defaultPath.getRelative(m.group(2)))); // filename is an absolute path
+ }
+ this.table = tableBuilder.build();
+ this.bufferLength = buffer.length;
+ this.defaultPath = defaultPath;
+ }
+
+ @Override
+ LineAndColumn getLineAndColumn(int offset) {
+ int line = -1;
+ for (int ii = 0, len = table.size(); ii < len; ii++) {
+ SingleHashLine hash = table.get(ii);
+ if (hash.offset > offset) {
+ break;
+ }
+ line = hash.line;
+ }
+ return new LineAndColumn(line, 1);
+ }
+
+ @Override
+ Path getPath(int offset) {
+ Path path = this.defaultPath;
+ for (int ii = 0, len = table.size(); ii < len; ii++) {
+ SingleHashLine hash = table.get(ii);
+ if (hash.offset > offset) {
+ break;
+ }
+ path = hash.path;
+ }
+ return path;
+ }
+
+ /**
+ * Returns 0, 0 for an unknown line
+ */
+ @Override
+ Pair<Integer, Integer> getOffsetsForLine(int line) {
+ for (int ii = 0, len = table.size(); ii < len; ii++) {
+ if (table.get(ii).line == line) {
+ return Pair.of(table.get(ii).offset, ii < len - 1
+ ? table.get(ii + 1).offset
+ : bufferLength);
+ }
+ }
+ return Pair.of(0, 0);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java b/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java
new file mode 100644
index 0000000..6a13ba8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Syntax node for lists comprehension expressions.
+ */
+public final class ListComprehension extends Expression {
+
+ private final Expression elementExpression;
+ // This cannot be a map, because we need to both preserve order _and_ allow duplicate identifiers.
+ private final List<Map.Entry<Ident, Expression>> lists;
+
+ /**
+ * [elementExpr (for var in listExpr)+]
+ */
+ public ListComprehension(Expression elementExpression) {
+ this.elementExpression = elementExpression;
+ lists = new ArrayList<Map.Entry<Ident, Expression>>();
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ if (lists.size() == 0) {
+ return convert(new ArrayList<>(), env);
+ }
+
+ List<Map.Entry<Ident, Iterable<?>>> listValues = Lists.newArrayListWithCapacity(lists.size());
+ int size = 1;
+ for (Map.Entry<Ident, Expression> list : lists) {
+ Object listValueObject = list.getValue().eval(env);
+ final Iterable<?> listValue = EvalUtils.toIterable(listValueObject, getLocation());
+ int listSize = EvalUtils.size(listValue);
+ if (listSize == 0) {
+ return convert(new ArrayList<>(), env);
+ }
+ size *= listSize;
+ listValues.add(Maps.<Ident, Iterable<?>>immutableEntry(list.getKey(), listValue));
+ }
+ List<Object> resultList = Lists.newArrayListWithCapacity(size);
+ evalLists(env, listValues, resultList);
+ return convert(resultList, env);
+ }
+
+ private Object convert(List<Object> list, Environment env) throws EvalException {
+ if (env.isSkylarkEnabled()) {
+ return SkylarkList.list(list, getLocation());
+ } else {
+ return list;
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('[').append(elementExpression);
+ for (Map.Entry<Ident, Expression> list : lists) {
+ sb.append(" for ").append(list.getKey()).append(" in ").append(list.getValue());
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+
+ public Expression getElementExpression() {
+ return elementExpression;
+ }
+
+ public void add(Ident ident, Expression listExpression) {
+ lists.add(Maps.immutableEntry(ident, listExpression));
+ }
+
+ public List<Map.Entry<Ident, Expression>> getLists() {
+ return lists;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ /**
+ * Evaluates element expression over all combinations of list element values.
+ *
+ * <p>Iterates over all elements in outermost list (list at index 0) and
+ * updates the value of the list variable in the environment on each
+ * iteration. If there are no other lists to iterate over added evaluation
+ * of the element expression to the result. Otherwise calls itself recursively
+ * with all the lists except the outermost.
+ */
+ private void evalLists(Environment env, List<Map.Entry<Ident, Iterable<?>>> listValues,
+ List<Object> result) throws EvalException, InterruptedException {
+ Map.Entry<Ident, Iterable<?>> listValue = listValues.get(0);
+ for (Object listElement : listValue.getValue()) {
+ env.update(listValue.getKey().getName(), listElement);
+ if (listValues.size() == 1) {
+ result.add(elementExpression.eval(env));
+ } else {
+ evalLists(env, listValues.subList(1, listValues.size()), result);
+ }
+ }
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ for (Map.Entry<Ident, Expression> list : lists) {
+ // TODO(bazel-team): Get the type of elements
+ SkylarkType type = list.getValue().validate(env);
+ env.checkIterable(type, getLocation());
+ env.update(list.getKey().getName(), SkylarkType.UNKNOWN, getLocation());
+ }
+ elementExpression.validate(env);
+ return SkylarkType.of(SkylarkList.class);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java
new file mode 100644
index 0000000..9437135
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java
@@ -0,0 +1,128 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Syntax node for list and tuple literals.
+ *
+ * (Note that during evaluation, both list and tuple values are represented by
+ * java.util.List objects, the only difference between them being whether or not
+ * they are mutable.)
+ */
+public final class ListLiteral extends Expression {
+
+ /**
+ * Types of the ListLiteral.
+ */
+ public static enum Kind {LIST, TUPLE}
+
+ private final Kind kind;
+
+ private final List<Expression> exprs;
+
+ private ListLiteral(Kind kind, List<Expression> exprs) {
+ this.kind = kind;
+ this.exprs = exprs;
+ }
+
+ public static ListLiteral makeList(List<Expression> exprs) {
+ return new ListLiteral(Kind.LIST, exprs);
+ }
+
+ public static ListLiteral makeTuple(List<Expression> exprs) {
+ return new ListLiteral(Kind.TUPLE, exprs);
+ }
+
+ /**
+ * Returns the list of expressions for each element of the tuple.
+ */
+ public List<Expression> getElements() {
+ return exprs;
+ }
+
+ /**
+ * Returns true if this list is a tuple (a hash table, immutable list).
+ */
+ public boolean isTuple() {
+ return kind == Kind.TUPLE;
+ }
+
+ private static char startChar(Kind kind) {
+ switch(kind) {
+ case LIST: return '[';
+ case TUPLE: return '(';
+ }
+ return '[';
+ }
+
+ private static char endChar(Kind kind) {
+ switch(kind) {
+ case LIST: return ']';
+ case TUPLE: return ')';
+ }
+ return ']';
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(startChar(kind));
+ String sep = "";
+ for (Expression e : exprs) {
+ sb.append(sep);
+ sb.append(e);
+ sep = ", ";
+ }
+ sb.append(endChar(kind));
+ return sb.toString();
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ List<Object> result = new ArrayList<>();
+ for (Expression expr : exprs) {
+ // Convert NPEs to EvalExceptions.
+ if (expr == null) {
+ throw new EvalException(getLocation(), "null expression in " + this);
+ }
+ result.add(expr.eval(env));
+ }
+ if (env.isSkylarkEnabled()) {
+ return isTuple()
+ ? SkylarkList.tuple(result) : SkylarkList.list(result, getLocation());
+ } else {
+ return EvalUtils.makeSequence(result, isTuple());
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType type = SkylarkType.UNKNOWN;
+ if (!isTuple()) {
+ for (Expression expr : exprs) {
+ SkylarkType nextType = expr.validate(env);
+ type = type.infer(nextType, "list literal", expr.getLocation(), getLocation());
+ }
+ }
+ return SkylarkType.of(SkylarkList.class, type.getType());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Literal.java b/src/main/java/com/google/devtools/build/lib/syntax/Literal.java
new file mode 100644
index 0000000..9289081
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Literal.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Generic base class for primitive literals.
+ */
+public abstract class Literal<T> extends Expression {
+
+ protected final T value;
+
+ protected Literal(T value) {
+ this.value = value;
+ }
+
+ /**
+ * Returns the value of this literal.
+ */
+ public T getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return value.toString();
+ }
+
+ @Override
+ Object eval(Environment env) {
+ return value;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
new file mode 100644
index 0000000..6873995
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * Syntax node for an import statement.
+ */
+public final class LoadStatement extends Statement {
+
+ private final ImmutableList<Ident> symbols;
+ private final PathFragment importPath;
+
+ /**
+ * Constructs an import statement.
+ */
+ LoadStatement(String path, List<Ident> symbols) {
+ this.symbols = ImmutableList.copyOf(symbols);
+ this.importPath = new PathFragment(path + ".bzl");
+ }
+
+ public ImmutableList<Ident> getSymbols() {
+ return symbols;
+ }
+
+ public PathFragment getImportPath() {
+ return importPath;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("load(\"%s\", %s)", importPath, Joiner.on(", ").join(symbols));
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ for (Ident i : symbols) {
+ try {
+ if (i.getName().startsWith("_")) {
+ throw new EvalException(getLocation(), "symbol '" + i + "' is private and cannot "
+ + "be imported");
+ }
+ env.importSymbol(getImportPath(), i.getName());
+ } catch (Environment.NoSuchVariableException | Environment.LoadFailedException e) {
+ throw new EvalException(getLocation(), e.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): implement semantical check.
+ for (Ident symbol : symbols) {
+ env.update(symbol.getName(), SkylarkType.UNKNOWN, getLocation());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java
new file mode 100644
index 0000000..0427157
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java
@@ -0,0 +1,187 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract implementation of Function for functions that accept a mixture of
+ * positional and keyword parameters, as in Python.
+ */
+public abstract class MixedModeFunction extends AbstractFunction {
+
+ // Nomenclature:
+ // "Parameters" are formal parameters of a function definition.
+ // "Arguments" are actual parameters supplied at the call site.
+
+ // Number of regular named parameters (excluding *p and **p) in the
+ // equivalent Python function definition).
+ private final List<String> parameters;
+
+ // Number of leading "parameters" which are mandatory
+ private final int numMandatoryParameters;
+
+ // True if this function requires all arguments to be named
+ // TODO(bazel-team): replace this by a count of arguments before the * with optional arg,
+ // in the style Python 3 or PEP 3102.
+ private final boolean onlyNamedArguments;
+
+ // Location of the function definition, or null for builtin functions.
+ protected final Location location;
+
+ /**
+ * Constructs an instance of Function that supports Python-style mixed-mode
+ * parameter passing.
+ *
+ * @param parameters a list of named parameters
+ * @param numMandatoryParameters the number of leading parameters which are
+ * considered mandatory; the remaining ones may be omitted, in which
+ * case they will have the default value of null.
+ */
+ public MixedModeFunction(String name,
+ Iterable<String> parameters,
+ int numMandatoryParameters,
+ boolean onlyNamedArguments) {
+ this(name, parameters, numMandatoryParameters, onlyNamedArguments, null);
+ }
+
+ protected MixedModeFunction(String name,
+ Iterable<String> parameters,
+ int numMandatoryParameters,
+ boolean onlyNamedArguments,
+ Location location) {
+ super(name);
+ this.parameters = ImmutableList.copyOf(parameters);
+ this.numMandatoryParameters = numMandatoryParameters;
+ this.onlyNamedArguments = onlyNamedArguments;
+ this.location = location;
+ }
+
+ @Override
+ public Object call(List<Object> args,
+ Map<String, Object> kwargs,
+ FuncallExpression ast,
+ Environment env)
+ throws EvalException, InterruptedException {
+
+ // ast is null when called from Java (as there's no Skylark call site).
+ Location loc = ast == null ? location : ast.getLocation();
+ if (onlyNamedArguments && args.size() > 0) {
+ throw new EvalException(loc,
+ getSignature() + " does not accept positional arguments");
+ }
+
+ if (kwargs == null) {
+ kwargs = ImmutableMap.<String, Object>of();
+ }
+
+ int numParams = parameters.size();
+ int numArgs = args.size();
+ Object[] namedArguments = new Object[numParams];
+
+ // first, positional arguments:
+ if (numArgs > numParams) {
+ throw new EvalException(loc,
+ "too many positional arguments in call to " + getSignature());
+ }
+ for (int ii = 0; ii < numArgs; ++ii) {
+ namedArguments[ii] = args.get(ii);
+ }
+
+ // TODO(bazel-team): here, support *varargs splicing
+
+ // second, keyword arguments:
+ for (Map.Entry<String, Object> entry : kwargs.entrySet()) {
+ String keyword = entry.getKey();
+ int pos = parameters.indexOf(keyword);
+ if (pos == -1) {
+ throw new EvalException(loc,
+ "unexpected keyword '" + keyword
+ + "' in call to " + getSignature());
+ } else {
+ if (namedArguments[pos] != null) {
+ throw new EvalException(loc, getSignature()
+ + " got multiple values for keyword argument '" + keyword + "'");
+ }
+ namedArguments[pos] = kwargs.get(keyword);
+ }
+ }
+
+ // third, defaults:
+ for (int ii = 0; ii < numMandatoryParameters; ++ii) {
+ if (namedArguments[ii] == null) {
+ throw new EvalException(loc,
+ getSignature() + " received insufficient arguments");
+ }
+ }
+ // (defaults are always null so nothing extra to do here.)
+
+ try {
+ return call(namedArguments, ast, env);
+ } catch (ConversionException | IllegalArgumentException | IllegalStateException
+ | ClassCastException e) {
+ throw new EvalException(loc, e.getMessage());
+ }
+ }
+
+ /**
+ * Like Function.call, but generalised to support Python-style mixed-mode
+ * keyword and positional parameter passing.
+ *
+ * @param args an array of argument values corresponding to the list
+ * of named parameters passed to the constructor.
+ */
+ protected Object call(Object[] args, FuncallExpression ast)
+ throws EvalException, ConversionException, InterruptedException {
+ throw new UnsupportedOperationException("Method not overridden");
+ }
+
+ /**
+ * Override this method instead of the one above, if you need to access
+ * the environment.
+ */
+ protected Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws EvalException, ConversionException, InterruptedException {
+ return call(args, ast);
+ }
+
+ /**
+ * Render this object in the form of an equivalent Python function signature.
+ */
+ public String getSignature() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(getName()).append('(');
+ int ii = 0;
+ int len = parameters.size();
+ for (; ii < len; ++ii) {
+ String parameter = parameters.get(ii);
+ if (ii > 0) {
+ sb.append(", ");
+ }
+ sb.append(parameter);
+ if (ii >= numMandatoryParameters) {
+ sb.append(" = null");
+ }
+ }
+ sb.append(')');
+ return sb.toString();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java
new file mode 100644
index 0000000..5a13e79
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * As syntax node for the not boolean operation.
+ */
+public class NotExpression extends Expression {
+
+ private final Expression expression;
+
+ public NotExpression(Expression expression) {
+ this.expression = expression;
+ }
+
+ Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ return !EvalUtils.toBoolean(expression.eval(env));
+ }
+
+ @Override
+ public String toString() {
+ return "not " + expression;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ // Don't need type check here since EvalUtils.toBoolean() converts everything.
+ expression.validate(env);
+ return SkylarkType.BOOL;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
new file mode 100644
index 0000000..628570e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Infix operators supported by the build language.
+ */
+public enum Operator {
+
+ AND("and"),
+ EQUALS_EQUALS("=="),
+ GREATER(">"),
+ GREATER_EQUALS(">="),
+ IN("in"),
+ LESS("<"),
+ LESS_EQUALS("<="),
+ MINUS("-"),
+ MULT("*"),
+ NOT("not"),
+ NOT_EQUALS("!="),
+ OR("or"),
+ PERCENT("%"),
+ PLUS("+");
+
+ private final String name;
+
+ private Operator(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
new file mode 100644
index 0000000..66c3c67
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -0,0 +1,1274 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral;
+import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Recursive descent parser for LL(2) BUILD language.
+ * Loosely based on Python 2 grammar.
+ * See https://docs.python.org/2/reference/grammar.html
+ *
+ */
+class Parser {
+
+ /**
+ * Combines the parser result into a single value object.
+ */
+ public static final class ParseResult {
+ /** The statements (rules, basically) from the parsed file. */
+ public final List<Statement> statements;
+
+ /** The comments from the parsed file. */
+ public final List<Comment> comments;
+
+ /** Whether the file contained any errors. */
+ public final boolean containsErrors;
+
+ public ParseResult(List<Statement> statements, List<Comment> comments, boolean containsErrors) {
+ // No need to copy here; when the object is created, the parser instance is just about to go
+ // out of scope and be garbage collected.
+ this.statements = Preconditions.checkNotNull(statements);
+ this.comments = Preconditions.checkNotNull(comments);
+ this.containsErrors = containsErrors;
+ }
+ }
+
+ private static final EnumSet<TokenKind> STATEMENT_TERMINATOR_SET =
+ EnumSet.of(TokenKind.EOF, TokenKind.NEWLINE);
+
+ private static final EnumSet<TokenKind> LIST_TERMINATOR_SET =
+ EnumSet.of(TokenKind.EOF, TokenKind.RBRACKET, TokenKind.SEMI);
+
+ private static final EnumSet<TokenKind> DICT_TERMINATOR_SET =
+ EnumSet.of(TokenKind.EOF, TokenKind.RBRACE, TokenKind.SEMI);
+
+ private static final EnumSet<TokenKind> EXPR_TERMINATOR_SET = EnumSet.of(
+ TokenKind.EOF,
+ TokenKind.COMMA,
+ TokenKind.COLON,
+ TokenKind.FOR,
+ TokenKind.PLUS,
+ TokenKind.MINUS,
+ TokenKind.PERCENT,
+ TokenKind.RPAREN,
+ TokenKind.RBRACKET);
+
+ private Token token; // current lookahead token
+ private Token pushedToken = null; // used to implement LL(2)
+
+ private static final boolean DEBUGGING = false;
+
+ private final Lexer lexer;
+ private final EventHandler eventHandler;
+ private final List<Comment> comments;
+ private final boolean parsePython;
+ /** Whether advanced language constructs are allowed */
+ private boolean skylarkMode = false;
+
+ private static final Map<TokenKind, Operator> binaryOperators =
+ new ImmutableMap.Builder<TokenKind, Operator>()
+ .put(TokenKind.AND, Operator.AND)
+ .put(TokenKind.EQUALS_EQUALS, Operator.EQUALS_EQUALS)
+ .put(TokenKind.GREATER, Operator.GREATER)
+ .put(TokenKind.GREATER_EQUALS, Operator.GREATER_EQUALS)
+ .put(TokenKind.IN, Operator.IN)
+ .put(TokenKind.LESS, Operator.LESS)
+ .put(TokenKind.LESS_EQUALS, Operator.LESS_EQUALS)
+ .put(TokenKind.MINUS, Operator.MINUS)
+ .put(TokenKind.NOT_EQUALS, Operator.NOT_EQUALS)
+ .put(TokenKind.OR, Operator.OR)
+ .put(TokenKind.PERCENT, Operator.PERCENT)
+ .put(TokenKind.PLUS, Operator.PLUS)
+ .put(TokenKind.STAR, Operator.MULT)
+ .build();
+
+ private static final Map<TokenKind, Operator> augmentedAssignmentMethods =
+ new ImmutableMap.Builder<TokenKind, Operator>()
+ .put(TokenKind.PLUS_EQUALS, Operator.PLUS) // += // TODO(bazel-team): other similar operators
+ .build();
+
+ /** Highest precedence goes last.
+ * Based on: http://docs.python.org/2/reference/expressions.html#operator-precedence
+ **/
+ private static final List<EnumSet<Operator>> operatorPrecedence = ImmutableList.of(
+ EnumSet.of(Operator.OR),
+ EnumSet.of(Operator.AND),
+ EnumSet.of(Operator.NOT),
+ EnumSet.of(Operator.EQUALS_EQUALS, Operator.NOT_EQUALS, Operator.LESS, Operator.LESS_EQUALS,
+ Operator.GREATER, Operator.GREATER_EQUALS, Operator.IN),
+ EnumSet.of(Operator.MINUS, Operator.PLUS),
+ EnumSet.of(Operator.MULT, Operator.PERCENT));
+
+ private Iterator<Token> tokens = null;
+ private int errorsCount;
+ private boolean recoveryMode; // stop reporting errors until next statement
+
+ private CachingPackageLocator locator;
+
+ private List<Path> includedFiles;
+
+ private static final String PREPROCESSING_NEEDED =
+ "Add \"# PYTHON-PREPROCESSING-REQUIRED\" on the first line of the file";
+
+ private Parser(Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator,
+ boolean parsePython) {
+ this.lexer = lexer;
+ this.eventHandler = eventHandler;
+ this.parsePython = parsePython;
+ this.tokens = lexer.getTokens().iterator();
+ this.comments = new ArrayList<Comment>();
+ this.locator = locator;
+ this.includedFiles = new ArrayList<Path>();
+ this.includedFiles.add(lexer.getFilename());
+ nextToken();
+ }
+
+ private Parser(Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator) {
+ this(lexer, eventHandler, locator, false /* parsePython */);
+ }
+
+ public Parser setSkylarkMode(boolean skylarkMode) {
+ this.skylarkMode = skylarkMode;
+ return this;
+ }
+
+ /**
+ * Entry-point to parser that parses a build file with comments. All errors
+ * encountered during parsing are reported via "reporter".
+ */
+ public static ParseResult parseFile(
+ Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator,
+ boolean parsePython) {
+ Parser parser = new Parser(lexer, eventHandler, locator, parsePython);
+ List<Statement> statements = parser.parseFileInput();
+ return new ParseResult(statements, parser.comments,
+ parser.errorsCount > 0 || lexer.containsErrors());
+ }
+
+ /**
+ * Entry-point to parser that parses a build file with comments. All errors
+ * encountered during parsing are reported via "reporter". Enable Skylark extensions
+ * that are not part of the core BUILD language.
+ */
+ public static ParseResult parseFileForSkylark(
+ Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator,
+ ValidationEnvironment validationEnvironment) {
+ Parser parser = new Parser(lexer, eventHandler, locator).setSkylarkMode(true);
+ List<Statement> statements = parser.parseFileInput();
+ boolean hasSemanticalErrors = false;
+ try {
+ for (Statement statement : statements) {
+ statement.validate(validationEnvironment);
+ }
+ } catch (EvalException e) {
+ eventHandler.handle(Event.error(e.getLocation(), e.getMessage()));
+ hasSemanticalErrors = true;
+ }
+ return new ParseResult(statements, parser.comments,
+ parser.errorsCount > 0 || lexer.containsErrors() || hasSemanticalErrors);
+ }
+
+ /**
+ * Entry-point to parser that parses a statement. All errors encountered
+ * during parsing are reported via "reporter".
+ */
+ @VisibleForTesting
+ public static Statement parseStatement(
+ Lexer lexer, EventHandler eventHandler) {
+ return new Parser(lexer, eventHandler, null).parseSmallStatement();
+ }
+
+ /**
+ * Entry-point to parser that parses an expression. All errors encountered
+ * during parsing are reported via "reporter". The expression may be followed
+ * by newline tokens.
+ */
+ @VisibleForTesting
+ public static Expression parseExpression(Lexer lexer, EventHandler eventHandler) {
+ Parser parser = new Parser(lexer, eventHandler, null);
+ Expression result = parser.parseExpression();
+ while (parser.token.kind == TokenKind.NEWLINE) {
+ parser.nextToken();
+ }
+ parser.expect(TokenKind.EOF);
+ return result;
+ }
+
+ private void addIncludedFiles(List<Path> files) {
+ this.includedFiles.addAll(files);
+ }
+
+ private void reportError(Location location, String message) {
+ errorsCount++;
+ // Limit the number of reported errors to avoid spamming output.
+ if (errorsCount <= 5) {
+ eventHandler.handle(Event.error(location, message));
+ }
+ }
+
+ private void syntaxError(Token token) {
+ if (!recoveryMode) {
+ String msg = token.kind == TokenKind.INDENT
+ ? "indentation error"
+ : "syntax error at '" + token + "'";
+ reportError(lexer.createLocation(token.left, token.right), msg);
+ recoveryMode = true;
+ }
+ }
+
+ // Consumes the current token. If it is not of the specified (expected)
+ // kind, reports a syntax error.
+ private boolean expect(TokenKind kind) {
+ boolean expected = token.kind == kind;
+ if (!expected) {
+ syntaxError(token);
+ }
+ nextToken();
+ return expected;
+ }
+
+ /**
+ * Consume tokens past the first token that has a kind that is in the set of
+ * teminatingTokens.
+ * @param terminatingTokens
+ * @return the end offset of the terminating token.
+ */
+ private int syncPast(EnumSet<TokenKind> terminatingTokens) {
+ Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF));
+ while (!terminatingTokens.contains(token.kind)) {
+ nextToken();
+ }
+ int end = token.right;
+ // read past the synchronization token
+ nextToken();
+ return end;
+ }
+
+ /**
+ * Consume tokens until we reach the first token that has a kind that is in
+ * the set of teminatingTokens.
+ * @param terminatingTokens
+ * @return the end offset of the terminating token.
+ */
+ private int syncTo(EnumSet<TokenKind> terminatingTokens) {
+ // EOF must be in the set to prevent an infinite loop
+ Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF));
+ // read past the problematic token
+ int previous = token.right;
+ nextToken();
+ int current = previous;
+ while (!terminatingTokens.contains(token.kind)) {
+ nextToken();
+ previous = current;
+ current = token.right;
+ }
+ return previous;
+ }
+
+ private void nextToken() {
+ if (pushedToken != null) {
+ token = pushedToken;
+ pushedToken = null;
+ } else {
+ if (token == null || token.kind != TokenKind.EOF) {
+ token = tokens.next();
+ // transparently handle comment tokens
+ while (token.kind == TokenKind.COMMENT) {
+ makeComment(token);
+ token = tokens.next();
+ }
+ }
+ }
+ if (DEBUGGING) {
+ System.err.print(token);
+ }
+ }
+
+ private void pushToken(Token tokenToPush) {
+ if (pushedToken != null) {
+ throw new IllegalStateException("Exceeded LL(2) lookahead!");
+ }
+ pushedToken = token;
+ token = tokenToPush;
+ }
+
+ // create an error expression
+ private Ident makeErrorExpression(int start, int end) {
+ return setLocation(new Ident("$error$"), start, end);
+ }
+
+ // Convenience wrapper around ASTNode.setLocation that returns the node.
+ private <NODE extends ASTNode> NODE
+ setLocation(NODE node, int startOffset, int endOffset) {
+ node.setLocation(lexer.createLocation(startOffset, endOffset));
+ return node;
+ }
+
+ // Another convenience wrapper method around ASTNode.setLocation
+ private <NODE extends ASTNode> NODE setLocation(NODE node, Location location) {
+ node.setLocation(location);
+ return node;
+ }
+
+ // Convenience method that uses end offset from the last node.
+ private <NODE extends ASTNode> NODE setLocation(NODE node, int startOffset, ASTNode lastNode) {
+ return setLocation(node, startOffset, lastNode.getLocation().getEndOffset());
+ }
+
+ // create a funcall expression
+ private Expression makeFuncallExpression(Expression receiver, Ident function,
+ List<Argument> args,
+ int start, int end) {
+ if (function.getLocation() == null) {
+ function = setLocation(function, start, end);
+ }
+ boolean seenKeywordArg = false;
+ boolean seenKwargs = false;
+ for (Argument arg : args) {
+ if (arg.isPositional()) {
+ if (seenKeywordArg || seenKwargs) {
+ reportError(arg.getLocation(), "syntax error: non-keyword arg after keyword arg");
+ return makeErrorExpression(start, end);
+ }
+ } else if (arg.isKwargs()) {
+ if (seenKwargs) {
+ reportError(arg.getLocation(), "there can be only one **kwargs argument");
+ return makeErrorExpression(start, end);
+ }
+ seenKwargs = true;
+ } else {
+ seenKeywordArg = true;
+ }
+ }
+
+ return setLocation(new FuncallExpression(receiver, function, args), start, end);
+ }
+
+ // arg ::= IDENTIFIER '=' expr
+ // | expr
+ private Argument parseFunctionCallArgument() {
+ int start = token.left;
+ if (token.kind == TokenKind.IDENTIFIER) {
+ Token identToken = token;
+ String name = (String) token.value;
+ Ident ident = setLocation(new Ident(name), start, token.right);
+ nextToken();
+ if (token.kind == TokenKind.EQUALS) { // it's a named argument
+ nextToken();
+ Expression expr = parseExpression();
+ return setLocation(new Argument(ident, expr), start, expr);
+ } else { // oops, back up!
+ pushToken(identToken);
+ }
+ }
+ // parse **expr
+ if (token.kind == TokenKind.STAR) {
+ expect(TokenKind.STAR);
+ expect(TokenKind.STAR);
+ Expression expr = parseExpression();
+ return setLocation(new Argument(null, expr, true), start, expr);
+ }
+ // parse a positional argument
+ Expression expr = parseExpression();
+ return setLocation(new Argument(expr), start, expr);
+ }
+
+ // arg ::= IDENTIFIER '=' expr
+ // | IDENTIFIER
+ private Argument parseFunctionDefArgument(boolean onlyOptional) {
+ int start = token.left;
+ Ident ident = parseIdent();
+ if (token.kind == TokenKind.EQUALS) { // there's a default value
+ nextToken();
+ Expression expr = parseExpression();
+ return setLocation(new Argument(ident, expr), start, expr);
+ } else if (onlyOptional) {
+ reportError(ident.getLocation(),
+ "Optional arguments are only allowed at the end of the argument list.");
+ }
+ return setLocation(new Argument(ident), start, ident);
+ }
+
+ // funcall_suffix ::= '(' arg_list? ')'
+ private Expression parseFuncallSuffix(int start, Expression receiver,
+ Ident function) {
+ List<Argument> args = Collections.emptyList();
+ expect(TokenKind.LPAREN);
+ int end;
+ if (token.kind == TokenKind.RPAREN) {
+ end = token.right;
+ nextToken(); // RPAREN
+ } else {
+ args = parseFunctionCallArguments(); // (includes optional trailing comma)
+ end = token.right;
+ expect(TokenKind.RPAREN);
+ }
+ return makeFuncallExpression(receiver, function, args, start, end);
+ }
+
+ // selector_suffix ::= '.' IDENTIFIER
+ // |'.' IDENTIFIER funcall_suffix
+ private Expression parseSelectorSuffix(int start, Expression receiver) {
+ expect(TokenKind.DOT);
+ if (token.kind == TokenKind.IDENTIFIER) {
+ Ident ident = parseIdent();
+ if (token.kind == TokenKind.LPAREN) {
+ return parseFuncallSuffix(start, receiver, ident);
+ } else {
+ return setLocation(new DotExpression(receiver, ident), start, token.right);
+ }
+ } else {
+ syntaxError(token);
+ int end = syncTo(EXPR_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ }
+
+ // arg_list ::= ( (arg ',')* arg ','? )?
+ private List<Argument> parseFunctionCallArguments() {
+ List<Argument> args = new ArrayList<>();
+ // terminating tokens for an arg list
+ while (token.kind != TokenKind.RPAREN) {
+ if (token.kind == TokenKind.EOF) {
+ syntaxError(token);
+ break;
+ }
+ args.add(parseFunctionCallArgument());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return args;
+ }
+
+ // expr_list ::= ( (expr ',')* expr ','? )?
+ private List<Expression> parseExprList() {
+ List<Expression> list = new ArrayList<>();
+ // terminating tokens for an expression list
+ while (token.kind != TokenKind.RPAREN && token.kind != TokenKind.RBRACKET) {
+ list.add(parseExpression());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return list;
+ }
+
+ // dict_entry_list ::= ( (dict_entry ',')* dict_entry ','? )?
+ private List<DictionaryEntryLiteral> parseDictEntryList() {
+ List<DictionaryEntryLiteral> list = new ArrayList<>();
+ // the terminating token for a dict entry list
+ while (token.kind != TokenKind.RBRACE) {
+ list.add(parseDictEntry());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return list;
+ }
+
+ // dict_entry ::= expression ':' expression
+ private DictionaryEntryLiteral parseDictEntry() {
+ int start = token.left;
+ Expression key = parseExpression();
+ expect(TokenKind.COLON);
+ Expression value = parseExpression();
+ return setLocation(new DictionaryEntryLiteral(key, value), start, value);
+ }
+
+ private ExpressionStatement mocksubincludeExpression(
+ String labelName, String file, Location location) {
+ List<Argument> args = new ArrayList<>();
+ args.add(setLocation(new Argument(new StringLiteral(labelName, '"')), location));
+ args.add(setLocation(new Argument(new StringLiteral(file, '"')), location));
+ Ident mockIdent = setLocation(new Ident("mocksubinclude"), location);
+ Expression funCall = new FuncallExpression(null, mockIdent, args);
+ return setLocation(new ExpressionStatement(funCall), location);
+ }
+
+ // parse a file from an include call
+ private void include(String labelName, List<Statement> list, Location location) {
+ if (locator == null) {
+ return;
+ }
+
+ try {
+ Label label = Label.parseAbsolute(labelName);
+ String packageName = label.getPackageFragment().getPathString();
+ Path packagePath = locator.getBuildFileForPackage(packageName);
+ if (packagePath == null) {
+ reportError(location, "Package '" + packageName + "' not found");
+ list.add(mocksubincludeExpression(labelName, "", location));
+ return;
+ }
+ Path path = packagePath.getParentDirectory();
+ Path file = path.getRelative(label.getName());
+
+ if (this.includedFiles.contains(file)) {
+ reportError(location, "Recursive inclusion of file '" + path + "'");
+ return;
+ }
+ ParserInputSource inputSource = ParserInputSource.create(file);
+
+ // Insert call to the mocksubinclude function to get the dependencies right.
+ list.add(mocksubincludeExpression(labelName, file.toString(), location));
+
+ Lexer lexer = new Lexer(inputSource, eventHandler, parsePython);
+ Parser parser = new Parser(lexer, eventHandler, locator, parsePython);
+ parser.addIncludedFiles(this.includedFiles);
+ list.addAll(parser.parseFileInput());
+ } catch (Label.SyntaxException e) {
+ reportError(location, "Invalid label '" + labelName + "'");
+ } catch (IOException e) {
+ reportError(location, "Include of '" + labelName + "' failed: " + e.getMessage());
+ list.add(mocksubincludeExpression(labelName, "", location));
+ }
+ }
+
+ // primary ::= INTEGER
+ // | STRING
+ // | STRING '.' IDENTIFIER funcall_suffix
+ // | IDENTIFIER
+ // | IDENTIFIER funcall_suffix
+ // | IDENTIFIER '.' selector_suffix
+ // | list_expression
+ // | '(' ')' // a tuple with zero elements
+ // | '(' expr ')' // a parenthesized expression
+ // | '(' expr ',' expr_list ')' // a tuple with n elements
+ // | dict_expression
+ // | '-' primary_with_suffix
+ private Expression parsePrimary() {
+ int start = token.left;
+ switch (token.kind) {
+ case INT: {
+ IntegerLiteral literal = new IntegerLiteral((Integer) token.value);
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ case STRING: {
+ String value = (String) token.value;
+ int end = token.right;
+ char quoteChar = lexer.charAt(start);
+ nextToken();
+ if (token.kind == TokenKind.STRING) {
+ reportError(lexer.createLocation(end, token.left),
+ "Implicit string concatenation is forbidden, use the + operator");
+ }
+ StringLiteral literal = new StringLiteral(value, quoteChar);
+ setLocation(literal, start, end);
+ return literal;
+ }
+ case IDENTIFIER: {
+ Ident ident = parseIdent();
+ if (token.kind == TokenKind.LPAREN) { // it's a function application
+ return parseFuncallSuffix(start, null, ident);
+ } else {
+ return ident;
+ }
+ }
+ case LBRACKET: { // it's a list
+ return parseListExpression();
+ }
+ case LBRACE: { // it's a dictionary
+ return parseDictExpression();
+ }
+ case LPAREN: {
+ nextToken();
+ // check for the empty tuple literal
+ if (token.kind == TokenKind.RPAREN) {
+ ListLiteral literal =
+ ListLiteral.makeTuple(Collections.<Expression>emptyList());
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ // parse the first expression
+ Expression expression = parseExpression();
+ if (token.kind == TokenKind.COMMA) { // it's a tuple
+ nextToken();
+ // parse the rest of the expression tuple
+ List<Expression> tuple = parseExprList();
+ // add the first expression to the front of the tuple
+ tuple.add(0, expression);
+ expect(TokenKind.RPAREN);
+ return setLocation(
+ ListLiteral.makeTuple(tuple), start, token.right);
+ }
+ setLocation(expression, start, token.right);
+ if (token.kind == TokenKind.RPAREN) {
+ nextToken();
+ return expression;
+ }
+ syntaxError(token);
+ int end = syncTo(EXPR_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ case MINUS: {
+ nextToken();
+
+ List<Argument> args = new ArrayList<>();
+ Expression expr = parsePrimaryWithSuffix();
+ args.add(setLocation(new Argument(expr), start, expr));
+ return makeFuncallExpression(null, new Ident("-"), args,
+ start, token.right);
+ }
+ default: {
+ syntaxError(token);
+ int end = syncTo(EXPR_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ }
+ }
+
+ // primary_with_suffix ::= primary selector_suffix*
+ // | primary substring_suffix
+ private Expression parsePrimaryWithSuffix() {
+ int start = token.left;
+ Expression receiver = parsePrimary();
+ while (true) {
+ if (token.kind == TokenKind.DOT) {
+ receiver = parseSelectorSuffix(start, receiver);
+ } else if (token.kind == TokenKind.LBRACKET) {
+ receiver = parseSubstringSuffix(start, receiver);
+ } else {
+ break;
+ }
+ }
+ return receiver;
+ }
+
+ // substring_suffix ::= '[' expression? ':' expression? ']'
+ private Expression parseSubstringSuffix(int start, Expression receiver) {
+ List<Argument> args = new ArrayList<>();
+ Expression startExpr;
+ Expression endExpr;
+
+ expect(TokenKind.LBRACKET);
+ int loc1 = token.left;
+ if (token.kind == TokenKind.COLON) {
+ startExpr = setLocation(new IntegerLiteral(0), token.left, token.right);
+ } else {
+ startExpr = parseExpression();
+ }
+ args.add(setLocation(new Argument(startExpr), loc1, startExpr));
+ // This is a dictionary access
+ if (token.kind == TokenKind.RBRACKET) {
+ expect(TokenKind.RBRACKET);
+ return makeFuncallExpression(receiver, new Ident("$index"), args,
+ start, token.right);
+ }
+ // This is a substring
+ expect(TokenKind.COLON);
+ int loc2 = token.left;
+ if (token.kind == TokenKind.RBRACKET) {
+ endExpr = setLocation(new IntegerLiteral(Integer.MAX_VALUE), token.left, token.right);
+ } else {
+ endExpr = parseExpression();
+ }
+ expect(TokenKind.RBRACKET);
+
+ args.add(setLocation(new Argument(endExpr), loc2, endExpr));
+ return makeFuncallExpression(receiver, new Ident("$substring"), args,
+ start, token.right);
+ }
+
+ // loop_variables ::= '(' variables ')'
+ // | variables
+ // variables ::= ident (',' ident)*
+ private Ident parseForLoopVariables() {
+ int start = token.left;
+ boolean hasParen = false;
+ if (token.kind == TokenKind.LPAREN) {
+ hasParen = true;
+ nextToken();
+ }
+
+ // TODO(bazel-team): allow multiple variables in the core Blaze language too.
+ Ident firstIdent = parseIdent();
+ boolean multipleVariables = false;
+
+ while (token.kind == TokenKind.COMMA) {
+ multipleVariables = true;
+ nextToken();
+ parseIdent();
+ }
+
+ if (hasParen) {
+ expect(TokenKind.RPAREN);
+ }
+
+ int end = token.right;
+ if (multipleVariables && !parsePython) {
+ reportError(lexer.createLocation(start, end),
+ "For loops with multiple variables are not yet supported. "
+ + PREPROCESSING_NEEDED);
+ }
+ return multipleVariables ? makeErrorExpression(start, end) : firstIdent;
+ }
+
+ // list_expression ::= '[' ']'
+ // |'[' expr ']'
+ // |'[' expr ',' expr_list ']'
+ // |'[' expr ('FOR' loop_variables 'IN' expr)+ ']'
+ private Expression parseListExpression() {
+ int start = token.left;
+ expect(TokenKind.LBRACKET);
+ if (token.kind == TokenKind.RBRACKET) { // empty List
+ ListLiteral literal =
+ ListLiteral.makeList(Collections.<Expression>emptyList());
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ Expression expression = parseExpression();
+ Preconditions.checkNotNull(expression,
+ "null element in list in AST at %s:%s", token.left, token.right);
+ switch (token.kind) {
+ case RBRACKET: { // singleton List
+ ListLiteral literal =
+ ListLiteral.makeList(Collections.singletonList(expression));
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ case FOR: { // list comprehension
+ ListComprehension listComprehension =
+ new ListComprehension(expression);
+ do {
+ nextToken();
+ Ident ident = parseForLoopVariables();
+ if (token.kind == TokenKind.IN) {
+ nextToken();
+ Expression listExpression = parseExpression();
+ listComprehension.add(ident, listExpression);
+ } else {
+ break;
+ }
+ if (token.kind == TokenKind.RBRACKET) {
+ setLocation(listComprehension, start, token.right);
+ nextToken();
+ return listComprehension;
+ }
+ } while (token.kind == TokenKind.FOR);
+
+ syntaxError(token);
+ int end = syncPast(LIST_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ case COMMA: {
+ nextToken();
+ List<Expression> list = parseExprList();
+ Preconditions.checkState(!list.contains(null),
+ "null element in list in AST at %s:%s", token.left, token.right);
+ list.add(0, expression);
+ if (token.kind == TokenKind.RBRACKET) {
+ ListLiteral literal = ListLiteral.makeList(list);
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ syntaxError(token);
+ int end = syncPast(LIST_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ default: {
+ syntaxError(token);
+ int end = syncPast(LIST_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ }
+ }
+
+ // dict_expression ::= '{' '}'
+ // |'{' dict_entry_list '}'
+ // |'{' dict_entry 'FOR' loop_variables 'IN' expr '}'
+ private Expression parseDictExpression() {
+ int start = token.left;
+ expect(TokenKind.LBRACE);
+ if (token.kind == TokenKind.RBRACE) { // empty List
+ DictionaryLiteral literal =
+ new DictionaryLiteral(ImmutableList.<DictionaryEntryLiteral>of());
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ DictionaryEntryLiteral entry = parseDictEntry();
+ if (token.kind == TokenKind.FOR) {
+ // Dict comprehension
+ nextToken();
+ Ident loopVar = parseForLoopVariables();
+ expect(TokenKind.IN);
+ Expression listExpression = parseExpression();
+ expect(TokenKind.RBRACE);
+ return setLocation(new DictComprehension(
+ entry.getKey(), entry.getValue(), loopVar, listExpression), start, token.right);
+ }
+ List<DictionaryEntryLiteral> entries = new ArrayList<>();
+ entries.add(entry);
+ if (token.kind == TokenKind.COMMA) {
+ expect(TokenKind.COMMA);
+ entries.addAll(parseDictEntryList());
+ }
+ if (token.kind == TokenKind.RBRACE) {
+ DictionaryLiteral literal = new DictionaryLiteral(entries);
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ syntaxError(token);
+ int end = syncPast(DICT_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+
+ private Ident parseIdent() {
+ if (token.kind != TokenKind.IDENTIFIER) {
+ syntaxError(token);
+ return makeErrorExpression(token.left, token.right);
+ }
+ Ident ident = new Ident(((String) token.value));
+ setLocation(ident, token.left, token.right);
+ nextToken();
+ return ident;
+ }
+
+ // binop_expression ::= binop_expression OP binop_expression
+ // | parsePrimaryWithSuffix
+ // This function takes care of precedence between operators (see operatorPrecedence for
+ // the order), and it assumes left-to-right associativity.
+ private Expression parseBinOpExpression(int prec) {
+ int start = token.left;
+ Expression expr = parseExpression(prec + 1);
+ // The loop is not strictly needed, but it prevents risks of stack overflow. Depth is
+ // limited to number of different precedence levels (operatorPrecedence.size()).
+ for (;;) {
+ if (!binaryOperators.containsKey(token.kind)) {
+ return expr;
+ }
+ Operator operator = binaryOperators.get(token.kind);
+ if (!operatorPrecedence.get(prec).contains(operator)) {
+ return expr;
+ }
+ nextToken();
+ Expression secondary = parseExpression(prec + 1);
+ expr = optimizeBinOpExpression(operator, expr, secondary);
+ setLocation(expr, start, secondary);
+ }
+ }
+
+ // Optimize binary expressions.
+ // string literal + string literal can be concatenated into one string literal
+ // so we don't have to do the expensive string concatenation at runtime.
+ private Expression optimizeBinOpExpression(
+ Operator operator, Expression expr, Expression secondary) {
+ if (operator == Operator.PLUS) {
+ if (expr instanceof StringLiteral && secondary instanceof StringLiteral) {
+ StringLiteral left = (StringLiteral) expr;
+ StringLiteral right = (StringLiteral) secondary;
+ if (left.getQuoteChar() == right.getQuoteChar()) {
+ return new StringLiteral(left.getValue() + right.getValue(), left.getQuoteChar());
+ }
+ }
+ }
+ return new BinaryOperatorExpression(operator, expr, secondary);
+ }
+
+ private Expression parseExpression() {
+ return parseExpression(0);
+ }
+
+ private Expression parseExpression(int prec) {
+ if (prec >= operatorPrecedence.size()) {
+ return parsePrimaryWithSuffix();
+ }
+ if (token.kind == TokenKind.NOT && operatorPrecedence.get(prec).contains(Operator.NOT)) {
+ return parseNotExpression(prec);
+ }
+ return parseBinOpExpression(prec);
+ }
+
+ // not_expr :== 'not' expr
+ private Expression parseNotExpression(int prec) {
+ int start = token.left;
+ expect(TokenKind.NOT);
+ Expression expression = parseExpression(prec + 1);
+ NotExpression notExpression = new NotExpression(expression);
+ return setLocation(notExpression, start, token.right);
+ }
+
+ // file_input ::= ('\n' | stmt)* EOF
+ private List<Statement> parseFileInput() {
+ List<Statement> list = new ArrayList<>();
+ while (token.kind != TokenKind.EOF) {
+ if (token.kind == TokenKind.NEWLINE) {
+ expect(TokenKind.NEWLINE);
+ } else {
+ parseTopLevelStatement(list);
+ }
+ }
+ return list;
+ }
+
+ // load(STRING (COMMA STRING)*)
+ private void parseLoad(List<Statement> list) {
+ int start = token.left;
+ if (token.kind != TokenKind.STRING) {
+ expect(TokenKind.STRING);
+ return;
+ }
+ String path = (String) token.value;
+ nextToken();
+ expect(TokenKind.COMMA);
+
+ List<Ident> symbols = new ArrayList<>();
+ if (token.kind == TokenKind.STRING) {
+ symbols.add(new Ident((String) token.value));
+ }
+ expect(TokenKind.STRING);
+ while (token.kind == TokenKind.COMMA) {
+ expect(TokenKind.COMMA);
+ if (token.kind == TokenKind.STRING) {
+ symbols.add(new Ident((String) token.value));
+ }
+ expect(TokenKind.STRING);
+ }
+ expect(TokenKind.RPAREN);
+ list.add(setLocation(new LoadStatement(path, symbols), start, token.left));
+ }
+
+ private void parseTopLevelStatement(List<Statement> list) {
+ // In Python grammar, there is no "top-level statement" and imports are
+ // considered as "small statements". We are a bit stricter than Python here.
+ int start = token.left;
+
+ // Check if there is an include
+ if (token.kind == TokenKind.IDENTIFIER) {
+ Token identToken = token;
+ Ident ident = parseIdent();
+
+ if (ident.getName().equals("include") && token.kind == TokenKind.LPAREN && !skylarkMode) {
+ expect(TokenKind.LPAREN);
+ if (token.kind == TokenKind.STRING) {
+ include((String) token.value, list, lexer.createLocation(start, token.right));
+ }
+ expect(TokenKind.STRING);
+ expect(TokenKind.RPAREN);
+ return;
+ } else if (ident.getName().equals("load") && token.kind == TokenKind.LPAREN) {
+ expect(TokenKind.LPAREN);
+ parseLoad(list);
+ return;
+ }
+ pushToken(identToken); // push the ident back to parse it as a statement
+ }
+ parseStatement(list, true);
+ }
+
+ // simple_stmt ::= small_stmt (';' small_stmt)* ';'? NEWLINE
+ private void parseSimpleStatement(List<Statement> list) {
+ list.add(parseSmallStatement());
+
+ while (token.kind == TokenKind.SEMI) {
+ nextToken();
+ if (token.kind == TokenKind.NEWLINE) {
+ break;
+ }
+ list.add(parseSmallStatement());
+ }
+ expect(TokenKind.NEWLINE);
+ // This is a safe place to recover: There is a new line at top-level
+ // and the parser is at the end of a statement.
+ recoveryMode = false;
+ }
+
+ // small_stmt ::= assign_stmt
+ // | expr
+ // | RETURN expr
+ // assign_stmt ::= expr ('=' | augassign) expr
+ // augassign ::= ('+=' )
+ // Note that these are in Python, but not implemented here (at least for now):
+ // '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |'<<=' | '>>=' | '**=' | '//='
+ // Semantic difference from Python:
+ // In Skylark, x += y is simple syntactic sugar for x = x + y.
+ // In Python, x += y is more or less equivalent to x = x + y, but if a method is defined
+ // on x.__iadd__(y), then it takes precedence, and in the case of lists it side-effects
+ // the original list (it doesn't do that on tuples); if no such method is defined it falls back
+ // to the x.__add__(y) method that backs x + y. In Skylark, we don't support this side-effect.
+ // Note also that there is a special casing to translate 'ident[key] = value'
+ // to 'ident = ident + {key: value}'. This is needed to support the pure version of Python-like
+ // dictionary assignment syntax.
+ private Statement parseSmallStatement() {
+ int start = token.left;
+ if (token.kind == TokenKind.RETURN) {
+ return parseReturnStatement();
+ }
+ Expression expression = parseExpression();
+ if (token.kind == TokenKind.EQUALS) {
+ nextToken();
+ Expression rvalue = parseExpression();
+ if (expression instanceof FuncallExpression) {
+ FuncallExpression func = (FuncallExpression) expression;
+ if (func.getFunction().getName().equals("$index") && func.getObject() instanceof Ident) {
+ // Special casing to translate 'ident[key] = value' to 'ident = ident + {key: value}'
+ // Note that the locations of these extra expressions are fake.
+ Preconditions.checkArgument(func.getArguments().size() == 1);
+ DictionaryLiteral dictRValue = setLocation(new DictionaryLiteral(ImmutableList.of(
+ setLocation(new DictionaryEntryLiteral(func.getArguments().get(0).getValue(), rvalue),
+ start, token.right))), start, token.right);
+ BinaryOperatorExpression binExp = setLocation(new BinaryOperatorExpression(
+ Operator.PLUS, func.getObject(), dictRValue), start, token.right);
+ return setLocation(new AssignmentStatement(func.getObject(), binExp), start, token.right);
+ }
+ }
+ return setLocation(new AssignmentStatement(expression, rvalue), start, rvalue);
+ } else if (augmentedAssignmentMethods.containsKey(token.kind)) {
+ Operator operator = augmentedAssignmentMethods.get(token.kind);
+ nextToken();
+ Expression operand = parseExpression();
+ int end = operand.getLocation().getEndOffset();
+ return setLocation(new AssignmentStatement(expression,
+ setLocation(new BinaryOperatorExpression(
+ operator, expression, operand), start, end)),
+ start, end);
+ } else {
+ return setLocation(new ExpressionStatement(expression), start, expression);
+ }
+ }
+
+ // if_stmt ::= IF expr ':' suite [ELIF expr ':' suite]* [ELSE ':' suite]?
+ private void parseIfStatement(List<Statement> list) {
+ int start = token.left;
+ List<ConditionalStatements> thenBlocks = new ArrayList<>();
+ thenBlocks.add(parseConditionalStatements(TokenKind.IF));
+ while (token.kind == TokenKind.ELIF) {
+ thenBlocks.add(parseConditionalStatements(TokenKind.ELIF));
+ }
+ List<Statement> elseBlock = new ArrayList<>();
+ if (token.kind == TokenKind.ELSE) {
+ expect(TokenKind.ELSE);
+ expect(TokenKind.COLON);
+ parseSuite(elseBlock);
+ }
+ Statement stmt = new IfStatement(thenBlocks, elseBlock);
+ list.add(setLocation(stmt, start, token.right));
+ }
+
+ // cond_stmts ::= [EL]IF expr ':' suite
+ private ConditionalStatements parseConditionalStatements(TokenKind tokenKind) {
+ int start = token.left;
+ expect(tokenKind);
+ Expression expr = parseExpression();
+ expect(TokenKind.COLON);
+ List<Statement> thenBlock = new ArrayList<>();
+ parseSuite(thenBlock);
+ ConditionalStatements stmt = new ConditionalStatements(expr, thenBlock);
+ return setLocation(stmt, start, token.right);
+ }
+
+ // for_stmt ::= FOR IDENTIFIER IN expr ':' suite
+ private void parseForStatement(List<Statement> list) {
+ int start = token.left;
+ expect(TokenKind.FOR);
+ Ident ident = parseIdent();
+ expect(TokenKind.IN);
+ Expression collection = parseExpression();
+ expect(TokenKind.COLON);
+ List<Statement> block = new ArrayList<>();
+ parseSuite(block);
+ Statement stmt = new ForStatement(ident, collection, block);
+ list.add(setLocation(stmt, start, token.right));
+ }
+
+ // def foo(bar1, bar2):
+ private void parseFunctionDefStatement(List<Statement> list) {
+ int start = token.left;
+ expect(TokenKind.DEF);
+ Ident ident = parseIdent();
+ expect(TokenKind.LPAREN);
+ // parsing the function arguments, at this point only identifiers
+ // TODO(bazel-team): support proper arguments with default values and kwargs
+ List<Argument> args = parseFunctionDefArguments();
+ expect(TokenKind.RPAREN);
+ expect(TokenKind.COLON);
+ List<Statement> block = new ArrayList<>();
+ parseSuite(block);
+ FunctionDefStatement stmt = new FunctionDefStatement(ident, args, block);
+ list.add(setLocation(stmt, start, token.right));
+ }
+
+ private List<Argument> parseFunctionDefArguments() {
+ List<Argument> args = new ArrayList<>();
+ Set<String> argNames = new HashSet<>();
+ boolean onlyOptional = false;
+ while (token.kind != TokenKind.RPAREN) {
+ Argument arg = parseFunctionDefArgument(onlyOptional);
+ if (arg.hasValue()) {
+ onlyOptional = true;
+ }
+ args.add(arg);
+ if (argNames.contains(arg.getArgName())) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "duplicate argument name in function definition");
+ }
+ argNames.add(arg.getArgName());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return args;
+ }
+
+ // suite ::= simple_stmt
+ // | NEWLINE INDENT stmt+ OUTDENT
+ private void parseSuite(List<Statement> list) {
+ if (token.kind == TokenKind.NEWLINE) {
+ expect(TokenKind.NEWLINE);
+ if (token.kind != TokenKind.INDENT) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "expected an indented block");
+ return;
+ }
+ expect(TokenKind.INDENT);
+ while (token.kind != TokenKind.OUTDENT && token.kind != TokenKind.EOF) {
+ parseStatement(list, false);
+ }
+ expect(TokenKind.OUTDENT);
+ } else {
+ Statement stmt = parseSmallStatement();
+ list.add(stmt);
+ expect(TokenKind.NEWLINE);
+ }
+ }
+
+ // skipSuite does not check that the code is syntactically correct, it
+ // just skips based on indentation levels.
+ private void skipSuite() {
+ if (token.kind == TokenKind.NEWLINE) {
+ expect(TokenKind.NEWLINE);
+ if (token.kind != TokenKind.INDENT) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "expected an indented block");
+ return;
+ }
+ expect(TokenKind.INDENT);
+
+ // Don't try to parse all the Python syntax, just skip the block
+ // until the corresponding outdent token.
+ int depth = 1;
+ while (depth > 0) {
+ // Because of the way the lexer works, this should never happen
+ Preconditions.checkState(token.kind != TokenKind.EOF);
+
+ if (token.kind == TokenKind.INDENT) {
+ depth++;
+ }
+ if (token.kind == TokenKind.OUTDENT) {
+ depth--;
+ }
+ nextToken();
+ }
+
+ } else {
+ // the block ends at the newline token
+ // e.g. if x == 3: print "three"
+ syncTo(STATEMENT_TERMINATOR_SET);
+ }
+ }
+
+ // stmt ::= simple_stmt
+ // | compound_stmt
+ private void parseStatement(List<Statement> list, boolean isTopLevel) {
+ if (token.kind == TokenKind.DEF && skylarkMode) {
+ if (!isTopLevel) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "nested functions are not allowed. Move the function to top-level");
+ }
+ parseFunctionDefStatement(list);
+ } else if (token.kind == TokenKind.IF && skylarkMode) {
+ parseIfStatement(list);
+ } else if (token.kind == TokenKind.FOR && skylarkMode) {
+ if (isTopLevel) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "for loops are not allowed on top-level. Put it into a function");
+ }
+ parseForStatement(list);
+ } else if (token.kind == TokenKind.IF
+ || token.kind == TokenKind.ELSE
+ || token.kind == TokenKind.FOR
+ || token.kind == TokenKind.CLASS
+ || token.kind == TokenKind.DEF
+ || token.kind == TokenKind.TRY) {
+ skipBlock();
+ } else {
+ parseSimpleStatement(list);
+ }
+ }
+
+ // return_stmt ::= RETURN expr
+ private ReturnStatement parseReturnStatement() {
+ int start = token.left;
+ expect(TokenKind.RETURN);
+ Expression expression = parseExpression();
+ return setLocation(new ReturnStatement(expression), start, expression);
+ }
+
+ // block ::= ('if' | 'for' | 'class') expr ':' suite
+ private void skipBlock() {
+ int start = token.left;
+ Token blockToken = token;
+ syncTo(EnumSet.of(TokenKind.COLON, TokenKind.EOF)); // skip over expression or name
+ if (!parsePython) {
+ reportError(lexer.createLocation(start, token.right), "syntax error at '"
+ + blockToken + "': This Python-style construct is not supported. "
+ + PREPROCESSING_NEEDED);
+ }
+ expect(TokenKind.COLON);
+ skipSuite();
+ }
+
+ // create a comment node
+ private void makeComment(Token token) {
+ comments.add(setLocation(new Comment((String) token.value), token.left, token.right));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java b/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java
new file mode 100644
index 0000000..488c762
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java
@@ -0,0 +1,112 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.hash.HashCode;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An abstraction for reading input from a file or taking it as a pre-cooked
+ * char[] or String.
+ */
+public abstract class ParserInputSource {
+
+ protected ParserInputSource() {}
+
+ /**
+ * Returns the content of the input source.
+ */
+ public abstract char [] getContent();
+
+ /**
+ * Returns the path of the input source. Note: Once constructed, this object
+ * will never re-read the content from path.
+ */
+ public abstract Path getPath();
+
+ /**
+ * Create an input source instance by (eagerly) reading from the file at
+ * path. The file is assumed to be ISO-8859-1 encoded and smaller than
+ * 2 Gigs - these assumptions are reasonable for BUILD files, which is
+ * all we care about here.
+ */
+ public static ParserInputSource create(Path path) throws IOException {
+ char[] content = FileSystemUtils.readContentAsLatin1(path);
+ if (path.getFileSize() > content.length) {
+ // This assertion is to help diagnose problems arising from the
+ // filesystem; see bugs and #859334 and #920195.
+ throw new IOException("Unexpected short read from file '" + path
+ + "' (expected " + path.getFileSize() + ", got " + content.length + " bytes)");
+ }
+ return create(content, path);
+ }
+
+ /**
+ * Create an input source from the given content, and associate path with
+ * this source. Path will be used in error messages etc. but we will *never*
+ * attempt to read the content from path.
+ */
+ public static ParserInputSource create(String content, Path path) {
+ return create(content.toCharArray(), path);
+ }
+
+ /**
+ * Create an input source from the given content, and associate path with
+ * this source. Path will be used in error messages etc. but we will *never*
+ * attempt to read the content from path.
+ */
+ public static ParserInputSource create(final char[] content, final Path path) {
+ return new ParserInputSource() {
+
+ @Override
+ public char[] getContent() {
+ return content;
+ }
+
+ @Override
+ public Path getPath() {
+ return path;
+ }
+ };
+ }
+
+ /**
+ * Create an input source from the given input stream, and associate path
+ * with this source. 'path' will be used in error messages, etc, but will
+ * not (in general) be used to to read the content from path.
+ *
+ * (The exception is the case in which Python pre-processing is required; the
+ * path will be used to provide the input to the Python pre-processor.
+ * Arguably, we should just send the content as input to the subprocess
+ * instead of using the path, but it's not clear it's worth the effort.)
+ */
+ public static ParserInputSource create(InputStream in, Path path) throws IOException {
+ try {
+ return create(new String(FileSystemUtils.readContentAsLatin1(in)), path);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Returns a hash code calculated from the string content of this file.
+ */
+ public String contentHashCode() throws IOException {
+ return HashCode.fromBytes(getPath().getMD5Digest()).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java
new file mode 100644
index 0000000..07032c2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+/**
+ * A wrapper Statement class for return expressions.
+ */
+public class ReturnStatement extends Statement {
+
+ /**
+ * Exception sent by the return statement, to be caught by the function body.
+ */
+ public class ReturnException extends EvalException {
+ Object value;
+
+ public ReturnException(Location location, Object value) {
+ super(location, "Return statements must be inside a function");
+ this.value = value;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+ }
+
+ private final Expression returnExpression;
+
+ public ReturnStatement(Expression returnExpression) {
+ this.returnExpression = returnExpression;
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ throw new ReturnException(returnExpression.getLocation(), returnExpression.eval(env));
+ }
+
+ Expression getReturnExpression() {
+ return returnExpression;
+ }
+
+ @Override
+ public String toString() {
+ return "return " + returnExpression;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): save the return type in the environment, to type-check functions.
+ SkylarkFunctionType fct = env.getCurrentFunction();
+ if (fct == null) {
+ throw new EvalException(getLocation(), "Return statements must be inside a function");
+ }
+ SkylarkType resultType = returnExpression.validate(env);
+ fct.setReturnType(resultType, getLocation());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java b/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java
new file mode 100644
index 0000000..4fb3bdb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.Map;
+
+/**
+ * The value passed to a select({...}) statement, e.g.:
+ *
+ * <pre>
+ * rule(
+ * name = 'myrule',
+ * deps = select({
+ * 'a': [':adep'],
+ * 'b': [':bdep'],
+ * })
+ * </pre>
+ */
+public final class SelectorValue {
+ Map<?, ?> dictionary;
+
+ public SelectorValue(Map<?, ?> dictionary) {
+ this.dictionary = dictionary;
+ }
+
+ public Map<?, ?> getDictionary() {
+ return dictionary;
+ }
+
+ @Override
+ public String toString() {
+ return "selector({...})";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java
new file mode 100644
index 0000000..a2f0d1b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * An annotation to mark built-in keyword argument methods accessible from Skylark.
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SkylarkBuiltin {
+
+ String name();
+
+ String doc();
+
+ Param[] mandatoryParams() default {};
+
+ Param[] optionalParams() default {};
+
+ boolean hidden() default false;
+
+ Class<?> objectType() default Object.class;
+
+ Class<?> returnType() default Object.class;
+
+ boolean onlyLoadingPhase() default false;
+
+ /**
+ * An annotation for parameters of Skylark built-in functions.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Param {
+
+ String name();
+
+ String doc();
+
+ Class<?> type() default Object.class;
+
+ Class<?> generic1() default Object.class;
+
+ boolean callbackEnabled() default false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java
new file mode 100644
index 0000000..ae6987f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * A marker interface for Java methods which can be called from Skylark.
+ */
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SkylarkCallable {
+ String name() default "";
+
+ String doc();
+
+ boolean hidden() default false;
+
+ boolean structField() default false;
+
+ boolean allowReturnNones() default false;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java
new file mode 100644
index 0000000..2e94be8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+
+
+/**
+ * A helper class for calling Skylark functions from Java.
+ */
+public class SkylarkCallbackFunction {
+
+ private final UserDefinedFunction callback;
+ private final FuncallExpression ast;
+ private final SkylarkEnvironment funcallEnv;
+
+ public SkylarkCallbackFunction(UserDefinedFunction callback, FuncallExpression ast,
+ SkylarkEnvironment funcallEnv) {
+ this.callback = callback;
+ this.ast = ast;
+ this.funcallEnv = funcallEnv;
+ }
+
+ public Object call(ClassObject ctx, Object... arguments) throws EvalException {
+ try {
+ return callback.call(
+ ImmutableList.<Object>builder().add(ctx).add(arguments).build(), null, ast, funcallEnv);
+ } catch (InterruptedException | ClassCastException
+ | IllegalArgumentException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java
new file mode 100644
index 0000000..7e6f414
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java
@@ -0,0 +1,253 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * The environment for Skylark.
+ */
+public class SkylarkEnvironment extends Environment {
+
+ /**
+ * This set contains the variable names of all the successful lookups from the global
+ * environment. This is necessary because if in a function definition something
+ * reads a global variable after which a local variable with the same name is assigned an
+ * Exception needs to be thrown.
+ */
+ private final Set<String> readGlobalVariables = new HashSet<>();
+
+ private ImmutableList<String> stackTrace;
+
+ @Nullable private String fileContentHashCode;
+
+ /**
+ * Creates a Skylark Environment for function calling, from the global Environment of the
+ * caller Environment (which must be a Skylark Environment).
+ */
+ public static SkylarkEnvironment createEnvironmentForFunctionCalling(
+ Environment callerEnv, SkylarkEnvironment definitionEnv,
+ UserDefinedFunction function) throws EvalException {
+ if (callerEnv.getStackTrace().contains(function.getName())) {
+ throw new EvalException(function.getLocation(), "Recursion was detected when calling '"
+ + function.getName() + "' from '" + Iterables.getLast(callerEnv.getStackTrace()) + "'");
+ }
+ ImmutableList<String> stackTrace = new ImmutableList.Builder<String>()
+ .addAll(callerEnv.getStackTrace())
+ .add(function.getName())
+ .build();
+ SkylarkEnvironment childEnv =
+ // Always use the caller Environment's EventHandler. We cannot assume that the
+ // definition Environment's EventHandler is still working properly.
+ new SkylarkEnvironment(definitionEnv, stackTrace, callerEnv.eventHandler);
+ try {
+ for (String varname : callerEnv.propagatingVariables) {
+ childEnv.updateAndPropagate(varname, callerEnv.lookup(varname));
+ }
+ } catch (NoSuchVariableException e) {
+ // This should never happen.
+ throw new IllegalStateException(e);
+ }
+ childEnv.disabledVariables = callerEnv.disabledVariables;
+ childEnv.disabledNameSpaces = callerEnv.disabledNameSpaces;
+ return childEnv;
+ }
+
+ private SkylarkEnvironment(SkylarkEnvironment definitionEnv, ImmutableList<String> stackTrace,
+ EventHandler eventHandler) {
+ super(definitionEnv.getGlobalEnvironment());
+ this.stackTrace = stackTrace;
+ this.eventHandler = Preconditions.checkNotNull(eventHandler,
+ "EventHandler cannot be null in an Environment which calls into Skylark");
+ }
+
+ /**
+ * Creates a global SkylarkEnvironment.
+ */
+ public SkylarkEnvironment(EventHandler eventHandler, String astFileContentHashCode) {
+ super();
+ stackTrace = ImmutableList.of();
+ this.eventHandler = eventHandler;
+ this.fileContentHashCode = astFileContentHashCode;
+ }
+
+ @VisibleForTesting
+ public SkylarkEnvironment(EventHandler eventHandler) {
+ this(eventHandler, null);
+ }
+
+ public SkylarkEnvironment(SkylarkEnvironment globalEnv) {
+ super(globalEnv);
+ stackTrace = ImmutableList.of();
+ this.eventHandler = globalEnv.eventHandler;
+ }
+
+ @Override
+ public ImmutableList<String> getStackTrace() {
+ return stackTrace;
+ }
+
+ /**
+ * Clones this Skylark global environment.
+ */
+ public SkylarkEnvironment cloneEnv(EventHandler eventHandler) {
+ Preconditions.checkArgument(isGlobalEnvironment());
+ SkylarkEnvironment newEnv = new SkylarkEnvironment(eventHandler, this.fileContentHashCode);
+ for (Entry<String, Object> entry : env.entrySet()) {
+ newEnv.env.put(entry.getKey(), entry.getValue());
+ }
+ for (Map.Entry<Class<?>, Map<String, Function>> functionMap : functions.entrySet()) {
+ newEnv.functions.put(functionMap.getKey(), functionMap.getValue());
+ }
+ return newEnv;
+ }
+
+ /**
+ * Returns the global environment. Only works for Skylark environments. For the global Skylark
+ * environment this method returns this Environment.
+ */
+ public SkylarkEnvironment getGlobalEnvironment() {
+ // If there's a parent that's the global environment, otherwise this is.
+ return parent != null ? (SkylarkEnvironment) parent : this;
+ }
+
+ /**
+ * Returns true if this is a Skylark global environment.
+ */
+ public boolean isGlobalEnvironment() {
+ return parent == null;
+ }
+
+ /**
+ * Returns true if varname has been read as a global variable.
+ */
+ public boolean hasBeenReadGlobalVariable(String varname) {
+ return readGlobalVariables.contains(varname);
+ }
+
+ @Override
+ public boolean isSkylarkEnabled() {
+ return true;
+ }
+
+ /**
+ * @return the value from the environment whose name is "varname".
+ * @throws NoSuchVariableException if the variable is not defined in the environment.
+ */
+ @Override
+ public Object lookup(String varname) throws NoSuchVariableException {
+ if (disabledVariables.contains(varname)) {
+ throw new NoSuchVariableException(varname);
+ }
+ Object value = env.get(varname);
+ if (value == null) {
+ if (parent != null && parent.hasVariable(varname)) {
+ readGlobalVariables.add(varname);
+ return parent.lookup(varname);
+ }
+ throw new NoSuchVariableException(varname);
+ }
+ return value;
+ }
+
+ /**
+ * Like <code>lookup(String)</code>, but instead of throwing an exception in
+ * the case where "varname" is not defined, "defaultValue" is returned instead.
+ */
+ @Override
+ public Object lookup(String varname, Object defaultValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Updates the value of variable "varname" in the environment, corresponding
+ * to an AssignmentStatement.
+ */
+ @Override
+ public void update(String varname, Object value) {
+ Preconditions.checkNotNull(value, "update(value == null)");
+ env.put(varname, value);
+ }
+
+ /**
+ * Returns the class of the variable or null if the variable does not exist. This function
+ * works only in the local Environment, it doesn't check the global Environment.
+ */
+ public Class<?> getVariableType(String varname) {
+ Object variable = env.get(varname);
+ return variable != null ? EvalUtils.getSkylarkType(variable.getClass()) : null;
+ }
+
+ /**
+ * Removes the functions and the modules (i.e. the symbol of the module from the top level
+ * Environment and the functions attached to it) from the Environment which should be present
+ * only during the loading phase.
+ */
+ public void disableOnlyLoadingPhaseObjects() {
+ List<String> objectsToRemove = new ArrayList<>();
+ List<Class<?>> modulesToRemove = new ArrayList<>();
+ for (Map.Entry<String, Object> entry : env.entrySet()) {
+ Object object = entry.getValue();
+ if (object instanceof SkylarkFunction) {
+ if (((SkylarkFunction) object).isOnlyLoadingPhase()) {
+ objectsToRemove.add(entry.getKey());
+ }
+ } else if (object.getClass().isAnnotationPresent(SkylarkModule.class)) {
+ if (object.getClass().getAnnotation(SkylarkModule.class).onlyLoadingPhase()) {
+ objectsToRemove.add(entry.getKey());
+ modulesToRemove.add(entry.getValue().getClass());
+ }
+ }
+ }
+ for (String symbol : objectsToRemove) {
+ disabledVariables.add(symbol);
+ }
+ for (Class<?> moduleClass : modulesToRemove) {
+ disabledNameSpaces.add(moduleClass);
+ }
+ }
+
+ public void handleEvent(Event event) {
+ eventHandler.handle(event);
+ }
+
+ /**
+ * Returns a hash code calculated from the hash code of this Environment and the
+ * transitive closure of other Environments it loads.
+ */
+ public String getTransitiveFileContentHashCode() {
+ Fingerprint fingerprint = new Fingerprint();
+ fingerprint.addString(Preconditions.checkNotNull(fileContentHashCode));
+ // Calculate a new hash from the hash of the loaded Environments.
+ for (SkylarkEnvironment env : importedExtensions.values()) {
+ fingerprint.addString(env.getTransitiveFileContentHashCode());
+ }
+ return fingerprint.hexDigestAndReset();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java
new file mode 100644
index 0000000..bd2cc83
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java
@@ -0,0 +1,317 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.EvalException.EvalExceptionWithJavaCause;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * A function class for Skylark built in functions. Supports mandatory and optional arguments.
+ * All usable arguments have to be specified. In case of ambiguous arguments (a parameter is
+ * specified as positional and keyword arguments in the function call) an exception is thrown.
+ */
+public abstract class SkylarkFunction extends AbstractFunction {
+
+ private ImmutableList<String> parameters;
+ private ImmutableMap<String, SkylarkBuiltin.Param> parameterTypes;
+ private int mandatoryParamNum;
+ private boolean configured = false;
+ private Class<?> objectType;
+ private boolean onlyLoadingPhase;
+
+ /**
+ * Creates a SkylarkFunction with the given name.
+ */
+ public SkylarkFunction(String name) {
+ super(name);
+ }
+
+ /**
+ * Configures the parameter of this Skylark function using the annotation.
+ */
+ @VisibleForTesting
+ public void configure(SkylarkBuiltin annotation) {
+ Preconditions.checkState(!configured);
+ Preconditions.checkArgument(getName().equals(annotation.name()),
+ getName() + " != " + annotation.name());
+ mandatoryParamNum = 0;
+ ImmutableList.Builder<String> paramListBuilder = ImmutableList.builder();
+ ImmutableMap.Builder<String, SkylarkBuiltin.Param> paramTypeBuilder = ImmutableMap.builder();
+ for (SkylarkBuiltin.Param param : annotation.mandatoryParams()) {
+ paramListBuilder.add(param.name());
+ paramTypeBuilder.put(param.name(), param);
+ mandatoryParamNum++;
+ }
+ for (SkylarkBuiltin.Param param : annotation.optionalParams()) {
+ paramListBuilder.add(param.name());
+ paramTypeBuilder.put(param.name(), param);
+ }
+ parameters = paramListBuilder.build();
+ parameterTypes = paramTypeBuilder.build();
+ this.objectType = annotation.objectType().equals(Object.class) ? null : annotation.objectType();
+ this.onlyLoadingPhase = annotation.onlyLoadingPhase();
+ configured = true;
+ }
+
+ /**
+ * Returns true if the SkylarkFunction is configured.
+ */
+ public boolean isConfigured() {
+ return configured;
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return objectType;
+ }
+
+ public boolean isOnlyLoadingPhase() {
+ return onlyLoadingPhase;
+ }
+
+ @Override
+ public Object call(List<Object> args,
+ Map<String, Object> kwargs,
+ FuncallExpression ast,
+ Environment env)
+ throws EvalException, InterruptedException {
+
+ Preconditions.checkState(configured, "Function " + getName() + " was not configured");
+ try {
+ ImmutableMap.Builder<String, Object> arguments = new ImmutableMap.Builder<>();
+ if (objectType != null && !FuncallExpression.isNamespace(objectType)) {
+ arguments.put("self", args.remove(0));
+ }
+
+ int maxParamNum = parameters.size();
+ int paramNum = args.size() + kwargs.size();
+
+ if (paramNum < mandatoryParamNum) {
+ throw new EvalException(ast.getLocation(),
+ String.format("incorrect number of arguments (got %s, expected at least %s)",
+ paramNum, mandatoryParamNum));
+ } else if (paramNum > maxParamNum) {
+ throw new EvalException(ast.getLocation(),
+ String.format("incorrect number of arguments (got %s, expected at most %s)",
+ paramNum, maxParamNum));
+ }
+
+ for (int i = 0; i < mandatoryParamNum; i++) {
+ Preconditions.checkState(i < args.size() || kwargs.containsKey(parameters.get(i)),
+ String.format("missing mandatory parameter: %s", parameters.get(i)));
+ }
+
+ for (int i = 0; i < args.size(); i++) {
+ checkTypeAndAddArg(parameters.get(i), args.get(i), arguments, ast.getLocation());
+ }
+
+ for (Entry<String, Object> kwarg : kwargs.entrySet()) {
+ int idx = parameters.indexOf(kwarg.getKey());
+ if (idx < 0) {
+ throw new EvalException(ast.getLocation(),
+ String.format("unknown keyword argument: %s", kwarg.getKey()));
+ }
+ if (idx < args.size()) {
+ throw new EvalException(ast.getLocation(),
+ String.format("ambiguous argument: %s", kwarg.getKey()));
+ }
+ checkTypeAndAddArg(kwarg.getKey(), kwarg.getValue(), arguments, ast.getLocation());
+ }
+
+ return call(arguments.build(), ast, env);
+ } catch (ConversionException | IllegalArgumentException | IllegalStateException
+ | ClassCastException | ClassNotFoundException | ExecutionException e) {
+ if (e.getMessage() != null) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ } else {
+ // TODO(bazel-team): ideally this shouldn't happen, however we need this for debugging
+ throw new EvalExceptionWithJavaCause(ast.getLocation(), e);
+ }
+ }
+ }
+
+ private void checkTypeAndAddArg(String paramName, Object value,
+ ImmutableMap.Builder<String, Object> arguments, Location loc) throws EvalException {
+ SkylarkBuiltin.Param param = parameterTypes.get(paramName);
+ if (param.callbackEnabled() && Function.class.isAssignableFrom(value.getClass())) {
+ // If we pass a function as an argument we trust the Function implementation with the type
+ // check. It's OK since the function needs to be called manually anyway.
+ arguments.put(paramName, value);
+ return;
+ }
+ if (!(param.type().isAssignableFrom(value.getClass()))) {
+ throw new EvalException(loc, String.format("expected %s for '%s' but got %s instead\n"
+ + "%s.%s: %s",
+ EvalUtils.getDataTypeNameFromClass(param.type()), paramName,
+ EvalUtils.getDatatypeName(value), getName(), paramName, param.doc()));
+ }
+ if (param.type().equals(SkylarkList.class)) {
+ checkGeneric(paramName, param, value, ((SkylarkList) value).getGenericType(), loc);
+ } else if (param.type().equals(SkylarkNestedSet.class)) {
+ checkGeneric(paramName, param, value, ((SkylarkNestedSet) value).getGenericType(), loc);
+ }
+ arguments.put(paramName, value);
+ }
+
+ private void checkGeneric(String paramName, SkylarkBuiltin.Param param, Object value,
+ Class<?> genericType, Location loc) throws EvalException {
+ if (!genericType.equals(Object.class) && !param.generic1().isAssignableFrom(genericType)) {
+ String mainType = EvalUtils.getDataTypeNameFromClass(param.type());
+ throw new EvalException(loc, String.format(
+ "expected %s of %ss for '%s' but got %s of %ss instead\n%s.%s: %s",
+ mainType, EvalUtils.getDataTypeNameFromClass(param.generic1()),
+ paramName,
+ EvalUtils.getDatatypeName(value), EvalUtils.getDataTypeNameFromClass(genericType),
+ getName(), paramName, param.doc()));
+ }
+ }
+
+ /**
+ * The actual function call. All positional and keyword arguments are put in the
+ * arguments map.
+ */
+ protected abstract Object call(
+ Map<String, Object> arguments, FuncallExpression ast, Environment env) throws EvalException,
+ ConversionException,
+ IllegalArgumentException,
+ IllegalStateException,
+ ClassCastException,
+ ClassNotFoundException,
+ ExecutionException;
+
+ /**
+ * An intermediate class to provide a simpler interface for Skylark functions.
+ */
+ public abstract static class SimpleSkylarkFunction extends SkylarkFunction {
+
+ public SimpleSkylarkFunction(String name) {
+ super(name);
+ }
+
+ @Override
+ protected final Object call(
+ Map<String, Object> arguments, FuncallExpression ast, Environment env) throws EvalException,
+ ConversionException,
+ IllegalArgumentException,
+ IllegalStateException,
+ ClassCastException,
+ ExecutionException {
+ return call(arguments, ast.getLocation());
+ }
+
+ /**
+ * The actual function call. All positional and keyword arguments are put in the
+ * arguments map.
+ */
+ protected abstract Object call(Map<String, Object> arguments, Location loc)
+ throws EvalException,
+ ConversionException,
+ IllegalArgumentException,
+ IllegalStateException,
+ ClassCastException,
+ ExecutionException;
+ }
+
+ public static <TYPE> Iterable<TYPE> castList(Object obj, final Class<TYPE> type) {
+ if (obj == null) {
+ return ImmutableList.of();
+ }
+ return ((SkylarkList) obj).to(type);
+ }
+
+ public static <TYPE> Iterable<TYPE> castList(
+ Object obj, final Class<TYPE> type, final String what) throws ConversionException {
+ if (obj == null) {
+ return ImmutableList.of();
+ }
+ return Iterables.transform(Type.LIST.convert(obj, what),
+ new com.google.common.base.Function<Object, TYPE>() {
+ @Override
+ public TYPE apply(Object input) {
+ try {
+ return type.cast(input);
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(String.format(
+ "expected %s type for '%s' but got %s instead",
+ EvalUtils.getDataTypeNameFromClass(type), what,
+ EvalUtils.getDatatypeName(input)));
+ }
+ }
+ });
+ }
+
+ public static <KEY_TYPE, VALUE_TYPE> ImmutableMap<KEY_TYPE, VALUE_TYPE> toMap(
+ Iterable<Map.Entry<KEY_TYPE, VALUE_TYPE>> obj) {
+ ImmutableMap.Builder<KEY_TYPE, VALUE_TYPE> builder = ImmutableMap.builder();
+ for (Map.Entry<KEY_TYPE, VALUE_TYPE> entry : obj) {
+ builder.put(entry.getKey(), entry.getValue());
+ }
+ return builder.build();
+ }
+
+ public static <KEY_TYPE, VALUE_TYPE> Iterable<Map.Entry<KEY_TYPE, VALUE_TYPE>> castMap(Object obj,
+ final Class<KEY_TYPE> keyType, final Class<VALUE_TYPE> valueType, final String what) {
+ if (obj == null) {
+ return ImmutableList.of();
+ }
+ if (!(obj instanceof Map<?, ?>)) {
+ throw new IllegalArgumentException(String.format(
+ "expected a dictionary for %s but got %s instead",
+ what, EvalUtils.getDatatypeName(obj)));
+ }
+ return Iterables.transform(((Map<?, ?>) obj).entrySet(),
+ new com.google.common.base.Function<Map.Entry<?, ?>, Map.Entry<KEY_TYPE, VALUE_TYPE>>() {
+ // This is safe. We check the type of the key-value pairs for every entry in the Map.
+ // In Map.Entry the key always has the type of the first generic parameter, the
+ // value has the second.
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map.Entry<KEY_TYPE, VALUE_TYPE> apply(Map.Entry<?, ?> input) {
+ if (keyType.isAssignableFrom(input.getKey().getClass())
+ && valueType.isAssignableFrom(input.getValue().getClass())) {
+ return (Map.Entry<KEY_TYPE, VALUE_TYPE>) input;
+ }
+ throw new IllegalArgumentException(String.format(
+ "expected <%s, %s> type for '%s' but got <%s, %s> instead",
+ keyType.getSimpleName(), valueType.getSimpleName(), what,
+ EvalUtils.getDatatypeName(input.getKey()),
+ EvalUtils.getDatatypeName(input.getValue())));
+ }
+ });
+ }
+
+ // TODO(bazel-team): this is only used in SkylarkRuleConfgiuredTargetBuilder, fix typing for
+ // structs then remove this.
+ public static <TYPE> TYPE cast(Object elem, Class<TYPE> type, String what, Location loc)
+ throws EvalException {
+ try {
+ return type.cast(elem);
+ } catch (ClassCastException e) {
+ throw new EvalException(loc, String.format("expected %s for '%s' but got %s instead",
+ type.getSimpleName(), what, EvalUtils.getDatatypeName(elem)));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java
new file mode 100644
index 0000000..ef9fe10
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java
@@ -0,0 +1,373 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A class to handle lists and tuples in Skylark.
+ */
+@SkylarkModule(name = "list",
+ doc = "A language built-in type to support lists. Example of list literal:<br>"
+ + "<pre class=language-python>l = [1, 2, 3]</pre>"
+ + "Accessing elements is possible using indexing (starts from <code>0</code>):<br>"
+ + "<pre class=language-python>e = l[1] # e == 2</pre>"
+ + "Lists support the <code>+</code> operator to concatenate two lists. Example:<br>"
+ + "<pre class=language-python>l = [1, 2] + [3, 4] # l == [1, 2, 3, 4]\n"
+ + "l = [\"a\", \"b\"]\n"
+ + "l += [\"c\"] # l == [\"a\", \"b\", \"c\"]</pre>"
+ + "List elements have to be of the same type, <code>[1, 2, \"c\"]</code> results in an "
+ + "error. Lists - just like everything - are immutable, therefore <code>l[1] = \"a\""
+ + "</code> is not supported.")
+public abstract class SkylarkList implements Iterable<Object> {
+
+ private final boolean tuple;
+ private final Class<?> genericType;
+
+ private SkylarkList(boolean tuple, Class<?> genericType) {
+ this.tuple = tuple;
+ this.genericType = genericType;
+ }
+
+ /**
+ * The size of the list.
+ */
+ public abstract int size();
+
+ /**
+ * Returns true if the list is empty.
+ */
+ public abstract boolean isEmpty();
+
+ /**
+ * Returns the i-th element of the list.
+ */
+ public abstract Object get(int i);
+
+ /**
+ * Returns true if this list is a tuple.
+ */
+ public boolean isTuple() {
+ return tuple;
+ }
+
+ @VisibleForTesting
+ public Class<?> getGenericType() {
+ return genericType;
+ }
+
+ @Override
+ public String toString() {
+ return toList().toString();
+ }
+
+ // TODO(bazel-team): we should be very careful using this method. Check and remove
+ // auto conversions on the Java-Skylark interface if possible.
+ /**
+ * Converts this Skylark list to a Java list.
+ */
+ public abstract List<?> toList();
+
+ @SuppressWarnings("unchecked")
+ public <T> Iterable<T> to(Class<T> type) {
+ Preconditions.checkArgument(this == EMPTY_LIST || type.isAssignableFrom(genericType));
+ return (Iterable<T>) this;
+ }
+
+ private static final class EmptySkylarkList extends SkylarkList {
+ private EmptySkylarkList(boolean tuple) {
+ super(tuple, Object.class);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return ImmutableList.of().iterator();
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return true;
+ }
+
+ @Override
+ public Object get(int i) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<?> toList() {
+ return isTuple() ? ImmutableList.of() : Lists.newArrayList();
+ }
+
+ @Override
+ public String toString() {
+ return "[]";
+ }
+ }
+
+ /**
+ * An empty Skylark list.
+ */
+ public static final SkylarkList EMPTY_LIST = new EmptySkylarkList(false);
+
+ private static final class SimpleSkylarkList extends SkylarkList {
+ private final ImmutableList<Object> list;
+
+ private SimpleSkylarkList(ImmutableList<Object> list, boolean tuple, Class<?> genericType) {
+ super(tuple, genericType);
+ this.list = Preconditions.checkNotNull(list);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return list.iterator();
+ }
+
+ @Override
+ public int size() {
+ return list.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return list.isEmpty();
+ }
+
+ @Override
+ public Object get(int i) {
+ return list.get(i);
+ }
+
+ @Override
+ public List<?> toList() {
+ return isTuple() ? list : Lists.newArrayList(list);
+ }
+
+ @Override
+ public String toString() {
+ return list.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return list.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SimpleSkylarkList)) {
+ return false;
+ }
+ SimpleSkylarkList other = (SimpleSkylarkList) obj;
+ return other.list.equals(this.list);
+ }
+ }
+
+ /**
+ * A Skylark list to support lazy iteration (i.e. we only call iterator on the object this
+ * list masks when it's absolutely necessary). This is useful if iteration is expensive
+ * (e.g. NestedSet-s). Size(), get() and isEmpty() are expensive operations but
+ * concatenation is quick.
+ */
+ private static final class LazySkylarkList extends SkylarkList {
+ private final Iterable<Object> iterable;
+ private ImmutableList<Object> list = null;
+
+ private LazySkylarkList(Iterable<Object> iterable, boolean tuple, Class<?> genericType) {
+ super(tuple, genericType);
+ this.iterable = Preconditions.checkNotNull(iterable);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return iterable.iterator();
+ }
+
+ @Override
+ public int size() {
+ return Iterables.size(iterable);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return Iterables.isEmpty(iterable);
+ }
+
+ @Override
+ public Object get(int i) {
+ return getList().get(i);
+ }
+
+ @Override
+ public List<?> toList() {
+ return getList();
+ }
+
+ private ImmutableList<Object> getList() {
+ if (list == null) {
+ list = ImmutableList.copyOf(iterable);
+ }
+ return list;
+ }
+ }
+
+ /**
+ * A Skylark list to support quick concatenation of lists. Concatenation is O(1),
+ * size(), isEmpty() is O(n), get() is O(h).
+ */
+ private static final class ConcatenatedSkylarkList extends SkylarkList {
+ private final SkylarkList left;
+ private final SkylarkList right;
+
+ private ConcatenatedSkylarkList(
+ SkylarkList left, SkylarkList right, boolean tuple, Class<?> genericType) {
+ super(tuple, genericType);
+ this.left = Preconditions.checkNotNull(left);
+ this.right = Preconditions.checkNotNull(right);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return Iterables.concat(left, right).iterator();
+ }
+
+ @Override
+ public int size() {
+ // We shouldn't evaluate the size function until it's necessary, because it can be expensive
+ // for lazy lists (e.g. lists containing a NestedSet).
+ // TODO(bazel-team): make this class more clever to store the size and empty parameters
+ // for every non-LazySkylarkList member.
+ return left.size() + right.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return left.isEmpty() && right.isEmpty();
+ }
+
+ @Override
+ public Object get(int i) {
+ int leftSize = left.size();
+ if (i < leftSize) {
+ return left.get(i);
+ } else {
+ return right.get(i - leftSize);
+ }
+ }
+
+ @Override
+ public List<?> toList() {
+ return ImmutableList.<Object>builder().addAll(left).addAll(right).build();
+ }
+ }
+
+ /**
+ * Returns a Skylark list containing elements without a type check. Only use if all elements
+ * are of the same type.
+ */
+ public static SkylarkList list(Collection<?> elements, Class<?> genericType) {
+ if (elements.isEmpty()) {
+ return EMPTY_LIST;
+ }
+ return new SimpleSkylarkList(ImmutableList.copyOf(elements), false, genericType);
+ }
+
+ /**
+ * Returns a Skylark list containing elements without a type check and without creating
+ * an immutable copy. Therefore the iterable containing elements must be immutable
+ * (which is not checked here so callers must be extra careful). This way
+ * it's possibly to create a SkylarkList without requesting the original iterator. This
+ * can be useful for nested set - list conversions.
+ */
+ @SuppressWarnings("unchecked")
+ public static SkylarkList lazyList(Iterable<?> elements, Class<?> genericType) {
+ return new LazySkylarkList((Iterable<Object>) elements, false, genericType);
+ }
+
+ /**
+ * Returns a Skylark list containing elements. Performs type check and throws an exception
+ * in case the list contains elements of different type.
+ */
+ public static SkylarkList list(Collection<?> elements, Location loc) throws EvalException {
+ if (elements.isEmpty()) {
+ return EMPTY_LIST;
+ }
+ return new SimpleSkylarkList(
+ ImmutableList.copyOf(elements), false, getGenericType(elements, loc));
+ }
+
+ private static Class<?> getGenericType(Collection<?> elements, Location loc)
+ throws EvalException {
+ Class<?> genericType = elements.iterator().next().getClass();
+ for (Object element : elements) {
+ Class<?> type = element.getClass();
+ if (!EvalUtils.getSkylarkType(genericType).equals(EvalUtils.getSkylarkType(type))) {
+ throw new EvalException(loc, String.format(
+ "Incompatible types in list: found a %s but the first element is a %s",
+ EvalUtils.getDataTypeNameFromClass(type),
+ EvalUtils.getDataTypeNameFromClass(genericType)));
+ }
+ }
+ return genericType;
+ }
+
+ /**
+ * Returns a Skylark list created from Skylark lists left and right. Throws an exception
+ * if they are not of the same generic type.
+ */
+ public static SkylarkList concat(SkylarkList left, SkylarkList right, Location loc)
+ throws EvalException {
+ if (left.isTuple() != right.isTuple()) {
+ throw new EvalException(loc, "cannot concatenate lists and tuples");
+ }
+ if (left == EMPTY_LIST) {
+ return right;
+ }
+ if (right == EMPTY_LIST) {
+ return left;
+ }
+ if (!left.genericType.equals(right.genericType)) {
+ throw new EvalException(loc, String.format("cannot concatenate list of %s with list of %s",
+ EvalUtils.getDataTypeNameFromClass(left.genericType),
+ EvalUtils.getDataTypeNameFromClass(right.genericType)));
+ }
+ return new ConcatenatedSkylarkList(left, right, left.isTuple(), left.genericType);
+ }
+
+ /**
+ * Returns a Skylark tuple containing elements.
+ */
+ public static SkylarkList tuple(List<?> elements) {
+ // Tuple elements do not have to have the same type.
+ return new SimpleSkylarkList(ImmutableList.copyOf(elements), true, Object.class);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java
new file mode 100644
index 0000000..96421b2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation to mark Skylark modules or Skylark accessible Java data types.
+ * A Skylark modules always corresponds to exactly one Java class.
+ */
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SkylarkModule {
+
+ String name();
+
+ String doc();
+
+ boolean hidden() default false;
+
+ boolean namespace() default false;
+
+ boolean onlyLoadingPhase() default false;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java
new file mode 100644
index 0000000..17fc55f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java
@@ -0,0 +1,193 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A generic type safe NestedSet wrapper for Skylark.
+ */
+@SkylarkModule(name = "set",
+ doc = "A language built-in type to supports (nested) sets. "
+ + "Sets can be created using the global <code>set</code> function, and they "
+ + "support the <code>+</code> operator to extends and nest sets. Examples:<br>"
+ + "<pre class=language-python>s = set([1, 2])\n"
+ + "s += [3] # s == {1, 2, 3}\n"
+ + "s += set([4, 5]) # s == {1, 2, 3, {4, 5}}</pre>"
+ + "Note that in these examples <code>{..}</code> is not a valid literal to create sets. "
+ + "Sets have a fixed generic type, so <code>set([1]) + [\"a\"]</code> or "
+ + "<code>set([1]) + set([\"a\"])</code> results in an error.")
+@Immutable
+public final class SkylarkNestedSet implements Iterable<Object> {
+
+ private final Class<?> genericType;
+ @Nullable private final List<Object> items;
+ @Nullable private final List<NestedSet<Object>> transitiveItems;
+ private final NestedSet<?> set;
+
+ public SkylarkNestedSet(Order order, Object item, Location loc) throws EvalException {
+ this(order, Object.class, item, loc, new ArrayList<Object>(),
+ new ArrayList<NestedSet<Object>>());
+ }
+
+ public SkylarkNestedSet(SkylarkNestedSet left, Object right, Location loc) throws EvalException {
+ this(left.set.getOrder(), left.genericType, right, loc,
+ new ArrayList<Object>(checkItems(left.items, loc)),
+ new ArrayList<NestedSet<Object>>(checkItems(left.transitiveItems, loc)));
+ }
+
+ private static <T> T checkItems(T items, Location loc) throws EvalException {
+ // SkylarkNestedSets created directly from ordinary NestedSets (those were created in a
+ // native rule) don't have directly accessible items and transitiveItems, so we cannot
+ // add more elements to them.
+ if (items == null) {
+ throw new EvalException(loc, "Cannot add more elements to this set. Sets created in "
+ + "native rules cannot be left side operands of the + operator.");
+ }
+ return items;
+ }
+
+ // This is safe because of the type checking
+ @SuppressWarnings("unchecked")
+ private SkylarkNestedSet(Order order, Class<?> genericType, Object item, Location loc,
+ List<Object> items, List<NestedSet<Object>> transitiveItems) throws EvalException {
+
+ // Adding the item
+ if (item instanceof SkylarkNestedSet) {
+ SkylarkNestedSet nestedSet = (SkylarkNestedSet) item;
+ if (!nestedSet.isEmpty()) {
+ genericType = checkType(genericType, nestedSet.genericType, loc);
+ transitiveItems.add((NestedSet<Object>) nestedSet.set);
+ }
+ } else if (item instanceof SkylarkList) {
+ // TODO(bazel-team): we should check ImmutableList here but it screws up genrule at line 43
+ for (Object object : (SkylarkList) item) {
+ genericType = checkType(genericType, object.getClass(), loc);
+ items.add(object);
+ }
+ } else {
+ throw new EvalException(loc,
+ String.format("cannot add '%s'-s to nested sets", EvalUtils.getDatatypeName(item)));
+ }
+ this.genericType = Preconditions.checkNotNull(genericType, "type cannot be null");
+
+ // Initializing the real nested set
+ NestedSetBuilder<Object> builder = new NestedSetBuilder<Object>(order);
+ builder.addAll(items);
+ try {
+ for (NestedSet<Object> nestedSet : transitiveItems) {
+ builder.addTransitive(nestedSet);
+ }
+ } catch (IllegalStateException e) {
+ throw new EvalException(loc, e.getMessage());
+ }
+ this.set = builder.build();
+ this.items = ImmutableList.copyOf(items);
+ this.transitiveItems = ImmutableList.copyOf(transitiveItems);
+ }
+
+ /**
+ * Returns a type safe SkylarkNestedSet. Use this instead of the constructor if possible.
+ */
+ public static <T> SkylarkNestedSet of(Class<T> genericType, NestedSet<T> set) {
+ return new SkylarkNestedSet(genericType, set);
+ }
+
+ /**
+ * A not type safe constructor for SkylarkNestedSet. It's discouraged to use it unless type
+ * generic safety is guaranteed from the caller side.
+ */
+ SkylarkNestedSet(Class<?> genericType, NestedSet<?> set) {
+ // This is here for the sake of FuncallExpression.
+ this.genericType = Preconditions.checkNotNull(genericType, "type cannot be null");
+ this.set = Preconditions.checkNotNull(set, "set cannot be null");
+ this.items = null;
+ this.transitiveItems = null;
+ }
+
+ private static Class<?> checkType(Class<?> builderType, Class<?> itemType, Location loc)
+ throws EvalException {
+ if (Map.class.isAssignableFrom(itemType) || SkylarkList.class.isAssignableFrom(itemType)
+ || ClassObject.class.isAssignableFrom(itemType)) {
+ throw new EvalException(loc, String.format("nested set item is composite (type of %s)",
+ EvalUtils.getDataTypeNameFromClass(itemType)));
+ }
+ if (!EvalUtils.isSkylarkImmutable(itemType)) {
+ throw new EvalException(loc, String.format("nested set item is not immutable (type of %s)",
+ EvalUtils.getDataTypeNameFromClass(itemType)));
+ }
+ if (builderType.equals(Object.class)) {
+ return itemType;
+ }
+ if (!EvalUtils.getSkylarkType(builderType).equals(EvalUtils.getSkylarkType(itemType))) {
+ throw new EvalException(loc, String.format(
+ "nested set item is type of %s but the nested set accepts only %s-s",
+ EvalUtils.getDataTypeNameFromClass(itemType),
+ EvalUtils.getDataTypeNameFromClass(builderType)));
+ }
+ return builderType;
+ }
+
+ /**
+ * Returns the NestedSet embedded in this SkylarkNestedSet if it is of the parameter type.
+ */
+ // The precondition ensures generic type safety
+ @SuppressWarnings("unchecked")
+ public <T> NestedSet<T> getSet(Class<T> type) {
+ // Empty sets don't need have to have a type since they don't have items
+ if (set.isEmpty()) {
+ return (NestedSet<T>) set;
+ }
+ Preconditions.checkArgument(type.isAssignableFrom(genericType),
+ String.format("Expected %s as a type but got %s",
+ EvalUtils.getDataTypeNameFromClass(type),
+ EvalUtils.getDataTypeNameFromClass(genericType)));
+ return (NestedSet<T>) set;
+ }
+
+ // For some reason this cast is unsafe in Java
+ @SuppressWarnings("unchecked")
+ @Override
+ public Iterator<Object> iterator() {
+ return (Iterator<Object>) set.iterator();
+ }
+
+ public Collection<Object> toCollection() {
+ return ImmutableList.copyOf(set.toCollection());
+ }
+
+ public boolean isEmpty() {
+ return set.isEmpty();
+ }
+
+ @VisibleForTesting
+ public Class<?> getGenericType() {
+ return genericType;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java
new file mode 100644
index 0000000..04c345f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java
@@ -0,0 +1,307 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.events.Location;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.lang.reflect.WildcardType;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A class representing types available in Skylark.
+ */
+public class SkylarkType {
+
+ private static final class Global {}
+
+ public static final SkylarkType UNKNOWN = new SkylarkType(Object.class);
+ public static final SkylarkType NONE = new SkylarkType(Environment.NoneType.class);
+ public static final SkylarkType GLOBAL = new SkylarkType(Global.class);
+
+ public static final SkylarkType STRING = new SkylarkType(String.class);
+ public static final SkylarkType INT = new SkylarkType(Integer.class);
+ public static final SkylarkType BOOL = new SkylarkType(Boolean.class);
+
+ private final Class<?> type;
+
+ // TODO(bazel-team): Change this to SkylarkType and check generics of generics etc.
+ // Object.class is used for UNKNOWN.
+ private Class<?> generic1;
+
+ public static SkylarkType of(Class<?> type, Class<?> generic1) {
+ return new SkylarkType(type, generic1);
+ }
+
+ public static SkylarkType of(Class<?> type) {
+ if (type.equals(Object.class)) {
+ return SkylarkType.UNKNOWN;
+ } else if (type.equals(String.class)) {
+ return SkylarkType.STRING;
+ } else if (type.equals(Integer.class)) {
+ return SkylarkType.INT;
+ } else if (type.equals(Boolean.class)) {
+ return SkylarkType.BOOL;
+ }
+ return new SkylarkType(type);
+ }
+
+ private SkylarkType(Class<?> type, Class<?> generic1) {
+ this.type = Preconditions.checkNotNull(type);
+ this.generic1 = Preconditions.checkNotNull(generic1);
+ }
+
+ private SkylarkType(Class<?> type) {
+ this.type = Preconditions.checkNotNull(type);
+ this.generic1 = Object.class;
+ }
+
+ public Class<?> getType() {
+ return type;
+ }
+
+ Class<?> getGenericType1() {
+ return generic1;
+ }
+
+ /**
+ * Returns the stronger type of this and o if they are compatible. Stronger means that
+ * the more information is available, e.g. STRING is stronger than UNKNOWN and
+ * LIST<STRING> is stronger than LIST<UNKNOWN>. Note than there's no type
+ * hierarchy in Skylark.
+ * <p>If they are not compatible an EvalException is thrown.
+ */
+ SkylarkType infer(SkylarkType o, String name, Location thisLoc, Location originalLoc)
+ throws EvalException {
+ if (this == o) {
+ return this;
+ }
+ if (this == UNKNOWN || this.equals(SkylarkType.NONE)) {
+ return o;
+ }
+ if (o == UNKNOWN || o.equals(SkylarkType.NONE)) {
+ return this;
+ }
+ if (!type.equals(o.type)) {
+ throw new EvalException(thisLoc, String.format("bad %s: %s is incompatible with %s at %s",
+ name,
+ EvalUtils.getDataTypeNameFromClass(o.getType()),
+ EvalUtils.getDataTypeNameFromClass(this.getType()),
+ originalLoc));
+ }
+ if (generic1.equals(Object.class)) {
+ return o;
+ }
+ if (o.generic1.equals(Object.class)) {
+ return this;
+ }
+ if (!generic1.equals(o.generic1)) {
+ throw new EvalException(thisLoc, String.format("bad %s: incompatible generic variable types "
+ + "%s with %s",
+ name,
+ EvalUtils.getDataTypeNameFromClass(o.generic1),
+ EvalUtils.getDataTypeNameFromClass(this.generic1)));
+ }
+ return this;
+ }
+
+ boolean isStruct() {
+ return type.equals(ClassObject.class);
+ }
+
+ boolean isList() {
+ return SkylarkList.class.isAssignableFrom(type);
+ }
+
+ boolean isDict() {
+ return Map.class.isAssignableFrom(type);
+ }
+
+ boolean isSet() {
+ return Set.class.isAssignableFrom(type);
+ }
+
+ boolean isNset() {
+ // TODO(bazel-team): NestedSets are going to be a bit strange with 2 type info (validation
+ // and execution time). That can be cleaned up once we have complete type inference.
+ return SkylarkNestedSet.class.isAssignableFrom(type);
+ }
+
+ boolean isSimple() {
+ return !isStruct() && !isDict() && !isList() && !isNset() && !isSet();
+ }
+
+ @Override
+ public String toString() {
+ return this == UNKNOWN ? "Unknown" : EvalUtils.getDataTypeNameFromClass(type);
+ }
+
+ // hashCode() and equals() only uses the type field
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof SkylarkType)) {
+ return false;
+ }
+ SkylarkType o = (SkylarkType) other;
+ return this.type.equals(o.type);
+ }
+
+ @Override
+ public int hashCode() {
+ return type.hashCode();
+ }
+
+ /**
+ * A class representing the type of a Skylark function.
+ */
+ public static final class SkylarkFunctionType extends SkylarkType {
+
+ private final String name;
+ @Nullable private SkylarkType returnType;
+ @Nullable private Location returnTypeLoc;
+
+ public static SkylarkFunctionType of(String name) {
+ return new SkylarkFunctionType(name, null);
+ }
+
+ public static SkylarkFunctionType of(String name, SkylarkType returnType) {
+ return new SkylarkFunctionType(name, returnType);
+ }
+
+ private SkylarkFunctionType(String name, SkylarkType returnType) {
+ super(Function.class);
+ this.name = name;
+ this.returnType = returnType;
+ }
+
+ public SkylarkType getReturnType() {
+ return returnType;
+ }
+
+ /**
+ * Sets the return type of the function type if it's compatible with the existing return type.
+ * Note that setting NONE only has an effect if the return type hasn't been set previously.
+ */
+ public void setReturnType(SkylarkType newReturnType, Location newLoc) throws EvalException {
+ if (returnType == null) {
+ returnType = newReturnType;
+ returnTypeLoc = newLoc;
+ } else if (newReturnType != SkylarkType.NONE) {
+ returnType =
+ returnType.infer(newReturnType, "return type of " + name, newLoc, returnTypeLoc);
+ if (returnType == newReturnType) {
+ returnTypeLoc = newLoc;
+ }
+ }
+ }
+ }
+
+ private static boolean isTypeAllowedInSkylark(Object object) {
+ if (object instanceof NestedSet<?>) {
+ return false;
+ } else if (object instanceof List<?>) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Throws EvalException if the type of the object is not allowed to be present in Skylark.
+ */
+ static void checkTypeAllowedInSkylark(Object object, Location loc) throws EvalException {
+ if (!isTypeAllowedInSkylark(object)) {
+ throw new EvalException(loc,
+ "Type is not allowed in Skylark: "
+ + object.getClass().getSimpleName());
+ }
+ }
+
+ private static Class<?> getGenericTypeFromMethod(Method method) {
+ // This is where we can infer generic type information, so SkylarkNestedSets can be
+ // created in a safe way. Eventually we should probably do something with Lists and Maps too.
+ ParameterizedType t = (ParameterizedType) method.getGenericReturnType();
+ Type type = t.getActualTypeArguments()[0];
+ if (type instanceof Class) {
+ return (Class<?>) type;
+ }
+ if (type instanceof WildcardType) {
+ WildcardType wildcard = (WildcardType) type;
+ Type upperBound = wildcard.getUpperBounds()[0];
+ if (upperBound instanceof Class) {
+ // i.e. List<? extends SuperClass>
+ return (Class<?>) upperBound;
+ }
+ }
+ // It means someone annotated a method with @SkylarkCallable with no specific generic type info.
+ // We shouldn't annotate methods which return List<?> or List<T>.
+ throw new IllegalStateException("Cannot infer type from method signature " + method);
+ }
+
+ /**
+ * Converts an object retrieved from a Java method to a Skylark-compatible type.
+ */
+ static Object convertToSkylark(Object object, Method method) {
+ if (object instanceof NestedSet<?>) {
+ return new SkylarkNestedSet(getGenericTypeFromMethod(method), (NestedSet<?>) object);
+ } else if (object instanceof List<?>) {
+ return SkylarkList.list((List<?>) object, getGenericTypeFromMethod(method));
+ }
+ return object;
+ }
+
+ /**
+ * Converts an object to a Skylark-compatible type if possible.
+ */
+ public static Object convertToSkylark(Object object, Location loc) throws EvalException {
+ if (object instanceof List<?>) {
+ return SkylarkList.list((List<?>) object, loc);
+ }
+ return object;
+ }
+
+ /**
+ * Converts object from a Skylark-compatible wrapper type to its original type.
+ */
+ public static Object convertFromSkylark(Object value) {
+ if (value instanceof SkylarkList) {
+ return ((SkylarkList) value).toList();
+ }
+ return value;
+ }
+
+ /**
+ * Creates a SkylarkType from the SkylarkBuiltin annotation.
+ */
+ public static SkylarkType getReturnType(SkylarkBuiltin annotation) {
+ if (annotation.returnType().equals(Object.class)) {
+ return SkylarkType.UNKNOWN;
+ }
+ if (Function.class.isAssignableFrom(annotation.returnType())) {
+ return SkylarkFunctionType.of(annotation.name());
+ }
+ return SkylarkType.of(annotation.returnType());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Statement.java b/src/main/java/com/google/devtools/build/lib/syntax/Statement.java
new file mode 100644
index 0000000..ca89b1a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Statement.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Base class for all statements nodes in the AST.
+ */
+public abstract class Statement extends ASTNode {
+
+ /**
+ * Executes the statement in the specified build environment, which may be
+ * modified.
+ *
+ * @throws EvalException if execution of the statement could not be completed.
+ */
+ abstract void exec(Environment env) throws EvalException, InterruptedException;
+
+ /**
+ * Checks the semantics of the Statement using the SkylarkEnvironment according to
+ * the rules of the Skylark language. The SkylarkEnvironment can be used e.g. to check
+ * variable type collision, read only variables, detecting recursion, existence of
+ * built-in variables, functions, etc.
+ *
+ * <p>The semantical check should be performed after the Skylark extension is loaded
+ * (i.e. is syntactically correct) and before is executed. The point of the semantical check
+ * is to make sure (as much as possible) that no error can occur during execution (Skylark
+ * programmers get a "compile time" error). It should also check execution branches (e.g. in
+ * if statements) that otherwise might never get executed.
+ *
+ * @throws EvalException if the Statement has a semantical error.
+ */
+ abstract void validate(ValidationEnvironment env) throws EvalException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java
new file mode 100644
index 0000000..98d5045
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for a string literal.
+ */
+public final class StringLiteral extends Literal<String> {
+
+ private final char quoteChar;
+
+ public StringLiteral(String value, char quoteChar) {
+ super(value);
+ this.quoteChar = quoteChar;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder()
+ .append(quoteChar)
+ .append(value.replace(Character.toString(quoteChar), "\\" + quoteChar))
+ .append(quoteChar)
+ .toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ /**
+ * Gets the quote character that was used for this string. For example, if
+ * the string was 'hello, world!', then this method returns '\''.
+ *
+ * @return the character used to quote the string.
+ */
+ public char getQuoteChar() {
+ return quoteChar;
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ return SkylarkType.STRING;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java b/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java
new file mode 100644
index 0000000..5a95026
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral;
+import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A visitor for visiting the nodes in the syntax tree left to right, top to
+ * bottom.
+ */
+public class SyntaxTreeVisitor {
+
+ public void visit(ASTNode node) {
+ // dispatch to the node specific method
+ node.accept(this);
+ }
+
+ public void visitAll(List<? extends ASTNode> nodes) {
+ for (ASTNode node : nodes) {
+ visit(node);
+ }
+ }
+
+ // node specific visit methods
+ public void visit(Argument node) {
+ if (node.isNamed()) {
+ visit(node.getName());
+ }
+ if (node.hasValue()) {
+ visit(node.getValue());
+ }
+ }
+
+ public void visit(BuildFileAST node) {
+ visitAll(node.getStatements());
+ visitAll(node.getComments());
+ }
+
+ public void visit(BinaryOperatorExpression node) {
+ visit(node.getLhs());
+ visit(node.getRhs());
+ }
+
+ public void visit(FuncallExpression node) {
+ visit(node.getFunction());
+ visitAll(node.getArguments());
+ }
+
+ public void visit(Ident node) {
+ }
+
+ public void visit(ListComprehension node) {
+ visit(node.getElementExpression());
+ for (Map.Entry<Ident, Expression> list : node.getLists()) {
+ visit(list.getKey());
+ visit(list.getValue());
+ }
+ }
+
+ public void accept(DictComprehension node) {
+ visit(node.getKeyExpression());
+ visit(node.getValueExpression());
+ visit(node.getLoopVar());
+ visit(node.getListExpression());
+ }
+
+ public void visit(ListLiteral node) {
+ visitAll(node.getElements());
+ }
+
+ public void visit(IntegerLiteral node) {
+ }
+
+ public void visit(StringLiteral node) {
+ }
+
+ public void visit(AssignmentStatement node) {
+ visit(node.getLValue());
+ visit(node.getExpression());
+ }
+
+ public void visit(ExpressionStatement node) {
+ visit(node.getExpression());
+ }
+
+ public void visit(IfStatement node) {
+ for (ConditionalStatements stmt : node.getThenBlocks()) {
+ visit(stmt);
+ }
+ for (Statement stmt : node.getElseBlock()) {
+ visit(stmt);
+ }
+ }
+
+ public void visit(ConditionalStatements node) {
+ visit(node.getCondition());
+ for (Statement stmt : node.getStmts()) {
+ visit(stmt);
+ }
+ }
+
+ public void visit(FunctionDefStatement node) {
+ visit(node.getIdent());
+ for (Argument arg : node.getArgs()) {
+ visit(arg);
+ }
+ for (Statement stmt : node.getStatements()) {
+ visit(stmt);
+ }
+ }
+
+ public void visit(DictionaryLiteral node) {
+ for (DictionaryEntryLiteral entry : node.getEntries()) {
+ visit(entry);
+ }
+ }
+
+ public void visit(DictionaryEntryLiteral node) {
+ visit(node.getKey());
+ visit(node.getValue());
+ }
+
+ public void visit(NotExpression node) {
+ visit(node.getExpression());
+ }
+
+ public void visit(Comment node) {
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Token.java b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
new file mode 100644
index 0000000..e3bcfec
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * A Token represents an actual lexeme; that is, a lexical unit, its location in
+ * the input text, its lexical kind (TokenKind) and any associated value.
+ */
+public class Token {
+
+ public final TokenKind kind;
+ public final int left;
+ public final int right;
+ public final Object value;
+
+ public Token(TokenKind kind, int left, int right) {
+ this(kind, left, right, null);
+ }
+
+ public Token(TokenKind kind, int left, int right, Object value) {
+ this.kind = kind;
+ this.left = left;
+ this.right = right;
+ this.value = value;
+ }
+
+ /**
+ * Constructs an easy-to-read string representation of token, suitable for use
+ * in user error messages.
+ */
+ @Override
+ public String toString() {
+ // TODO(bazel-team): do proper escaping of string literals
+ return kind == TokenKind.STRING ? ("\"" + value + "\"")
+ : value == null ? kind.getPrettyName()
+ : value.toString();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
new file mode 100644
index 0000000..f6dad9f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * A TokenKind is an enumeration of each different kind of lexical symbol.
+ */
+public enum TokenKind {
+
+ AND("and"),
+ AS("as"),
+ CLASS("class"),
+ COLON(":"),
+ COMMA(","),
+ COMMENT("comment"),
+ DEF("def"),
+ DOT("."),
+ ELIF("elif"),
+ ELSE("else"),
+ EOF("EOF"),
+ EQUALS("="),
+ EQUALS_EQUALS("=="),
+ EXCEPT("except"),
+ FINALLY("finally"),
+ FOR("for"),
+ FROM("from"),
+ GREATER(">"),
+ GREATER_EQUALS(">="),
+ IDENTIFIER("identifier"),
+ IF("if"),
+ ILLEGAL("illegal character"),
+ IMPORT("import"),
+ IN("in"),
+ INDENT("indent"),
+ INT("integer"),
+ LBRACE("{"),
+ LBRACKET("["),
+ LESS("<"),
+ LESS_EQUALS("<="),
+ LPAREN("("),
+ MINUS("-"),
+ NEWLINE("newline"),
+ NOT("not"),
+ NOT_EQUALS("!="),
+ OR("or"),
+ OUTDENT("outdent"),
+ PERCENT("%"),
+ PLUS("+"),
+ PLUS_EQUALS("+="),
+ RBRACE("}"),
+ RBRACKET("]"),
+ RETURN("return"),
+ RPAREN(")"),
+ SEMI(";"),
+ STAR("*"),
+ STRING("string"),
+ TRY("try");
+
+ private final String prettyName;
+
+ private TokenKind(String prettyName) {
+ this.prettyName = prettyName;
+ }
+
+ /**
+ * Returns the pretty name for this token, for use in error messages for the user.
+ */
+ public String getPrettyName() {
+ return prettyName;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java
new file mode 100644
index 0000000..cd909a9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java
@@ -0,0 +1,115 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Location;
+
+/**
+ * The actual function registered in the environment. This function is defined in the
+ * parsed code using {@link FunctionDefStatement}.
+ */
+public class UserDefinedFunction extends MixedModeFunction {
+
+ private final ImmutableList<Argument> args;
+ private final ImmutableMap<String, Integer> argIndexes;
+ private final ImmutableMap<String, Object> defaultValues;
+ private final ImmutableList<Statement> statements;
+ private final SkylarkEnvironment definitionEnv;
+
+ private static ImmutableList<String> argumentToStringList(ImmutableList<Argument> args) {
+ Function<Argument, String> function = new Function<Argument, String>() {
+ @Override
+ public String apply(Argument id) {
+ return id.getArgName();
+ }
+ };
+ return ImmutableList.copyOf(Lists.transform(args, function));
+ }
+
+ private static int mandatoryArgNum(ImmutableList<Argument> args) {
+ int mandatoryArgNum = 0;
+ for (Argument arg : args) {
+ if (!arg.hasValue()) {
+ mandatoryArgNum++;
+ }
+ }
+ return mandatoryArgNum;
+ }
+
+ UserDefinedFunction(Ident function, ImmutableList<Argument> args,
+ ImmutableMap<String, Object> defaultValues,
+ ImmutableList<Statement> statements, SkylarkEnvironment definitionEnv) {
+ super(function.getName(), argumentToStringList(args), mandatoryArgNum(args), false,
+ function.getLocation());
+ this.args = args;
+ this.statements = statements;
+ this.definitionEnv = definitionEnv;
+ this.defaultValues = defaultValues;
+
+ ImmutableMap.Builder<String, Integer> argIndexes = new ImmutableMap.Builder<> ();
+ int i = 0;
+ for (Argument arg : args) {
+ if (!arg.isKwargs()) { // TODO(bazel-team): add varargs support?
+ argIndexes.put(arg.getArgName(), i++);
+ }
+ }
+ this.argIndexes = argIndexes.build();
+ }
+
+ public ImmutableList<Argument> getArgs() {
+ return args;
+ }
+
+ public Integer getArgIndex(String s) {
+ return argIndexes.get(s);
+ }
+
+ ImmutableMap<String, Object> getDefaultValues() {
+ return defaultValues;
+ }
+
+ ImmutableList<Statement> getStatements() {
+ return statements;
+ }
+
+ Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public Object call(Object[] namedArguments, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException {
+ SkylarkEnvironment functionEnv = SkylarkEnvironment.createEnvironmentForFunctionCalling(
+ env, definitionEnv, this);
+
+ // Registering the functions's arguments as variables in the local Environment
+ int i = 0;
+ for (Object arg : namedArguments) {
+ functionEnv.update(args.get(i++).getArgName(), arg);
+ }
+
+ try {
+ for (Statement stmt : statements) {
+ stmt.exec(functionEnv);
+ }
+ } catch (ReturnStatement.ReturnException e) {
+ return e.getValue();
+ }
+ return Environment.NONE;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
new file mode 100644
index 0000000..6afb96a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
@@ -0,0 +1,244 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+/**
+ * An Environment for the semantic checking of Skylark files.
+ *
+ * @see Statement#validate
+ * @see Expression#validate
+ */
+public class ValidationEnvironment {
+
+ private final ValidationEnvironment parent;
+
+ private Map<SkylarkType, Map<String, SkylarkType>> variableTypes = new HashMap<>();
+
+ private Map<String, Location> variableLocations = new HashMap<>();
+
+ private Set<String> readOnlyVariables = new HashSet<>();
+
+ // A stack of variable-sets which are read only but can be assigned in different
+ // branches of if-else statements.
+ private Stack<Set<String>> futureReadOnlyVariables = new Stack<>();
+
+ // The function we are currently validating.
+ private SkylarkFunctionType currentFunction;
+
+ // Whether this validation environment is not modified therefore clonable or not.
+ private boolean clonable;
+
+ public ValidationEnvironment(
+ ImmutableMap<SkylarkType, ImmutableMap<String, SkylarkType>> builtinVariableTypes) {
+ parent = null;
+ variableTypes = CollectionUtils.copyOf(builtinVariableTypes);
+ readOnlyVariables.addAll(builtinVariableTypes.get(SkylarkType.GLOBAL).keySet());
+ clonable = true;
+ }
+
+ private ValidationEnvironment(Map<SkylarkType, Map<String, SkylarkType>> builtinVariableTypes,
+ Set<String> readOnlyVariables) {
+ parent = null;
+ this.variableTypes = CollectionUtils.copyOf(builtinVariableTypes);
+ this.readOnlyVariables = new HashSet<>(readOnlyVariables);
+ clonable = false;
+ }
+
+ @Override
+ public ValidationEnvironment clone() {
+ Preconditions.checkState(clonable);
+ return new ValidationEnvironment(variableTypes, readOnlyVariables);
+ }
+
+ /**
+ * Creates a local ValidationEnvironment to validate user defined function bodies.
+ */
+ public ValidationEnvironment(ValidationEnvironment parent, SkylarkFunctionType currentFunction) {
+ this.parent = parent;
+ this.variableTypes.put(SkylarkType.GLOBAL, new HashMap<String, SkylarkType>());
+ this.currentFunction = currentFunction;
+ for (String var : parent.readOnlyVariables) {
+ if (!parent.variableLocations.containsKey(var)) {
+ // Mark built in global vars readonly. Variables defined in Skylark may be shadowed locally.
+ readOnlyVariables.add(var);
+ }
+ }
+ this.clonable = false;
+ }
+
+ /**
+ * Returns true if this ValidationEnvironment is top level i.e. has no parent.
+ */
+ public boolean isTopLevel() {
+ return parent == null;
+ }
+
+ /**
+ * Updates the variable type if the new type is "stronger" then the old one.
+ * The old and the new vartype has to be compatible, otherwise an EvalException is thrown.
+ * The new type is stronger if the old one doesn't exist or unknown.
+ */
+ public void update(String varname, SkylarkType newVartype, Location location)
+ throws EvalException {
+ checkReadonly(varname, location);
+ if (parent == null) { // top-level values are immutable
+ readOnlyVariables.add(varname);
+ if (!futureReadOnlyVariables.isEmpty()) {
+ // Currently validating an if-else statement
+ futureReadOnlyVariables.peek().add(varname);
+ }
+ }
+ SkylarkType oldVartype = variableTypes.get(SkylarkType.GLOBAL).get(varname);
+ if (oldVartype != null) {
+ newVartype = oldVartype.infer(newVartype, "variable '" + varname + "'",
+ location, variableLocations.get(varname));
+ }
+ variableTypes.get(SkylarkType.GLOBAL).put(varname, newVartype);
+ variableLocations.put(varname, location);
+ clonable = false;
+ }
+
+ private void checkReadonly(String varname, Location location) throws EvalException {
+ if (readOnlyVariables.contains(varname)) {
+ throw new EvalException(location, String.format("Variable %s is read only", varname));
+ }
+ }
+
+ public void checkIterable(SkylarkType type, Location loc) throws EvalException {
+ if (type == SkylarkType.UNKNOWN) {
+ // Until all the language is properly typed, we ignore Object types.
+ return;
+ }
+ if (!Iterable.class.isAssignableFrom(type.getType())
+ && !Map.class.isAssignableFrom(type.getType())
+ && !String.class.equals(type.getType())) {
+ throw new EvalException(loc,
+ "type '" + EvalUtils.getDataTypeNameFromClass(type.getType()) + "' is not iterable");
+ }
+ }
+
+ /**
+ * Returns true if the symbol exists in the validation environment.
+ */
+ public boolean hasSymbolInEnvironment(String varname) {
+ return variableTypes.get(SkylarkType.GLOBAL).containsKey(varname)
+ || topLevel().variableTypes.get(SkylarkType.GLOBAL).containsKey(varname);
+ }
+
+ /**
+ * Returns the type of the existing variable.
+ */
+ public SkylarkType getVartype(String varname) {
+ SkylarkType type = variableTypes.get(SkylarkType.GLOBAL).get(varname);
+ if (type == null && parent != null) {
+ type = parent.getVartype(varname);
+ }
+ return Preconditions.checkNotNull(type,
+ String.format("Variable %s is not found in the validation environment", varname));
+ }
+
+ public SkylarkFunctionType getCurrentFunction() {
+ return currentFunction;
+ }
+
+ /**
+ * Returns the return type of the function.
+ */
+ public SkylarkType getReturnType(String funcName, Location loc) throws EvalException {
+ return getReturnType(SkylarkType.GLOBAL, funcName, loc);
+ }
+
+ /**
+ * Returns the return type of the object function.
+ */
+ public SkylarkType getReturnType(SkylarkType objectType, String funcName, Location loc)
+ throws EvalException {
+ // All functions are registered in the top level ValidationEnvironment.
+ Map<String, SkylarkType> functions = topLevel().variableTypes.get(objectType);
+ // TODO(bazel-team): eventually not finding the return type should be a validation error,
+ // because it means the function doesn't exist. First we have to make sure that we register
+ // every possible function before.
+ if (functions != null) {
+ SkylarkType functionType = functions.get(funcName);
+ if (functionType != null && functionType != SkylarkType.UNKNOWN) {
+ if (!(functionType instanceof SkylarkFunctionType)) {
+ throw new EvalException(loc, (objectType == SkylarkType.GLOBAL ? "" : objectType + ".")
+ + funcName + " is not a function");
+ }
+ return ((SkylarkFunctionType) functionType).getReturnType();
+ }
+ }
+ return SkylarkType.UNKNOWN;
+ }
+
+ private ValidationEnvironment topLevel() {
+ return Preconditions.checkNotNull(parent == null ? this : parent);
+ }
+
+ /**
+ * Adds a user defined function to the validation environment is not exists.
+ */
+ public void updateFunction(String name, SkylarkFunctionType type, Location loc)
+ throws EvalException {
+ checkReadonly(name, loc);
+ if (variableTypes.get(SkylarkType.GLOBAL).containsKey(name)) {
+ throw new EvalException(loc, "function " + name + " already exists");
+ }
+ variableTypes.get(SkylarkType.GLOBAL).put(name, type);
+ clonable = false;
+ }
+
+ /**
+ * Starts a session with temporarily disabled readonly checking for variables between branches.
+ * This is useful to validate control flows like if-else when we know that certain parts of the
+ * code cannot both be executed.
+ */
+ public void startTemporarilyDisableReadonlyCheckSession() {
+ futureReadOnlyVariables.add(new HashSet<String>());
+ clonable = false;
+ }
+
+ /**
+ * Finishes the session with temporarily disabled readonly checking.
+ */
+ public void finishTemporarilyDisableReadonlyCheckSession() {
+ Set<String> variables = futureReadOnlyVariables.pop();
+ readOnlyVariables.addAll(variables);
+ if (!futureReadOnlyVariables.isEmpty()) {
+ futureReadOnlyVariables.peek().addAll(variables);
+ }
+ clonable = false;
+ }
+
+ /**
+ * Finishes a branch of temporarily disabled readonly checking.
+ */
+ public void finishTemporarilyDisableReadonlyCheckBranch() {
+ readOnlyVariables.removeAll(futureReadOnlyVariables.peek());
+ clonable = false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/Directories.java b/src/main/java/com/google/devtools/build/lib/unix/Directories.java
new file mode 100644
index 0000000..b6c3cda
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/Directories.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.shell.AbnormalTerminationException;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Provides utility methods for working with directories in a Unix environment.
+ */
+public final class Directories {
+
+ /**
+ * Deletes a file or directory and all contents recursively, like {@code rm -rf file}.
+ *
+ * <p>If the file argument is a symbolic link, the link will be deleted but not the target of the
+ * link. If the argument is a directory, symbolic links within the directory will not be followed.
+ * If the argument does not exist, throws a FileNotFoundException.
+ *
+ * @param file the file or directory to delete
+ * @throws FileNotFoundException if the file or directory does not exist
+ * @throws IOException if an I/O error occurs
+ */
+ public static void deleteRecursively(File file) throws IOException {
+ deleteRecursivelyImpl(file, true);
+ }
+
+ /**
+ * Deletes a file or directory and all contents recursively, like {@code rm -rf file}, if it
+ * exists.
+ *
+ * <p>If the file argument is a symbolic link, the link will be deleted but not the target of the
+ * link. If the argument is a directory, symbolic links within the directory will not be followed.
+ *
+ * @param file the file or directory to delete
+ * @return {@code true} if the file or directory was deleted by this method; {@code false} if the
+ * file or directory could not be deleted because it did not exist
+ * @throws IOException if an I/O error occurs
+ */
+ public static boolean deleteRecursivelyIfExists(File file) throws IOException {
+ return deleteRecursivelyImpl(file, false);
+ }
+
+ private static boolean deleteRecursivelyImpl(File file, boolean failIfFileDoesNotExist)
+ throws IOException {
+ if (!file.exists()) {
+ if (failIfFileDoesNotExist) {
+ throw new FileNotFoundException(file.getPath());
+ } else {
+ return false;
+ }
+ }
+ String filePath = file.getPath();
+ if (!filePath.isEmpty() && filePath.charAt(0) == '-') {
+ filePath = "./" + filePath;
+ }
+ try {
+ new Command(new String[] {"/bin/rm", "-rf", filePath}).execute();
+ } catch (AbnormalTerminationException e) {
+ String message =
+ e.getResult().getTerminationStatus() + ": " + new String(e.getResult().getStderr());
+ throw new IOException(message, e);
+ } catch (CommandException e) {
+ throw new IOException(e);
+ }
+ return true;
+ }
+
+ private Directories() {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/ErrnoFileStatus.java b/src/main/java/com/google/devtools/build/lib/unix/ErrnoFileStatus.java
new file mode 100644
index 0000000..fa0c3a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/ErrnoFileStatus.java
@@ -0,0 +1,94 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.UnixJniLoader;
+
+
+/**
+ * A subsclass of FileStatus which contains an errno.
+ * If there is an error, all other data fields are undefined.
+ */
+public class ErrnoFileStatus extends FileStatus {
+
+ private final int errno;
+
+ // These constants are passed in from JNI via ErrnoConstants.
+ public static final int ENOENT;
+ public static final int EACCES;
+ public static final int ELOOP;
+ public static final int ENOTDIR;
+ public static final int ENAMETOOLONG;
+
+ static {
+ ErrnoConstants constants = ErrnoConstants.getErrnoConstants();
+ ENOENT = constants.ENOENT;
+ EACCES = constants.EACCES;
+ ELOOP = constants.ELOOP;
+ ENOTDIR = constants.ENOTDIR;
+ ENAMETOOLONG = constants.ENAMETOOLONG;
+ }
+
+ /**
+ * Constructs a ErrnoFileSatus instance. (Called only from JNI code.)
+ */
+ private ErrnoFileStatus(int st_mode, int st_atime, int st_atimensec, int st_mtime,
+ int st_mtimensec, int st_ctime, int st_ctimensec, long st_size,
+ int st_dev, long st_ino) {
+ super(st_mode, st_atime, st_atimensec, st_mtime, st_mtimensec, st_ctime, st_ctimensec, st_size,
+ st_dev, st_ino);
+ this.errno = 0;
+ }
+
+ /**
+ * Constructs a ErrnoFileSatus instance. (Called only from JNI code.)
+ */
+ private ErrnoFileStatus(int errno) {
+ super(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ this.errno = errno;
+ }
+
+ public int getErrno() {
+ return errno;
+ }
+
+ public boolean hasError() {
+ // errno = 0 means the operation succeeded.
+ return errno != 0;
+ }
+
+ // Used to transfer the constants from native to java code.
+ private static class ErrnoConstants {
+
+ // These are set in JNI.
+ private int ENOENT;
+ private int EACCES;
+ private int ELOOP;
+ private int ENOTDIR;
+ private int ENAMETOOLONG;
+
+ public static ErrnoConstants getErrnoConstants() {
+ ErrnoConstants constants = new ErrnoConstants();
+ constants.initErrnoConstants();
+ return constants;
+ }
+
+ static {
+ UnixJniLoader.loadJni();
+ }
+
+ private native void initErrnoConstants();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/FileAccessException.java b/src/main/java/com/google/devtools/build/lib/unix/FileAccessException.java
new file mode 100644
index 0000000..9ea4c6e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/FileAccessException.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.unix;
+
+import java.io.IOException;
+
+/**
+ * An IOException subclass that is thrown when a POSIX filesystem call returns
+ * an EACCES errno. The message is generally "Permission denied".
+ */
+public class FileAccessException extends IOException {
+ /**
+ * Constructs a <code>FileAccessException</code> with <code>null</code>
+ * as its error detail message.
+ */
+ public FileAccessException() {
+ super();
+ }
+
+ /**
+ * Constructs an <code>FileAccessException</code> with the specified detail
+ * message. The error message string <code>s</code> can later be
+ * retrieved by the <code>{@link java.lang.Throwable#getMessage}</code>
+ * method of class <code>java.lang.Throwable</code>.
+ *
+ * @param s the detail message.
+ */
+ public FileAccessException(String s) {
+ super(s);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/FileStatus.java b/src/main/java/com/google/devtools/build/lib/unix/FileStatus.java
new file mode 100644
index 0000000..10f4d99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/FileStatus.java
@@ -0,0 +1,262 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+/**
+ * <p>Equivalent to UNIX's "struct stat", a FileStatus instance contains
+ * various bits of metadata about a directory entry.
+ *
+ * <p>The Java SDK provides access to some but not all of the information
+ * available via the stat(2) and lstat(2) syscalls, but often requires that
+ * multiple calls be made to obtain it. By reifying stat buffers as Java
+ * objects and providing a wrapper around the stat/lstat calls, we give client
+ * applications access to the richer file metadata and enable a reduction in
+ * the number of system calls, which is critical for high-performance tools.
+ *
+ * <p>This class is optimized for memory usage. Operations that are not yet
+ * required for any client are intentionally unimplemented to save space.
+ * Currently, we only support these fields: st_mode, st_size, st_atime,
+ * st_atimensec, st_mtime, st_mtimensec, st_ctime, st_ctimensec, st_dev, st_ino.
+ * Methods that require other fields throw UnsupportedOperationException.
+ */
+public class FileStatus {
+
+ private final int st_mode;
+ private final int st_atime; // (unsigned)
+ private final int st_atimensec; // (unsigned)
+ private final int st_mtime; // (unsigned)
+ private final int st_mtimensec; // (unsigned)
+ private final int st_ctime; // (unsigned)
+ private final int st_ctimensec; // (unsigned)
+ private final long st_size;
+ private final int st_dev;
+ private final long st_ino;
+
+ /**
+ * Constructs a FileStatus instance. (Called only from JNI code.)
+ */
+ protected FileStatus(int st_mode, int st_atime, int st_atimensec, int st_mtime, int st_mtimensec,
+ int st_ctime, int st_ctimensec, long st_size, int st_dev, long st_ino) {
+ this.st_mode = st_mode;
+ this.st_atime = st_atime;
+ this.st_atimensec = st_atimensec;
+ this.st_mtime = st_mtime;
+ this.st_mtimensec = st_mtimensec;
+ this.st_ctime = st_ctime;
+ this.st_ctimensec = st_ctimensec;
+ this.st_size = st_size;
+ this.st_dev = st_dev;
+ this.st_ino = st_ino;
+ }
+
+ /**
+ * Returns the device number of this inode.
+ */
+ public int getDeviceNumber() {
+ return st_dev;
+ }
+
+ /**
+ * Returns the number of this inode. Inode numbers are (usually) unique for
+ * a given device.
+ */
+ public long getInodeNumber() {
+ return st_ino;
+ }
+
+ /**
+ * Returns true iff this file is a regular file.
+ */
+ public boolean isRegularFile() {
+ return (st_mode & S_IFMT) == S_IFREG;
+ }
+
+ /**
+ * Returns true iff this file is a directory.
+ */
+ public boolean isDirectory() {
+ return (st_mode & S_IFMT) == S_IFDIR;
+ }
+
+ /**
+ * Returns true iff this file is a symbolic link.
+ */
+ public boolean isSymbolicLink() {
+ return (st_mode & S_IFMT) == S_IFLNK;
+ }
+
+ /**
+ * Returns true iff this file is a character device.
+ */
+ public boolean isCharacterDevice() {
+ return (st_mode & S_IFMT) == S_IFCHR;
+ }
+
+ /**
+ * Returns true iff this file is a block device.
+ */
+ public boolean isBlockDevice() {
+ return (st_mode & S_IFMT) == S_IFBLK;
+ }
+
+ /**
+ * Returns true iff this file is a FIFO.
+ */
+ public boolean isFIFO() {
+ return (st_mode & S_IFMT) == S_IFIFO;
+ }
+
+ /**
+ * Returns true iff this file is a UNIX-domain socket.
+ */
+ public boolean isSocket() {
+ return (st_mode & S_IFMT) == S_IFSOCK;
+ }
+
+ /**
+ * Returns true iff this file has its "set UID" bit set.
+ */
+ public boolean isSetUserId() {
+ return (st_mode & S_ISUID) != 0;
+ }
+
+ /**
+ * Returns true iff this file has its "set GID" bit set.
+ */
+ public boolean isSetGroupId() {
+ return (st_mode & S_ISGID) != 0;
+ }
+
+ /**
+ * Returns true iff this file has its "sticky" bit set. See UNIX manuals for
+ * explanation.
+ */
+ public boolean isSticky() {
+ return (st_mode & S_ISVTX) != 0;
+ }
+
+ /**
+ * Returns the user/group/other permissions part of the mode bits (i.e.
+ * st_mode masked with 0777), interpreted according to longstanding UNIX
+ * tradition.
+ */
+ public int getPermissions() {
+ return st_mode & S_IRWXA;
+ }
+
+ /**
+ * Returns the total size, in bytes, of this file.
+ */
+ public long getSize() {
+ return st_size;
+ }
+
+ /**
+ * Returns the last access time of this file (seconds since UNIX epoch).
+ */
+ public long getLastAccessTime() {
+ return unsignedIntToLong(st_atime);
+ }
+
+ /**
+ * Returns the fractional part of the last access time of this file (nanoseconds).
+ */
+ public long getFractionalLastAccessTime() {
+ return unsignedIntToLong(st_atimensec);
+ }
+
+ /**
+ * Returns the last modified time of this file (seconds since UNIX epoch).
+ */
+ public long getLastModifiedTime() {
+ return unsignedIntToLong(st_mtime);
+ }
+
+ /**
+ * Returns the fractional part of the last modified time of this file (nanoseconds).
+ */
+ public long getFractionalLastModifiedTime() {
+ return unsignedIntToLong(st_mtimensec);
+ }
+
+ /**
+ * Returns the last change time of this file (seconds since UNIX epoch).
+ */
+ public long getLastChangeTime() {
+ return unsignedIntToLong(st_ctime);
+ }
+
+ /**
+ * Returns the fractional part of the last change time of this file (nanoseconds).
+ */
+ public long getFractionalLastChangeTime() {
+ return unsignedIntToLong(st_ctimensec);
+ }
+
+ ////////////////////////////////////////////////////////////////////////
+
+ @Override
+ public String toString() {
+ return String.format("FileStatus(mode=0%06o,size=%d,mtime=%d)",
+ st_mode, st_size, st_mtime);
+ }
+
+ @Override
+ public int hashCode() {
+ return st_mode;
+ }
+
+ ////////////////////////////////////////////////////////////////////////
+ // Platform-specific details. These fields are public so that they can
+ // be used from other packages. See POSIX and/or Linux manuals for details.
+ //
+ // These need to be kept in sync with the native code and system call
+ // interface. (The unit tests ensure that.) Of course, this decoding could
+ // be done in the JNI code to ensure maximum portability, but (a) we don't
+ // expect we'll need that any time soon, and (b) that would require eager
+ // rather than on-demand bitmunging of all attributes. In any case, it's not
+ // part of the interface so it can be easily changed later if necessary.
+
+ public static final int S_IFMT = 0170000; // mask: filetype bitfields
+ public static final int S_IFSOCK = 0140000; // socket
+ public static final int S_IFLNK = 0120000; // symbolic link
+ public static final int S_IFREG = 0100000; // regular file
+ public static final int S_IFBLK = 0060000; // block device
+ public static final int S_IFDIR = 0040000; // directory
+ public static final int S_IFCHR = 0020000; // character device
+ public static final int S_IFIFO = 0010000; // fifo
+ public static final int S_ISUID = 0004000; // set UID bit
+ public static final int S_ISGID = 0002000; // set GID bit (see below)
+ public static final int S_ISVTX = 0001000; // sticky bit (see below)
+ public static final int S_IRWXA = 00777; // mask: all permissions
+ public static final int S_IRWXU = 00700; // mask: file owner permissions
+ public static final int S_IRUSR = 00400; // owner has read permission
+ public static final int S_IWUSR = 00200; // owner has write permission
+ public static final int S_IXUSR = 00100; // owner has execute permission
+ public static final int S_IRWXG = 00070; // mask: group permissions
+ public static final int S_IRGRP = 00040; // group has read permission
+ public static final int S_IWGRP = 00020; // group has write permission
+ public static final int S_IXGRP = 00010; // group has execute permission
+ public static final int S_IRWXO = 00007; // mask: other permissions
+ public static final int S_IROTH = 00004; // others have read permission
+ public static final int S_IWOTH = 00002; // others have write permisson
+ public static final int S_IXOTH = 00001; // others have execute permission
+
+ public static final int S_IEXEC = 00111; // owner, group, world execute
+
+ static long unsignedIntToLong(int i) {
+ return (i & 0x7FFFFFFF) - (long) (i & 0x80000000);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/FilesystemUtils.java b/src/main/java/com/google/devtools/build/lib/unix/FilesystemUtils.java
new file mode 100644
index 0000000..462ed9c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/FilesystemUtils.java
@@ -0,0 +1,442 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.unix;
+
+import com.google.common.hash.HashCode;
+import com.google.devtools.build.lib.UnixJniLoader;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+/**
+ * Utility methods for access to UNIX filesystem calls not exposed by the Java
+ * SDK. Exception messages are selected to be consistent with those generated
+ * by the java.io package where appropriate--see package javadoc for details.
+ */
+public final class FilesystemUtils {
+
+ private FilesystemUtils() {}
+
+ /**
+ * Returns true iff the file identified by 'path' is a symbolic link. Has
+ * similar semantics to POSIX stat(2) syscall, with all errors being mapped to
+ * a false return.
+ *
+ * @param path the file of interest
+ * @return true iff path exists, is accessible and is a symlink
+ */
+ public static boolean isSymbolicLink(File path) {
+ try {
+ return lstat(path.toString()).isSymbolicLink();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+
+ /**
+ * Returns true iff the file identified by 'path' is a directory. Has
+ * similar semantics to POSIX stat(2) syscall, with all errors being mapped to
+ * a false return.
+ *
+ * @param path the file of interest
+ * @return true iff path exists, is accessible and is a symlink
+ */
+ public static boolean isDirectory(String path) {
+ try {
+ return lstat(path).isDirectory();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+
+ /**
+ * Marks the file or directory {@code path} as executable. (Non-atomic)
+ *
+ * @see File#setReadOnly
+ *
+ * @param path the file of interest
+ * @throws FileAccessException if path can't be accessed
+ * @throws FileNotFoundException if path doesn't exist
+ * @throws IOException for other filesystem or path errors
+ */
+ public static void setExecutable(File path) throws IOException {
+ String p = path.toString();
+ chmod(p, stat(p).getPermissions() | FileStatus.S_IEXEC);
+ }
+
+ /**
+ * Marks the file or directory {@code path} as owner writable. (Non-atomic)
+ *
+ * @see File#setReadOnly
+ *
+ * @param path the file of interest
+ * @throws FileAccessException if path can't be accessed
+ * @throws FileNotFoundException if path doesn't exist
+ * @throws IOException for other filesystem or path errors
+ */
+ public static void setWritable(File path) throws IOException {
+ String p = path.toString();
+ chmod(p, stat(p).getPermissions() | FileStatus.S_IWUSR);
+ }
+
+ /**
+ * Changes permissions of a file.
+ *
+ * @param path the file whose mode to change.
+ * @param mode the mode bits within 07777, interpreted according to
+ * long-standing UNIX tradition.
+ * @throws IOException if the chmod() syscall failed.
+ */
+ public static void chmod(File path, int mode) throws IOException {
+ int mask = FileStatus.S_ISUID |
+ FileStatus.S_ISGID |
+ FileStatus.S_ISVTX |
+ FileStatus.S_IRWXA;
+ chmod(path.toString(), mode & mask);
+ }
+
+ /*
+ * Native-based implementation
+ */
+
+ static {
+ if (!java.nio.charset.Charset.defaultCharset().name().equals("ISO-8859-1")) {
+ // Defer the Logger call, so we don't deadlock if this is called from Logger
+ // initialization code.
+ new Thread() {
+ @Override
+ public void run() {
+ // wait (if necessary) until the logging system is initialized
+ synchronized (LogManager.getLogManager()) {}
+ Logger.getLogger("com.google.devtools.build.lib.unix.FilesystemUtils").log(Level.FINE,
+ "WARNING: Default character set is not latin1; java.io.File and " +
+ "com.google.devtools.build.lib.unix.FilesystemUtils will represent some filenames " +
+ "differently.");
+ }
+ }.start();
+ }
+ UnixJniLoader.loadJni();
+ }
+
+ /**
+ * Native wrapper around Linux readlink(2) call.
+ *
+ * @param path the file of interest
+ * @return the pathname to which the symbolic link 'path' links
+ * @throws IOException iff the readlink() call failed
+ */
+ public static native String readlink(String path) throws IOException;
+
+ /**
+ * Native wrapper around POSIX chmod(2) syscall: Changes the file access
+ * permissions of 'path' to 'mode'.
+ *
+ * @param path the file of interest
+ * @param mode the POSIX type and permission mode bits to set
+ * @throws IOException iff the chmod() call failed.
+ */
+ public static native void chmod(String path, int mode) throws IOException;
+
+ /**
+ * Native wrapper around POSIX symlink(2) syscall.
+ *
+ * @param oldpath the file to link to
+ * @param newpath the new path for the link
+ * @throws IOException iff the symlink() syscall failed.
+ */
+ public static native void symlink(String oldpath, String newpath)
+ throws IOException;
+
+ /**
+ * Native wrapper around POSIX stat(2) syscall.
+ *
+ * @param path the file to stat.
+ * @return a FileStatus instance containing the metadata.
+ * @throws IOException if the stat() syscall failed.
+ */
+ public static native FileStatus stat(String path) throws IOException;
+
+ /**
+ * Native wrapper around POSIX lstat(2) syscall.
+ *
+ * @param path the file to lstat.
+ * @return a FileStatus instance containing the metadata.
+ * @throws IOException if the lstat() syscall failed.
+ */
+ public static native FileStatus lstat(String path) throws IOException;
+
+ /**
+ * Native wrapper around POSIX stat(2) syscall.
+ *
+ * @param path the file to stat.
+ * @return an ErrnoFileStatus instance containing the metadata.
+ * If there was an error, the return value's hasError() method
+ * will return true, and all stat information is undefined.
+ */
+ public static native ErrnoFileStatus errnoStat(String path);
+
+ /**
+ * Native wrapper around POSIX lstat(2) syscall.
+ *
+ * @param path the file to lstat.
+ * @return an ErrnoFileStatus instance containing the metadata.
+ * If there was an error, the return value's hasError() method
+ * will return true, and all stat information is undefined.
+ */
+ public static native ErrnoFileStatus errnoLstat(String path);
+
+ /**
+ * Native wrapper around POSIX utime(2) syscall.
+ *
+ * Note: negative file times are interpreted as unsigned time_t.
+ *
+ * @param path the file whose times to change.
+ * @param now if true, ignore actime/modtime parameters and use current time.
+ * @param actime the file access time in seconds since the UNIX epoch.
+ * @param modtime the file modification time in seconds since the UNIX epoch.
+ * @throws IOException if the utime() syscall failed.
+ */
+ public static native void utime(String path, boolean now,
+ int actime, int modtime) throws IOException;
+
+ /**
+ * Native wrapper around POSIX mkdir(2) syscall.
+ *
+ * Caveat: errno==EEXIST is mapped to the return value "false", not
+ * IOException. It requires an additional stat() to determine if mkdir
+ * failed because the directory already exists.
+ *
+ * @param path the directory to create.
+ * @param mode the mode with which to create the directory.
+ * @return true if the directory was successfully created; false if the
+ * system call returned EEXIST because some kind of a file (not necessarily
+ * a directory) already exists.
+ * @throws IOException if the mkdir() syscall failed for any other reason.
+ */
+ public static native boolean mkdir(String path, int mode)
+ throws IOException;
+
+ /**
+ * Native wrapper around POSIX opendir(2)/readdir(3)/closedir(3) syscall.
+ *
+ * @param path the directory to read.
+ * @return the list of directory entries in the order they were returned by
+ * the system, excluding "." and "..".
+ * @throws IOException if the call to opendir failed for any reason.
+ */
+ public static String[] readdir(String path) throws IOException {
+ return readdir(path, ReadTypes.NONE).names;
+ }
+
+ /**
+ * An enum for specifying now the types of the individual entries returned by
+ * {@link #readdir(String, ReadTypes)} is to be returned.
+ */
+ public enum ReadTypes {
+ NONE('n'), // Do not read types
+ NOFOLLOW('d'), // Do not follow symlinks
+ FOLLOW('f'); // Follow symlinks; never returns "SYMLINK" and returns "UNKNOWN" when dangling
+
+ private final char code;
+
+ private ReadTypes(char code) {
+ this.code = code;
+ }
+
+ private char getCode() {
+ return code;
+ }
+ }
+
+ /**
+ * A compound return type for readdir(), analogous to struct dirent[] in C. A low memory profile
+ * is critical for this class, as instances are expected to be kept around for caching for
+ * potentially a long time.
+ */
+ public static final class Dirents {
+
+ /**
+ * The type of the directory entry.
+ */
+ public enum Type {
+ FILE,
+ DIRECTORY,
+ SYMLINK,
+ UNKNOWN;
+
+ private static Type forChar(char c) {
+ if (c == 'f') {
+ return Type.FILE;
+ } else if (c == 'd') {
+ return Type.DIRECTORY;
+ } else if (c == 's') {
+ return Type.SYMLINK;
+ } else {
+ return Type.UNKNOWN;
+ }
+ }
+ }
+
+ /** The names of the entries in a directory. */
+ private final String[] names;
+ /**
+ * An optional (nullable) array of entry types, corresponding positionally
+ * to the "names" field. The types are:
+ * 'd': a subdirectory
+ * 'f': a regular file
+ * 's': a symlink (only returned with {@code NOFOLLOW})
+ * '?': anything else
+ * Note that unlike libc, this implementation of readdir() follows
+ * symlinks when determining these types.
+ *
+ * <p>This is intentionally a byte array rather than a array of enums to save memory.
+ */
+ private final byte[] types;
+
+ /** called from JNI */
+ public Dirents(String[] names, byte[] types) {
+ this.names = names;
+ this.types = types;
+ }
+
+ public int size() {
+ return names.length;
+ }
+
+ public boolean hasTypes() {
+ return types != null;
+ }
+
+ public String getName(int i) {
+ return names[i];
+ }
+
+ public Type getType(int i) {
+ return Type.forChar((char) types[i]);
+ }
+ }
+
+ /**
+ * Native wrapper around POSIX opendir(2)/readdir(3)/closedir(3) syscall.
+ *
+ * @param path the directory to read.
+ * @param readTypes How the types of individual entries should be returned. If {@code NONE},
+ * the "types" field in the result will be null.
+ * @return a Dirents object, containing "names", the list of directory entries
+ * (excluding "." and "..") in the order they were returned by the system,
+ * and "types", an array of entry types (file, directory, etc) corresponding
+ * positionally to "names".
+ * @throws IOException if the call to opendir failed for any reason.
+ */
+ public static Dirents readdir(String path, ReadTypes readTypes) throws IOException {
+ // Passing enums to native code is possible, but onerous; we use a char instead.
+ return readdir(path, readTypes.getCode());
+ }
+
+ private static native Dirents readdir(String path, char typeCode)
+ throws IOException;
+
+ /**
+ * Native wrapper around POSIX rename(2) syscall.
+ *
+ * @param oldpath the source location.
+ * @param newpath the destination location.
+ * @throws IOException if the rename failed for any reason.
+ */
+ public static native void rename(String oldpath, String newpath)
+ throws IOException;
+
+ /**
+ * Native wrapper around POSIX remove(3) C library call.
+ *
+ * @param path the file or directory to remove.
+ * @return true iff the file was actually deleted by this call.
+ * @throws IOException if the remove failed, but the file was present prior to the call.
+ */
+ public static native boolean remove(String path) throws IOException;
+
+ /********************************************************************
+ * *
+ * Linux extended file attributes *
+ * *
+ ********************************************************************/
+
+ /**
+ * Native wrapper around Linux getxattr(2) syscall.
+ *
+ * @param path the file whose extended attribute is to be returned.
+ * @param name the name of the extended attribute key.
+ * @return the value of the extended attribute associated with 'path', if
+ * any, or null if no such attribute is defined (ENODATA).
+ * @throws IOException if the call failed for any other reason.
+ */
+ public static native byte[] getxattr(String path, String name)
+ throws IOException;
+
+ /**
+ * Native wrapper around Linux lgetxattr(2) syscall. (Like getxattr, but
+ * does not follow symbolic links.)
+ *
+ * @param path the file whose extended attribute is to be returned.
+ * @param name the name of the extended attribute key.
+ * @return the value of the extended attribute associated with 'path', if
+ * any, or null if no such attribute is defined (ENODATA).
+ * @throws IOException if the call failed for any other reason.
+ */
+ public static native byte[] lgetxattr(String path, String name)
+ throws IOException;
+
+ /**
+ * Returns the MD5 digest of the specified file, following symbolic links.
+ *
+ * @param path the file whose MD5 digest is required.
+ * @return the MD5 digest, as a 16-byte array.
+ * @throws IOException if the call failed for any reason.
+ */
+ static native byte[] md5sumAsBytes(String path) throws IOException;
+
+ /**
+ * Returns the MD5 digest of the specified file, following symbolic links.
+ *
+ * @param path the file whose MD5 digest is required.
+ * @return the MD5 digest, as a {@link HashCode}
+ * @throws IOException if the call failed for any reason.
+ */
+ public static HashCode md5sum(String path) throws IOException {
+ return HashCode.fromBytes(md5sumAsBytes(path));
+ }
+
+ /**
+ * Removes entire directory tree. Doesn't follow symlinks.
+ *
+ * @param path the file or directory to remove.
+ * @throws IOException if the remove failed.
+ */
+ public static void rmTree(String path) throws IOException {
+ if (isDirectory(path)) {
+ String[] contents = readdir(path);
+ for (String entry : contents) {
+ rmTree(path + "/" + entry);
+ }
+ }
+ remove(path.toString());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalClientSocket.java b/src/main/java/com/google/devtools/build/lib/unix/LocalClientSocket.java
new file mode 100644
index 0000000..46980da
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/LocalClientSocket.java
@@ -0,0 +1,117 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.SocketException;
+
+/**
+ * <p>An implementation of client Socket for local (AF_UNIX) sockets.
+ *
+ * <p>This class intentionally doesn't extend java.net.Socket although it
+ * has some similarity to it. The java.net class hierarchy is a terrible mess
+ * and is inextricably coupled to the Internet Protocol.
+ *
+ * <p>This code is not intended to be portable to non-UNIX platforms.
+ */
+public class LocalClientSocket extends LocalSocket {
+
+ /**
+ * Constructs an unconnected local client socket.
+ *
+ * @throws IOException if the socket could not be created.
+ */
+ public LocalClientSocket() throws IOException {
+ super();
+ }
+
+ /**
+ * Constructs a client socket and connects it to the specified address.
+ *
+ * @throws IOException if either of the socket/connect operations failed.
+ */
+ public LocalClientSocket(LocalSocketAddress address) throws IOException {
+ super();
+ connect(address);
+ }
+
+ /**
+ * Connect to the specified server. Blocks until the server accepts the
+ * connection.
+ *
+ * @throws IOException if the connection failed.
+ */
+ public synchronized void connect(LocalSocketAddress address)
+ throws IOException {
+ checkNotClosed();
+ if (state == State.CONNECTED) {
+ throw new SocketException("socket is already connected");
+ }
+ connect(fd, address.getName().toString()); // JNI
+ this.address = address;
+ this.state = State.CONNECTED;
+ }
+
+ /**
+ * Returns the input stream for reading from the server.
+ *
+ * @param closeSocket close the socket when this input stream is closed.
+ * @throws IOException if there was a problem.
+ */
+ public synchronized InputStream getInputStream(final boolean closeSocket) throws IOException {
+ checkConnected();
+ checkInputNotShutdown();
+ return new FileInputStream(fd) {
+ @Override
+ public void close() throws IOException {
+ if (closeSocket) {
+ LocalClientSocket.this.close();
+ }
+ }
+ };
+ }
+
+ /**
+ * Returns the input stream for reading from the server.
+ *
+ * @throws IOException if there was a problem.
+ */
+ public synchronized InputStream getInputStream() throws IOException {
+ return getInputStream(false);
+ }
+
+ /**
+ * Returns the output stream for writing to the server.
+ *
+ * @throws IOException if there was a problem.
+ */
+ public synchronized OutputStream getOutputStream() throws IOException {
+ checkConnected();
+ checkOutputNotShutdown();
+ return new FileOutputStream(fd) {
+ @Override public void close() {
+ // Don't close the file descriptor.
+ }
+ };
+ }
+
+ @Override
+ public String toString() {
+ return "LocalClientSocket(" + address + ")";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalServerSocket.java b/src/main/java/com/google/devtools/build/lib/unix/LocalServerSocket.java
new file mode 100644
index 0000000..4eb1265
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/LocalServerSocket.java
@@ -0,0 +1,173 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+
+/**
+ * <p>An implementation of ServerSocket for local (AF_UNIX) sockets.
+ *
+ * <p>This class intentionally doesn't extend java.net.ServerSocket although it
+ * has some similarity to it. The java.net class hierarchy is a terrible mess
+ * and is inextricably coupled to the Internet Protocol.
+ *
+ * <p>This code is not intended to be portable to non-UNIX platforms.
+ */
+public class LocalServerSocket extends LocalSocket {
+
+ // Socket timeout in milliseconds. No timeout by default.
+ private long soTimeoutMillis = 0;
+
+ /**
+ * Constructs an unbound local server socket.
+ */
+ public LocalServerSocket() throws IOException {
+ super();
+ }
+
+ /**
+ * Constructs a server socket, binds it to the specified address, and
+ * listens for incoming connections with the specified backlog.
+ *
+ * @throws IOException if any of the socket/bind/listen operations failed.
+ */
+ public LocalServerSocket(LocalSocketAddress address, int backlog)
+ throws IOException {
+ this();
+ bind(address);
+ listen(backlog);
+ }
+
+ /**
+ * Constructs a server socket, binds it to the specified address, and begin
+ * listening for incoming connections using the default backlog.
+ *
+ * @throws IOException if any of the socket/bind/listen operations failed.
+ */
+ public LocalServerSocket(LocalSocketAddress address) throws IOException {
+ this(address, 50);
+ }
+
+ /**
+ * Specifies the timeout in milliseconds for accept(). Setting it to
+ * zero means an indefinite timeout.
+ */
+ public void setSoTimeout(long timeoutMillis) {
+ soTimeoutMillis = timeoutMillis;
+ }
+
+ /**
+ * Returns the current timeout in milliseconds.
+ */
+ public long getSoTimeout() {
+ return soTimeoutMillis;
+ }
+
+ /**
+ * Binds the specified address to this socket. The socket must be unbound.
+ * This causes the filesystem entry to appear.
+ *
+ * @throws IOException if the bind failed.
+ */
+ public synchronized void bind(LocalSocketAddress address)
+ throws IOException {
+ if (address == null) {
+ throw new NullPointerException("address");
+ }
+ checkNotClosed();
+ if (state != State.NEW) {
+ throw new SocketException("socket is already bound to an address");
+ }
+ bind(fd, address.getName().toString()); // JNI
+ this.address = address;
+ this.state = State.BOUND;
+ }
+
+ /**
+ * Listen for incoming connections on a socket using the specfied backlog.
+ * The socket must be bound but not already listening.
+ *
+ * @throws IOException if the listen failed.
+ */
+ public synchronized void listen(int backlog) throws IOException {
+ if (backlog < 1) {
+ throw new IllegalArgumentException("backlog=" + backlog);
+ }
+ checkNotClosed();
+ if (address == null) {
+ throw new SocketException("socket has no address bound");
+ }
+ if (state == State.LISTENING) {
+ throw new SocketException("socket is already listening");
+ }
+ listen(fd, backlog); // JNI
+ this.state = State.LISTENING;
+ }
+
+ /**
+ * Blocks until a connection is made to this socket and accepts it, returning
+ * a new socket connected to the client.
+ *
+ * @return the new socket connected to the client.
+ * @throws IOException if an error occurs when waiting for a connection.
+ * @throws SocketTimeoutException if a timeout was previously set with
+ * setSoTimeout and the timeout has been reached.
+ * @throws InterruptedIOException if the thread is interrupted when the
+ * method is blocked.
+ */
+ public synchronized Socket accept()
+ throws IOException, SocketTimeoutException, InterruptedIOException {
+ if (state != State.LISTENING) {
+ throw new SocketException("socket is not in listening state");
+ }
+
+ // Throws a SocketTimeoutException if timeout.
+ if (soTimeoutMillis != 0) {
+ poll(fd, soTimeoutMillis); // JNI
+ }
+
+ FileDescriptor clientFd = new FileDescriptor();
+ accept(fd, clientFd); // JNI
+ final LocalSocketImpl impl = new LocalSocketImpl(clientFd);
+ return new Socket(impl) {
+ @Override
+ public boolean isConnected() {
+ return true;
+ }
+ @Override
+ public synchronized void close() throws IOException {
+ if (isClosed()) {
+ return;
+ } else {
+ super.close();
+ // Workaround for the fact that super.created==false because we
+ // created the impl ourselves. As a result, super.close() doesn't
+ // call impl.close(). *Sigh*, java.net is horrendous.
+ // (Perhaps we should dispense with Socket/SocketImpl altogether?)
+ impl.close();
+ }
+ }
+ };
+ }
+
+ @Override
+ public String toString() {
+ return "LocalServerSocket(" + address + ")";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalSocket.java b/src/main/java/com/google/devtools/build/lib/unix/LocalSocket.java
new file mode 100644
index 0000000..c9d1c91
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/LocalSocket.java
@@ -0,0 +1,217 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.UnixJniLoader;
+
+import java.io.Closeable;
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+
+/**
+ * Abstract superclass for client and server local sockets.
+ */
+abstract class LocalSocket implements Closeable {
+
+ protected enum State {
+ NEW,
+ BOUND, // server only
+ LISTENING, // server only
+ CONNECTED, // client only
+ CLOSED,
+ }
+
+ protected LocalSocketAddress address = null;
+ protected FileDescriptor fd = new FileDescriptor();
+ protected State state;
+ protected boolean inputShutdown = false;
+ protected boolean outputShutdown = false;
+
+ /**
+ * Constructs an unconnected local socket.
+ */
+ protected LocalSocket() throws IOException {
+ socket(fd);
+ if (!fd.valid()) {
+ throw new IOException("Couldn't create socket!");
+ }
+ this.state = State.NEW;
+ }
+
+ /**
+ * Returns the address of the endpoint this socket is bound to.
+ *
+ * @return a <code>SocketAddress</code> representing the local endpoint of
+ * this socket.
+ */
+ public LocalSocketAddress getLocalSocketAddress() {
+ return address;
+ }
+
+ /**
+ * Closes this socket. This operation is idempotent.
+ *
+ * To be consistent with Java Socket, the shutdown states of the socket are
+ * not changed. This makes it easier to port applications between Socket and
+ * LocalSocket.
+ *
+ * @throws IOException if an I/O error occurred when closing the socket.
+ */
+ @Override
+ public synchronized void close() throws IOException {
+ if (state == State.CLOSED) {
+ return;
+ }
+ // Closes the file descriptor if it has not been closed by the
+ // input/output streams.
+ if (!fd.valid()) {
+ throw new IllegalStateException("LocalSocket.close(-1)");
+ }
+ close(fd);
+ if (fd.valid()) {
+ throw new IllegalStateException("LocalSocket.close() did not set fd to -1");
+ }
+ this.state = State.CLOSED;
+ }
+
+ /**
+ * Returns the closed state of the ServerSocket.
+ *
+ * @return true if the socket has been closed
+ */
+ public synchronized boolean isClosed() {
+ // If the file descriptor has been closed by the input/output
+ // streams, marks the socket as closed too.
+ return state == State.CLOSED;
+ }
+
+ /**
+ * Returns the connected state of the ClientSocket.
+ *
+ * @return true if the socket is currently connected.
+ */
+ public synchronized boolean isConnected() {
+ return state == State.CONNECTED;
+ }
+
+ protected synchronized void checkConnected() throws SocketException {
+ if (!isConnected()) {
+ throw new SocketException("Transport endpoint is not connected");
+ }
+ }
+
+ protected synchronized void checkNotClosed() throws SocketException {
+ if (isClosed()) {
+ throw new SocketException("socket is closed");
+ }
+ }
+
+ /**
+ * Returns the shutdown state of the input channel.
+ *
+ * @return true is the input channel of the socket is shutdown.
+ */
+ public synchronized boolean isInputShutdown() {
+ return inputShutdown;
+ }
+
+ /**
+ * Returns the shutdown state of the output channel.
+ *
+ * @return true is the input channel of the socket is shutdown.
+ */
+ public synchronized boolean isOutputShutdown() {
+ return outputShutdown;
+ }
+
+ protected synchronized void checkInputNotShutdown() throws SocketException {
+ if (isInputShutdown()) {
+ throw new SocketException("Socket input is shutdown");
+ }
+ }
+
+ protected synchronized void checkOutputNotShutdown() throws SocketException {
+ if (isOutputShutdown()) {
+ throw new SocketException("Socket output is shutdown");
+ }
+ }
+
+ static final int SHUT_RD = 0; // Mapped to BSD SHUT_RD in JNI.
+ static final int SHUT_WR = 1; // Mapped to BSD SHUT_WR in JNI.
+
+ public synchronized void shutdownInput() throws IOException {
+ checkNotClosed();
+ checkConnected();
+ checkInputNotShutdown();
+ inputShutdown = true;
+ shutdown(fd, SHUT_RD);
+ }
+
+ public synchronized void shutdownOutput() throws IOException {
+ checkNotClosed();
+ checkConnected();
+ checkOutputNotShutdown();
+ outputShutdown = true;
+ shutdown(fd, SHUT_WR);
+ }
+
+ ////////////////////////////////////////////////////////////////////////
+ // JNI:
+
+ static {
+ UnixJniLoader.loadJni();
+ }
+
+ // The native calls below are thin wrappers around linux system calls. The
+ // semantics remains the same except for poll(). See the comments for the
+ // method.
+ //
+ // Note: FileDescriptor is a box for a mutable integer that is visible only
+ // to native code.
+
+ // Generic operations:
+ protected static native void socket(FileDescriptor server)
+ throws IOException;
+ static native void close(FileDescriptor server)
+ throws IOException;
+ /**
+ * Shut down part of a full-duplex connection
+ * @param code Must be either SHUT_RD or SHUT_WR
+ */
+ static native void shutdown(FileDescriptor fd, int code)
+ throws IOException;
+
+ /**
+ * This method checks waits for the given file descriptor to become available for read.
+ * If timeoutMillis passed and there is no activity, a SocketTimeoutException will be thrown.
+ */
+ protected static native void poll(FileDescriptor read, long timeoutMillis)
+ throws IOException, SocketTimeoutException, InterruptedIOException;
+
+ // Server operations:
+ protected static native void bind(FileDescriptor server, String filename)
+ throws IOException;
+ protected static native void listen(FileDescriptor server, int backlog)
+ throws IOException;
+ protected static native void accept(FileDescriptor server,
+ FileDescriptor client)
+ throws IOException;
+
+ // Client operations:
+ protected static native void connect(FileDescriptor client, String filename)
+ throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalSocketAddress.java b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketAddress.java
new file mode 100644
index 0000000..b92a04d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketAddress.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import java.io.File;
+import java.net.SocketAddress;
+
+/**
+ * An implementation of SocketAddress for naming local sockets, i.e. files in
+ * the UNIX file system.
+ */
+public class LocalSocketAddress extends SocketAddress {
+
+ private final File name;
+
+ /**
+ * Constructs a SocketAddress for the specified file.
+ */
+ public LocalSocketAddress(File name) {
+ this.name = name;
+ }
+
+ /**
+ * Returns the filename of this local socket address.
+ */
+ public File getName() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof LocalSocketAddress &&
+ ((LocalSocketAddress) other).name.equals(this.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalSocketImpl.java b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketImpl.java
new file mode 100644
index 0000000..aee473a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketImpl.java
@@ -0,0 +1,168 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.UnixJniLoader;
+
+import java.io.Closeable;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.SocketAddress;
+import java.net.SocketImpl;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A simple implementation of SocketImpl for sockets that wrap a UNIX
+ * file-descriptor. This SocketImpl assumes that the socket is already
+ * created, bound, connected and supports no socket options or out-of-band
+ * features. This is used to implement server-side accepted client sockets
+ * (i.e. those returned by {@link LocalServerSocket#accept}).
+ */
+class LocalSocketImpl extends SocketImpl {
+ private static final Logger logger =
+ Logger.getLogger(LocalSocketImpl.class.getName());
+
+ static {
+ UnixJniLoader.loadJni();
+ init();
+ }
+
+ // The logic here is a little twisted, to support JDK7 and JDK8.
+
+ // 1) In JDK7, the FileDescriptor class keeps a reference count of
+ // instances using the fd, and closes it when it goes to 0. The
+ // reference count is only decremented by the finalizer for a
+ // given class. When the call to close() happens, the fd is
+ // closed regardless of the current state of the refcount.
+ //
+ // 2) In JDK8, every instance that uses the fd registers a Closeable
+ // with the FileDescriptor. Since the FileDescriptor has a
+ // reference to every user, only when all of the users and the
+ // FileDescriptor get GC'd does the finalizer run. An explicit
+ // call to close() calls FileDescriptor.closeAll(), which
+ // force-closes all of the users.
+
+ // So, in our case:
+
+ // 1) ref() increments the refcount in JDK7, and registers with the
+ // FD in JDK8.
+
+ // 2) unref() decrements the refcount in JDK7, and does nothing in
+ // JDK8.
+
+ // 3) The finalizer decrements the refcount in JDK7, and simply
+ // calls close() in JDK8 (where we don't have to worry about
+ // multiple live users of the FD). The close() method itself is
+ // idempotent.
+
+ // 4) close() calls fd.closeAll in JDK8, which, in turn, calls
+ // closer.close(). In JDK7, close() calls closer.close()
+ // explicitly.
+ private static native void init();
+ private static native void ref(FileDescriptor fd, Closeable closeable);
+ private static native boolean unref(FileDescriptor fd);
+ private static native boolean close0(FileDescriptor fd, Closeable closeable);
+
+ private final boolean isInitialized;
+ private final Closeable closer = new Closeable() {
+ AtomicBoolean isClosed = new AtomicBoolean(false);
+ @Override public void close() throws IOException {
+ if (isClosed.compareAndSet(false, true)) {
+ LocalSocket.close(fd);
+ }
+ }
+ };
+
+ // Note to callers: if you pass a FD into this constructor, this
+ // instance is now responsible for closing it (in the sense of
+ // LocalSocket.close()). If some other instance tries to close it,
+ // then terrible things will happen.
+ LocalSocketImpl(FileDescriptor fd) {
+ this.fd = fd; // (inherited field)
+ ref(fd, closer);
+ isInitialized = true;
+ }
+
+ @Override protected void finalize() {
+ try {
+ if (isInitialized) {
+ if (!unref(fd)) {
+ // JDK8 codepath
+ close0(fd, closer);
+ }
+ }
+ } catch (Exception e) {
+ logger.log(Level.WARNING, "Unable to access FileDescriptor class - " +
+ "may cause a file descriptor leak", e);
+ }
+ }
+ @Override protected InputStream getInputStream() {
+ return new FileInputStream(getFileDescriptor());
+ }
+ @Override protected OutputStream getOutputStream() {
+ return new FileOutputStream(getFileDescriptor());
+ }
+ @Override protected void close() throws IOException {
+ if (fd.valid()) {
+ if (!close0(fd, closer)) {
+ // JDK7 codepath
+ closer.close();
+ }
+ }
+ }
+
+ // Unused:
+ @Override
+ public void setOption(int optID, Object value) {
+ throw new UnsupportedOperationException("setOption");
+ }
+ @Override
+ public Object getOption(int optID) {
+ throw new UnsupportedOperationException("getOption");
+ }
+ @Override protected void create(boolean stream) {
+ throw new UnsupportedOperationException("create");
+ }
+ @Override protected void connect(String host, int port) {
+ throw new UnsupportedOperationException("connect");
+ }
+ @Override protected void connect(InetAddress address, int port) {
+ throw new UnsupportedOperationException("connect2");
+ }
+ @Override protected void connect(SocketAddress address, int timeout) {
+ throw new UnsupportedOperationException("connect3");
+ }
+ @Override protected void bind(InetAddress host, int port) {
+ throw new UnsupportedOperationException("bind");
+ }
+ @Override protected void listen(int backlog) {
+ throw new UnsupportedOperationException("listen");
+ }
+ @Override protected void accept(SocketImpl s) {
+ throw new UnsupportedOperationException("accept");
+ }
+ @Override protected int available() {
+ throw new UnsupportedOperationException("available");
+ }
+ @Override protected void sendUrgentData(int i) {
+ throw new UnsupportedOperationException("sendUrgentData");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/ProcessUtils.java b/src/main/java/com/google/devtools/build/lib/unix/ProcessUtils.java
new file mode 100644
index 0000000..5288e17
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/ProcessUtils.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.UnixJniLoader;
+
+
+/**
+ * Various utilities related to UNIX processes.
+ */
+public final class ProcessUtils {
+
+ private ProcessUtils() {}
+
+ static {
+ UnixJniLoader.loadJni();
+ }
+
+ /**
+ * Native wrapper around POSIX getgid(2).
+ *
+ * @return the real group ID of the current process.
+ */
+ public static native int getgid();
+
+ /**
+ * Native wrapper around POSIX getpid(2) syscall.
+ *
+ * @return the process ID of this process.
+ */
+ public static native int getpid();
+
+ /**
+ * Native wrapper around POSIX getuid(2).
+ *
+ * @return the real user ID of the current process.
+ */
+ public static native int getuid();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/AbruptExitException.java b/src/main/java/com/google/devtools/build/lib/util/AbruptExitException.java
new file mode 100644
index 0000000..ff62a7e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/AbruptExitException.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+/**
+ * An exception thrown by various error conditions that are severe enough to halt the command (e.g.
+ * even a --keep_going build). These typically need to signal to the handling code what happened.
+ * Therefore, these exceptions contain a recommended ExitCode allowing the exception to "set" a
+ * returned numeric exit code.
+ *
+ * When an instance of this exception is thrown, Blaze will try to halt as soon as reasonably
+ * possible.
+ */
+public class AbruptExitException extends Exception {
+
+ private final ExitCode exitCode;
+
+ public AbruptExitException(String message, ExitCode exitCode) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ public AbruptExitException(String message, ExitCode exitCode, Throwable cause) {
+ super(message, cause);
+ this.exitCode = exitCode;
+ }
+
+ public AbruptExitException(ExitCode exitCode, Throwable cause) {
+ super(cause);
+ this.exitCode = exitCode;
+ }
+
+ public AbruptExitException(ExitCode exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ public ExitCode getExitCode() {
+ return exitCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/AbstractIndexer.java b/src/main/java/com/google/devtools/build/lib/util/AbstractIndexer.java
new file mode 100644
index 0000000..4b61fe6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/AbstractIndexer.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Abstract class for string indexers.
+ */
+abstract class AbstractIndexer implements StringIndexer {
+
+ /**
+ * Conversion from String to byte[].
+ */
+ protected static byte[] string2bytes(String string) {
+ return string.getBytes(UTF_8);
+ }
+
+ /**
+ * Conversion from byte[] to String.
+ */
+ protected static String bytes2string(byte[] bytes) {
+ return new String(bytes, UTF_8);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStream.java
new file mode 100644
index 0000000..6c6b878
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStream.java
@@ -0,0 +1,176 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A pass-thru {@link OutputStream} that strips ANSI control codes.
+ */
+public class AnsiStrippingOutputStream extends OutputStream {
+ // The idea is straightforward: the regexp for ANSI control codes is
+ // \x1b\[[;0-9]*[a-zA-Z] . Implementing it as a stream is a little ugly,
+ // though.
+
+ private enum State {
+ NORMAL,
+ AFTER_ESCAPE,
+ PARAMETER,
+ }
+
+ private byte[] outputBuffer;
+ private int outputBufferPos;
+
+ private static final int ESCAPE_BUFFER_LENGTH = 128;
+ private byte[] escapeCodeBuffer;
+ private int escapeCodeBufferPos;
+ private OutputStream output;
+ private State state;
+
+ public AnsiStrippingOutputStream(OutputStream output) {
+ this.output = output;
+ escapeCodeBuffer = new byte[ESCAPE_BUFFER_LENGTH];
+ escapeCodeBufferPos = 0;
+ state = State.NORMAL;
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ // As per the contract of OutputStream.write(int)
+ byte[] array = { (byte) (b & 0xff) };
+ write(array, 0, 1);
+ }
+
+ @Override
+ public synchronized void write(byte b[], int off, int len) throws IOException {
+ int i = 0;
+ if (state == State.NORMAL) {
+
+ // Avoid outputBuffer allocation entirely if that's possible
+ while ((i < len) && (b[off + i] != 0x1b)) {
+ i++;
+ }
+ if (i == len) {
+ output.write(b, off, len);
+ return;
+ }
+ }
+
+ // In the worst case, the contents of the escape buffer and the contents
+ // of the input buffer are both copied to the output, so the length of the
+ // output buffer should be the sum of the length of both these buffers.
+ outputBuffer = new byte[len + ESCAPE_BUFFER_LENGTH];
+ System.arraycopy(b, off, outputBuffer, 0, i);
+ outputBufferPos = i;
+
+ for (; i < len; i++) {
+ processByte(b[off + i]);
+ }
+
+ try {
+ output.write(outputBuffer, 0, outputBufferPos);
+ } finally {
+ outputBuffer = null; // Make it possible to garbage collect the array
+ }
+ }
+
+ private void processByte(byte b) {
+ switch (state) {
+ case NORMAL:
+ if (escapeCodeBufferPos != 0) {
+ throw new IllegalStateException();
+ }
+ if (b == 0x1b) {
+ state = State.AFTER_ESCAPE;
+ addByteToEscapeBuffer(b);
+ } else {
+ dumpByte(b);
+ }
+ break;
+
+ case AFTER_ESCAPE:
+ if (b == '[') {
+ state = State.PARAMETER;
+ addByteToEscapeBuffer(b);
+ } else if (b == 0x1b) {
+ dumpEscapeBuffer();
+ state = State.AFTER_ESCAPE;
+ addByteToEscapeBuffer(b);
+ } else {
+ dumpEscapeBuffer();
+ dumpByte(b);
+ state = State.NORMAL;
+ }
+ break;
+
+ case PARAMETER:
+ if ((b >= '0' && b <= '9') || b == ';') {
+ // Parameter continues
+ addByteToEscapeBuffer(b);
+ } else if ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')) {
+ // Found a control sequence, discard it and revert to normal state
+ discardEscapeBuffer();
+ state = State.NORMAL;
+ } else if (b == 0x1b) {
+ // Another escape sequence begins immediately after, and this is
+ // an illegal escape sequence
+ dumpEscapeBuffer();
+ state = State.AFTER_ESCAPE;
+ addByteToEscapeBuffer(b);
+ } else {
+ // Illegal control sequence, output it
+ dumpEscapeBuffer();
+ state = State.NORMAL;
+ }
+ break;
+ }
+ }
+
+ private void addByteToEscapeBuffer(byte b) {
+ escapeCodeBuffer[escapeCodeBufferPos++] = b;
+ if (escapeCodeBufferPos == ESCAPE_BUFFER_LENGTH) {
+ // Buffer full. Assume that no sane code emits an ANSI control code this
+ // long and revert to normal state.
+ dumpEscapeBuffer();
+ state = State.NORMAL;
+ }
+ }
+
+ private void discardEscapeBuffer() {
+ escapeCodeBufferPos = 0;
+ }
+
+ private void dumpByte(byte b) {
+ outputBuffer[outputBufferPos++] = b;
+ }
+
+ private void dumpEscapeBuffer() {
+ System.arraycopy(escapeCodeBuffer, 0,
+ outputBuffer, outputBufferPos, escapeCodeBufferPos);
+ outputBufferPos += escapeCodeBufferPos;
+ escapeCodeBufferPos = 0;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ output.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ output.close();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/BinaryPredicate.java b/src/main/java/com/google/devtools/build/lib/util/BinaryPredicate.java
new file mode 100644
index 0000000..c7709e2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/BinaryPredicate.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Predicate;
+
+import javax.annotation.Nullable;
+
+/**
+ * A two-argument version of {@link Predicate} that determines a true or false value for pairs of
+ * inputs.
+ *
+ * <p>Just as a {@link Predicate} is useful for filtering iterables of values, a {@link
+ * BinaryPredicate} is useful for filtering iterables of paired values, like {@link
+ * java.util.Map.Entry} or {@link Pair}.
+ *
+ * <p>See {@link Predicate} for implementation notes and advice.
+ */
+public interface BinaryPredicate<X, Y> {
+
+ /**
+ * Applies this {@link BinaryPredicate} to the given objects.
+ *
+ * @return the value of this predicate when applied to inputs {@code x, y}
+ */
+ boolean apply(@Nullable X x, @Nullable Y y);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/BlazeClock.java b/src/main/java/com/google/devtools/build/lib/util/BlazeClock.java
new file mode 100644
index 0000000..72806dd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/BlazeClock.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.JavaClock;
+
+/**
+ * Provides the clock implementation used by Blaze, which is {@link JavaClock}
+ * by default, but can be overridden at runtime. Note that if you set this
+ * clock, you also have to set the clock used by the Profiler.
+ */
+@ThreadSafe
+public abstract class BlazeClock {
+
+ private BlazeClock() {
+ }
+
+ private static volatile Clock instance = new JavaClock();
+
+ /**
+ * Returns singleton instance of the clock
+ */
+ public static Clock instance() {
+ return instance;
+ }
+
+ /**
+ * Overrides default clock instance.
+ */
+ public static synchronized void setClock(Clock clock) {
+ instance = clock;
+ }
+
+ public static long nanoTime() {
+ return instance().nanoTime();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CanonicalStringIndexer.java b/src/main/java/com/google/devtools/build/lib/util/CanonicalStringIndexer.java
new file mode 100644
index 0000000..618e88b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CanonicalStringIndexer.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+
+import java.util.Map;
+
+/**
+ * A string indexer backed by a map and reverse index lookup.
+ * Every unique string is stored in memory exactly once.
+ */
+@ThreadSafe
+public class CanonicalStringIndexer extends AbstractIndexer {
+
+ private static final int NOT_FOUND = -1;
+
+ // This is similar to (Synchronized) BiMap.
+ // These maps *must* be weakly threadsafe to ensure thread safety for string
+ // indexer as a whole. Specifically, mutating operations are serialized, but
+ // read-only operations may be executed concurrently with mutators.
+ private final Map<String, Integer> stringToInt;
+ private final Map<Integer, String> intToString;
+
+ /*
+ * Creates an indexer instance from two backing maps. These maps may be
+ * pre-initialized with data, but *must*:
+ * a. Support read-only operations done concurrently with mutations.
+ * Note that mutations will be serialized.
+ * b. Be reverse mappings of each other, if pre-initialized.
+ */
+ public CanonicalStringIndexer(Map<String, Integer> stringToInt,
+ Map<Integer, String> intToString) {
+ Preconditions.checkArgument(stringToInt.size() == intToString.size());
+ this.stringToInt = stringToInt;
+ this.intToString = intToString;
+ }
+
+
+ @Override
+ public synchronized void clear() {
+ stringToInt.clear();
+ intToString.clear();
+ }
+
+ @Override
+ public int size() {
+ return intToString.size();
+ }
+
+ @Override
+ public int getOrCreateIndex(String s) {
+ Integer i = stringToInt.get(s);
+ if (i == null) {
+ synchronized (this) {
+ // First, make sure another thread hasn't just added the entry:
+ i = stringToInt.get(s);
+ if (i != null) {
+ return i;
+ }
+
+ int ind = intToString.size();
+ s = StringCanonicalizer.intern(s);
+ stringToInt.put(s, ind);
+ intToString.put(ind, s);
+ return ind;
+ }
+ } else {
+ return i;
+ }
+ }
+
+ @Override
+ public int getIndex(String s) {
+ Integer i = stringToInt.get(s);
+ return (i == null) ? NOT_FOUND : i;
+ }
+
+ @Override
+ public synchronized boolean addString(String s) {
+ int originalSize = size();
+ getOrCreateIndex(s);
+ return (size() > originalSize);
+ }
+
+ @Override
+ public String getStringForIndex(int i) {
+ return intToString.get(i);
+ }
+
+ @Override
+ public synchronized String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("size = ").append(size()).append("\n");
+ for (Map.Entry<String, Integer> entry : stringToInt.entrySet()) {
+ builder.append(entry.getKey()).append(" <==> ").append(entry.getValue()).append("\n");
+ }
+ return builder.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Clock.java b/src/main/java/com/google/devtools/build/lib/util/Clock.java
new file mode 100644
index 0000000..878cb11
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/Clock.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+/**
+ * This class provides an interface for a pluggable clock.
+ */
+public interface Clock {
+
+ /**
+ * Returns the current time in milliseconds. The milliseconds are counted from midnight
+ * Jan 1, 1970.
+ */
+ long currentTimeMillis();
+
+ /**
+ * Returns the current time in nanoseconds. The nanoseconds are measured relative to some
+ * unknown, but fixed event. Unfortunately, a sequence of calls to this method is *not*
+ * guaranteed to return non-decreasing values, so callers should be tolerant to this behavior.
+ */
+ long nanoTime();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandBuilder.java b/src/main/java/com/google/devtools/build/lib/util/CommandBuilder.java
new file mode 100644
index 0000000..372802d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CommandBuilder.java
@@ -0,0 +1,176 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Implements OS aware {@link Command} builder. At this point only Linux, Mac
+ * and Windows XP are supported.
+ *
+ * <p>Builder will also apply heuristic to identify trivial cases where
+ * unix-like command lines could be automatically converted into the
+ * Windows-compatible form.
+ *
+ * <p>TODO(bazel-team): (2010) Some of the code here is very similar to the
+ * {@link com.google.devtools.build.lib.shell.Shell} class. This should be looked at.
+ */
+public final class CommandBuilder {
+
+ private static final List<String> SHELLS = ImmutableList.of("/bin/sh", "/bin/bash");
+
+ private static final Splitter ARGV_SPLITTER = Splitter.on(CharMatcher.anyOf(" \t"));
+
+ private final OS system;
+ private final List<String> argv = new ArrayList<>();
+ private final Map<String, String> env = new HashMap<>();
+ private File workingDir = null;
+ private boolean useShell = false;
+
+ public CommandBuilder() {
+ this(OS.getCurrent());
+ }
+
+ @VisibleForTesting
+ CommandBuilder(OS system) {
+ this.system = system;
+ }
+
+ public CommandBuilder addArg(String arg) {
+ Preconditions.checkNotNull(arg, "Argument must not be null");
+ argv.add(arg);
+ return this;
+ }
+
+ public CommandBuilder addArgs(Iterable<String> args) {
+ Preconditions.checkArgument(!Iterables.contains(args, null), "Arguments must not be null");
+ Iterables.addAll(argv, args);
+ return this;
+ }
+
+ public CommandBuilder addArgs(String... args) {
+ return addArgs(Arrays.asList(args));
+ }
+
+ public CommandBuilder addEnv(Map<String, String> env) {
+ Preconditions.checkNotNull(env);
+ this.env.putAll(env);
+ return this;
+ }
+
+ public CommandBuilder emptyEnv() {
+ env.clear();
+ return this;
+ }
+
+ public CommandBuilder setEnv(Map<String, String> env) {
+ emptyEnv();
+ addEnv(env);
+ return this;
+ }
+
+ public CommandBuilder setWorkingDir(Path path) {
+ Preconditions.checkNotNull(path);
+ workingDir = path.getPathFile();
+ return this;
+ }
+
+ public CommandBuilder useTempDir() {
+ workingDir = new File(System.getProperty("java.io.tmpdir"));
+ return this;
+ }
+
+ public CommandBuilder useShell(boolean useShell) {
+ this.useShell = useShell;
+ return this;
+ }
+
+ private boolean argvStartsWithSh() {
+ return argv.size() >= 2 && SHELLS.contains(argv.get(0)) && "-c".equals(argv.get(1));
+ }
+
+ private String[] transformArgvForLinux() {
+ // If command line already starts with "/bin/sh -c", ignore useShell attribute.
+ if (useShell && !argvStartsWithSh()) {
+ // c.g.io.base.shell.Shell.shellify() actually concatenates argv into the space-separated
+ // string here. Not sure why, but we will do the same.
+ return new String[] { "/bin/sh", "-c", Joiner.on(' ').join(argv) };
+ }
+ return argv.toArray(new String[argv.size()]);
+ }
+
+ private String[] transformArgvForWindows() {
+ List<String> modifiedArgv;
+ // Heuristic: replace "/bin/sh -c" with something more appropriate for Windows.
+ if (argvStartsWithSh()) {
+ useShell = true;
+ modifiedArgv = Lists.newArrayList(argv.subList(2, argv.size()));
+ } else {
+ modifiedArgv = Lists.newArrayList(argv);
+ }
+
+ if (!modifiedArgv.isEmpty()) {
+ // args can contain whitespace, so figure out the first word
+ String argv0 = modifiedArgv.get(0);
+ String command = ARGV_SPLITTER.split(argv0).iterator().next();
+
+ // Automatically enable CMD.EXE use if we are executing something else besides "*.exe" file.
+ if (!command.toLowerCase().endsWith(".exe")) {
+ useShell = true;
+ }
+ } else {
+ // This is degenerate "/bin/sh -c" case. We ensure that Windows behavior is identical
+ // to the Linux - call shell that will do nothing.
+ useShell = true;
+ }
+ if (useShell) {
+ // /S - strip first and last quotes and execute everything else as is.
+ // /E:ON - enable extended command set.
+ // /V:ON - enable delayed variable expansion
+ // /D - ignore AutoRun registry entries.
+ // /C - execute command. This must be the last option before the command itself.
+ return new String[] { "CMD.EXE", "/S", "/E:ON", "/V:ON", "/D", "/C",
+ "\"" + Joiner.on(' ').join(modifiedArgv) + "\"" };
+ } else {
+ return modifiedArgv.toArray(new String[argv.size()]);
+ }
+ }
+
+ public Command build() {
+ Preconditions.checkState(system != OS.UNKNOWN, "Unidentified operating system");
+ Preconditions.checkNotNull(workingDir, "Working directory must be set");
+ Preconditions.checkState(argv.size() > 0, "At least one argument is expected");
+
+ return new Command(
+ system == OS.WINDOWS ? transformArgvForWindows() : transformArgvForLinux(),
+ env, workingDir);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandDescriptionForm.java b/src/main/java/com/google/devtools/build/lib/util/CommandDescriptionForm.java
new file mode 100644
index 0000000..8d37275
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CommandDescriptionForm.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+/**
+ * Forms in which a command can be described by {@link CommandFailureUtils#describeCommand}.
+ */
+public enum CommandDescriptionForm {
+ /**
+ * A form that is usually suitable for identifying the command but not for
+ * re-executing it. The working directory and environment are not shown, and
+ * the arguments are truncated to a maximum of a few hundred bytes.
+ */
+ ABBREVIATED,
+
+ /**
+ * A form that is complete and suitable for a user to copy and paste into a
+ * shell. On Linux, the command is placed in a subshell so it has no side
+ * effects on the user's shell. On Windows, this is not implemented, but the
+ * side effects in question are less severe (no "exec").
+ */
+ COMPLETE,
+
+ /**
+ * A form that is complete and does not isolate side effects. Suitable for
+ * launch scripts, i.e., "blaze run --script_path".
+ */
+ COMPLETE_UNISOLATED,
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandFailureUtils.java b/src/main/java/com/google/devtools/build/lib/util/CommandFailureUtils.java
new file mode 100644
index 0000000..9178f985
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CommandFailureUtils.java
@@ -0,0 +1,252 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Utility methods for describing command failures.
+ * See also the CommandUtils class.
+ * Unlike that one, this class does not depend on Command;
+ * instead, it just manipulates command lines represented as
+ * Collection<String>.
+ */
+public class CommandFailureUtils {
+
+ // Interface that provides building blocks when describing command.
+ private interface DescribeCommandImpl {
+ void describeCommandBeginIsolate(StringBuilder message);
+ void describeCommandEndIsolate(StringBuilder message);
+ void describeCommandCwd(String cwd, StringBuilder message);
+ void describeCommandEnvPrefix(StringBuilder message);
+ void describeCommandEnvVar(StringBuilder message, Map.Entry<String, String> entry);
+ void describeCommandElement(StringBuilder message, String commandElement);
+ void describeCommandExec(StringBuilder message);
+ }
+
+ private static final class LinuxDescribeCommandImpl implements DescribeCommandImpl {
+
+ @Override
+ public void describeCommandBeginIsolate(StringBuilder message) {
+ message.append("(");
+ }
+
+ @Override
+ public void describeCommandEndIsolate(StringBuilder message) {
+ message.append(")");
+ }
+
+ @Override
+ public void describeCommandCwd(String cwd, StringBuilder message) {
+ message.append("cd ").append(ShellEscaper.escapeString(cwd)).append(" && \\\n ");
+ }
+
+ @Override
+ public void describeCommandEnvPrefix(StringBuilder message) {
+ message.append("env - \\\n ");
+ }
+
+ @Override
+ public void describeCommandEnvVar(StringBuilder message, Map.Entry<String, String> entry) {
+ message.append(ShellEscaper.escapeString(entry.getKey())).append('=')
+ .append(ShellEscaper.escapeString(entry.getValue())).append(" \\\n ");
+ }
+
+ @Override
+ public void describeCommandElement(StringBuilder message, String commandElement) {
+ message.append(ShellEscaper.escapeString(commandElement));
+ }
+
+ @Override
+ public void describeCommandExec(StringBuilder message) {
+ message.append("exec ");
+ }
+ }
+
+ // TODO(bazel-team): (2010) Add proper escaping. We can't use ShellUtils.shellEscape() as it is
+ // incompatible with CMD.EXE syntax, but something else might be needed.
+ private static final class WindowsDescribeCommandImpl implements DescribeCommandImpl {
+
+ @Override
+ public void describeCommandBeginIsolate(StringBuilder message) {
+ // TODO(bazel-team): Implement this.
+ }
+
+ @Override
+ public void describeCommandEndIsolate(StringBuilder message) {
+ // TODO(bazel-team): Implement this.
+ }
+
+ @Override
+ public void describeCommandCwd(String cwd, StringBuilder message) {
+ message.append("cd ").append(cwd).append("\n");
+ }
+
+ @Override
+ public void describeCommandEnvPrefix(StringBuilder message) { }
+
+ @Override
+ public void describeCommandEnvVar(StringBuilder message, Map.Entry<String, String> entry) {
+ message.append("SET ").append(entry.getKey()).append('=')
+ .append(entry.getValue()).append("\n ");
+ }
+
+ @Override
+ public void describeCommandElement(StringBuilder message, String commandElement) {
+ message.append(commandElement);
+ }
+
+ @Override
+ public void describeCommandExec(StringBuilder message) {
+ // TODO(bazel-team): Implement this if possible for greater efficiency.
+ }
+ }
+
+ private static final DescribeCommandImpl describeCommandImpl =
+ OS.getCurrent() == OS.WINDOWS ? new WindowsDescribeCommandImpl()
+ : new LinuxDescribeCommandImpl();
+
+ private CommandFailureUtils() {} // Prevent instantiation.
+
+ private static Comparator<Map.Entry<String, String>> mapEntryComparator =
+ new Comparator<Map.Entry<String, String>>() {
+ @Override
+ public int compare(Map.Entry<String, String> x, Map.Entry<String, String> y) {
+ // A map can never have two keys with the same value, so we only need to compare the keys.
+ return x.getKey().compareTo(y.getKey());
+ }
+ };
+
+ /**
+ * Construct a string that describes the command.
+ * Currently this returns a message of the form "foo bar baz",
+ * with shell meta-characters appropriately quoted and/or escaped,
+ * prefixed (if verbose is true) with an "env" command to set the environment.
+ *
+ * @param form Form of the command to generate; see the documentation of the
+ * {@link CommandDescriptionForm} values.
+ */
+ public static String describeCommand(CommandDescriptionForm form,
+ Collection<String> commandLineElements,
+ @Nullable Map<String, String> environment, @Nullable String cwd) {
+ Preconditions.checkNotNull(form);
+ final int APPROXIMATE_MAXIMUM_MESSAGE_LENGTH = 200;
+ StringBuilder message = new StringBuilder();
+ int size = commandLineElements.size();
+ int numberRemaining = size;
+ if (form == CommandDescriptionForm.COMPLETE) {
+ describeCommandImpl.describeCommandBeginIsolate(message);
+ }
+ if (form != CommandDescriptionForm.ABBREVIATED) {
+ if (cwd != null) {
+ describeCommandImpl.describeCommandCwd(cwd, message);
+ }
+ /*
+ * On Linux, insert an "exec" keyword to save a fork in "blaze run"
+ * generated scripts. If we use "env" as a wrapper, the "exec" needs to
+ * be applied to the entire "env" invocation.
+ *
+ * On Windows, this is a no-op.
+ */
+ describeCommandImpl.describeCommandExec(message);
+ /*
+ * Java does not provide any way to invoke a subprocess with the environment variables
+ * in a specified order. The order of environment variables in the 'environ' array
+ * (which is set by the 'envp' parameter to the execve() system call)
+ * is determined by the order of iteration on a HashMap constructed inside Java's
+ * ProcessBuilder class (in the ProcessEnvironment class), which is nondeterministic.
+ *
+ * Nevertheless, we *print* the environment variables here in sorted order, rather
+ * than in the potentially nondeterministic order that will be actually used.
+ * This is slightly dubious... in theory a process's behaviour could depend on the order
+ * of the environment variables passed to it. (For example, the order of environment
+ * variables in the environ array affects the output of '/usr/bin/env'.)
+ * However, in practice very few processes depend on the order of the environment
+ * variables, and using a deterministic sorted order here makes Blaze's output more
+ * deterministic and easier to read. So this seems the lesser of two evils... I think.
+ * Anyway, it's not like we have much choice... even if we wanted to, there's no way to
+ * print out the nondeterministic order that will actually be used, since there's
+ * no way to guarantee that the iteration over entrySet() here will return the same
+ * sequence as the iteration over entrySet() inside the ProcessBuilder class
+ * (in ProcessEnvironment.StringEnvironment.toEnvironmentBlock()).
+ */
+ if (environment != null) {
+ describeCommandImpl.describeCommandEnvPrefix(message);
+ for (Map.Entry<String, String> entry :
+ Ordering.from(mapEntryComparator).sortedCopy(environment.entrySet())) {
+ message.append(" ");
+ describeCommandImpl.describeCommandEnvVar(message, entry);
+ }
+ }
+ }
+ for (String commandElement : commandLineElements) {
+ if (form == CommandDescriptionForm.ABBREVIATED &&
+ message.length() + commandElement.length() > APPROXIMATE_MAXIMUM_MESSAGE_LENGTH) {
+ message.append(
+ " ... (remaining " + numberRemaining + " argument(s) skipped)");
+ break;
+ } else {
+ if (numberRemaining < size) {
+ message.append(' ');
+ }
+ describeCommandImpl.describeCommandElement(message, commandElement);
+ numberRemaining--;
+ }
+ }
+ if (form == CommandDescriptionForm.COMPLETE) {
+ describeCommandImpl.describeCommandEndIsolate(message);
+ }
+ return message.toString();
+ }
+
+ /**
+ * Construct an error message that describes a failed command invocation.
+ * Currently this returns a message of the form "error executing command foo
+ * bar baz".
+ */
+ public static String describeCommandError(boolean verbose,
+ Collection<String> commandLineElements,
+ Map<String, String> env, String cwd) {
+ CommandDescriptionForm form = verbose
+ ? CommandDescriptionForm.COMPLETE
+ : CommandDescriptionForm.ABBREVIATED;
+ return "error executing command " + (verbose ? "\n " : "")
+ + describeCommand(form, commandLineElements, env, cwd);
+ }
+
+ /**
+ * Construct an error message that describes a failed command invocation.
+ * Currently this returns a message of the form "foo failed: error executing
+ * command /dir/foo bar baz".
+ */
+ public static String describeCommandFailure(boolean verbose,
+ Collection<String> commandLineElements,
+ Map<String, String> env, String cwd) {
+ String commandName = commandLineElements.iterator().next();
+ // Extract the part of the command name after the last "/", if any.
+ String shortCommandName = new File(commandName).getName();
+ return shortCommandName + " failed: " +
+ describeCommandError(verbose, commandLineElements, env, cwd);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandUtils.java b/src/main/java/com/google/devtools/build/lib/util/CommandUtils.java
new file mode 100644
index 0000000..e6c0011
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CommandUtils.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.shell.AbnormalTerminationException;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.shell.CommandResult;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Utility methods relating to the {@link Command} class.
+ */
+public class CommandUtils {
+
+ private CommandUtils() {} // Prevent instantiation.
+
+ private static Collection<String> commandLine(Command command) {
+ return Arrays.asList(command.getCommandLineElements());
+ }
+
+ private static Map<String, String> env(Command command) {
+ return command.getEnvironmentVariables();
+ }
+
+ private static String cwd(Command command) {
+ return command.getWorkingDirectory() == null ? null : command.getWorkingDirectory().getPath();
+ }
+
+ /**
+ * Construct an error message that describes a failed command invocation.
+ * Currently this returns a message of the form "error executing command foo
+ * bar baz".
+ */
+ public static String describeCommandError(boolean verbose, Command command) {
+ return CommandFailureUtils.describeCommandError(verbose, commandLine(command), env(command),
+ cwd(command));
+ }
+
+ /**
+ * Construct an error message that describes a failed command invocation.
+ * Currently this returns a message of the form "foo failed: error executing
+ * command /dir/foo bar baz".
+ */
+ public static String describeCommandFailure(boolean verbose, Command command) {
+ return CommandFailureUtils.describeCommandFailure(verbose, commandLine(command), env(command),
+ cwd(command));
+ }
+
+ /**
+ * Construct an error message that describes a failed command invocation.
+ * Currently this returns a message of the form "foo failed: error executing
+ * command /dir/foo bar baz: exception message", with the
+ * command's stdout and stderr output appended if available.
+ */
+ public static String describeCommandFailure(boolean verbose, CommandException exception) {
+ String message = describeCommandFailure(verbose, exception.getCommand()) + ": "
+ + exception.getMessage();
+ if (exception instanceof AbnormalTerminationException) {
+ CommandResult result = ((AbnormalTerminationException) exception).getResult();
+ try {
+ return message + "\n" +
+ new String(result.getStdout()) +
+ new String(result.getStderr());
+ } catch (IllegalStateException e) {
+ // This can happen if the command didn't save stdout/stderr,
+ // so ignore this exception and fall through to the ordinary case.
+ }
+ }
+ return message;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CompactStringIndexer.java b/src/main/java/com/google/devtools/build/lib/util/CompactStringIndexer.java
new file mode 100644
index 0000000..698758d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/CompactStringIndexer.java
@@ -0,0 +1,546 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.util.ArrayList;
+
+/**
+ * Provides memory-efficient bidirectional mapping String <-> unique integer.
+ * Uses byte-wise compressed prefix trie internally.
+ * <p>
+ * Class allows index retrieval for the given string, addition of the new
+ * index and string retrieval for the given index. It also allows efficient
+ * serialization of the internal data structures.
+ * <p>
+ * Internally class stores list of nodes with each node containing byte[]
+ * representation of compressed trie node:
+ * <pre>
+ * varint32 parentIndex; // index of the parent node
+ * varint32 keylen; // length of the node key
+ * byte[keylen] key; // node key data
+ * repeated jumpEntry { // Zero or more jump entries, referencing child nodes
+ * byte key // jump key (first byte of the child node key)
+ * varint32 nodeIndex // child index
+ * }
+ * <p>
+ * Note that jumpEntry key byte is actually duplicated in the child node
+ * instance. This is done to improve performance of the index->string
+ * lookup (so we can avoid jump table parsing during this lookup).
+ * <p>
+ * Root node of the trie must have parent id pointing to itself.
+ * <p>
+ * TODO(bazel-team): (2010) Consider more fine-tuned locking mechanism - e.g.
+ * distinguishing between read and write locks.
+ */
+@ThreadSafe
+public class CompactStringIndexer extends AbstractIndexer {
+
+ private static final int NOT_FOUND = -1;
+
+ private ArrayList<byte[]> nodes; // Compressed prefix trie nodes.
+ private int rootId; // Root node id.
+
+ /*
+ * Creates indexer instance.
+ */
+ public CompactStringIndexer (int expectedCapacity) {
+ Preconditions.checkArgument(expectedCapacity > 0);
+ nodes = Lists.newArrayListWithExpectedSize(expectedCapacity);
+ rootId = NOT_FOUND;
+ }
+
+ /**
+ * Allocates new node index. Must be called only from
+ * synchronized methods.
+ */
+ private int allocateIndex() {
+ nodes.add(null);
+ return nodes.size() - 1;
+ }
+
+ /**
+ * Replaces given node record with the new one. Must be called only from
+ * synchronized methods.
+ * <p>
+ * Subclasses can override this method to be notified when an update actually
+ * takes place.
+ */
+ @ThreadCompatible
+ protected void updateNode(int index, byte[] content) {
+ nodes.set(index, content);
+ }
+
+ /**
+ * Returns parent id for the given node content.
+ *
+ * @return parent node id
+ */
+ private int getParentId(byte[] content) {
+ int[] intHolder = new int[1];
+ VarInt.getVarInt(content, 0, intHolder);
+ return intHolder[0];
+ }
+
+ /**
+ * Creates new node using specified key suffix. Must be called from
+ * synchronized methods.
+ *
+ * @param parentNode parent node id
+ * @param key original key that is being added to the indexer
+ * @param offset node key offset in the original key.
+ *
+ * @return new node id corresponding to the given key
+ */
+ private int createNode(int parentNode, byte[] key, int offset) {
+ int index = allocateIndex();
+
+ int len = key.length - offset;
+ Preconditions.checkState(len >= 0);
+
+ // Content consists of parent id, key length and key. There are no jump records.
+ byte[] content = new byte[VarInt.varIntSize(parentNode) + VarInt.varIntSize(len) + len];
+ // Add parent id.
+ int contentOffset = VarInt.putVarInt(parentNode, content, 0);
+ // Add node key length.
+ contentOffset = VarInt.putVarInt(len, content, contentOffset);
+ // Add node key content.
+ System.arraycopy(key, offset, content, contentOffset, len);
+
+ updateNode(index, content);
+ return index;
+ }
+
+ /**
+ * Updates jump entry index in the given node.
+ *
+ * @param node node id to update
+ * @param oldIndex old jump entry index
+ * @param newIndex updated jump entry index
+ */
+ private void updateJumpEntry(int node, int oldIndex, int newIndex) {
+ byte[] content = nodes.get(node);
+ int[] intHolder = new int[1];
+ int offset = VarInt.getVarInt(content, 0, intHolder); // parent id
+ offset = VarInt.getVarInt(content, offset, intHolder); // key length
+ offset += intHolder[0]; // Offset now points to the first jump entry.
+ while (offset < content.length) {
+ int next = VarInt.getVarInt(content, offset + 1, intHolder); // jump index
+ if (intHolder[0] == oldIndex) {
+ // Substitute oldIndex value with newIndex.
+ byte[] newContent =
+ new byte[content.length + VarInt.varIntSize(newIndex) - VarInt.varIntSize(oldIndex)];
+ System.arraycopy(content, 0, newContent, 0, offset + 1);
+ offset = VarInt.putVarInt(newIndex, newContent, offset + 1);
+ System.arraycopy(content, next, newContent, offset, content.length - next);
+ updateNode(node, newContent);
+ return;
+ } else {
+ offset = next;
+ }
+ }
+ StringBuilder builder = new StringBuilder().append("Index ").append(oldIndex)
+ .append(" is not present in the node ").append(node).append(", ");
+ dumpNodeContent(builder, content);
+ throw new IllegalArgumentException(builder.toString());
+ }
+
+ /**
+ * Creates new branch node content at the predefined location, splitting
+ * prefix from the given node and optionally adding another child node
+ * jump entry.
+ *
+ * @param originalNode node that will be split
+ * @param newBranchNode new branch node id
+ * @param splitOffset offset at which to split original node key
+ * @param indexKey optional additional jump key
+ * @param childIndex optional additional jump index. Optional jump entry will
+ * be skipped if this index is set to NOT_FOUND.
+ */
+ private void createNewBranchNode(int originalNode, int newBranchNode, int splitOffset,
+ byte indexKey, int childIndex) {
+ byte[] content = nodes.get(originalNode);
+ int[] intHolder = new int[1];
+ int keyOffset = VarInt.getVarInt(content, 0, intHolder); // parent id
+
+ // If original node is a root node, new branch node will become new root. So set parent id
+ // appropriately (for root node it is set to the node's own id).
+ int parentIndex = (originalNode == intHolder[0] ? newBranchNode : intHolder[0]);
+
+ keyOffset = VarInt.getVarInt(content, keyOffset, intHolder); // key length
+ Preconditions.checkState(intHolder[0] >= splitOffset);
+ // Calculate new content size.
+ int newSize = VarInt.varIntSize(parentIndex)
+ + VarInt.varIntSize(splitOffset) + splitOffset
+ + 1 + VarInt.varIntSize(originalNode)
+ + (childIndex != NOT_FOUND ? 1 + VarInt.varIntSize(childIndex) : 0);
+ // New content consists of parent id, new key length, truncated key and two jump records.
+ byte[] newContent = new byte[newSize];
+ // Add parent id.
+ int contentOffset = VarInt.putVarInt(parentIndex, newContent, 0);
+ // Add adjusted key length.
+ contentOffset = VarInt.putVarInt(splitOffset, newContent, contentOffset);
+ // Add truncated key content and first jump key.
+ System.arraycopy(content, keyOffset, newContent, contentOffset, splitOffset + 1);
+ // Add index for the first jump key.
+ contentOffset = VarInt.putVarInt(originalNode, newContent, contentOffset + splitOffset + 1);
+ // If requested, add additional jump entry.
+ if (childIndex != NOT_FOUND) {
+ // Add second jump key.
+ newContent[contentOffset] = indexKey;
+ // Add index for the second jump key.
+ VarInt.putVarInt(childIndex, newContent, contentOffset + 1);
+ }
+ updateNode(newBranchNode, newContent);
+ }
+
+ /**
+ * Inject newly created branch node into the trie data structure. Method
+ * will update parent node jump entry to point to the new branch node (or
+ * will update root id if branch node becomes new root) and will truncate
+ * key prefix from the original node that was split (that prefix now
+ * resides in the branch node).
+ *
+ * @param originalNode node that will be split
+ * @param newBranchNode new branch node id
+ * @param commonPrefixLength how many bytes should be split into the new branch node.
+ */
+ private void injectNewBranchNode(int originalNode, int newBranchNode, int commonPrefixLength) {
+ byte[] content = nodes.get(originalNode);
+
+ int parentId = getParentId(content);
+ if (originalNode == parentId) {
+ rootId = newBranchNode; // update root index
+ } else {
+ updateJumpEntry(parentId, originalNode, newBranchNode);
+ }
+
+ // Truncate prefix from the original node and set its parent to the our new branch node.
+ int[] intHolder = new int[1];
+ int suffixOffset = VarInt.getVarInt(content, 0, intHolder); // parent id
+ suffixOffset = VarInt.getVarInt(content, suffixOffset, intHolder); // key length
+ int len = intHolder[0] - commonPrefixLength;
+ Preconditions.checkState(len >= 0);
+ suffixOffset += commonPrefixLength;
+ // New content consists of parent id, new key length and duplicated key suffix.
+ byte[] newContent = new byte[VarInt.varIntSize(newBranchNode) + VarInt.varIntSize(len) +
+ (content.length - suffixOffset)];
+ // Add parent id.
+ int contentOffset = VarInt.putVarInt(newBranchNode, newContent, 0);
+ // Add new key length.
+ contentOffset = VarInt.putVarInt(len, newContent, contentOffset);
+ // Add key and jump table.
+ System.arraycopy(content, suffixOffset, newContent, contentOffset,
+ content.length - suffixOffset);
+ updateNode(originalNode, newContent);
+ }
+
+ /**
+ * Adds new child node (that uses specified key suffix) to the given
+ * current node.
+ * Example:
+ * <pre>
+ * Had "ab". Adding "abcd".
+ *
+ * 1:"ab",'c'->2
+ * 1:"ab" -> \
+ * 2:"cd"
+ * </pre>
+ */
+ private int addChildNode(int parentNode, byte[] key, int keyOffset) {
+ int child = createNode(parentNode, key, keyOffset);
+
+ byte[] content = nodes.get(parentNode);
+ // Add jump table entry to the parent node.
+ int entryOffset = content.length;
+ // New content consists of original content and additional jump record.
+ byte[] newContent = new byte[entryOffset + 1 + VarInt.varIntSize(child)];
+ // Copy original content.
+ System.arraycopy(content, 0, newContent, 0, entryOffset);
+ // Add jump key.
+ newContent[entryOffset] = key[keyOffset];
+ // Add jump index.
+ VarInt.putVarInt(child, newContent, entryOffset + 1);
+
+ updateNode(parentNode, newContent);
+ return child;
+ }
+
+ /**
+ * Splits node into two at the specified offset.
+ * Example:
+ * <pre>
+ * Had "abcd". Adding "ab".
+ *
+ * 2:"ab",'c'->1
+ * 1:"abcd" -> \
+ * 1:"cd"
+ * </pre>
+ */
+ private int splitNodeSuffix(int nodeToSplit, int commonPrefixLength) {
+ int newBranchNode = allocateIndex();
+ // Create new node with truncated key.
+ createNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength, (byte) 0, NOT_FOUND);
+ injectNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength);
+
+ return newBranchNode;
+ }
+
+ /**
+ * Splits node into two at the specified offset and adds another leaf.
+ * Example:
+ * <pre>
+ * Had "abcd". Adding "abef".
+ *
+ * 3:"ab",'c'->1,'e'->2
+ * 1:"abcd" -> / \
+ * 1:"cd" 2:"ef"
+ * </pre>
+ */
+ private int addBranch(int nodeToSplit, byte[] key, int offset, int commonPrefixLength) {
+ int newBranchNode = allocateIndex();
+ int child = createNode(newBranchNode, key, offset + commonPrefixLength);
+ // Create new node with the truncated key and reference to the new child node.
+ createNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength,
+ key[offset + commonPrefixLength], child);
+ injectNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength);
+
+ return child;
+ }
+
+ private int findOrCreateIndexInternal(int node, byte[] key, int offset,
+ boolean createIfNotFound) {
+ byte[] content = nodes.get(node);
+ int[] intHolder = new int[1];
+ int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id
+ contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length
+ int skyKeyLen = intHolder[0];
+ int remainingKeyLen = key.length - offset;
+ int minKeyLen = remainingKeyLen > skyKeyLen ? skyKeyLen : remainingKeyLen;
+
+ // Compare given key/offset content with the node key. Skip first key byte for recursive
+ // calls - this byte is equal to the byte in the jump entry and was already compared.
+ for (int i = (offset > 0 ? 1 : 0); i < minKeyLen; i++) { // compare key
+ if (key[offset + i] != content[contentOffset + i]) {
+ // Mismatch found somewhere in the middle of the node key. If requested, node
+ // should be split and another leaf added for the new key.
+ return createIfNotFound ? addBranch(node, key, offset, i) : NOT_FOUND;
+ }
+ }
+
+ if (remainingKeyLen > minKeyLen) {
+ // Node key matched portion of the key - find appropriate jump entry. If found - recursion.
+ // If not - mismatch (we will add new child node if requested).
+ contentOffset += skyKeyLen;
+ while (contentOffset < content.length) {
+ if (key[offset + skyKeyLen] == content[contentOffset]) { // compare index value
+ VarInt.getVarInt(content, contentOffset + 1, intHolder);
+ // Found matching jump entry - recursively compare the child.
+ return findOrCreateIndexInternal(intHolder[0], key, offset + skyKeyLen,
+ createIfNotFound);
+ } else {
+ // Jump entry key does not match. Skip rest of the entry data.
+ contentOffset = VarInt.getVarInt(content, contentOffset + 1, intHolder);
+ }
+ }
+ // There are no matching jump entries - report mismatch or create a new leaf if necessary.
+ return createIfNotFound ? addChildNode(node, key, offset + skyKeyLen) : NOT_FOUND;
+ } else if (skyKeyLen > minKeyLen) {
+ // Key suffix is a subset of the node key. Report mismatch or split the node if requested).
+ return createIfNotFound ? splitNodeSuffix(node, minKeyLen) : NOT_FOUND;
+ } else {
+ // Node key exactly matches key suffix - return associated index value.
+ return node;
+ }
+ }
+
+ private synchronized int findOrCreateIndex(byte[] key, boolean createIfNotFound) {
+ if (rootId == NOT_FOUND) {
+ // Root node does not seem to exist - create it if needed.
+ if (createIfNotFound) {
+ rootId = createNode(0, key, 0);
+ Preconditions.checkState(rootId == 0);
+ return 0;
+ } else {
+ return NOT_FOUND;
+ }
+ }
+ return findOrCreateIndexInternal(rootId, key, 0, createIfNotFound);
+ }
+
+ private byte[] reconstructKeyInternal(int node, int suffixSize) {
+ byte[] content = nodes.get(node);
+ Preconditions.checkNotNull(content);
+ int[] intHolder = new int[1];
+ int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id
+ int parentNode = intHolder[0];
+ contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length
+ int len = intHolder[0];
+ byte[] key;
+ if (node != parentNode) {
+ // We haven't reached root node yet. Make a recursive call, adjusting suffix length.
+ key = reconstructKeyInternal(parentNode, suffixSize + len);
+ } else {
+ // We are in a root node. Finally allocate array for the key. It will be filled up
+ // on our way back from recursive call tree.
+ key = new byte[suffixSize + len];
+ }
+ // Fill appropriate portion of the full key with the node key content.
+ System.arraycopy(content, contentOffset, key, key.length - suffixSize - len, len);
+ return key;
+ }
+
+ private byte[] reconstructKey(int node) {
+ return reconstructKeyInternal(node, 0);
+ }
+
+ /* (non-Javadoc)
+ * @see com.google.devtools.build.lib.util.StringIndexer#clear()
+ */
+ @Override
+ public synchronized void clear() {
+ nodes.clear();
+ }
+
+ /* (non-Javadoc)
+ * @see com.google.devtools.build.lib.util.StringIndexer#size()
+ */
+ @Override
+ public synchronized int size() {
+ return nodes.size();
+ }
+
+ protected int getOrCreateIndexForBytes(byte[] bytes) {
+ return findOrCreateIndex(bytes, true);
+ }
+
+ protected synchronized boolean addBytes(byte[] bytes) {
+ int count = nodes.size();
+ int index = getOrCreateIndexForBytes(bytes);
+ return index >= count;
+ }
+
+ protected int getIndexForBytes(byte[] bytes) {
+ return findOrCreateIndex(bytes, false);
+ }
+
+ /* (non-Javadoc)
+ * @see com.google.devtools.build.lib.util.StringIndexer#getOrCreateIndex(java.lang.String)
+ */
+ @Override
+ public int getOrCreateIndex(String s) {
+ return getOrCreateIndexForBytes(string2bytes(s));
+ }
+
+ /* (non-Javadoc)
+ * @see com.google.devtools.build.lib.util.StringIndexer#getIndex(java.lang.String)
+ */
+ @Override
+ public int getIndex(String s) {
+ return getIndexForBytes(string2bytes(s));
+ }
+
+ /* (non-Javadoc)
+ * @see com.google.devtools.build.lib.util.StringIndexer#addString(java.lang.String)
+ */
+ @Override
+ public boolean addString(String s) {
+ return addBytes(string2bytes(s));
+ }
+
+ protected synchronized byte[] getBytesForIndex(int i) {
+ Preconditions.checkArgument(i >= 0);
+ if (i >= nodes.size()) {
+ return null;
+ }
+ return reconstructKey(i);
+ }
+
+ /* (non-Javadoc)
+ * @see com.google.devtools.build.lib.util.StringIndexer#getStringForIndex(int)
+ */
+ @Override
+ public String getStringForIndex(int i) {
+ byte[] bytes = getBytesForIndex(i);
+ return bytes != null ? bytes2string(bytes) : null;
+ }
+
+ private void dumpNodeContent(StringBuilder builder, byte[] content) {
+ int[] intHolder = new int[1];
+ int offset = VarInt.getVarInt(content, 0, intHolder);
+ builder.append("parent: ").append(intHolder[0]);
+ offset = VarInt.getVarInt(content, offset, intHolder);
+ int len = intHolder[0];
+ builder.append(", len: ").append(len).append(", key: \"")
+ .append(new String(content, offset, len, UTF_8)).append('"');
+ offset += len;
+ while (offset < content.length) {
+ builder.append(", '").append(new String(content, offset, 1, UTF_8)).append("': ");
+ offset = VarInt.getVarInt(content, offset + 1, intHolder);
+ builder.append(intHolder[0]);
+ }
+ builder.append(", size: ").append(content.length);
+ }
+
+ private int dumpContent(StringBuilder builder, int node, int indent, boolean[] seen) {
+ for(int i = 0; i < indent; i++) {
+ builder.append(" ");
+ }
+ builder.append(node).append(": ");
+ if (node >= nodes.size()) {
+ builder.append("OUT_OF_BOUNDS\n");
+ return 0;
+ } else if (seen[node]) {
+ builder.append("ALREADY_SEEN\n");
+ return 0;
+ }
+ seen[node] = true;
+ byte[] content = nodes.get(node);
+ if (content == null) {
+ builder.append("NULL\n");
+ return 0;
+ }
+ dumpNodeContent(builder, content);
+ builder.append("\n");
+ int contentSize = content.length;
+
+ int[] intHolder = new int[1];
+ int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id
+ contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length
+ contentOffset += intHolder[0];
+ while (contentOffset < content.length) {
+ contentOffset = VarInt.getVarInt(content, contentOffset + 1, intHolder);
+ contentSize += dumpContent(builder, intHolder[0], indent + 1, seen);
+ }
+ return contentSize;
+ }
+
+ @Override
+ public synchronized String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("size = ").append(nodes.size()).append("\n");
+ if (nodes.size() > 0) {
+ int contentSize = dumpContent(builder, rootId, 0, new boolean[nodes.size()]);
+ builder.append("contentSize = ").append(contentSize).append("\n");
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/DependencySet.java b/src/main/java/com/google/devtools/build/lib/util/DependencySet.java
new file mode 100644
index 0000000..788037d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/DependencySet.java
@@ -0,0 +1,225 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Representation of a set of file dependencies for a given output file. There
+ * are generally one input dependency and a bunch of include dependencies. The
+ * files are stored as {@code PathFragment}s and may be relative or absolute.
+ * <p>
+ * The serialized format read and written is equivalent and compatible with the
+ * ".d" file produced by the -MM for a given out (.o) file.
+ * <p>
+ * The file format looks like:
+ *
+ * <pre>
+ * {outfile}: \
+ * {infile} \
+ * {include} \
+ * ... \
+ * {include}
+ * </pre>
+ *
+ * @see "http://gcc.gnu.org/onlinedocs/gcc-4.2.1/gcc/Preprocessor-Options.html#Preprocessor-Options"
+ */
+public final class DependencySet {
+
+ private static final Pattern DOTD_MERGED_LINE_SEPARATOR = Pattern.compile("\\\\[\n\r]+");
+ private static final Pattern DOTD_LINE_SEPARATOR = Pattern.compile("[\n\r]+");
+ private static final Pattern DOTD_DEP = Pattern.compile("(?:[^\\s\\\\]++|\\\\ |\\\\)+");
+
+ /**
+ * The set of dependent files that this DependencySet embodies. May be
+ * relative or absolute PathFragments. A tree set is used to ensure that we
+ * write them out in a consistent order.
+ */
+ private final Collection<PathFragment> dependencies = new ArrayList<>();
+
+ private final Path root;
+ private String outputFileName;
+
+ /**
+ * Get output file name for which dependencies are included in this DependencySet.
+ */
+ public String getOutputFileName() {
+ return outputFileName;
+ }
+
+ public void setOutputFileName(String outputFileName) {
+ this.outputFileName = outputFileName;
+ }
+
+ /**
+ * Constructs a new empty DependencySet instance.
+ */
+ public DependencySet(Path root) {
+ this.root = root;
+ }
+
+ /**
+ * Gets an unmodifiable view of the set of dependencies in PathFragment form
+ * from this DependencySet instance.
+ */
+ public Collection<PathFragment> getDependencies() {
+ return Collections.unmodifiableCollection(dependencies);
+ }
+
+ /**
+ * Adds a given collection of dependencies in Path form to this DependencySet
+ * instance. Paths are converted to root-relative
+ */
+ public void addDependencies(Collection<Path> deps) {
+ for (Path d : deps) {
+ addDependency(d.relativeTo(root));
+ }
+ }
+
+ /**
+ * Adds a given dependency in PathFragment form to this DependencySet
+ * instance.
+ */
+ public void addDependency(PathFragment dep) {
+ dependencies.add(Preconditions.checkNotNull(dep));
+ }
+
+ /**
+ * Reads a dotd file into this DependencySet instance.
+ */
+ public DependencySet read(Path dotdFile) throws IOException {
+ return process(FileSystemUtils.readContent(dotdFile));
+ }
+
+ /**
+ * Parses a .d file.
+ *
+ * <p>Performance-critical! In large C++ builds there are lots of .d files to read, and some of
+ * them reach into hundreds of kilobytes.
+ */
+ public DependencySet process(byte[] content) {
+ // true if there is a CR in the input.
+ boolean cr = content.length > 0 && content[0] == '\r';
+ // true if there is more than one line in the input, not counting \-wrapped lines.
+ boolean multiline = false;
+
+ byte prevByte = ' ';
+ for (int i = 1; i < content.length; i++) {
+ byte b = content[i];
+ if (cr || b == '\r') {
+ // CR found, abort since our little loop here does not deal with CR/LFs.
+ cr = true;
+ break;
+ }
+ if (b == '\n') {
+ // Merge lines wrapped using backslashes.
+ if (prevByte == '\\') {
+ content[i] = ' ';
+ content[i - 1] = ' ';
+ } else {
+ multiline = true;
+ }
+ }
+ prevByte = b;
+ }
+
+ if (!cr && content.length > 0 && content[content.length - 1] == '\n') {
+ content[content.length - 1] = ' ';
+ }
+
+ String s = new String(content, StandardCharsets.UTF_8);
+ if (cr) {
+ s = DOTD_MERGED_LINE_SEPARATOR.matcher(s).replaceAll(" ").trim();
+ multiline = true;
+ }
+ return process(s, multiline);
+ }
+
+ private DependencySet process(String contents, boolean multiline) {
+ String[] lines;
+ if (!multiline) {
+ // Microoptimization: skip the usually unnecessary expensive-ish splitting step if there is
+ // only one target. This saves about 20% of CPU time.
+ lines = new String[] { contents };
+ } else {
+ lines = DOTD_LINE_SEPARATOR.split(contents);
+ }
+
+ for (String line : lines) {
+ // Split off output file name.
+ int pos = line.indexOf(':');
+ if (pos == -1) {
+ continue;
+ }
+ outputFileName = line.substring(0, pos);
+
+ String deps = line.substring(pos + 1);
+
+ Matcher m = DOTD_DEP.matcher(deps);
+ while (m.find()) {
+ String token = m.group();
+ // Process escaped spaces.
+ if (token.contains("\\ ")) {
+ token = token.replace("\\ ", " ");
+ }
+ dependencies.add(new PathFragment(token).normalize());
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Writes this DependencySet object for a specified output file under the root
+ * dir, and with a given suffix.
+ */
+ public void write(Path outFile, String suffix) throws IOException {
+ Path dotdFile =
+ outFile.getRelative(FileSystemUtils.replaceExtension(outFile.asFragment(), suffix));
+
+ PrintStream out = new PrintStream(dotdFile.getOutputStream());
+ try {
+ out.print(outFile.relativeTo(root) + ": ");
+ for (PathFragment d : dependencies) {
+ out.print(" \\\n " + d.getPathString()); // should already be root relative
+ }
+ out.println();
+ } finally {
+ out.close();
+ }
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof DependencySet
+ && ((DependencySet) other).dependencies.equals(dependencies);
+ }
+
+ @Override
+ public int hashCode() {
+ return dependencies.hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ExitCode.java b/src/main/java/com/google/devtools/build/lib/util/ExitCode.java
new file mode 100644
index 0000000..8307538
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ExitCode.java
@@ -0,0 +1,181 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Objects;
+
+import java.util.Collection;
+import java.util.HashMap;
+
+/**
+ * <p>Anything marked FAILURE is generally from a problem with the source code
+ * under consideration. In these cases, a re-run in an identical client should
+ * produce an identical return code all things being constant.
+ *
+ * <p>Anything marked as an ERROR is generally a problem unrelated to the
+ * source code itself. It is either something wrong with the user's command
+ * line or the user's machine or environment.
+ *
+ * <p>Note that these exit codes should be kept consistent with the codes
+ * returned by Blaze's launcher in //devtools/blaze/main:blaze.cc
+ */
+public class ExitCode {
+ // Tracks all exit codes defined here and elsewhere in Bazel.
+ private static final HashMap<Integer, ExitCode> exitCodeRegistry = new HashMap<>();
+
+ public static final ExitCode SUCCESS = ExitCode.create(0, "SUCCESS");
+ public static final ExitCode BUILD_FAILURE = ExitCode.create(1, "BUILD_FAILURE");
+ public static final ExitCode PARSING_FAILURE = ExitCode.createUnregistered(1, "PARSING_FAILURE");
+ public static final ExitCode COMMAND_LINE_ERROR = ExitCode.create(2, "COMMAND_LINE_ERROR");
+ public static final ExitCode TESTS_FAILED = ExitCode.create(3, "TESTS_FAILED");
+ public static final ExitCode PARTIAL_ANALYSIS_FAILURE =
+ ExitCode.createUnregistered(3, "PARTIAL_ANALYSIS_FAILURE");
+ public static final ExitCode NO_TESTS_FOUND = ExitCode.create(4, "NO_TESTS_FOUND");
+ public static final ExitCode RUN_FAILURE = ExitCode.create(6, "RUN_FAILURE");
+ public static final ExitCode ANALYSIS_FAILURE = ExitCode.create(7, "ANALYSIS_FAILURE");
+ public static final ExitCode INTERRUPTED = ExitCode.create(8, "INTERRUPTED");
+ public static final ExitCode OOM_ERROR = ExitCode.createInfrastructureFailure(33, "OOM_ERROR");
+ public static final ExitCode LOCAL_ENVIRONMENTAL_ERROR =
+ ExitCode.createInfrastructureFailure(36, "LOCAL_ENVIRONMENTAL_ERROR");
+ public static final ExitCode BLAZE_INTERNAL_ERROR =
+ ExitCode.createInfrastructureFailure(37, "BLAZE_INTERNAL_ERROR");
+ public static final ExitCode RESERVED = ExitCode.createInfrastructureFailure(40, "RESERVED");
+ /*
+ exit codes [50..60] and 253 are reserved for site specific wrappers to Bazel.
+ */
+
+ /**
+ * Creates and returns an ExitCode. Requires a unique exit code number.
+ *
+ * @param code the int value for this exit code
+ * @param name a human-readable description
+ */
+ public static ExitCode create(int code, String name) {
+ return new ExitCode(code, name, /*infrastructureFailure=*/false, /*register=*/true);
+ }
+
+ /**
+ * Creates and returns an ExitCode that represents an infrastructure failure.
+ *
+ * @param code the int value for this exit code
+ * @param name a human-readable description
+ */
+ public static ExitCode createInfrastructureFailure(int code, String name) {
+ return new ExitCode(code, name, /*infrastructureFailure=*/true, /*register=*/true);
+ }
+
+ /**
+ * Creates and returns an ExitCode that has the same numeric code as another ExitCode. This is to
+ * allow the duplicate error codes listed above to be registered, but is private to prevent other
+ * users from creating duplicate error codes in the future.
+ *
+ * @param code the int value for this exit code
+ * @param name a human-readable description
+ */
+ private static ExitCode createUnregistered(int code, String name) {
+ return new ExitCode(code, name, /*infrastructureFailure=*/false, /*register=*/false);
+ }
+
+ /**
+ * Add the given exit code to the registry.
+ *
+ * @param exitCode the exit code to register
+ * @throws IllegalStateException if the numeric exit code is already in the registry.
+ */
+ private static void register(ExitCode exitCode) {
+ synchronized (exitCodeRegistry) {
+ int codeNum = exitCode.getNumericExitCode();
+ if (exitCodeRegistry.containsKey(codeNum)) {
+ throw new IllegalStateException(
+ "Exit code " + codeNum + " (" + exitCode.name + ") already registered");
+ }
+ exitCodeRegistry.put(codeNum, exitCode);
+ }
+ }
+
+ /**
+ * Returns all registered ExitCodes.
+ */
+ public static Collection<ExitCode> values() {
+ synchronized (exitCodeRegistry) {
+ return exitCodeRegistry.values();
+ }
+ }
+
+ private final int numericExitCode;
+ private final String name;
+ private final boolean infrastructureFailure;
+
+ /**
+ * Whenever a new exit code is created, it is registered (to prevent exit codes with identical
+ * numeric codes from being created). However, there are some exit codes in this file that have
+ * duplicate numeric codes, so these are not registered.
+ */
+ private ExitCode(int exitCode, String name, boolean infrastructureFailure, boolean register) {
+ this.numericExitCode = exitCode;
+ this.name = name;
+ this.infrastructureFailure = infrastructureFailure;
+ if (register) {
+ ExitCode.register(this);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(numericExitCode, name, infrastructureFailure);
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (object instanceof ExitCode) {
+ ExitCode that = (ExitCode) object;
+ return this.numericExitCode == that.numericExitCode
+ && this.name.equals(that.name)
+ && this.infrastructureFailure == that.infrastructureFailure;
+ }
+ return false;
+ }
+
+ /**
+ * Returns the human-readable name for this exit code. Not guaranteed to be stable, use the
+ * numeric exit code for that.
+ */
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ /**
+ * Returns the error's int value.
+ */
+ public int getNumericExitCode() {
+ return numericExitCode;
+ }
+
+ /**
+ * Returns the human-readable name.
+ */
+ public String name() {
+ return name;
+ }
+
+ /**
+ * Returns true if the current exit code represents a failure of Blaze infrastructure,
+ * vs. a build failure.
+ */
+ public boolean isInfrastructureFailure() {
+ return infrastructureFailure;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/FileType.java b/src/main/java/com/google/devtools/build/lib/util/FileType.java
new file mode 100644
index 0000000..c91b17b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/FileType.java
@@ -0,0 +1,278 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A base class for FileType matchers.
+ */
+@Immutable
+public abstract class FileType implements Predicate<String> {
+ // A special file type
+ public static final FileType NO_EXTENSION = new FileType() {
+ @Override
+ public boolean apply(String filename) {
+ return filename.lastIndexOf('.') == -1;
+ }
+ };
+
+ public static FileType of(final String ext) {
+ return new FileType() {
+ @Override
+ public boolean apply(String filename) {
+ return filename.endsWith(ext);
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.of(ext);
+ }
+ };
+ }
+
+ public static FileType of(final Iterable<String> extensions) {
+ return new FileType() {
+ @Override
+ public boolean apply(String filename) {
+ for (String ext : extensions) {
+ if (filename.endsWith(ext)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.copyOf(extensions);
+ }
+ };
+ }
+
+ public static FileType of(final String... extensions) {
+ return of(Arrays.asList(extensions));
+ }
+
+ @Override
+ public String toString() {
+ return getExtensions().toString();
+ }
+
+ /**
+ * Returns true if the the filename matches. The filename should be a basename (the filename
+ * component without a path) for performance reasons.
+ */
+ @Override
+ public abstract boolean apply(String filename);
+
+ /**
+ * Get a list of filename extensions this matcher handles. The first entry in the list (if
+ * available) is the primary extension that code can use to construct output file names.
+ * The list can be empty for some matchers.
+ *
+ * @return a list of filename extensions
+ */
+ public List<String> getExtensions() {
+ return ImmutableList.of();
+ }
+
+ /** Return true if a file name is matched by the FileType */
+ public boolean matches(String filename) {
+ int slashIndex = filename.lastIndexOf('/');
+ if (slashIndex != -1) {
+ filename = filename.substring(slashIndex + 1);
+ }
+ return apply(filename);
+ }
+
+ /** Return true if a file referred by path is matched by the FileType */
+ public boolean matches(Path path) {
+ return apply(path.getBaseName());
+ }
+
+ /** Return true if a file referred by fragment is matched by the FileType */
+ public boolean matches(PathFragment fragment) {
+ return apply(fragment.getBaseName());
+ }
+
+ // Check FileTypes
+
+ /**
+ * An interface for entities that have a filename.
+ */
+ public interface HasFilename {
+ /**
+ * Returns the filename of this entity.
+ */
+ String getFilename();
+ }
+
+ /**
+ * Checks whether an Iterable<? extends HasFileType> contains any of the specified file types.
+ *
+ * <p>At least one FileType must be specified.
+ */
+ public static <T extends HasFilename> boolean contains(final Iterable<T> items,
+ FileType... fileTypes) {
+ Preconditions.checkState(fileTypes.length > 0, "Must specify at least one file type");
+ final FileTypeSet fileTypeSet = FileTypeSet.of(fileTypes);
+ for (T item : items) {
+ if (fileTypeSet.matches(item.getFilename())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Checks whether a HasFileType is any of the specified file types.
+ *
+ * <p>At least one FileType must be specified.
+ */
+ public static <T extends HasFilename> boolean contains(T item, FileType... fileTypes) {
+ return FileTypeSet.of(fileTypes).matches(item.getFilename());
+ }
+
+
+ private static <T extends HasFilename> Predicate<T> typeMatchingPredicateFor(
+ final FileType matchingType) {
+ return new Predicate<T>() {
+ @Override
+ public boolean apply(T item) {
+ return matchingType.matches(item.getFilename());
+ }
+ };
+ }
+
+ private static <T extends HasFilename> Predicate<T> typeMatchingPredicateFor(
+ final FileTypeSet matchingTypes) {
+ return new Predicate<T>() {
+ @Override
+ public boolean apply(T item) {
+ return matchingTypes.matches(item.getFilename());
+ }
+ };
+ }
+
+ private static <T extends HasFilename> Predicate<T> typeMatchingPredicateFrom(
+ final Predicate<String> fileTypePredicate) {
+ return new Predicate<T>() {
+ @Override
+ public boolean apply(T item) {
+ return fileTypePredicate.apply(item.getFilename());
+ }
+ };
+ }
+
+ /**
+ * A filter for Iterable<? extends HasFileType> that returns only those whose FileType matches the
+ * specified Predicate.
+ */
+ public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items,
+ final Predicate<String> predicate) {
+ return Iterables.filter(items, typeMatchingPredicateFrom(predicate));
+ }
+
+ /**
+ * A filter for Iterable<? extends HasFileType> that returns only those of the specified file
+ * types.
+ */
+ public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items,
+ FileType... fileTypes) {
+ return filter(items, FileTypeSet.of(fileTypes));
+ }
+
+ /**
+ * A filter for Iterable<? extends HasFileType> that returns only those of the specified file
+ * types.
+ */
+ public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items,
+ FileTypeSet fileTypes) {
+ return Iterables.filter(items, typeMatchingPredicateFor(fileTypes));
+ }
+
+ /**
+ * A filter for Iterable<? extends HasFileType> that returns only those of the specified file
+ * type.
+ */
+ public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items,
+ FileType fileType) {
+ return Iterables.filter(items, typeMatchingPredicateFor(fileType));
+ }
+
+ /**
+ * A filter for Iterable<? extends HasFileType> that returns everything except the specified file
+ * type.
+ */
+ public static <T extends HasFilename> Iterable<T> except(final Iterable<T> items,
+ FileType fileType) {
+ return Iterables.filter(items, Predicates.not(typeMatchingPredicateFor(fileType)));
+ }
+
+
+ /**
+ * A filter for List<? extends HasFileType> that returns only those of the specified file types.
+ * The result is a mutable list, computed eagerly; see {@link #filter} for a lazy variant.
+ */
+ public static <T extends HasFilename> List<T> filterList(final Iterable<T> items,
+ FileType... fileTypes) {
+ if (fileTypes.length > 0) {
+ return filterList(items, FileTypeSet.of(fileTypes));
+ } else {
+ return new ArrayList<>();
+ }
+ }
+
+ /**
+ * A filter for List<? extends HasFileType> that returns only those of the specified file type.
+ * The result is a mutable list, computed eagerly.
+ */
+ public static <T extends HasFilename> List<T> filterList(final Iterable<T> items,
+ final FileType fileType) {
+ List<T> result = new ArrayList<>();
+ for (T item : items) {
+ if (fileType.matches(item.getFilename())) {
+ result.add(item);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * A filter for List<? extends HasFileType> that returns only those of the specified file types.
+ * The result is a mutable list, computed eagerly.
+ */
+ public static <T extends HasFilename> List<T> filterList(final Iterable<T> items,
+ final FileTypeSet fileTypeSet) {
+ List<T> result = new ArrayList<>();
+ for (T item : items) {
+ if (fileTypeSet.matches(item.getFilename())) {
+ result.add(item);
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/FileTypeSet.java b/src/main/java/com/google/devtools/build/lib/util/FileTypeSet.java
new file mode 100644
index 0000000..694e877
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/FileTypeSet.java
@@ -0,0 +1,139 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A set of FileTypes for grouped matching.
+ */
+@Immutable
+public class FileTypeSet implements Predicate<String> {
+ private final ImmutableSet<FileType> types;
+
+ /** A set that matches all files. */
+ public static final FileTypeSet ANY_FILE =
+ new FileTypeSet() {
+ @Override
+ public String toString() {
+ return "any files";
+ }
+ @Override
+ public boolean matches(String filename) {
+ return true;
+ }
+ @Override
+ public List<String> getExtensions() {
+ return ImmutableList.<String>of();
+ }
+ };
+
+ /** A predicate that matches no files. */
+ public static final FileTypeSet NO_FILE =
+ new FileTypeSet(ImmutableList.<FileType>of()) {
+ @Override
+ public String toString() {
+ return "no files";
+ }
+ @Override
+ public boolean matches(String filename) {
+ return false;
+ }
+ };
+
+ private FileTypeSet() {
+ this.types = null;
+ }
+
+ private FileTypeSet(FileType... fileTypes) {
+ this.types = ImmutableSet.copyOf(fileTypes);
+ }
+
+ private FileTypeSet(Iterable<FileType> fileTypes) {
+ this.types = ImmutableSet.copyOf(fileTypes);
+ }
+
+ /**
+ * Returns a set that matches only the provided {@code fileTypes}.
+ *
+ * <p>If {@code fileTypes} is empty, the returned predicate will match no files.
+ */
+ public static FileTypeSet of(FileType... fileTypes) {
+ if (fileTypes.length == 0) {
+ return FileTypeSet.NO_FILE;
+ } else {
+ return new FileTypeSet(fileTypes);
+ }
+ }
+
+ /**
+ * Returns a set that matches only the provided {@code fileTypes}.
+ *
+ * <p>If {@code fileTypes} is empty, the returned predicate will match no files.
+ */
+ public static FileTypeSet of(Iterable<FileType> fileTypes) {
+ if (Iterables.isEmpty(fileTypes)) {
+ return FileTypeSet.NO_FILE;
+ } else {
+ return new FileTypeSet(fileTypes);
+ }
+ }
+
+ /** Returns true if the filename can be matched by any FileType in this set. */
+ public boolean matches(String filename) {
+ int slashIndex = filename.lastIndexOf('/');
+ if (slashIndex != -1) {
+ filename = filename.substring(slashIndex + 1);
+ }
+ for (FileType type : types) {
+ if (type.apply(filename)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Returns true if this predicate matches nothing. */
+ public boolean isNone() {
+ return this == FileTypeSet.NO_FILE;
+ }
+
+ @Override
+ public boolean apply(String filename) {
+ return matches(filename);
+ }
+
+ /** Returns the list of possible file extensions for this file type. Can be empty. */
+ public List<String> getExtensions() {
+ List<String> extensions = new ArrayList<>();
+ for (FileType type : types) {
+ extensions.addAll(type.getExtensions());
+ }
+ return extensions;
+ }
+
+ @Override
+ public String toString() {
+ return StringUtil.joinEnglishList(getExtensions());
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java
new file mode 100644
index 0000000..e4c0876
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java
@@ -0,0 +1,319 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Simplified wrapper for MD5 message digests. See also
+ * com.google.math.crypto.MD5HMAC for a similar interface.
+ *
+ * @see java.security.MessageDigest
+ */
+public final class Fingerprint {
+
+ private final MessageDigest md;
+
+ /**
+ * Creates and initializes a new MD5 object; if this fails, Java must be
+ * installed incorrectly.
+ */
+ public Fingerprint() {
+ try {
+ md = MessageDigest.getInstance("md5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("MD5 not available");
+ }
+ }
+
+ /**
+ * Completes the hash computation by doing final operations, e.g., padding.
+ *
+ * <p>This method has the side-effect of resetting the underlying digest computer.
+ *
+ * @return the MD5 digest as a 16-byte array
+ * @see java.security.MessageDigest#digest()
+ */
+ public byte[] digestAndReset() {
+ return md.digest();
+ }
+
+ /**
+ * Completes the hash computation and returns the digest as a string.
+ *
+ * <p>This method has the side-effect of resetting the underlying digest computer.
+ *
+ * @return the MD5 digest as a 32-character string of hexadecimal digits
+ * @see com.google.math.crypto.MD5HMAC#toString()
+ */
+ public String hexDigestAndReset() {
+ return hexDigest(digestAndReset());
+ }
+
+ /**
+ * Returns a string representation of an MD5 digest.
+ *
+ * @param digest the MD5 digest, perhaps from a previous call to digest
+ * @return the digest as a 32-character string of hexadecimal digits
+ */
+ public static String hexDigest(byte[] digest) {
+ StringBuilder b = new StringBuilder(32);
+ for (int i = 0; i < digest.length; i++) {
+ int n = digest[i];
+ b.append("0123456789abcdef".charAt((n >> 4) & 0xF));
+ b.append("0123456789abcdef".charAt(n & 0xF));
+ }
+ return b.toString();
+ }
+
+ /**
+ * Override of Object.toString to return a string for the MD5 digest without
+ * finalizing the digest computation. Calling hexDigest() instead will
+ * finalize the digest computation.
+ *
+ * @return the string returned by hexDigest()
+ */
+ @Override
+ public String toString() {
+ try {
+ // MD5 does support cloning, so this should not fail
+ return hexDigest(((MessageDigest) md.clone()).digest());
+ } catch (CloneNotSupportedException e) {
+ // MessageDigest does not support cloning,
+ // so just return the toString() on the MessageDigest.
+ return md.toString();
+ }
+ }
+
+ /**
+ * Updates the digest with 0 or more bytes.
+ *
+ * @param input the array of bytes with which to update the digest
+ * @see java.security.MessageDigest#update(byte[])
+ */
+ public Fingerprint addBytes(byte[] input) {
+ md.update(input);
+ return this;
+ }
+
+ /**
+ * Updates the digest with the specified number of bytes starting at offset.
+ *
+ * @param input the array of bytes with which to update the digest
+ * @param offset the offset into the array
+ * @param len the number of bytes to use
+ * @see java.security.MessageDigest#update(byte[], int, int)
+ */
+ public Fingerprint addBytes(byte[] input, int offset, int len) {
+ md.update(input, offset, len);
+ return this;
+ }
+
+ /**
+ * Updates the digest with a boolean value.
+ */
+ public Fingerprint addBoolean(boolean input) {
+ addBytes(new byte[] { (byte) (input ? 1 : 0) });
+ return this;
+ }
+
+ /**
+ * Updates the digest with the little-endian bytes of a given int value.
+ *
+ * @param input the integer with which to update the digest
+ */
+ public Fingerprint addInt(int input) {
+ md.update(new byte[] {
+ (byte) input,
+ (byte) (input >> 8),
+ (byte) (input >> 16),
+ (byte) (input >> 24),
+ });
+
+ return this;
+ }
+
+ /**
+ * Updates the digest with the little-endian bytes of a given long value.
+ *
+ * @param input the long with which to update the digest
+ */
+ public Fingerprint addLong(long input) {
+ md.update(new byte[]{
+ (byte) input,
+ (byte) (input >> 8),
+ (byte) (input >> 16),
+ (byte) (input >> 24),
+ (byte) (input >> 32),
+ (byte) (input >> 40),
+ (byte) (input >> 48),
+ (byte) (input >> 56),
+ });
+
+ return this;
+ }
+
+ /**
+ * Updates the digest with a UUID.
+ *
+ * @param uuid the UUID with which to update the digest. Must not be null.
+ */
+ public Fingerprint addUUID(UUID uuid) {
+ addLong(uuid.getLeastSignificantBits());
+ addLong(uuid.getMostSignificantBits());
+ return this;
+ }
+
+ /**
+ * Updates the digest with a String using its length plus its UTF8 encoded bytes.
+ *
+ * @param input the String with which to update the digest
+ * @see java.security.MessageDigest#update(byte[])
+ */
+ public Fingerprint addString(String input) {
+ byte[] bytes = input.getBytes(UTF_8);
+ addInt(bytes.length);
+ md.update(bytes);
+ return this;
+ }
+
+ /**
+ * Updates the digest with a String using its length and content.
+ *
+ * @param input the String with which to update the digest
+ * @see java.security.MessageDigest#update(byte[])
+ */
+ public Fingerprint addStringLatin1(String input) {
+ addInt(input.length());
+ byte[] bytes = new byte[input.length()];
+ for (int i = 0; i < input.length(); i++) {
+ bytes[i] = (byte) input.charAt(i);
+ }
+ md.update(bytes);
+ return this;
+ }
+
+ /**
+ * Updates the digest with a Path.
+ *
+ * @param input the Path with which to update the digest.
+ */
+ public Fingerprint addPath(Path input) {
+ addStringLatin1(input.getPathString());
+ return this;
+ }
+
+ /**
+ * Updates the digest with a Path.
+ *
+ * @param input the Path with which to update the digest.
+ */
+ public Fingerprint addPath(PathFragment input) {
+ addStringLatin1(input.getPathString());
+ return this;
+ }
+
+ /**
+ * Updates the digest with inputs by iterating over them and invoking
+ * {@code #addString(String)} on each element.
+ *
+ * @param inputs the inputs with which to update the digest
+ */
+ public Fingerprint addStrings(Iterable<String> inputs) {
+ addInt(Iterables.size(inputs));
+ for (String input : inputs) {
+ addString(input);
+ }
+
+ return this;
+ }
+
+ /**
+ * Updates the digest with inputs by iterating over them and invoking
+ * {@code #addString(String)} on each element.
+ *
+ * @param inputs the inputs with which to update the digest
+ */
+ public Fingerprint addStrings(String... inputs) {
+ addInt(inputs.length);
+ for (String input : inputs) {
+ addString(input);
+ }
+
+ return this;
+ }
+
+ /**
+ * Updates the digest with inputs which are pairs in a map, by iterating over
+ * the map entries and invoking {@code #addString(String)} on each key and
+ * value.
+ *
+ * @param inputs the inputs in a map with which to update the digest
+ */
+ public Fingerprint addStringMap(Map<String, String> inputs) {
+ addInt(inputs.size());
+ for (Map.Entry<String, String> entry : inputs.entrySet()) {
+ addString(entry.getKey());
+ addString(entry.getValue());
+ }
+
+ return this;
+ }
+
+ /**
+ * Updates the digest with a list of paths by iterating over them and
+ * invoking {@link #addPath(PathFragment)} on each element.
+ *
+ * @param inputs the paths with which to update the digest
+ */
+ public Fingerprint addPaths(Iterable<PathFragment> inputs) {
+ addInt(Iterables.size(inputs));
+ for (PathFragment path : inputs) {
+ addPath(path);
+ }
+
+ return this;
+ }
+
+ /**
+ * Reset the Fingerprint for additional use as though previous digesting had not been done.
+ */
+ public void reset() {
+ md.reset();
+ }
+
+ // -------- Convenience methods ----------------------------
+
+ /**
+ * Computes the hex digest from a String using UTF8 encoding and returning
+ * the hexDigest().
+ *
+ * @param input the String from which to compute the digest
+ */
+ public static String md5Digest(String input) {
+ Fingerprint f = new Fingerprint();
+ f.addBytes(input.getBytes(UTF_8));
+ return f.hexDigestAndReset();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/GroupedList.java b/src/main/java/com/google/devtools/build/lib/util/GroupedList.java
new file mode 100644
index 0000000..2bf956d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/GroupedList.java
@@ -0,0 +1,344 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.collect.CompactHashSet;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Encapsulates a list of lists. Is intended to be used in "batch" mode -- to set the value of a
+ * GroupedList, users should first construct a {@link GroupedListHelper}, add elements to it, and
+ * then {@link #append} the helper to a new GroupedList instance. The generic type T <i>must not</i>
+ * be a {@link List}.
+ *
+ * <p>Despite the "list" name, it is an error for the same element to appear multiple times in the
+ * list. Users are responsible for not trying to add the same element to a GroupedList twice.
+ */
+public class GroupedList<T> implements Iterable<Iterable<T>> {
+ // Total number of items in the list. At least elements.size(), but might be larger if there are
+ // any nested lists.
+ private int size = 0;
+ // Items in this GroupedList. Each element is either of type T or List<T>.
+ // Non-final only for #remove.
+ private List<Object> elements;
+
+ public GroupedList() {
+ // We optimize for small lists.
+ this.elements = new ArrayList<>(1);
+ }
+
+ // Only for use when uncompressing a GroupedList.
+ private GroupedList(int size, List<Object> elements) {
+ this.size = size;
+ this.elements = new ArrayList<>(elements);
+ }
+
+ /** Appends the list constructed in helper to this list. */
+ public void append(GroupedListHelper<T> helper) {
+ Preconditions.checkState(helper.currentGroup == null, "%s %s", this, helper);
+ // Do a check to make sure we don't have lists here. Note that if helper.elements is empty,
+ // Iterables.getFirst will return null, and null is not instanceof List.
+ Preconditions.checkState(!(Iterables.getFirst(helper.elements, null) instanceof List),
+ "Cannot make grouped list of lists: %s", helper);
+ elements.addAll(helper.groupedList);
+ size += helper.size();
+ }
+
+ /**
+ * Removes the elements in toRemove from this list. Takes time proportional to the size of the
+ * list, so should not be called often.
+ */
+ public void remove(Set<T> toRemove) {
+ elements = remove(elements, toRemove);
+ size -= toRemove.size();
+ }
+
+ /** Returns the number of elements in this list. */
+ public int size() {
+ return size;
+ }
+
+ /** Returns true if this list contains no elements. */
+ public boolean isEmpty() {
+ return elements.isEmpty();
+ }
+
+ private static final Object EMPTY_LIST = new Object();
+
+ public Object compress() {
+ switch (size()) {
+ case 0:
+ return EMPTY_LIST;
+ case 1:
+ return Iterables.getOnlyElement(elements);
+ default:
+ return elements.toArray();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public Set<T> toSet() {
+ ImmutableSet.Builder<T> builder = ImmutableSet.builder();
+ for (Object obj : elements) {
+ if (obj instanceof List) {
+ builder.addAll((List<T>) obj);
+ } else {
+ builder.add((T) obj);
+ }
+ }
+ return builder.build();
+ }
+
+ private static int sizeOf(Object obj) {
+ return obj instanceof List ? ((List<?>) obj).size() : 1;
+ }
+
+ public static <E> GroupedList<E> create(Object compressed) {
+ if (compressed == EMPTY_LIST) {
+ return new GroupedList<>();
+ }
+ if (compressed.getClass().isArray()) {
+ List<Object> elements = new ArrayList<>();
+ int size = 0;
+ for (Object item : (Object[]) compressed) {
+ size += sizeOf(item);
+ elements.add(item);
+ }
+ return new GroupedList<>(size, elements);
+ }
+ // Just a single element.
+ return new GroupedList<>(1, ImmutableList.<Object>of(compressed));
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (this.getClass() != other.getClass()) {
+ return false;
+ }
+ GroupedList<?> that = (GroupedList<?>) other;
+ return elements.equals(that.elements);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("elements", elements)
+ .add("size", size).toString();
+
+ }
+
+ /**
+ * Iterator that returns the next group in elements for each call to {@link #next}. A custom
+ * iterator is needed here because, to optimize memory, we store single-element lists as elements
+ * internally, and so they must be wrapped before they're returned.
+ */
+ private class GroupedIterator implements Iterator<Iterable<T>> {
+ private final Iterator<Object> iter = elements.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @SuppressWarnings("unchecked") // Cast of Object to List<T> or T.
+ @Override
+ public Iterable<T> next() {
+ Object obj = iter.next();
+ if (obj instanceof List) {
+ return (List<T>) obj;
+ }
+ return ImmutableList.of((T) obj);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public Iterator<Iterable<T>> iterator() {
+ return new GroupedIterator();
+ }
+
+ /**
+ * Removes everything in toRemove from the list of lists, elements. Called both by GroupedList and
+ * GroupedListHelper.
+ */
+ private static <E> List<Object> remove(List<Object> elements, Set<E> toRemove) {
+ int removedCount = 0;
+ // elements.size is an upper bound of the needed size. Since normally removal happens just
+ // before the list is finished and compressed, optimizing this size isn't a concern.
+ List<Object> newElements = new ArrayList<>(elements.size());
+ for (Object obj : elements) {
+ if (obj instanceof List) {
+ ImmutableList.Builder<E> newGroup = new ImmutableList.Builder<>();
+ @SuppressWarnings("unchecked")
+ List<E> oldGroup = (List<E>) obj;
+ for (E elt : oldGroup) {
+ if (toRemove.contains(elt)) {
+ removedCount++;
+ } else {
+ newGroup.add(elt);
+ }
+ }
+ ImmutableList<E> group = newGroup.build();
+ addItem(group, newElements);
+ } else {
+ if (toRemove.contains(obj)) {
+ removedCount++;
+ } else {
+ newElements.add(obj);
+ }
+ }
+ }
+ Preconditions.checkState(removedCount == toRemove.size(),
+ "%s %s %s %s", removedCount, removedCount, elements, newElements);
+ return newElements;
+ }
+
+ private static void addItem(Collection<?> item, List<Object> elements) {
+ switch (item.size()) {
+ case 0:
+ return;
+ case 1:
+ elements.add(Iterables.getOnlyElement(item));
+ return;
+ default:
+ elements.add(ImmutableList.copyOf(item));
+ }
+ }
+
+ /**
+ * Builder-like object for GroupedLists. An already-existing grouped list is appended to by
+ * constructing a helper, mutating it, and then appending that helper to the grouped list.
+ */
+ public static class GroupedListHelper<E> implements Iterable<E> {
+ // Non-final only for removal.
+ private List<Object> groupedList;
+ private List<E> currentGroup = null;
+ private final Set<E> elements = CompactHashSet.create();
+
+ private GroupedListHelper(GroupedList<E> groupedList) {
+ this.groupedList = new ArrayList<>(groupedList.elements);
+ for (Iterable<E> group : groupedList) {
+ Iterables.addAll(elements, group);
+ }
+ }
+
+ public GroupedListHelper() {
+ // Optimize for short lists.
+ groupedList = new ArrayList<>(1);
+ }
+
+ /**
+ * Add an element to this list. If in a group, will be added to the current group. Otherwise,
+ * goes in a group of its own.
+ */
+ public void add(E elt) {
+ Preconditions.checkState(elements.add(elt), "%s %s", elt, this);
+ if (currentGroup == null) {
+ groupedList.add(elt);
+ } else {
+ currentGroup.add(elt);
+ }
+ }
+
+ /**
+ * Remove all elements of toRemove from this list. It is a fatal error if any elements of
+ * toRemove are not present. Takes time proportional to the size of the list, so should not be
+ * called often.
+ */
+ public void remove(Set<E> toRemove) {
+ groupedList = GroupedList.remove(groupedList, toRemove);
+ int oldSize = size();
+ elements.removeAll(toRemove);
+ Preconditions.checkState(oldSize == size() + toRemove.size(),
+ "%s %s %s", oldSize, toRemove, this);
+ }
+
+ /**
+ * Starts a group. All elements added until {@link #endGroup} will be in the same group. Each
+ * call of {@link #startGroup} must be paired with a following {@link #endGroup} call.
+ */
+ public void startGroup() {
+ Preconditions.checkState(currentGroup == null, this);
+ currentGroup = new ArrayList<>();
+ }
+
+ private void addList(Collection<E> group) {
+ addItem(group, groupedList);
+ }
+
+ /** Ends a group started with {@link #startGroup}. */
+ public void endGroup() {
+ Preconditions.checkNotNull(currentGroup);
+ addList(currentGroup);
+ currentGroup = null;
+ }
+
+ /** Returns true if elt is present in the list. */
+ public boolean contains(E elt) {
+ return elements.contains(elt);
+ }
+
+ private int size() {
+ return elements.size();
+ }
+
+ /** Returns true if list is empty. */
+ public boolean isEmpty() {
+ return elements.isEmpty();
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return elements.iterator();
+ }
+
+ /** Create a GroupedListHelper from a collection of elements, all put in the same group.*/
+ public static <F> GroupedListHelper<F> create(Collection<F> elements) {
+ GroupedListHelper<F> helper = new GroupedListHelper<>();
+ helper.addList(elements);
+ helper.elements.addAll(elements);
+ Preconditions.checkState(helper.elements.size() == elements.size(),
+ "%s %s", helper, elements);
+ return helper;
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("groupedList", groupedList)
+ .add("elements", elements)
+ .add("currentGroup", currentGroup).toString();
+
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/IncludeScanningUtil.java b/src/main/java/com/google/devtools/build/lib/util/IncludeScanningUtil.java
new file mode 100644
index 0000000..24f55e4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/IncludeScanningUtil.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Static utilities for include scanning.
+ */
+public class IncludeScanningUtil {
+ private IncludeScanningUtil() {
+ }
+
+ private static final String INCLUDES_SUFFIX = ".includes";
+ public static final PathFragment GREPPED_INCLUDES =
+ new PathFragment(Constants.PRODUCT_NAME + "-out/_grepped_includes");
+
+ /**
+ * Returns the exec-root relative output path for grepped includes.
+ *
+ * @param srcExecPath the exec-root relative path of the source file.
+ */
+ public static PathFragment getExecRootRelativeOutputPath(PathFragment srcExecPath) {
+ return GREPPED_INCLUDES.getRelative(getRootRelativeOutputPath(srcExecPath));
+ }
+
+ /**
+ * Returns the root relative output path for grepped includes.
+ *
+ * @param srcExecPath the exec-root relative path of the source file.
+ */
+ public static PathFragment getRootRelativeOutputPath(PathFragment srcExecPath) {
+ return srcExecPath.replaceName(srcExecPath.getBaseName() + INCLUDES_SUFFIX);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/JavaClock.java b/src/main/java/com/google/devtools/build/lib/util/JavaClock.java
new file mode 100644
index 0000000..bdd1116
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/JavaClock.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+/**
+ * Class provides a simple clock implementation used by the tool. By default it uses {@link System}
+ * class.
+ */
+public class JavaClock implements Clock {
+
+ public JavaClock() {
+ }
+
+ @Override
+ public long currentTimeMillis() {
+ return System.currentTimeMillis();
+ }
+
+ @Override
+ public long nanoTime() {
+ // Note that some JVM implementations of System#nanoTime don't yield a non-decreasing
+ // sequence of values.
+ return System.nanoTime();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/LazyString.java b/src/main/java/com/google/devtools/build/lib/util/LazyString.java
new file mode 100644
index 0000000..0e037b2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/LazyString.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+/**
+ * This class serves as a base implementation for a {@code CharSequence}
+ * that delay string construction (mostly till the execution phase).
+ *
+ * They are not full implementations, they lack {@code #charAt(int)} and
+ * {@code #subSequence(int, int)}.
+ */
+public abstract class LazyString implements CharSequence {
+
+ @Override
+ public char charAt(int index) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int length() {
+ return toString().length();
+ }
+
+ @Override
+ public CharSequence subSequence(int start, int end) {
+ throw new UnsupportedOperationException();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/LoggingUtil.java b/src/main/java/com/google/devtools/build/lib/util/LoggingUtil.java
new file mode 100644
index 0000000..5170727
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/LoggingUtil.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+/**
+ * Logging utilities for sending log messages to a remote service. Log messages
+ * will not be output anywhere else, including the terminal and blaze clients.
+ */
+@ThreadSafety.ThreadSafe
+public final class LoggingUtil {
+ // TODO(bazel-team): this class is a thin wrapper around Logger and could probably be discarded.
+ private static Future<Logger> remoteLogger;
+
+ /**
+ * Installs the remote logger.
+ *
+ * <p>This can only be called once, and the caller should not keep the
+ * reference to the logger.
+ *
+ * @param logger The logger future. Must have already started.
+ */
+ public static synchronized void installRemoteLogger(Future<Logger> logger) {
+ Preconditions.checkState(remoteLogger == null);
+ remoteLogger = logger;
+ }
+
+ /** Returns the installed logger, or null if none is installed. */
+ public static synchronized Logger getRemoteLogger() {
+ try {
+ return (remoteLogger == null) ? null : Uninterruptibles.getUninterruptibly(remoteLogger);
+ } catch (ExecutionException e) {
+ throw new RuntimeException("Unexpected error initializing remote logging", e);
+ }
+ }
+
+ /**
+ * @see #logToRemote(Level, String, Throwable, String...).
+ */
+ public static void logToRemote(Level level, String msg, Throwable trace) {
+ Logger logger = getRemoteLogger();
+ if (logger != null) {
+ logger.log(level, msg, trace);
+ }
+ }
+
+ /**
+ * Log a message to the remote backend. This is done out of thread, so this
+ * method is non-blocking.
+ *
+ * @param level The severity level. Non null.
+ * @param msg The log message. Non null.
+ * @param trace The stack trace. May be null.
+ * @param values Additional values to upload.
+ */
+ public static void logToRemote(Level level, String msg, Throwable trace,
+ String... values) {
+ Logger logger = getRemoteLogger();
+ if (logger != null) {
+ LogRecord logRecord = new LogRecord(level, msg);
+ logRecord.setThrown(trace);
+ logRecord.setParameters(values);
+ logger.log(logRecord);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/NetUtil.java b/src/main/java/com/google/devtools/build/lib/util/NetUtil.java
new file mode 100644
index 0000000..498da77
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/NetUtil.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Various utility methods for network related stuff.
+ */
+public final class NetUtil {
+
+ private NetUtil() {
+ }
+
+ /**
+ * Returns the short hostname or <code>unknown</code> if the host name could
+ * not be determined.
+ */
+ public static String findShortHostName() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ return "unknown";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/OS.java b/src/main/java/com/google/devtools/build/lib/util/OS.java
new file mode 100644
index 0000000..b19bd7e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/OS.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+/**
+ * An operating system.
+ */
+public enum OS {
+ DARWIN,
+ LINUX,
+ WINDOWS,
+ UNKNOWN;
+
+ /**
+ * The current operating system.
+ */
+ public static OS getCurrent() {
+ return HOST_SYSTEM;
+ }
+ // We inject a the OS name through blaze.os, so we can have
+ // some coverage for Windows specific code on Linux.
+ private static String getOsName() {
+ String override = System.getProperty("blaze.os");
+ return override == null ? System.getProperty("os.name") : override;
+ }
+
+ private static final OS HOST_SYSTEM =
+ "Mac OS X".equals(getOsName()) ? OS.DARWIN : (
+ "Linux".equals(getOsName()) ? OS.LINUX : (
+ getOsName().contains("Windows") ? OS.WINDOWS : OS.UNKNOWN));
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/util/OptionsUtils.java b/src/main/java/com/google/devtools/build/lib/util/OptionsUtils.java
new file mode 100644
index 0000000..11bf94f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/OptionsUtils.java
@@ -0,0 +1,154 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Blaze-specific option utilities.
+ */
+public final class OptionsUtils {
+
+ /**
+ * Returns a string representation of the non-hidden specified options; option values are
+ * shell-escaped.
+ */
+ public static String asShellEscapedString(Iterable<UnparsedOptionValueDescription> optionsList) {
+ StringBuffer result = new StringBuffer();
+ for (UnparsedOptionValueDescription option : optionsList) {
+ if (option.isHidden()) {
+ continue;
+ }
+ if (result.length() != 0) {
+ result.append(' ');
+ }
+ String value = option.getUnparsedValue();
+ if (option.isBooleanOption()) {
+ boolean isEnabled = false;
+ try {
+ isEnabled = new Converters.BooleanConverter().convert(value);
+ } catch (OptionsParsingException e) {
+ throw new RuntimeException("Unexpected parsing exception", e);
+ }
+ result.append(isEnabled ? "--" : "--no").append(option.getName());
+ } else {
+ result.append("--").append(option.getName());
+ if (value != null) { // Can be null for Void options.
+ result.append("=").append(ShellEscaper.escapeString(value));
+ }
+ }
+ }
+ return result.toString();
+ }
+
+ /**
+ * Returns a string representation of the non-hidden explicitly or implicitly
+ * specified options; option values are shell-escaped.
+ */
+ public static String asShellEscapedString(OptionsProvider options) {
+ return asShellEscapedString(options.asListOfUnparsedOptions());
+ }
+
+ /**
+ * Returns a string representation of the non-hidden explicitly or implicitly
+ * specified options, filtering out any sensitive options; option values are
+ * shell-escaped.
+ */
+ public static String asFilteredShellEscapedString(OptionsProvider options,
+ Iterable<UnparsedOptionValueDescription> optionsList) {
+ return asShellEscapedString(optionsList);
+ }
+
+ /**
+ * Returns a string representation of the non-hidden explicitly or implicitly
+ * specified options, filtering out any sensitive options; option values are
+ * shell-escaped.
+ */
+ public static String asFilteredShellEscapedString(OptionsProvider options) {
+ return asFilteredShellEscapedString(options, options.asListOfUnparsedOptions());
+ }
+
+ /**
+ * Converter from String to PathFragment.
+ */
+ public static class PathFragmentConverter
+ implements Converter<PathFragment> {
+
+ @Override
+ public PathFragment convert(String input) {
+ return new PathFragment(input);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a path";
+ }
+ }
+
+ /**
+ * Converter from String to PathFragment.
+ *
+ * <p>Complains if the path is not absolute.
+ */
+ public static class AbsolutePathFragmentConverter
+ implements Converter<PathFragment> {
+
+ @Override
+ public PathFragment convert(String input) throws OptionsParsingException {
+ PathFragment pathFragment = new PathFragment(input);
+ if (!pathFragment.isAbsolute()) {
+ throw new OptionsParsingException("Expected absolute path, found " + input);
+ }
+ return pathFragment;
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "an absolute path";
+ }
+ }
+
+ /**
+ * Converts from a colon-separated list of strings into a list of PathFragment instances.
+ */
+ public static class PathFragmentListConverter
+ implements Converter<List<PathFragment>> {
+
+ @Override
+ public List<PathFragment> convert(String input) {
+ List<PathFragment> list = new ArrayList<>();
+ for (String piece : input.split(":")) {
+ if (!piece.equals("")) {
+ list.add(new PathFragment(piece));
+ }
+ }
+ return Collections.unmodifiableList(list);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a colon-separated list of paths";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/OsUtils.java b/src/main/java/com/google/devtools/build/lib/util/OsUtils.java
new file mode 100644
index 0000000..fa12f59
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/OsUtils.java
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * Operating system-specific utilities.
+ */
+public final class OsUtils {
+
+ private static final String EXECUTABLE_EXTENSION = OS.getCurrent() == OS.WINDOWS ? ".exe" : "";
+
+ // Utility class.
+ private OsUtils() {
+ }
+
+ /**
+ * Returns the extension used for executables on the current platform (.exe
+ * for Windows, empty string for others).
+ */
+ public static String executableExtension() {
+ return EXECUTABLE_EXTENSION;
+ }
+
+ /**
+ * Loads JNI libraries, if necessary under the current platform.
+ */
+ public static void maybeForceJNI(PathFragment installBase) {
+ if (jniLibsAvailable()) {
+ forceJNI(installBase);
+ }
+ }
+
+ private static boolean jniLibsAvailable() {
+ // JNI libraries work fine on Windows, but at the moment we are not using any.
+ return OS.getCurrent() != OS.WINDOWS;
+ }
+
+ // Force JNI linking at a moment when we have 'installBase' handy, and print
+ // an informative error if it fails.
+ private static void forceJNI(PathFragment installBase) {
+ try {
+ ProcessUtils.getpid(); // force JNI initialization
+ } catch (UnsatisfiedLinkError t) {
+ System.err.println("JNI initialization failed: " + t.getMessage() + ". "
+ + "Possibly your installation has been corrupted; "
+ + "if this problem persists, try 'rm -fr " + installBase + "'.");
+ throw t;
+ }
+ }
+
+ /**
+ * Returns the PID of the current process, or -1 if not available.
+ */
+ public static int getpid() {
+ if (jniLibsAvailable()) {
+ return ProcessUtils.getpid();
+ }
+ return -1;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Pair.java b/src/main/java/com/google/devtools/build/lib/util/Pair.java
new file mode 100644
index 0000000..a377c3c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/Pair.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Function;
+
+import java.util.Comparator;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * An immutable, semantic-free ordered pair of nullable values. Avoid using it in public APIs.
+ */
+public final class Pair<A, B> {
+
+ /**
+ * Creates a new pair containing the given elements in order.
+ */
+ public static <A, B> Pair<A, B> of(@Nullable A first, @Nullable B second) {
+ return new Pair<A, B>(first, second);
+ }
+
+ /**
+ * The first element of the pair.
+ */
+ @Nullable
+ public final A first;
+
+ /**
+ * The second element of the pair.
+ */
+ @Nullable
+ public final B second;
+
+ /**
+ * Constructor. It is usually easier to call {@link #of}.
+ */
+ public Pair(@Nullable A first, @Nullable B second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ @Nullable
+ public A getFirst() {
+ return first;
+ }
+
+ @Nullable
+ public B getSecond() {
+ return second;
+ }
+
+ @Override
+ public String toString() {
+ return "(" + first + ", " + second + ")";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Pair)) {
+ return false;
+ }
+ Pair<?, ?> p = (Pair<?, ?>) o;
+ return Objects.equals(first, p.first) && Objects.equals(second, p.second);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(first, second);
+ }
+
+ /**
+ * A function that maps to the first element in a pair.
+ */
+ public static <A, B> Function<Pair<A, B>, A> firstFunction() {
+ return new Function<Pair<A, B>, A>() {
+ @Override
+ public A apply(Pair<A, B> pair) {
+ return pair.first;
+ }
+ };
+ }
+
+ /**
+ * A function that maps to the second element in a pair.
+ */
+ public static <A, B> Function<Pair<A, B>, B> secondFunction() {
+ return new Function<Pair<A, B>, B>() {
+ @Override
+ public B apply(Pair<A, B> pair) {
+ return pair.second;
+ }
+ };
+ }
+
+ /**
+ * A comparator that compares pairs by comparing the first element.
+ */
+ public static <T extends Comparable<T>, B> Comparator<Pair<T, B>> compareByFirst() {
+ return new Comparator<Pair<T, B>>() {
+ @Override
+ public int compare(Pair<T, B> o1, Pair<T, B> o2) {
+ return o1.first.compareTo(o2.first);
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/PathFragmentFilter.java b/src/main/java/com/google/devtools/build/lib/util/PathFragmentFilter.java
new file mode 100644
index 0000000..34f6cd2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/PathFragmentFilter.java
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Handles options that specify list of included/excluded directories.
+ * Validates whether path is included in that filter.
+ *
+ * Excluded directories always take precedence over included ones (path depth
+ * and order are not important).
+ */
+public class PathFragmentFilter implements Serializable {
+ private final List<PathFragment> inclusions;
+ private final List<PathFragment> exclusions;
+
+ /**
+ * Converts from a colon-separated list of of paths with optional '-' prefix into the
+ * PathFragmentFilter:
+ * [-]path1[,[-]path2]...
+ *
+ * Order of paths is not important. Empty entries are ignored. '-' marks an excluded path.
+ */
+ public static class PathFragmentFilterConverter implements Converter<PathFragmentFilter> {
+
+ @Override
+ public PathFragmentFilter convert(String input) {
+ List<PathFragment> inclusionList = new ArrayList<>();
+ List<PathFragment> exclusionList = new ArrayList<>();
+
+ for (String piece : Splitter.on(',').split(input)) {
+ if (piece.length() > 1 && piece.startsWith("-")) {
+ exclusionList.add(new PathFragment(piece.substring(1)));
+ } else if (piece.length() > 0) {
+ inclusionList.add(new PathFragment(piece));
+ }
+ }
+
+ // TODO(bazel-team): (2010) Both lists could be optimized not to include unnecessary
+ // entries - e.g. entry 'a/b/c' is not needed if 'a/b' is also specified and 'a/b' on
+ // inclusion list is not needed if 'a' or 'a/b' is on exclusion list.
+ return new PathFragmentFilter(inclusionList, exclusionList);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a comma-separated list of paths with prefix '-' specifying excluded paths";
+ }
+
+ }
+
+ /**
+ * Creates new PathFragmentFilter using provided inclusion and exclusion path lists.
+ */
+ public PathFragmentFilter(List<PathFragment> inclusions, List<PathFragment> exclusions) {
+ this.inclusions = ImmutableList.copyOf(inclusions);
+ this.exclusions = ImmutableList.copyOf(exclusions);
+ }
+
+ /**
+ * @return true iff path is included (it is not on the exclusion list and
+ * it is either on the inclusion list or inclusion list is empty).
+ */
+ public boolean isIncluded(PathFragment path) {
+ for (PathFragment excludedPath : exclusions) {
+ if (path.startsWith(excludedPath)) {
+ return false;
+ }
+ }
+ for (PathFragment includedPath : inclusions) {
+ if (path.startsWith(includedPath)) {
+ return true;
+ }
+ }
+ return inclusions.isEmpty(); // If inclusion filter is not specified, path is included.
+ }
+
+ @Override
+ public String toString() {
+ List<String> list = Lists.newArrayListWithExpectedSize(inclusions.size() + exclusions.size());
+ for (PathFragment path : inclusions) {
+ list.add(path.getPathString());
+ }
+ for (PathFragment path : exclusions) {
+ list.add("-" + path.getPathString());
+ }
+ return Joiner.on(',').join(list);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/PersistentMap.java b/src/main/java/com/google/devtools/build/lib/util/PersistentMap.java
new file mode 100644
index 0000000..7fd4b6d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/PersistentMap.java
@@ -0,0 +1,486 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.collect.ForwardingMap;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * A map that is backed by persistent storage. It uses two files on disk for
+ * this: The first file contains all the entries and gets written when invoking
+ * the {@link #save()} method. The second file contains a journal of all entries
+ * that were added to or removed from the map since constructing the instance of
+ * the map or the last invocation of {@link #save()} and gets written after each
+ * update of the map although sub-classes are free to implement their own
+ * journal update strategy.
+ *
+ * <p><b>Ceci n'est pas un Map</b>. Strictly speaking, the {@link Map}
+ * interface doesn't permit the possibility of failure. This class uses
+ * persistence; persistence means I/O, and I/O means the possibility of
+ * failure. Therefore the semantics of this may deviate from the Map contract
+ * in failure cases. In particular, updates are not guaranteed to succeed.
+ * However, I/O failures are guaranteed to be reported upon the subsequent call
+ * to a method that throws {@code IOException} such as {@link #save}.
+ *
+ * <p>To populate the map entries using the previously persisted entries call
+ * {@link #load()} prior to invoking any other map operation.
+ * <p>
+ * Like {@link Hashtable} but unlike {@link HashMap}, this class does
+ * <em>not</em> allow <tt>null</tt> to be used as a key or a value.
+ * <p>
+ * IO failures during reading or writing the map entries to disk may result in
+ * {@link AssertionError} getting thrown from the failing method.
+ * <p>
+ * The implementation of the map is not synchronized. If access from multiple
+ * threads is required it must be synchronized using an external object.
+ * <p>
+ * The constructor allows passing in a version number that gets written to the
+ * files on disk and checked before reading from disk. Files with an
+ * incompatible version number will be ignored. This allows the client code to
+ * change the persistence format without polluting the file system name space.
+ */
+public abstract class PersistentMap<K, V> extends ForwardingMap<K, V> {
+
+ private static final int MAGIC = 0x20071105;
+
+ private static final int ENTRY_MAGIC = 0xfe;
+
+ private final int version;
+ private final Path mapFile;
+ private final Path journalFile;
+ private final Map<K, V> journal;
+ private DataOutputStream journalOut;
+
+ /**
+ * 'dirty' is true when the in-memory representation of the map is more recent
+ * than the on-disk representation.
+ */
+ private boolean dirty;
+
+ /**
+ * If non-null, contains the message from an {@code IOException} thrown by a
+ * previously failed write. This error is deferred until the next call to a
+ * method which is able to throw an exception.
+ */
+ private String deferredIOFailure = null;
+
+ /**
+ * 'loaded' is true when the in-memory representation is at least as recent as
+ * the on-disk representation.
+ */
+ private boolean loaded;
+
+ private final Map<K, V> delegate;
+
+ /**
+ * Creates a new PersistentMap instance using the specified backing map.
+ *
+ * @param version the version tag. Changing the version tag allows updating
+ * the on disk format. The map will never read from a file that was
+ * written using a different version tag.
+ * @param map the backing map to use for this PersistentMap.
+ * @param mapFile the file to save the map entries to.
+ * @param journalFile the journal file to write entries between invocations of
+ * {@link #save()}.
+ */
+ public PersistentMap(int version, Map<K, V> map, Path mapFile, Path journalFile) {
+ this.version = version;
+ journal = new LinkedHashMap<>();
+ this.mapFile = mapFile;
+ this.journalFile = journalFile;
+ delegate = map;
+ }
+
+ @Override protected Map<K, V> delegate() {
+ return delegate;
+ }
+
+ @Override
+ public V put(K key, V value) {
+ if (key == null) {
+ throw new NullPointerException();
+ }
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ V previous = delegate().put(key, value);
+ journal.put(key, value);
+ markAsDirty();
+ return previous;
+ }
+
+ /**
+ * Marks the map as dirty and potentially writes updated entries to the
+ * journal.
+ */
+ private void markAsDirty() {
+ dirty = true;
+ if (updateJournal()) {
+ writeJournal();
+ }
+ }
+
+ /**
+ * Determines if the journal should be updated. The default implementation
+ * always returns 'true', but subclasses are free to override this to
+ * implement their own journal updating strategy. For example it is possible
+ * to implement an update at most every five seconds using the following code:
+ *
+ * <pre>
+ * private long nextUpdate;
+ * protected boolean updateJournal() {
+ * long time = System.currentTimeMillis();
+ * if (time > nextUpdate) {
+ * nextUpdate = time + 5 * 1000;
+ * return true;
+ * }
+ * return false;
+ * }
+ * </pre>
+ */
+ protected boolean updateJournal() {
+ return true;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public V remove(Object object) {
+ V previous = delegate().remove(object);
+ if (previous != null) {
+ // we know that 'object' must be an instance of K, because the
+ // remove call succeeded, i.e. 'object' was mapped to 'previous'.
+ journal.put((K) object, null); // unchecked
+ markAsDirty();
+ }
+ return previous;
+ }
+
+ /**
+ * Updates the persistent journal by writing all entries to the
+ * {@link #journalOut} stream and clearing the in memory journal.
+ */
+ private void writeJournal() {
+ try {
+ if (journalOut == null) {
+ journalOut = createMapFile(journalFile);
+ }
+ writeEntries(journalOut, journal);
+ journalOut.flush();
+ journal.clear();
+ } catch (IOException e) {
+ this.deferredIOFailure = e.getMessage() + " during journal append";
+ }
+ }
+
+ protected void forceFlush() {
+ if (dirty) {
+ writeJournal();
+ }
+ }
+
+ /**
+ * Load the previous written map entries from disk.
+ *
+ * @param failFast if true, throw IOException rather than silently ignoring.
+ * @throws IOException
+ */
+ public void load(boolean failFast) throws IOException {
+ if (!loaded) {
+ loadEntries(mapFile, failFast);
+ if (journalFile.exists()) {
+ try {
+ loadEntries(journalFile, failFast);
+ } catch (IOException e) {
+ if (failFast) {
+ throw e;
+ }
+ //Else: ignore any errors reading the journal file as it may contain
+ //partial entries.
+ }
+ // Force the map to be dirty, so that we can save it to disk.
+ dirty = true;
+ save(/*fullSave=*/true);
+ } else {
+ dirty = false;
+ }
+ loaded = true;
+ }
+ }
+
+ /**
+ * Load the previous written map entries from disk.
+ *
+ * @throws IOException
+ */
+ public void load() throws IOException {
+ load(/*throwOnLoadFailure=*/false);
+ }
+
+ @Override
+ public void clear() {
+ super.clear();
+ markAsDirty();
+ try {
+ save();
+ } catch (IOException e) {
+ this.deferredIOFailure = e.getMessage() + " during map write";
+ }
+ }
+
+ /**
+ * Saves all the entries of this map to disk and deletes the journal file.
+ *
+ * @throws IOException if there was an I/O error during this call, or any previous call since the
+ * last save().
+ */
+ public long save() throws IOException {
+ return save(false);
+ }
+
+ /**
+ * Saves all the entries of this map to disk and deletes the journal file.
+ *
+ * @param fullSave if true, always write the full cache to disk, without the
+ * journal.
+ * @throws IOException if there was an I/O error during this call, or any
+ * previous call since the last save().
+ */
+ private long save(boolean fullSave) throws IOException {
+ /* Report a previously failing I/O operation. */
+ if (deferredIOFailure != null) {
+ try {
+ throw new IOException(deferredIOFailure);
+ } finally {
+ deferredIOFailure = null;
+ }
+ }
+ if (dirty) {
+ if (!fullSave && keepJournal()) {
+ forceFlush();
+ journalOut.close();
+ journalOut = null;
+ return journalSize() + cacheSize();
+ } else {
+ dirty = false;
+ Path mapTemp =
+ mapFile.getRelative(FileSystemUtils.replaceExtension(mapFile.asFragment(), ".tmp"));
+ try {
+ saveEntries(delegate(), mapTemp);
+ mapTemp.renameTo(mapFile);
+ } finally {
+ mapTemp.delete();
+ }
+ clearJournal();
+ journalFile.delete();
+ return cacheSize();
+ }
+ } else {
+ return cacheSize();
+ }
+ }
+
+ protected final long journalSize() throws IOException {
+ return journalFile.exists() ? journalFile.getFileSize() : 0;
+ }
+
+ protected final long cacheSize() throws IOException {
+ return mapFile.exists() ? mapFile.getFileSize() : 0;
+ }
+
+ /**
+ * If true, keep the journal during the save(). The journal is flushed, but
+ * the map file is not touched. This may be useful in cases where the journal
+ * is much smaller than the map.
+ */
+ protected boolean keepJournal() {
+ return false;
+ }
+
+ private void clearJournal() throws IOException {
+ journal.clear();
+ if (journalOut != null) {
+ journalOut.close();
+ journalOut = null;
+ }
+ }
+
+ private void loadEntries(Path mapFile, boolean failFast) throws IOException {
+ if (!mapFile.exists()) {
+ return;
+ }
+ DataInputStream in =
+ new DataInputStream(new BufferedInputStream(mapFile.getInputStream()));
+ try {
+ long fileSize = mapFile.getFileSize();
+ if (fileSize < (16)) {
+ if (failFast) {
+ throw new IOException(mapFile + " is too short: Only " + fileSize + " bytes");
+ } else {
+ return;
+ }
+ }
+ if (in.readLong() != MAGIC) { // not a PersistentMap
+ if (failFast) {
+ throw new IOException("Unexpected format");
+ }
+ return;
+ }
+ if (in.readLong() != version) { // PersistentMap version incompatible
+ if (failFast) {
+ throw new IOException("Unexpected format");
+ }
+ return;
+ }
+ readEntries(in, failFast);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Saves the entries in the specified map into the specified file.
+ *
+ * @param map the map to be written into the file.
+ * @param mapFile the file the map is written to.
+ * @throws IOException
+ */
+ private void saveEntries(Map<K, V> map, Path mapFile) throws IOException {
+ DataOutputStream out = createMapFile(mapFile);
+ writeEntries(out, map);
+ out.close();
+ }
+
+ /**
+ * Creates the specified file and returns the DataOuputStream suitable for writing entries.
+ *
+ * @param mapFile the file the map is written to.
+ * @return the DataOutputStream that was can be used for saving the map to the file.
+ * @throws IOException
+ */
+ private DataOutputStream createMapFile(Path mapFile) throws IOException {
+ FileSystemUtils.createDirectoryAndParents(mapFile.getParentDirectory());
+ DataOutputStream out =
+ new DataOutputStream(new BufferedOutputStream(mapFile.getOutputStream()));
+ out.writeLong(MAGIC);
+ out.writeLong(version);
+ return out;
+ }
+
+ /**
+ * Writes the Map entries to the specified DataOutputStream.
+ *
+ * @param out the DataOutputStream to write the Map entries to.
+ * @param map the Map containing the entries to be written to the
+ * DataOutputStream.
+ * @throws IOException
+ */
+ private void writeEntries(DataOutputStream out, Map<K, V> map)
+ throws IOException {
+ for (Map.Entry<K, V> entry : map.entrySet()) {
+ out.writeByte(ENTRY_MAGIC);
+ writeKey(entry.getKey(), out);
+ V value = entry.getValue();
+ boolean isEntry = (value != null);
+ out.writeBoolean(isEntry);
+ if (isEntry) {
+ writeValue(value, out);
+ }
+ }
+ }
+
+ /**
+ * Reads the Map entries from the specified DataInputStream.
+ *
+ * @param failFast if true, throw IOException if entries are in an unexpected
+ * format.
+ * @param in the DataInputStream to read the Map entries from.
+ * @throws IOException
+ */
+ private void readEntries(DataInputStream in, boolean failFast) throws IOException {
+ Map<K, V> map = delegate();
+ while (hasEntries(in, failFast)) {
+ K key = readKey(in);
+ boolean isEntry = in.readBoolean();
+ if (isEntry) {
+ V value = readValue(in);
+ map.put(key, value);
+ } else {
+ map.remove(key);
+ }
+ }
+ }
+
+ private boolean hasEntries(DataInputStream in, boolean failFast) throws IOException {
+ if (in.available() <= 0) {
+ return false;
+ } else if (!(in.readUnsignedByte() == ENTRY_MAGIC)) {
+ if (failFast) {
+ throw new IOException("Corrupted entry separator");
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Writes a key of this map into the specified DataOutputStream.
+ *
+ * @param key the key to write to the DataOutputStream.
+ * @param out the DataOutputStream to write the entry to.
+ * @throws IOException
+ */
+ protected abstract void writeKey(K key, DataOutputStream out)
+ throws IOException;
+
+ /**
+ * Writes a value of this map into the specified DataOutputStream.
+ *
+ * @param value the value to write to the DataOutputStream.
+ * @param out the DataOutputStream to write the entry to.
+ * @throws IOException
+ */
+ protected abstract void writeValue(V value, DataOutputStream out)
+ throws IOException;
+
+ /**
+ * Reads an entry of this map from the specified DataInputStream.
+ *
+ * @param in the DataOutputStream to read the entry from.
+ * @return the entry that was read from the DataInputStream.
+ * @throws IOException
+ */
+ protected abstract K readKey(DataInputStream in) throws IOException;
+
+ /**
+ * Reads an entry of this map from the specified DataInputStream.
+ *
+ * @param in the DataOutputStream to read the entry from.
+ * @return the entry that was read from the DataInputStream.
+ * @throws IOException
+ */
+ protected abstract V readValue(DataInputStream in) throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ProcMeminfoParser.java b/src/main/java/com/google/devtools/build/lib/util/ProcMeminfoParser.java
new file mode 100644
index 0000000..44c1112
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ProcMeminfoParser.java
@@ -0,0 +1,121 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CharMatcher;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Parse and return information from /proc/meminfo.
+ */
+public class ProcMeminfoParser {
+
+ public static final String FILE = "/proc/meminfo";
+
+ private final Map<String, Long> memInfo;
+
+ /**
+ * Populates memory information by reading /proc/meminfo.
+ * @throws IOException if reading the file failed.
+ */
+ public ProcMeminfoParser() throws IOException {
+ this(FILE);
+ }
+
+ @VisibleForTesting
+ public ProcMeminfoParser(String fileName) throws IOException {
+ List<String> lines = Files.readLines(new File(fileName), Charset.defaultCharset());
+ ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder();
+ for (String line : lines) {
+ int colon = line.indexOf(":");
+ String keyword = line.substring(0, colon);
+ String valString = line.substring(colon + 1);
+ try {
+ long val = Long.parseLong(CharMatcher.inRange('0', '9').retainFrom(valString));
+ builder.put(keyword, val);
+ } catch (NumberFormatException e) {
+ // Ignore: we'll fail later if somebody tries to capture this value.
+ }
+ }
+ memInfo = builder.build();
+ }
+
+ /**
+ * Gets a named field in KB.
+ */
+ public long getRamKb(String keyword) {
+ Long val = memInfo.get(keyword);
+ if (val == null) {
+ throw new IllegalArgumentException("Can't locate " + keyword + " in the /proc/meminfo");
+ }
+ return val;
+ }
+
+ /**
+ * Return the total physical memory.
+ */
+ public long getTotalKb() {
+ return getRamKb("MemTotal");
+ }
+
+ /**
+ * Return the inactive memory.
+ */
+ public long getInactiveKb() {
+ return getRamKb("Inactive");
+ }
+
+ /**
+ * Return the active memory.
+ */
+ public long getActiveKb() {
+ return getRamKb("Active");
+ }
+
+ /**
+ * Return the slab memory.
+ */
+ public long getSlabKb() {
+ return getRamKb("Slab");
+ }
+
+ /**
+ * Convert KB to MB.
+ */
+ public static double kbToMb(long kb) {
+ return kb / 1E3;
+ }
+
+ /**
+ * Calculates amount of free RAM from /proc/meminfo content by using
+ * MemTotal - (Active + 0.3*InActive + 0.8*Slab) formula.
+ * Assumption here is that we allow Blaze to use all memory except when
+ * used by active pages, 30% of the inactive pages (since they may become
+ * active at any time) and 80% of memory used by kernel slab heap (since we
+ * want to keep most of the slab heap in the memory but do not want it to
+ * consume all available free memory).
+ */
+ public long getFreeRamKb() {
+ return getTotalKb() - getActiveKb() - (long)(getInactiveKb() * 0.3) - (long)(getSlabKb() * 0.8);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ProcessUtils.java b/src/main/java/com/google/devtools/build/lib/util/ProcessUtils.java
new file mode 100644
index 0000000..ec68736
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ProcessUtils.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+/**
+ * OS Process related utilities.
+ *
+ * <p>Default implementation forwards all requests to
+ * {@link com.google.devtools.build.lib.unix.ProcessUtils}. The default implementation
+ * can be overridden by {@code #setImplementation(ProcessUtilsImpl)} method.
+ */
+@ThreadSafe
+public final class ProcessUtils {
+
+ /**
+ * Describes implementation to which all {@code ProcessUtils} requests are
+ * forwarded.
+ */
+ public interface ProcessUtilsImpl {
+ /** @see ProcessUtils#getgid() */
+ int getgid();
+
+ /** @see ProcessUtils#getpid() */
+ int getpid();
+
+ /** @see ProcessUtils#getuid() */
+ int getuid();
+ }
+
+ private volatile static ProcessUtilsImpl implementation = new ProcessUtilsImpl() {
+
+ @Override
+ public int getgid() {
+ return com.google.devtools.build.lib.unix.ProcessUtils.getgid();
+ }
+
+ @Override
+ public int getpid() {
+ return com.google.devtools.build.lib.unix.ProcessUtils.getpid();
+ }
+
+ @Override
+ public int getuid() {
+ return com.google.devtools.build.lib.unix.ProcessUtils.getuid();
+ }
+ };
+
+ private ProcessUtils() {
+ // prevent construction.
+ }
+
+ /**
+ * @return the real group ID of the current process.
+ */
+ public static int getgid() {
+ return implementation.getgid();
+ }
+
+ /**
+ * @return the process ID of this process.
+ */
+ public static int getpid() {
+ return implementation.getpid();
+ }
+
+ /**
+ * @return the real user ID of the current process.
+ */
+ public static int getuid() {
+ return implementation.getuid();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/RegexFilter.java b/src/main/java/com/google/devtools/build/lib/util/RegexFilter.java
new file mode 100644
index 0000000..d7c6834
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/RegexFilter.java
@@ -0,0 +1,167 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Joiner;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Handles options that specify list of included/excluded regex expressions.
+ * Validates whether string is included in that filter.
+ *
+ * String is considered to be included into the filter if it does not match
+ * any of the excluded regex expressions and if it matches at least one
+ * included regex expression.
+ */
+public class RegexFilter implements Serializable {
+ private final Pattern inclusionPattern;
+ private final Pattern exclusionPattern;
+ private final int hashCode;
+
+ /**
+ * Converts from a colon-separated list of regex expressions with optional
+ * -/+ prefix into the RegexFilter. Colons prefixed with backslash are
+ * considered to be part of regex definition and not a delimiter between
+ * separate regex expressions.
+ *
+ * Order of expressions is not important. Empty entries are ignored.
+ * '-' marks an excluded expression.
+ */
+ public static class RegexFilterConverter
+ implements Converter<RegexFilter> {
+
+ @Override
+ public RegexFilter convert(String input) throws OptionsParsingException {
+ List<String> inclusionList = new ArrayList<>();
+ List<String> exclusionList = new ArrayList<>();
+
+ for (String piece : input.split("(?<!\\\\),")) { // Split on ',' but not on '\,'
+ piece = piece.replace("\\,", ",");
+ boolean isExcluded = piece.startsWith("-");
+ if (isExcluded || piece.startsWith("+")) {
+ piece = piece.substring(1);
+ }
+ if (piece.length() > 0) {
+ (isExcluded ? exclusionList : inclusionList).add(piece);
+ }
+ }
+
+ try {
+ return new RegexFilter(inclusionList, exclusionList);
+ } catch (PatternSyntaxException e) {
+ throw new OptionsParsingException("Failed to build valid regular expression: "
+ + e.getMessage());
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a comma-separated list of regex expressions with prefix '-' specifying"
+ + " excluded paths";
+ }
+
+ }
+
+ /**
+ * Creates new RegexFilter using provided inclusion and exclusion path lists.
+ */
+ public RegexFilter(List<String> inclusions, List<String> exclusions) {
+ inclusionPattern = convertRegexListToPattern(inclusions);
+ exclusionPattern = convertRegexListToPattern(exclusions);
+ hashCode = Objects.hash(inclusions, exclusions);
+ }
+
+ /**
+ * Converts list of regex expressions into one compiled regex expression.
+ */
+ private static Pattern convertRegexListToPattern(List<String> regexList) {
+ if (regexList.size() == 0) {
+ return null;
+ }
+ // Wrap each individual regex in the independent group, combine them using '|' and
+ // wrap in the non-capturing group.
+ return Pattern.compile("(?:(?>" + Joiner.on(")|(?>").join(regexList) + "))");
+ }
+
+ /**
+ * @return true iff given string is included (it is does not match exclusion
+ * pattern (if any) and matches inclusionPatter (if any).
+ */
+ public boolean isIncluded(String value) {
+ if (exclusionPattern != null && exclusionPattern.matcher(value).find()) {
+ return false;
+ }
+ if (inclusionPattern == null) {
+ return true;
+ }
+ return inclusionPattern.matcher(value).find();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ if (inclusionPattern != null) {
+ builder.append(inclusionPattern.pattern().replace(",", "\\,"));
+ if (exclusionPattern != null) {
+ builder.append(",");
+ }
+ }
+ if (exclusionPattern != null) {
+ builder.append("-");
+ builder.append(exclusionPattern.pattern().replace(",", "\\,"));
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof RegexFilter)) {
+ return false;
+ }
+
+ RegexFilter otherFilter = (RegexFilter) other;
+ if (this.exclusionPattern == null ^ otherFilter.exclusionPattern == null) {
+ return false;
+ }
+ if (this.inclusionPattern == null ^ otherFilter.inclusionPattern == null) {
+ return false;
+ }
+ if (this.exclusionPattern != null && !this.exclusionPattern.pattern().equals(
+ otherFilter.exclusionPattern.pattern())) {
+ return false;
+ }
+ if (this.inclusionPattern != null && !this.inclusionPattern.pattern().equals(
+ otherFilter.inclusionPattern.pattern())) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ResourceFileLoader.java b/src/main/java/com/google/devtools/build/lib/util/ResourceFileLoader.java
new file mode 100644
index 0000000..ce26c6c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ResourceFileLoader.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.io.ByteStreams;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A little utility to load resources (property files) from jars or
+ * the classpath. Recommended for longer texts that do not fit nicely into
+ * a piece of Java code - e.g. a template for a lengthy email.
+ */
+public final class ResourceFileLoader {
+
+ private ResourceFileLoader() {}
+
+ /**
+ * Loads a text resource that is located in a directory on the Java classpath that
+ * corresponds to the package of <code>relativeToClass</code> using UTF8 encoding.
+ * E.g.
+ * <code>loadResource(Class.forName("com.google.foo.Foo", "bar.txt"))</code>
+ * will look for <code>com/google/foo/bar.txt</code> in the classpath.
+ */
+ public static String loadResource(Class<?> relativeToClass, String resourceName)
+ throws IOException {
+ ClassLoader loader = relativeToClass.getClassLoader();
+ // TODO(bazel-team): use relativeToClass.getPackage().getName().
+ String className = relativeToClass.getName();
+ String packageName = className.substring(0, className.lastIndexOf('.'));
+ String path = packageName.replace('.', '/');
+ String resource = path + '/' + resourceName;
+ InputStream stream = loader.getResourceAsStream(resource);
+ if (stream == null) {
+ throw new IOException(resourceName + " not found.");
+ }
+ try {
+ return new String(ByteStreams.toByteArray(stream), UTF_8);
+ } finally {
+ stream.close();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ResourceUsage.java b/src/main/java/com/google/devtools/build/lib/util/ResourceUsage.java
new file mode 100644
index 0000000..55807f2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ResourceUsage.java
@@ -0,0 +1,353 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
+
+import com.sun.management.OperatingSystemMXBean;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.util.Iterator;
+
+/**
+ * Provides methods to measure the current resource usage of the current
+ * process. Also provides some convenience methods to obtain several system
+ * characteristics, like number of processors , total memory, etc.
+ */
+public final class ResourceUsage {
+
+ /*
+ * Use com.sun.management.OperatingSystemMXBean instead of
+ * java.lang.management.OperatingSystemMXBean because the latter does not
+ * support getTotalPhysicalMemorySize() and getFreePhysicalMemorySize().
+ */
+ private static final OperatingSystemMXBean OS_BEAN =
+ (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+
+ private static final MemoryMXBean MEM_BEAN = ManagementFactory.getMemoryMXBean();
+ private static final Splitter WHITESPACE_SPLITTER = Splitter.on(CharMatcher.WHITESPACE);
+
+ /**
+ * Calculates an estimate of the current total CPU usage and the CPU usage of
+ * the process in percent measured from the two given measurements. The
+ * returned CPU usages rea average values for the time between the two
+ * measurements. The returned array contains the total CPU usage at index 0
+ * and the CPU usage of the measured process at index 1.
+ */
+ public static float[] calculateCurrentCpuUsage(Measurement oldMeasurement,
+ Measurement newMeasurement) {
+ if (oldMeasurement == null) {
+ return new float[2];
+ }
+ long idleJiffies =
+ newMeasurement.getTotalCpuIdleTimeInJiffies()
+ - oldMeasurement.getTotalCpuIdleTimeInJiffies();
+ long oldProcessJiffies =
+ oldMeasurement.getCpuUtilizationInJiffies()[0]
+ + oldMeasurement.getCpuUtilizationInJiffies()[1];
+ long newProcessJiffies =
+ newMeasurement.getCpuUtilizationInJiffies()[0]
+ + newMeasurement.getCpuUtilizationInJiffies()[1];
+ long processJiffies = newProcessJiffies - oldProcessJiffies;
+ long elapsedTimeJiffies =
+ newMeasurement.getTimeInJiffies() - oldMeasurement.getTimeInJiffies();
+ int processors = getAvailableProcessors();
+ // TODO(bazel-team): Sometimes smaller then zero. Not sure why.
+ double totalUsage = Math.max(0, 1.0D - (double) idleJiffies / elapsedTimeJiffies / processors);
+ double usage = Math.max(0, (double) processJiffies / elapsedTimeJiffies / processors);
+ return new float[] {(float) totalUsage * 100, (float) usage * 100};
+ }
+
+ private ResourceUsage() {
+ }
+
+ /**
+ * Returns the number of processors available to the Java virtual machine.
+ */
+ public static int getAvailableProcessors() {
+ return OS_BEAN.getAvailableProcessors();
+ }
+
+ /**
+ * Returns the total physical memory in bytes.
+ */
+ public static long getTotalPhysicalMemorySize() {
+ return OS_BEAN.getTotalPhysicalMemorySize();
+ }
+
+ /**
+ * Returns the operating system architecture.
+ */
+ public static String getOsArchitecture() {
+ return OS_BEAN.getArch();
+ }
+
+ /**
+ * Returns the operating system name.
+ */
+ public static String getOsName() {
+ return OS_BEAN.getName();
+ }
+
+ /**
+ * Returns the operating system version.
+ */
+ public static String getOsVersion() {
+ return OS_BEAN.getVersion();
+ }
+
+ /**
+ * Returns the initial size of heap memory in bytes.
+ *
+ * @see MemoryMXBean#getHeapMemoryUsage()
+ */
+ public static long getHeapMemoryInit() {
+ return MEM_BEAN.getHeapMemoryUsage().getInit();
+ }
+
+ /**
+ * Returns the initial size of non heap memory in bytes.
+ *
+ * @see MemoryMXBean#getNonHeapMemoryUsage()
+ */
+ public static long getNonHeapMemoryInit() {
+ return MEM_BEAN.getNonHeapMemoryUsage().getInit();
+ }
+
+ /**
+ * Returns the maximum size of heap memory in bytes.
+ *
+ * @see MemoryMXBean#getHeapMemoryUsage()
+ */
+ public static long getHeapMemoryMax() {
+ return MEM_BEAN.getHeapMemoryUsage().getMax();
+ }
+
+ /**
+ * Returns the maximum size of non heap memory in bytes.
+ *
+ * @see MemoryMXBean#getNonHeapMemoryUsage()
+ */
+ public static long getNonHeapMemoryMax() {
+ return MEM_BEAN.getNonHeapMemoryUsage().getMax();
+ }
+
+ /**
+ * Returns a measurement of the current resource usage of the current process.
+ */
+ public static Measurement measureCurrentResourceUsage() {
+ return measureCurrentResourceUsage("self");
+ }
+
+ /**
+ * Returns a measurement of the current resource usage of the process with the
+ * given process id.
+ *
+ * @param processId the process id or <code>self</code> for the current
+ * process.
+ */
+ public static Measurement measureCurrentResourceUsage(String processId) {
+ return new Measurement(MEM_BEAN.getHeapMemoryUsage().getUsed(), MEM_BEAN.getHeapMemoryUsage()
+ .getCommitted(), MEM_BEAN.getNonHeapMemoryUsage().getUsed(), MEM_BEAN
+ .getNonHeapMemoryUsage().getCommitted(), (float) OS_BEAN.getSystemLoadAverage(), OS_BEAN
+ .getFreePhysicalMemorySize(), getCurrentTotalIdleTimeInJiffies(),
+ getCurrentCpuUtilizationInJiffies(processId));
+ }
+
+ /**
+ * Returns the current total idle time of the processors since system boot.
+ * Reads /proc/stat to obtain this information.
+ */
+ private static long getCurrentTotalIdleTimeInJiffies() {
+ try {
+ File file = new File("/proc/stat");
+ String content = Files.toString(file, US_ASCII);
+ String value = Iterables.get(WHITESPACE_SPLITTER.split(content), 5);
+ return Long.parseLong(value);
+ } catch (NumberFormatException | IOException e) {
+ return 0L;
+ }
+ }
+
+ /**
+ * Returns the current cpu utilization of the current process with the given
+ * id in jiffies. The returned array contains the following information: The
+ * 1st entry is the number of jiffies that the process has executed in user
+ * mode, and the 2nd entry is the number of jiffies that the process has
+ * executed in kernel mode. Reads /proc/self/stat to obtain this information.
+ *
+ * @param processId the process id or <code>self</code> for the current
+ * process.
+ */
+ private static long[] getCurrentCpuUtilizationInJiffies(String processId) {
+ try {
+ File file = new File("/proc/" + processId + "/stat");
+ if (file.isDirectory()) {
+ return new long[2];
+ }
+ Iterator<String> stat = WHITESPACE_SPLITTER.split(
+ Files.toString(file, US_ASCII)).iterator();
+ for (int i = 0; i < 13; ++i) {
+ stat.next();
+ }
+ long token13 = Long.parseLong(stat.next());
+ long token14 = Long.parseLong(stat.next());
+ return new long[] { token13, token14 };
+ } catch (NumberFormatException e) {
+ return new long[2];
+ } catch (IOException e) {
+ return new long[2];
+ }
+ }
+
+ /**
+ * A snapshot of the resource usage of the current process at a point in time.
+ */
+ public static class Measurement {
+
+ private final long timeInNanos;
+ private final long heapMemoryUsed;
+ private final long heapMemoryCommitted;
+ private final long nonHeapMemoryUsed;
+ private final long nonHeapMemoryCommitted;
+ private final float loadAverageLastMinute;
+ private final long freePhysicalMemory;
+ private final long totalCpuIdleTimeInJiffies;
+ private final long[] cpuUtilizationInJiffies;
+
+ public Measurement(long heapMemoryUsed, long heapMemoryCommitted, long nonHeapMemoryUsed,
+ long nonHeapMemoryCommitted, float loadAverageLastMinute, long freePhysicalMemory,
+ long totalCpuIdleTimeInJiffies, long[] cpuUtilizationInJiffies) {
+ super();
+ timeInNanos = System.nanoTime();
+ this.heapMemoryUsed = heapMemoryUsed;
+ this.heapMemoryCommitted = heapMemoryCommitted;
+ this.nonHeapMemoryUsed = nonHeapMemoryUsed;
+ this.nonHeapMemoryCommitted = nonHeapMemoryCommitted;
+ this.loadAverageLastMinute = loadAverageLastMinute;
+ this.freePhysicalMemory = freePhysicalMemory;
+ this.totalCpuIdleTimeInJiffies = totalCpuIdleTimeInJiffies;
+ this.cpuUtilizationInJiffies = cpuUtilizationInJiffies;
+ }
+
+ /**
+ * Returns the time of the measurement in jiffies.
+ */
+ public long getTimeInJiffies() {
+ return timeInNanos / 10000000;
+ }
+
+ /**
+ * Returns the time of the measurement in ms.
+ */
+ public long getTimeInMs() {
+ return timeInNanos / 1000000;
+ }
+
+ /**
+ * Returns the amount of used heap memory in bytes at the time of
+ * measurement.
+ *
+ * @see MemoryMXBean#getHeapMemoryUsage()
+ */
+ public long getHeapMemoryUsed() {
+ return heapMemoryUsed;
+ }
+
+ /**
+ * Returns the amount of used non heap memory in bytes at the time of
+ * measurement.
+ *
+ * @see MemoryMXBean#getNonHeapMemoryUsage()
+ */
+ public long getHeapMemoryCommitted() {
+ return heapMemoryCommitted;
+ }
+
+ /**
+ * Returns the amount of memory in bytes that is committed for the Java
+ * virtual machine to use for the heap at the time of measurement.
+ *
+ * @see MemoryMXBean#getHeapMemoryUsage()
+ */
+ public long getNonHeapMemoryUsed() {
+ return nonHeapMemoryUsed;
+ }
+
+ /**
+ * Returns the amount of memory in bytes that is committed for the Java
+ * virtual machine to use for non heap memory at the time of measurement.
+ *
+ * @see MemoryMXBean#getNonHeapMemoryUsage()
+ */
+ public long getNonHeapMemoryCommitted() {
+ return nonHeapMemoryCommitted;
+ }
+
+ /**
+ * Returns the system load average for the last minute at the time of
+ * measurement.
+ *
+ * @see OperatingSystemMXBean#getSystemLoadAverage()
+ */
+ public float getLoadAverageLastMinute() {
+ return loadAverageLastMinute;
+ }
+
+ /**
+ * Returns the free physical memmory in bytes at the time of measurement.
+ */
+ public long getFreePhysicalMemory() {
+ return freePhysicalMemory;
+ }
+
+ /**
+ * Returns the current total cpu idle since system boot in jiffies.
+ */
+ public long getTotalCpuIdleTimeInJiffies() {
+ return totalCpuIdleTimeInJiffies;
+ }
+
+ /**
+ * Returns the current cpu utilization of the current process in jiffies.
+ * The returned array contains the following information: The 1st entry is
+ * the number of jiffies that the process has executed in user mode, and the
+ * 2nd entry is the number of jiffies that the process has executed in
+ * kernel mode. Reads /proc/self/stat to obtain this information.
+ */
+ public long[] getCpuUtilizationInJiffies() {
+ return cpuUtilizationInJiffies;
+ }
+
+ /**
+ * Returns the current cpu utilization of the current process in ms. The
+ * returned array contains the following information: The 1st entry is the
+ * number of ms that the process has executed in user mode, and the 2nd
+ * entry is the number of ms that the process has executed in kernel mode.
+ * Reads /proc/self/stat to obtain this information.
+ */
+ public long[] getCpuUtilizationInMs() {
+ return new long[] {cpuUtilizationInJiffies[0] * 10, cpuUtilizationInJiffies[1] * 10};
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java b/src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java
new file mode 100644
index 0000000..fd23443
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java
@@ -0,0 +1,202 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.common.escape.CharEscaperBuilder;
+import com.google.common.escape.Escaper;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+
+import java.io.IOException;
+
+/**
+ * Utility class to escape strings for use with shell commands.
+ *
+ * <p>Escaped strings may safely be inserted into shell commands. Escaping is
+ * only done if necessary. Strings containing only shell-neutral characters
+ * will not be escaped.
+ *
+ * <p>This is a replacement for {@code ShellUtils.shellEscape(String)} and
+ * {@code ShellUtils.prettyPrintArgv(java.util.List)} (see
+ * {@link com.google.devtools.build.lib.shell.ShellUtils}). Its advantage is the use
+ * of standard building blocks from the {@code com.google.common.base}
+ * package, such as {@link Joiner} and {@link CharMatcher}, making this class
+ * more efficient and reliable than {@code ShellUtils}.
+ *
+ * <p>The behavior is slightly different though: this implementation will
+ * defensively escape non-ASCII letters and digits, whereas
+ * {@code shellEscape} does not.
+ */
+@Immutable
+public final class ShellEscaper extends Escaper {
+ // Note: extending Escaper may seem desirable, but is in fact harmful.
+ // The class would then need to implement escape(Appendable), returning an Appendable
+ // that escapes everything it receives. In case of shell escaping, we most often join
+ // string parts on spaces, using a Joiner. Spaces are escaped characters. Using the
+ // Appendable returned by escape(Appendable) would escape these spaces too, which
+ // is unwanted.
+
+ public static final ShellEscaper INSTANCE = new ShellEscaper();
+
+ private static final Function<String, String> AS_FUNCTION = INSTANCE.asFunction();
+
+ private static final Joiner SPACE_JOINER = Joiner.on(' ');
+ private static final Escaper STRONGQUOTE_ESCAPER =
+ new CharEscaperBuilder().addEscape('\'', "'\\''").toEscaper();
+ private static final CharMatcher SAFECHAR_MATCHER =
+ CharMatcher.anyOf("@%-_+:,./")
+ .or(CharMatcher.inRange('0', '9')) // We can't use CharMatcher.JAVA_LETTER_OR_DIGIT,
+ .or(CharMatcher.inRange('a', 'z')) // that would also accept non-ASCII digits and
+ .or(CharMatcher.inRange('A', 'Z')); // letters.
+
+ /**
+ * Escapes a string by adding strong (single) quotes around it if necessary.
+ *
+ * <p>A string is not escaped iff it only contains safe characters.
+ * The following characters are safe:
+ * <ul>
+ * <li>ASCII letters and digits: [a-zA-Z0-9]
+ * <li>shell-neutral characters: at symbol (@), percent symbol (%),
+ * dash/minus sign (-), underscore (_), plus sign (+), colon (:),
+ * comma(,), period (.) and slash (/).
+ * </ul>
+ *
+ * <p>A string is escaped iff it contains at least one non-safe character.
+ * Escaped strings are created by replacing every occurrence of single
+ * quotes with the string '\'' and enclosing the result in a pair of
+ * single quotes.
+ *
+ * <p>Examples:
+ * <ul>
+ * <li>"{@code foo}" becomes "{@code foo}" (remains the same)
+ * <li>"{@code +bar}" becomes "{@code +bar}" (remains the same)
+ * <li>"" becomes "{@code''}" (empty string becomes a pair of strong quotes)
+ * <li>"{@code $BAZ}" becomes "{@code '$BAZ'}"
+ * <li>"{@code quote'd}" becomes "{@code 'quote'\''d'}"
+ * </ul>
+ */
+ @Override
+ public String escape(String unescaped) {
+ final String s = unescaped.toString();
+ if (s.isEmpty()) {
+ // Empty string is a special case: needs to be quoted to ensure that it
+ // gets treated as a separate argument.
+ return "''";
+ } else {
+ return SAFECHAR_MATCHER.matchesAllOf(s)
+ ? s
+ : "'" + STRONGQUOTE_ESCAPER.escape(s) + "'";
+ }
+ }
+
+ public static String escapeString(String unescaped) {
+ return INSTANCE.escape(unescaped);
+ }
+
+ /**
+ * Transforms the input {@code Iterable} of unescaped strings to an
+ * {@code Iterable} of escaped ones. The escaping is done lazily.
+ */
+ public static Iterable<String> escapeAll(Iterable<? extends String> unescaped) {
+ return Iterables.transform(unescaped, AS_FUNCTION);
+ }
+
+ /**
+ * Escapes all strings in {@code argv} individually and joins them on
+ * single spaces into {@code out}. The result is appended directly into
+ * {@code out}, without adding a separator.
+ *
+ * <p>This method works as if by invoking
+ * {@link #escapeJoinAll(Appendable, Iterable, Joiner)} with
+ * {@code Joiner.on(' ')}.
+ *
+ * @param out what the result will be appended to
+ * @param argv the strings to escape and join
+ * @return the same reference as {@code out}, now containing the the
+ * joined, escaped fragments
+ * @throws IOException if an I/O error occurs while appending
+ */
+ public static Appendable escapeJoinAll(Appendable out, Iterable<? extends String> argv)
+ throws IOException {
+ return SPACE_JOINER.appendTo(out, escapeAll(argv));
+ }
+
+ /**
+ * Escapes all strings in {@code argv} individually and joins them into
+ * {@code out} using the specified {@link Joiner}. The result is appended
+ * directly into {@code out}, without adding a separator.
+ *
+ * <p>The resulting strings are the same as if escaped one by one using
+ * {@link #escapeString(String)}.
+ *
+ * <p>Example: if the joiner is {@code Joiner.on('|')}, then the input
+ * {@code ["abc", "de'f"]} will be escaped as "{@code abc|'de'\''f'}".
+ * If {@code out} initially contains "{@code 123}", then the returned
+ * {@code Appendable} will contain "{@code 123abc|'de'\''f'}".
+ *
+ * @param out what the result will be appended to
+ * @param argv the strings to escape and join
+ * @param joiner the {@link Joiner} to use to join the escaped strings
+ * @return the same reference as {@code out}, now containing the the
+ * joined, escaped fragments
+ * @throws IOException if an I/O error occurs while appending
+ */
+ public static Appendable escapeJoinAll(Appendable out, Iterable<? extends String> argv,
+ Joiner joiner) throws IOException {
+ return joiner.appendTo(out, escapeAll(argv));
+ }
+
+ /**
+ * Escapes all strings in {@code argv} individually and joins them on
+ * single spaces, then returns the resulting string.
+ *
+ * <p>This method works as if by invoking
+ * {@link #escapeJoinAll(Iterable, Joiner)} with {@code Joiner.on(' ')}.
+ *
+ * <p>Example: {@code ["abc", "de'f"]} will be escaped and joined as
+ * "abc 'de'\''f'".
+ *
+ * @param argv the strings to escape and join
+ * @return the string of escaped and joined input elements
+ */
+ public static String escapeJoinAll(Iterable<? extends String> argv) {
+ return SPACE_JOINER.join(escapeAll(argv));
+ }
+
+ /**
+ * Escapes all strings in {@code argv} individually and joins them using
+ * the specified {@link Joiner}, then returns the resulting string.
+ *
+ * <p>The resulting strings are the same as if escaped one by one using
+ * {@link #escapeString(String)}.
+ *
+ * <p>Example: if the joiner is {@code Joiner.on('|')}, then the input
+ * {@code ["abc", "de'f"]} will be escaped and joined as "abc|'de'\''f'".
+ *
+ * @param argv the strings to escape and join
+ * @param joiner the {@link Joiner} to use to join the escaped strings
+ * @return the string of escaped and joined input elements
+ */
+ public static String escapeJoinAll(Iterable<? extends String> argv, Joiner joiner) {
+ return joiner.join(escapeAll(argv));
+ }
+
+ private ShellEscaper() {
+ // Utility class - do not instantiate.
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringCanonicalizer.java b/src/main/java/com/google/devtools/build/lib/util/StringCanonicalizer.java
new file mode 100644
index 0000000..7bdbe7e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/StringCanonicalizer.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+
+/**
+ * Static singleton holder for the string interning pool. Doesn't use {@link String#intern}
+ * because that consumes permgen space.
+ */
+public final class StringCanonicalizer {
+
+ private static final Interner<String> interner = Interners.newWeakInterner();
+
+ private StringCanonicalizer() {
+ }
+
+ /**
+ * Interns a String.
+ */
+ public final static String intern(String arg) {
+ return interner.intern(arg);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringIndexer.java b/src/main/java/com/google/devtools/build/lib/util/StringIndexer.java
new file mode 100644
index 0000000..cf345d2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/StringIndexer.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+/**
+ * An object that provides bidirectional String <-> unique integer mapping.
+ */
+public interface StringIndexer {
+
+ /**
+ * Removes all mappings.
+ */
+ public void clear();
+
+ /**
+ * @return some measure of the size of the index.
+ */
+ public int size();
+
+ /**
+ * Creates new mapping for the given string if necessary and returns
+ * string index. Also, as a side effect, zero or more additional mappings
+ * may be created for various prefixes of the given string.
+ *
+ * @return a unique index.
+ */
+ public int getOrCreateIndex(String s);
+
+ /**
+ * @return a unique index for the given string or -1 if string
+ * was not added.
+ */
+ public int getIndex(String s);
+
+ /**
+ * Creates mapping for the given string if necessary.
+ * Also, as a side effect, zero or more additional mappings may be
+ * created for various prefixes of the given string.
+ *
+ * @return true if new mapping was created, false if mapping already existed.
+ */
+ public boolean addString(String s);
+
+ /**
+ * @return string associated with the given index or null if
+ * mapping does not exist.
+ */
+ public String getStringForIndex(int i);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringTrie.java b/src/main/java/com/google/devtools/build/lib/util/StringTrie.java
new file mode 100644
index 0000000..4744c7e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/StringTrie.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Preconditions;
+
+
+/**
+ * A trie that operates on path segments of an input string instead of individual characters.
+ *
+ * <p>Only accepts strings that contain only low-ASCII characters (0-127)
+ *
+ * @param <T> the type of the values
+ */
+public class StringTrie<T> {
+ private static final int RANGE = 128;
+
+ @SuppressWarnings("unchecked")
+ private static class Node<T> {
+ private Node() {
+ children = new Node[RANGE];
+ }
+
+ private T value;
+ private Node<T> children[];
+ }
+
+ private final Node<T> root;
+
+ public StringTrie() {
+ root = new Node<T>();
+ }
+
+ /**
+ * Puts a value in the trie.
+ */
+ public void put(CharSequence key, T value) {
+ Node<T> current = root;
+
+ for (int i = 0; i < key.length(); i++) {
+ char ch = key.charAt(i);
+ Preconditions.checkState(ch < RANGE);
+ Node<T> next = current.children[ch];
+ if (next == null) {
+ next = new Node<T>();
+ current.children[ch] = next;
+ }
+
+ current = next;
+ }
+
+ current.value = value;
+ }
+
+ /**
+ * Gets a value from the trie. If there is an entry with the same key, that will be returned,
+ * otherwise, the value corresponding to the key that matches the longest prefix of the input.
+ */
+ public T get(String key) {
+ Node<T> current = root;
+ T lastValue = current.value;
+
+ for (int i = 0; i < key.length(); i++) {
+ char ch = key.charAt(i);
+ Preconditions.checkState(ch < RANGE);
+
+ current = current.children[ch];
+ if (current == null) {
+ break;
+ }
+
+ if (current.value != null) {
+ lastValue = current.value;
+ }
+ }
+
+ return lastValue;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringUtil.java b/src/main/java/com/google/devtools/build/lib/util/StringUtil.java
new file mode 100644
index 0000000..40f7ec1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/StringUtil.java
@@ -0,0 +1,175 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Various utility methods operating on strings.
+ */
+public class StringUtil {
+ /**
+ * Creates a comma-separated list of words as in English.
+ *
+ * <p>Example: ["a", "b", "c"] -> "a, b or c".
+ */
+ public static String joinEnglishList(Iterable<?> choices) {
+ return joinEnglishList(choices, "or", "");
+ }
+
+ /**
+ * Creates a comma-separated list of words as in English with the given last-separator.
+ *
+ * <p>Example with lastSeparator="then": ["a", "b", "c"] -> "a, b then c".
+ */
+ public static String joinEnglishList(Iterable<?> choices, String lastSeparator) {
+ return joinEnglishList(choices, lastSeparator, "");
+ }
+
+ /**
+ * Creates a comma-separated list of words as in English with the given last-separator and quotes.
+ *
+ * <p>Example with lastSeparator="then", quote="'": ["a", "b", "c"] -> "'a', 'b' then 'c'".
+ */
+ public static String joinEnglishList(Iterable<?> choices, String lastSeparator, String quote) {
+ StringBuilder buf = new StringBuilder();
+ for (Iterator<?> ii = choices.iterator(); ii.hasNext(); ) {
+ Object choice = ii.next();
+ if (buf.length() > 0) {
+ buf.append(ii.hasNext() ? "," : " " + lastSeparator);
+ buf.append(" ");
+ }
+ buf.append(quote).append(choice).append(quote);
+ }
+ return buf.length() == 0 ? "nothing" : buf.toString();
+ }
+
+ /**
+ * Split a single space-separated string into a List of values.
+ *
+ * <p>Individual values are canonicalized such that within and
+ * across calls to this method, equal values point to the same
+ * object.
+ *
+ * <p>If the input is null, return an empty list.
+ *
+ * @param in space-separated list of values, eg "value1 value2".
+ */
+ public static List<String> splitAndInternString(String in) {
+ List<String> result = new ArrayList<>();
+ if (in == null) {
+ return result;
+ }
+ for (String val : Splitter.on(" ").omitEmptyStrings().split(in)) {
+ // Note that splitter returns a substring(), effectively
+ // retaining the entire "in" String. Make an explicit copy here
+ // to avoid that memory pitfall. Further, because there may be
+ // many concurrent submissions that touch the same files,
+ // attempt to use a single reference for equal strings via the
+ // deduplicator.
+ result.add(StringCanonicalizer.intern(new String(val)));
+ }
+ return result;
+ }
+
+ /**
+ * Lists items up to a given limit, then prints how many were omitted.
+ */
+ public static StringBuilder listItemsWithLimit(StringBuilder appendTo, int limit,
+ Collection<?> items) {
+ Preconditions.checkState(limit > 0);
+ Joiner.on(", ").appendTo(appendTo, Iterables.limit(items, limit));
+ if (items.size() > limit) {
+ appendTo.append(" ...(omitting ")
+ .append(items.size() - limit)
+ .append(" more item(s))");
+ }
+ return appendTo;
+ }
+
+ /**
+ * Returns the ordinal representation of the number.
+ */
+ public static String ordinal(int number) {
+ switch (number) {
+ case 1:
+ return "1st";
+ case 2:
+ return "2nd";
+ case 3:
+ return "3rd";
+ default:
+ return number + "th";
+ }
+ }
+
+ /**
+ * Appends a prefix and a suffix to each of the Strings.
+ */
+ public static Iterable<String> append(Iterable<String> values, final String prefix,
+ final String suffix) {
+ return Iterables.transform(values, new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ return prefix + input + suffix;
+ }
+ });
+ }
+
+ /**
+ * Indents the specified string by the given number of characters.
+ *
+ * <p>The beginning of the string before the first newline is not indented.
+ */
+ public static String indent(String input, int depth) {
+ StringBuilder prefix = new StringBuilder();
+ prefix.append("\n");
+ for (int i = 0; i < depth; i++) {
+ prefix.append(" ");
+ }
+
+ return input.replace("\n", prefix);
+ }
+
+ /**
+ * Strips a suffix from a string. If the string does not end with the suffix, returns null.
+ */
+ public static String stripSuffix(String input, String suffix) {
+ return input.endsWith(suffix)
+ ? input.substring(0, input.length() - suffix.length())
+ : null;
+ }
+
+ /**
+ * Capitalizes the first character of a string.
+ */
+ public static String capitalize(String input) {
+ if (input.isEmpty()) {
+ return input;
+ }
+
+ char first = input.charAt(0);
+ char capitalized = Character.toUpperCase(first);
+ return first == capitalized ? input : capitalized + input.substring(1);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringUtilities.java b/src/main/java/com/google/devtools/build/lib/util/StringUtilities.java
new file mode 100644
index 0000000..9ac1d35
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/StringUtilities.java
@@ -0,0 +1,207 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.escape.CharEscaperBuilder;
+import com.google.common.escape.Escaper;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Various utility methods operating on strings.
+ */
+public class StringUtilities {
+
+ private static final Joiner NEWLINE_JOINER = Joiner.on('\n');
+
+ private static final Escaper KEY_ESCAPER = new CharEscaperBuilder()
+ .addEscape('!', "!!")
+ .addEscape('<', "!<")
+ .addEscape('>', "!>")
+ .toEscaper();
+
+ private static final Escaper CONTROL_CHAR_ESCAPER = new CharEscaperBuilder()
+ .addEscape('\r', "\\r")
+ .addEscapes(new char[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, /*13=\r*/
+ 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 127}, "<?>")
+ .toEscaper();
+
+ /**
+ * Java doesn't have multiline string literals, so having to join a bunch
+ * of lines is a very common problem. So, here's a static method that we
+ * can static import in such situations.
+ */
+ public static String joinLines(String... lines) {
+ return NEWLINE_JOINER.join(lines);
+ }
+
+ /**
+ * A corollary to {@link #joinLines(String[])} for collections.
+ */
+ public static String joinLines(Collection<String> lines) {
+ return NEWLINE_JOINER.join(lines);
+ }
+
+ /**
+ * combineKeys(x1, ..., xn):
+ * Computes a string that encodes the sequence
+ * x1, ..., xn. Distinct sequences map to distinct strings.
+ *
+ * The encoding is intended to be vaguely human-readable.
+ */
+ public static String combineKeys(Iterable<String> parts) {
+ final StringBuilder buf = new StringBuilder(128);
+ for (String part : parts) {
+ // We enclose each part in angle brackets to separate them. Some
+ // trickiness is required to ensure that the result is unique (distinct
+ // sequences map to distinct strings): we escape any angle bracket
+ // characters in the parts by preceding them with an escape character
+ // (we use "!") and we also need to escape any escape characters.
+ buf.append('<');
+ buf.append(KEY_ESCAPER.escape(part));
+ buf.append('>');
+ }
+ return buf.toString();
+ }
+
+ /**
+ * combineKeys(x1, ..., xn):
+ * Computes a string that encodes the sequence
+ * x1, ..., xn. Distinct sequences map to distinct strings.
+ *
+ * The encoding is intended to be vaguely human-readable.
+ */
+ public static String combineKeys(String... parts) {
+ return combineKeys(ImmutableList.copyOf(parts));
+ }
+
+ /**
+ * Replaces all occurrences of 'literal' in 'input' with 'replacement'.
+ * Like {@link String#replaceAll(String, String)} but for literal Strings
+ * instead of regular expression patterns.
+ *
+ * @param input the input String
+ * @param literal the literal String to replace in 'input'.
+ * @param replacement the replacement String to replace 'literal' in 'input'.
+ * @return the 'input' String with all occurrences of 'literal' replaced with
+ * 'replacement'.
+ */
+ public static String replaceAllLiteral(String input, String literal,
+ String replacement) {
+ int literalLength = literal.length();
+ if (literalLength == 0) {
+ return input;
+ }
+ StringBuilder result = new StringBuilder(
+ input.length() + replacement.length());
+ int start = 0;
+ int index = 0;
+
+ while ((index = input.indexOf(literal, start)) >= 0) {
+ result.append(input.substring(start, index));
+ result.append(replacement);
+ start = index + literalLength;
+ }
+ result.append(input.substring(start));
+ return result.toString();
+ }
+
+ /**
+ * Creates a simple key-value table of the form
+ *
+ * <pre>
+ * key: some value
+ * another key: some other value
+ * yet another key: and so on ...
+ * </pre>
+ *
+ * The return value will not include a final {@code "\n"}.
+ */
+ public static String layoutTable(Map<String, String> data) {
+ List<String> tableLines = new ArrayList<>();
+ for (Map.Entry<String, String> entry : data.entrySet()) {
+ tableLines.add(entry.getKey() + ": " + entry.getValue());
+ }
+ return NEWLINE_JOINER.join(tableLines);
+ }
+
+ /**
+ * Returns an easy-to-read string approximation of a number of bytes,
+ * e.g. "21MB". Note, these are IEEE units, i.e. decimal not binary powers.
+ */
+ public static String prettyPrintBytes(long bytes) {
+ if (bytes < 1E4) { // up to 10KB
+ return bytes + "B";
+ } else if (bytes < 1E7) { // up to 10MB
+ return ((int) (bytes / 1E3)) + "KB";
+ } else if (bytes < 1E11) { // up to 100GB
+ return ((int) (bytes / 1E6)) + "MB";
+ } else {
+ return ((int) (bytes / 1E9)) + "GB";
+ }
+ }
+
+ /**
+ * Returns true if 'source' contains 'target' as a sub-array.
+ */
+ public static boolean containsSubarray(char[] source, char[] target) {
+ if (target.length > source.length) {
+ return false;
+ }
+ for (int i = 0; i < source.length - target.length + 1; i++) {
+ boolean matches = true;
+ for (int j = 0; j < target.length; j++) {
+ if (source[i + j] != target[j]) {
+ matches = false;
+ break;
+ }
+ }
+ if (matches) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Replace control characters with visible strings.
+ * @return the sanitized string.
+ */
+ public static String sanitizeControlChars(String message) {
+ return CONTROL_CHAR_ESCAPER.escape(message);
+ }
+
+ /**
+ * Converts a Java style function name to a Python style function name the following way:
+ * every upper case character gets replaced with an underscore and its lower case counterpart.
+ * <p>E.g. fooBar --> foo_bar
+ */
+ public static String toPythonStyleFunctionName(String javaStyleFunctionName) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < javaStyleFunctionName.length(); i++) {
+ char c = javaStyleFunctionName.charAt(i);
+ if (Character.isUpperCase(c)) {
+ sb.append('_').append(Character.toLowerCase(c));
+ } else {
+ sb.append(c);
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ThreadUtils.java b/src/main/java/com/google/devtools/build/lib/util/ThreadUtils.java
new file mode 100644
index 0000000..7b8ebed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/ThreadUtils.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Utility methods relating to threads and stack traces.
+ */
+public class ThreadUtils {
+ private static final Logger LOG = Logger.getLogger(ThreadUtils.class.getName());
+
+ private ThreadUtils() {
+ }
+
+ /** Write a thread dump to the blaze.INFO log if interrupt took too long. */
+ public static void warnAboutSlowInterrupt() {
+ LOG.warning("Interrupt took too long. Dumping thread state.");
+ for (Map.Entry <Thread, StackTraceElement[]> e : Thread.getAllStackTraces().entrySet()) {
+ Thread t = e.getKey();
+ LOG.warning("\"" + t.getName() + "\"" + " "
+ + " Thread id=" + t.getId() + " " + t.getState());
+ for (StackTraceElement line : e.getValue()) {
+ LOG.warning("\t" + line);
+ }
+ LOG.warning("");
+ }
+ LoggingUtil.logToRemote(Level.WARNING, "Slow interrupt", new SlowInterruptException());
+ }
+
+ private static final class SlowInterruptException extends RuntimeException {
+ public SlowInterruptException() {
+ super("Slow interruption...");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/TimeUtilities.java b/src/main/java/com/google/devtools/build/lib/util/TimeUtilities.java
new file mode 100644
index 0000000..689744a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/TimeUtilities.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+/**
+ * Various utility methods operating on time values.
+ */
+public class TimeUtilities {
+
+ private TimeUtilities() {
+ }
+
+ /**
+ * Converts time to the user-friendly string representation.
+ *
+ * @param timeInNs The length of time in nanoseconds.
+ */
+ public static String prettyTime(long timeInNs) {
+ double ms = timeInNs / 1000000.0;
+ if (ms < 10.0) {
+ return String.format("%.2f ms", ms);
+ } else if (ms < 100.0) {
+ return String.format("%.1f ms", ms);
+ } else if (ms < 1000.0) {
+ return String.format("%.0f ms", ms);
+ }
+ return String.format("%.3f s", ms / 1000.0);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/UserUtils.java b/src/main/java/com/google/devtools/build/lib/util/UserUtils.java
new file mode 100644
index 0000000..93e2a66
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/UserUtils.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util;
+
+import com.google.common.base.Strings;
+
+import java.util.Map;
+
+/**
+ * User information utility methods.
+ */
+public final class UserUtils {
+
+ private static final String ORIGINATING_USER_KEY = "BLAZE_ORIGINATING_USER";
+
+ private UserUtils() {
+ // prohibit instantiation
+ }
+
+ private static class Holder {
+ static final String userName = System.getProperty("user.name");
+ }
+
+ /**
+ * Returns the user name as provided by system property 'user.name'.
+ */
+ public static String getUserName() {
+ return Holder.userName;
+ }
+
+ /**
+ * Returns the originating user for this build from the command-line or the environment.
+ */
+ public static String getOriginatingUser(String originatingUser,
+ Map<String, String> clientEnv) {
+ if (!Strings.isNullOrEmpty(originatingUser)) {
+ return originatingUser;
+ }
+
+ if (!Strings.isNullOrEmpty(clientEnv.get(ORIGINATING_USER_KEY))) {
+ return clientEnv.get(ORIGINATING_USER_KEY);
+ }
+
+ return UserUtils.getUserName();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/VarInt.java b/src/main/java/com/google/devtools/build/lib/util/VarInt.java
new file mode 100644
index 0000000..fd5daab
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/VarInt.java
@@ -0,0 +1,286 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+/**
+ * Common methods to encode and decode varints and varlongs into ByteBuffers and
+ * arrays.
+ */
+public class VarInt {
+
+ /**
+ * Maximum encoded size of 32-bit positive integers (in bytes)
+ */
+ public static final int MAX_VARINT_SIZE = 5;
+
+ /**
+ * maximum encoded size of 64-bit longs, and negative 32-bit ints (in bytes)
+ */
+ public static final int MAX_VARLONG_SIZE = 10;
+
+ private VarInt() { }
+
+ /** Returns the encoding size in bytes of its input value.
+ * @param i the integer to be measured
+ * @return the encoding size in bytes of its input value
+ */
+ public static int varIntSize(int i) {
+ int result = 0;
+ do {
+ result++;
+ i >>>= 7;
+ } while (i != 0);
+ return result;
+ }
+
+ /**
+ * Reads a varint from src, places its values into the first element of
+ * dst and returns the offset in to src of the first byte after the varint.
+ *
+ * @param src source buffer to retrieve from
+ * @param offset offset within src
+ * @param dst the resulting int value
+ * @return the updated offset after reading the varint
+ */
+ public static int getVarInt(byte[] src, int offset, int[] dst) {
+ int result = 0;
+ int shift = 0;
+ int b;
+ do {
+ if (shift >= 32) {
+ // Out of range
+ throw new IndexOutOfBoundsException("varint too long");
+ }
+ // Get 7 bits from next byte
+ b = src[offset++];
+ result |= (b & 0x7F) << shift;
+ shift += 7;
+ } while ((b & 0x80) != 0);
+ dst[0] = result;
+ return offset;
+ }
+
+ /**
+ * Encodes an integer in a variable-length encoding, 7 bits per byte, into a
+ * destination byte[], following the protocol buffer convention.
+ *
+ * @param v the int value to write to sink
+ * @param sink the sink buffer to write to
+ * @param offset the offset within sink to begin writing
+ * @return the updated offset after writing the varint
+ */
+ public static int putVarInt(int v, byte[] sink, int offset) {
+ do {
+ // Encode next 7 bits + terminator bit
+ int bits = v & 0x7F;
+ v >>>= 7;
+ byte b = (byte) (bits + ((v != 0) ? 0x80 : 0));
+ sink[offset++] = b;
+ } while (v != 0);
+ return offset;
+ }
+
+ /**
+ * Reads a varint from the current position of the given ByteBuffer and
+ * returns the decoded value as 32 bit integer.
+ *
+ * <p>The position of the buffer is advanced to the first byte after the
+ * decoded varint.
+ *
+ * @param src the ByteBuffer to get the var int from
+ * @return The integer value of the decoded varint
+ */
+ public static int getVarInt(ByteBuffer src) {
+ int tmp;
+ if ((tmp = src.get()) >= 0) {
+ return tmp;
+ }
+ int result = tmp & 0x7f;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 7;
+ } else {
+ result |= (tmp & 0x7f) << 7;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 14;
+ } else {
+ result |= (tmp & 0x7f) << 14;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 21;
+ } else {
+ result |= (tmp & 0x7f) << 21;
+ result |= (tmp = src.get()) << 28;
+ while (tmp < 0) {
+ // We get into this loop only in the case of overflow.
+ // By doing this, we can call getVarInt() instead of
+ // getVarLong() when we only need an int.
+ tmp = src.get();
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Encodes an integer in a variable-length encoding, 7 bits per byte, to a
+ * ByteBuffer sink.
+ * @param v the value to encode
+ * @param sink the ByteBuffer to add the encoded value
+ */
+ public static void putVarInt(int v, ByteBuffer sink) {
+ while (true) {
+ int bits = v & 0x7f;
+ v >>>= 7;
+ if (v == 0) {
+ sink.put((byte) bits);
+ return;
+ }
+ sink.put((byte) (bits | 0x80));
+ }
+ }
+
+ /**
+ * Reads a varint from the given InputStream and returns the decoded value
+ * as an int.
+ *
+ * @param inputStream the InputStream to read from
+ */
+ public static int getVarInt(InputStream inputStream) throws IOException {
+ int result = 0;
+ int shift = 0;
+ int b;
+ do {
+ if (shift >= 32) {
+ // Out of range
+ throw new IndexOutOfBoundsException("varint too long");
+ }
+ // Get 7 bits from next byte
+ b = inputStream.read();
+ result |= (b & 0x7F) << shift;
+ shift += 7;
+ } while ((b & 0x80) != 0);
+ return result;
+ }
+
+ /**
+ * Encodes an integer in a variable-length encoding, 7 bits per byte, and
+ * writes it to the given OutputStream.
+ *
+ * @param v the value to encode
+ * @param outputStream the OutputStream to write to
+ */
+ public static void putVarInt(int v, OutputStream outputStream) throws IOException {
+ byte[] bytes = new byte[varIntSize(v)];
+ putVarInt(v, bytes, 0);
+ outputStream.write(bytes);
+ }
+
+ /**
+ * Returns the encoding size in bytes of its input value.
+ *
+ * @param v the long to be measured
+ * @return the encoding size in bytes of a given long value.
+ */
+ public static int varLongSize(long v) {
+ int result = 0;
+ do {
+ result++;
+ v >>>= 7;
+ } while (v != 0);
+ return result;
+ }
+
+ /**
+ * Reads an up to 64 bit long varint from the current position of the
+ * given ByteBuffer and returns the decoded value as long.
+ *
+ * <p>The position of the buffer is advanced to the first byte after the
+ * decoded varint.
+ *
+ * @param src the ByteBuffer to get the var int from
+ * @return The integer value of the decoded long varint
+ */
+ public static long getVarLong(ByteBuffer src) {
+ long tmp;
+ if ((tmp = src.get()) >= 0) {
+ return tmp;
+ }
+ long result = tmp & 0x7f;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 7;
+ } else {
+ result |= (tmp & 0x7f) << 7;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 14;
+ } else {
+ result |= (tmp & 0x7f) << 14;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 21;
+ } else {
+ result |= (tmp & 0x7f) << 21;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 28;
+ } else {
+ result |= (tmp & 0x7f) << 28;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 35;
+ } else {
+ result |= (tmp & 0x7f) << 35;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 42;
+ } else {
+ result |= (tmp & 0x7f) << 42;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 49;
+ } else {
+ result |= (tmp & 0x7f) << 49;
+ if ((tmp = src.get()) >= 0) {
+ result |= tmp << 56;
+ } else {
+ result |= (tmp & 0x7f) << 56;
+ result |= ((long) src.get()) << 63;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Encodes a long integer in a variable-length encoding, 7 bits per byte, to a
+ * ByteBuffer sink.
+ * @param v the value to encode
+ * @param sink the ByteBuffer to add the encoded value
+ */
+ public static void putVarLong(long v, ByteBuffer sink) {
+ while (true) {
+ int bits = ((int) v) & 0x7f;
+ v >>>= 7;
+ if (v == 0) {
+ sink.put((byte) bits);
+ return;
+ }
+ sink.put((byte) (bits | 0x80));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminal.java b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminal.java
new file mode 100644
index 0000000..93bc12a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminal.java
@@ -0,0 +1,198 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A class which encapsulates the fancy curses-type stuff that you can do using
+ * standard ANSI terminal control sequences.
+ */
+public class AnsiTerminal {
+ private static final byte[] ESC = {27, (byte) '['};
+ private static final byte BEL = 7;
+ private static final byte UP = (byte) 'A';
+ private static final byte ERASE_LINE = (byte) 'K';
+ private static final byte SET_GRAPHICS = (byte) 'm';
+ private static final byte TEXT_BOLD = (byte) '1';
+ private static final byte[] SET_TERM_TITLE = {27, (byte) ']', (byte) '0', (byte) ';'};
+
+ public static final String FG_BLACK = "30";
+ public static final String FG_RED = "31";
+ public static final String FG_GREEN = "32";
+ public static final String FG_YELLOW = "33";
+ public static final String FG_BLUE = "34";
+ public static final String FG_MAGENTA = "35";
+ public static final String FG_CYAN = "36";
+ public static final String FG_GRAY = "37";
+ public static final String BG_BLACK = "40";
+ public static final String BG_RED = "41";
+ public static final String BG_GREEN = "42";
+ public static final String BG_YELLOW = "43";
+ public static final String BG_BLUE = "44";
+ public static final String BG_MAGENTA = "45";
+ public static final String BG_CYAN = "46";
+ public static final String BG_GRAY = "47";
+
+ public static byte[] CR = { 13 };
+
+ private final OutputStream out;
+
+ /**
+ * Creates an AnsiTerminal object wrapping an output stream which is going to
+ * be displayed in an ANSI compatible terminal or shell window.
+ *
+ * @param out the output stream
+ */
+ public AnsiTerminal(OutputStream out) {
+ this.out = out;
+ }
+
+ /**
+ * Moves the cursor upwards by a specified number of lines. This will not
+ * cause any scrolling if it tries to move above the top of the terminal
+ * window.
+ */
+ public void cursorUp(int numLines) throws IOException {
+ writeBytes(ESC, ("" + numLines).getBytes(), new byte[] { UP });
+ }
+
+ /**
+ * Clear the current terminal line from the cursor position to the end.
+ */
+ public void clearLine() throws IOException {
+ writeEscapeSequence(ERASE_LINE);
+ }
+
+ /**
+ * Makes any text output to the terminal appear in bold.
+ */
+ public void textBold() throws IOException {
+ writeEscapeSequence(TEXT_BOLD, SET_GRAPHICS);
+ }
+
+ /**
+ * Set the color of the foreground or background of the terminal.
+ *
+ * @param color one of the foreground or background color
+ * constants
+ */
+ public void setTextColor(String color) throws IOException {
+ writeBytes(ESC, color.getBytes(), new byte[] { SET_GRAPHICS });
+ }
+
+ /**
+ * Resets the terminal colors and fonts to defaults.
+ */
+ public void resetTerminal() throws IOException {
+ writeEscapeSequence((byte)'0', (byte)'m');
+ }
+
+ /**
+ * Makes text print on the terminal in red.
+ */
+ public void textRed() throws IOException {
+ setTextColor(FG_RED);
+ }
+
+ /**
+ * Makes text print on the terminal in blue.
+ */
+ public void textBlue() throws IOException {
+ setTextColor(FG_BLUE);
+ }
+
+ /**
+ * Makes text print on the terminal in red.
+ */
+ public void textGreen() throws IOException {
+ setTextColor(FG_GREEN);
+ }
+
+ /**
+ * Makes text print on the terminal in red.
+ */
+ public void textMagenta() throws IOException {
+ setTextColor(FG_MAGENTA);
+ }
+
+ /**
+ * Set the terminal title.
+ */
+ public void setTitle(String title) throws IOException {
+ writeBytes(SET_TERM_TITLE, title.getBytes(), new byte[] { BEL });
+ }
+
+ /**
+ * Writes a string to the terminal using the current font, color and cursor
+ * position settings.
+ *
+ * @param text the text to write
+ */
+ public void writeString(String text) throws IOException {
+ out.write(text.getBytes());
+ }
+
+ /**
+ * Writes a byte sequence to the terminal using the current font, color and
+ * cursor position settings.
+ *
+ * @param bytes the bytes to write
+ */
+ public void writeBytes(byte[] bytes) throws IOException {
+ out.write(bytes);
+ }
+
+ /**
+ * Utility method which makes it easier to generate the control sequences for
+ * the terminal.
+ *
+ * @param bytes bytes which should be prefixed with the terminal escape
+ * sequence to produce a valid control sequence
+ */
+ private void writeEscapeSequence(byte... bytes) throws IOException {
+ writeBytes(ESC, bytes);
+ }
+
+ /**
+ * Utility method for generating control sequences. Takes a collection of byte
+ * arrays, which contain the components of a control sequence, concatenates
+ * them, and prints them to the terminal.
+ *
+ * @param stuff the byte arrays that make up the sequence to be sent to the
+ * terminal
+ */
+ private void writeBytes(byte[]... stuff) throws IOException {
+ for (byte[] bytes : stuff) {
+ out.write(bytes);
+ }
+ }
+
+ /**
+ * Sends a carriage return to the terminal.
+ */
+ public void cr() throws IOException {
+ writeBytes(CR);
+ }
+
+ /**
+ * Flushes the underlying stream.
+ * This class does not do any buffering of its own, but the underlying
+ * OutputStream may.
+ */
+ public void flush() throws IOException {
+ out.flush();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinter.java b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinter.java
new file mode 100644
index 0000000..726c5dd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinter.java
@@ -0,0 +1,156 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.EnumSet;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+/**
+ * Allows to print "colored" strings by parsing predefined string keywords,
+ * which, depending on the useColor value are either replaced with ANSI terminal
+ * coloring sequences (as defined by the {@link AnsiTerminal} class) or stripped.
+ *
+ * Supported keywords are defined by the enum {@link AnsiTerminalPrinter.Mode}.
+ * Following keywords are supported:
+ * INFO - switches color to green.
+ * ERROR - switches color to bold red.
+ * WARNING - switches color to magenta.
+ * NORMAL - resets terminal to the default state.
+ *
+ * Each keyword is starts with prefix "{#" followed by the enum constant name
+ * and suffix "#}". Keywords should not be inserted manually - provided enum
+ * constants should be used instead.
+ */
+@ThreadCompatible
+public class AnsiTerminalPrinter {
+
+ private static final String MODE_PREFIX = "{#";
+ private static final String MODE_SUFFIX = "#}";
+
+ // Mode pattern must match MODE_PREFIX and do lookahead for the rest of the
+ // mode string.
+ private static final String MODE_PATTERN = "\\{\\#(?=[A-Z]+\\#\\})";
+
+ /**
+ * List of supported coloring modes for the {@link AnsiTerminalPrinter}.
+ */
+ public static enum Mode {
+ INFO, // green
+ ERROR, // bold red
+ WARNING, // magenta
+ DEFAULT; // default color
+
+ @Override
+ public String toString() {
+ return MODE_PREFIX + name() + MODE_SUFFIX;
+ }
+ }
+
+ private static final Logger LOG = Logger.getLogger(AnsiTerminalPrinter.class.getName());
+ private static final EnumSet<Mode> MODES = EnumSet.allOf(Mode.class);
+ private static final Pattern PATTERN = Pattern.compile(MODE_PATTERN);
+
+ private final OutputStream stream;
+ private final PrintWriter writer;
+ private final AnsiTerminal terminal;
+ private boolean useColor;
+ private Mode lastMode = Mode.DEFAULT;
+
+ /**
+ * Creates new instance using provided OutputStream and sets coloring logic
+ * for that instance.
+ */
+ public AnsiTerminalPrinter(OutputStream out, boolean useColor) {
+ this.useColor = useColor;
+ terminal = new AnsiTerminal(out);
+ writer = new PrintWriter(out, true);
+ stream = out;
+ }
+
+ /**
+ * Writes the specified string to the output stream while injecting coloring
+ * sequences when appropriate mode keyword is found and flushes.
+ *
+ * List of supported mode keywords is defined by the enum {@link Mode}.
+ *
+ * See class documentation for details.
+ */
+ public void print(String str) {
+ for (String part : PATTERN.split(str)) {
+ int index = part.indexOf(MODE_SUFFIX);
+ // Mode name will contain at least one character, so suffix index
+ // must be at least 1. If it isn't then there is no match.
+ if (index > 1) {
+ for (Mode mode : MODES) {
+ if (index == mode.name().length() && part.startsWith(mode.name())) {
+ setupTerminal(mode);
+ part = part.substring(index + MODE_SUFFIX.length());
+ break;
+ }
+ }
+ }
+ writer.print(part);
+ writer.flush();
+ }
+ }
+
+ public void printLn(String str) {
+ print(str + "\n");
+ }
+
+ /**
+ * Returns the underlying OutputStream.
+ */
+ public OutputStream getOutputStream() {
+ return stream;
+ }
+
+ /**
+ * Injects coloring escape sequences if output should be colored and mode
+ * has been changed.
+ */
+ private void setupTerminal(Mode mode) {
+ if (!useColor) {
+ return;
+ }
+ try {
+ if (lastMode != mode) {
+ terminal.resetTerminal();
+ lastMode = mode;
+ if (mode == Mode.DEFAULT) {
+ return; // Terminal is already reset - nothing else to do.
+ } else if (mode == Mode.INFO) {
+ terminal.textGreen();
+ } else if (mode == Mode.ERROR) {
+ terminal.textRed();
+ terminal.textBold();
+ } else if (mode == Mode.WARNING) {
+ terminal.textMagenta();
+ }
+ }
+ } catch (IOException e) {
+ // AnsiTerminal state is now considered to be inconsistent - coloring
+ // should be disabled to prevent future use of AnsiTerminal instance.
+ LOG.warning("Disabling coloring due to " + e.getMessage());
+ useColor = false;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/DelegatingOutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/DelegatingOutErr.java
new file mode 100644
index 0000000..83ccf2f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/DelegatingOutErr.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An {@link OutErr} specialization that supports subscribing / removing
+ * sinks, using {@link #addSink(OutErr)} and {@link #removeSink(OutErr)}.
+ * A sink is a destination to which the {@link DelegatingOutErr} will write.
+ *
+ * Also, we can hook up {@link System#out} / {@link System#err} as sources.
+ */
+public final class DelegatingOutErr extends OutErr {
+
+ /**
+ * Create a new instance to which no sinks have subscribed (basically just
+ * like a {@code /dev/null}.
+ */
+ public DelegatingOutErr() {
+ super(new DelegatingOutputStream(), new DelegatingOutputStream());
+ }
+
+
+ private final DelegatingOutputStream outSink() {
+ return (DelegatingOutputStream) getOutputStream();
+ }
+
+ private final DelegatingOutputStream errSink() {
+ return (DelegatingOutputStream) getErrorStream();
+ }
+
+ /**
+ * Add a sink, that is, after calling this method, {@code outErrSink} will
+ * receive all output / errors written to {@code this} object.
+ */
+ public void addSink(OutErr outErrSink) {
+ outSink().addSink(outErrSink.getOutputStream());
+ errSink().addSink(outErrSink.getErrorStream());
+ }
+
+ /**
+ * Remove the sink, that is, after calling this method, {@code outErrSink}
+ * will no longer receive output / errors written to {@code this} object.
+ */
+ public void removeSink(OutErr outErrSink) {
+ outSink().removeSink(outErrSink.getOutputStream());
+ errSink().removeSink(outErrSink.getErrorStream());
+ }
+
+ private static class DelegatingOutputStream extends OutputStream {
+
+ private final List<OutputStream> sinks = new ArrayList<>();
+
+ public void addSink(OutputStream sink) {
+ sinks.add(sink);
+ }
+
+ public void removeSink(OutputStream sink) {
+ sinks.remove(sink);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ for (OutputStream sink : sinks) {
+ sink.write(b);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ for (OutputStream sink : sinks) {
+ sink.close();
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ for (OutputStream sink : sinks) {
+ sink.flush();
+ }
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ for (OutputStream sink : sinks) {
+ sink.write(b, off, len);
+ }
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ for (OutputStream sink : sinks) {
+ sink.write(b);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/FileOutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/FileOutErr.java
new file mode 100644
index 0000000..4f9aecf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/FileOutErr.java
@@ -0,0 +1,404 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+
+/**
+ * An implementation of {@link OutErr} that captures all out/err output into
+ * a file for stdout and a file for stderr. The files are only created if any
+ * output is made.
+ * The OutErr assumes that the directory that will contain the output file
+ * must exist.
+ *
+ * You should not use this object from multiple different threads.
+ */
+@ThreadSafety.ThreadCompatible
+public class FileOutErr extends OutErr {
+
+ /**
+ * Create a new FileOutErr that will write its input,
+ * if any, to the files specified by stdout/stderr.
+ *
+ * No other process may write to the files,
+ *
+ * @param stdout The file for the stdout of this outErr
+ * @param stderr The file for the stderr of this outErr
+ */
+ public FileOutErr(Path stdout, Path stderr) {
+ super(new FileRecordingOutputStream(stdout), new FileRecordingOutputStream(stderr));
+ }
+
+ /**
+ * Creates a new FileOutErr that writes its input
+ * to the file specified by output. Both stdout/stderr will
+ * be copied into the single file.
+ *
+ * @param output The file for the both stdout and stderr of this outErr.
+ */
+ public FileOutErr(Path output) {
+ // We don't need to create a synchronized funnel here, like in the OutErr -- The
+ // respective functions in the FileRecordingOutputStream take care of locking.
+ this(new FileRecordingOutputStream(output));
+ }
+
+ /**
+ * Creates a new FileOutErr that discards its input. Useful
+ * for testing purposes.
+ */
+ @VisibleForTesting
+ public FileOutErr() {
+ this(new NullFileRecordingOutputStream());
+ }
+
+ private FileOutErr(OutputStream stream) {
+ // We need this function to duplicate the single new object into both arguments
+ // of the super-constructor.
+ super(stream, stream);
+ }
+
+ /**
+ * Returns true if any output was recorded.
+ */
+ public boolean hasRecordedOutput() {
+ return getFileOutputStream().hasRecordedOutput() || getFileErrorStream().hasRecordedOutput();
+ }
+
+ /**
+ * Returns true if output was recorded on stdout.
+ */
+ public boolean hasRecordedStdout() {
+ return getFileOutputStream().hasRecordedOutput();
+ }
+
+ /**
+ * Returns true if output was recorded on stderr.
+ */
+ public boolean hasRecordedStderr() {
+ return getFileErrorStream().hasRecordedOutput();
+ }
+
+ /**
+ * Returns the file this OutErr uses to buffer stdout
+ *
+ * The user must ensure that no other process is writing to the
+ * files at time of creation.
+ *
+ * @return the path object with the contents of stdout
+ */
+ public Path getOutputFile() {
+ return getFileOutputStream().getFile();
+ }
+
+ /**
+ * Returns the file this OutErr uses to buffer stderr.
+ *
+ * @return the path object with the contents of stderr
+ */
+ public Path getErrorFile() {
+ return getFileErrorStream().getFile();
+ }
+
+ /**
+ * Interprets the captured out content as an {@code ISO-8859-1} encoded
+ * string.
+ */
+ public String outAsLatin1() {
+ return getFileOutputStream().getRecordedOutput();
+ }
+
+ /**
+ * Interprets the captured err content as an {@code ISO-8859-1} encoded
+ * string.
+ */
+ public String errAsLatin1() {
+ return getFileErrorStream().getRecordedOutput();
+ }
+
+ /**
+ * Writes the captured out content to the given output stream,
+ * avoiding keeping the entire contents in memory.
+ */
+ public void dumpOutAsLatin1(OutputStream out) {
+ getFileOutputStream().dumpOut(out);
+ }
+
+ /**
+ * Writes the captured out content to the given output stream,
+ * avoiding keeping the entire contents in memory.
+ */
+ public void dumpErrAsLatin1(OutputStream out) {
+ getFileErrorStream().dumpOut(out);
+ }
+
+ private AbstractFileRecordingOutputStream getFileOutputStream() {
+ return (AbstractFileRecordingOutputStream) getOutputStream();
+ }
+
+ private AbstractFileRecordingOutputStream getFileErrorStream() {
+ return (AbstractFileRecordingOutputStream) getErrorStream();
+ }
+
+ /**
+ * An abstract supertype for the two other inner classes in this type
+ * to implement streams that can write to a file.
+ */
+ private abstract static class AbstractFileRecordingOutputStream extends OutputStream {
+
+ /**
+ * Returns true if this FileRecordingOutputStream has encountered an error.
+ *
+ * @return true there was an error, false otherwise.
+ */
+ abstract boolean hadError();
+
+ /**
+ * Returns the file this FileRecordingOutputStream is writing to.
+ */
+ abstract Path getFile();
+
+ /**
+ * Returns true if the FileOutErr has stored output.
+ */
+ abstract boolean hasRecordedOutput();
+
+ /**
+ * Returns the output this AbstractFileOutErr has recorded.
+ */
+ abstract String getRecordedOutput();
+
+ /**
+ * Writes the output to the given output stream,
+ * avoiding keeping the entire contents in memory.
+ */
+ abstract void dumpOut(OutputStream out);
+ }
+
+ /**
+ * An output stream that pretends to capture all its output into a file,
+ * but instead discards it.
+ */
+ private static class NullFileRecordingOutputStream extends AbstractFileRecordingOutputStream {
+
+ NullFileRecordingOutputStream() {
+ }
+
+ @Override
+ boolean hadError() {
+ return false;
+ }
+
+ @Override
+ Path getFile() {
+ return null;
+ }
+
+ @Override
+ boolean hasRecordedOutput() {
+ return false;
+ }
+
+ @Override
+ String getRecordedOutput() {
+ return "";
+ }
+
+ @Override
+ void dumpOut(OutputStream out) {
+ return;
+ }
+
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ }
+
+ @Override
+ public void write(int b) {
+ }
+
+ @Override
+ public void write(byte[] b) {
+ }
+ }
+
+
+ /**
+ * An output stream that captures all output into a file.
+ * The file is created only if output is received.
+ *
+ * The user must take care that nobody else is writing to the
+ * file that is backing the output stream.
+ *
+ * The write() methods of type are synchronized to ensure
+ * that writes from different threads are not mixed up.
+ *
+ * The outputStream is here only for the benefit of the pumping
+ * IO we're currently using for execution - Once that is gone,
+ * we can remove this output stream and fold its code into the
+ * FileOutErr.
+ */
+ @ThreadSafety.ThreadCompatible
+ private static class FileRecordingOutputStream extends AbstractFileRecordingOutputStream {
+
+ private final Path outputFile;
+ OutputStream outputStream;
+ String error;
+
+ FileRecordingOutputStream(Path outputFile) {
+ this.outputFile = outputFile;
+ }
+
+ @Override
+ boolean hadError() {
+ return error != null;
+ }
+
+ @Override
+ Path getFile() {
+ return outputFile;
+ }
+
+ private OutputStream getOutputStream() throws IOException {
+ // you should hold the lock before you invoke this method
+ if (outputStream == null) {
+ outputStream = outputFile.getOutputStream();
+ }
+ return outputStream;
+ }
+
+ private boolean hasOutputStream() {
+ return outputStream != null;
+ }
+
+ /**
+ * Called whenever the FileRecordingOutputStream finds an error.
+ */
+ private void recordError(IOException exception) {
+ String newErrorText = exception.getMessage();
+ error = (error == null) ? newErrorText : error + "\n" + newErrorText;
+ }
+
+ @Override
+ boolean hasRecordedOutput() {
+ if (hadError()) {
+ return true;
+ }
+ if (!outputFile.exists()) {
+ return false;
+ }
+ try {
+ return outputFile.getFileSize() > 0;
+ } catch (IOException ex) {
+ recordError(ex);
+ return true;
+ }
+ }
+
+ @Override
+ String getRecordedOutput() {
+ StringBuilder result = new StringBuilder();
+ try {
+ if (getFile().exists()) {
+ result.append(FileSystemUtils.readContentAsLatin1(getFile()));
+ }
+ } catch (IOException ex) {
+ recordError(ex);
+ }
+
+ if (hadError()) {
+ result.append(error);
+ }
+ return result.toString();
+ }
+
+ @Override
+ void dumpOut(OutputStream out) {
+ InputStream in = null;
+ try {
+ if (getFile().exists()) {
+ in = new FileInputStream(getFile().getPathString());
+ ByteStreams.copy(in, out);
+ }
+ } catch (IOException ex) {
+ recordError(ex);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ // Ignore.
+ }
+ }
+ }
+
+ if (hadError()) {
+ PrintStream ps = new PrintStream(out);
+ ps.print(error);
+ ps.flush();
+ }
+ }
+
+ @Override
+ public synchronized void write(byte[] b, int off, int len) {
+ if (len > 0) {
+ try {
+ getOutputStream().write(b, off, len);
+ } catch (IOException ex) {
+ recordError(ex);
+ }
+ }
+ }
+
+ @Override
+ public synchronized void write(int b) {
+ try {
+ getOutputStream().write(b);
+ } catch (IOException ex) {
+ recordError(ex);
+ }
+ }
+
+ @Override
+ public synchronized void write(byte[] b) throws IOException {
+ if (b.length > 0) {
+ getOutputStream().write(b);
+ }
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ if (hasOutputStream()) {
+ getOutputStream().flush();
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (hasOutputStream()) {
+ getOutputStream().close();
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/FileWatcher.java b/src/main/java/com/google/devtools/build/lib/util/io/FileWatcher.java
new file mode 100644
index 0000000..9355cc3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/FileWatcher.java
@@ -0,0 +1,111 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * The FileWatcher dumps the contents of a files into an OutErr.
+ * It then stays active and dumps any content to the OutErr that is
+ * added to the file, until it is told to stop and all output has
+ * been dumped.
+ *
+ * This is useful to emulate streaming test output.
+ */
+@ThreadSafe
+public class FileWatcher extends Thread {
+
+ // How often we check for updates in the file we watch. (in ms)
+ private static final int WATCH_INTERVAL = 100;
+
+ private final Path outputFile;
+ private final OutErr output;
+ private volatile boolean finishPumping;
+ private long toSkip = 0;
+
+ /**
+ * Creates a FileWatcher that will dump any output that is appended to
+ * outputFile onto output. If skipExisting is true, the watcher will not dump
+ * any output that is in outputFile when we construct the watcher. If
+ * skipExisting is false, already existing output will be dumped, too.
+ *
+ * @param outputFile the File to watch
+ * @param output the outErr to dump the files contents to
+ * @param skipExisting whether to dump already existing output or not.
+ */
+ public FileWatcher(Path outputFile, OutErr output, boolean skipExisting) throws IOException {
+ super("Streaming Test Output Pump");
+ this.outputFile = outputFile;
+ this.output = output;
+ finishPumping = false;
+
+ if (outputFile.exists() && skipExisting) {
+ toSkip = outputFile.getFileSize();
+ }
+ }
+
+ /**
+ * Tells the FileWatcher to stop pumping output and finish.
+ * The FileWatcher will only finish until there is no data left to display.
+ * This means that it is rarely a good idea to unconditionally wait for the
+ * FileWatcher thread to terminate -- Instead, it is better to have a timeout.
+ */
+ @ThreadSafe
+ public void stopPumping() {
+ finishPumping = true;
+ }
+
+ @Override
+ public void run() {
+
+ try {
+
+ // Wait until the file exists, or we have to abort.
+ while (!outputFile.exists() && !finishPumping) {
+ Thread.sleep(WATCH_INTERVAL);
+ }
+
+ // Check that we did not have abort before the file was created.
+ if (outputFile.exists()) {
+ try (InputStream inputStream = outputFile.getInputStream();
+ OutputStream outputStream = output.getOutputStream();) {
+ byte[] buffer = new byte[1024];
+ while (!finishPumping || (inputStream.available() != 0)) {
+ if (inputStream.available() != 0) {
+ if (toSkip > 0) {
+ toSkip -= inputStream.skip(toSkip);
+ } else {
+ int read = inputStream.read(buffer);
+ if (read > 0) {
+ outputStream.write(buffer, 0, read);
+ }
+ }
+ } else {
+ Thread.sleep(WATCH_INTERVAL);
+ }
+ }
+ }
+ }
+ } catch (IOException ex) {
+ output.printOutLn("Failure reading or writing: " + ex.getMessage());
+ } catch (InterruptedException ex) {
+ // Don't do anything.
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/LineFlushingOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/io/LineFlushingOutputStream.java
new file mode 100644
index 0000000..a5a10cf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/LineFlushingOutputStream.java
@@ -0,0 +1,92 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * This stream maintains a buffer, which it flushes upon encountering bytes
+ * that might be new line characters. This stream implements {@link #close()}
+ * as {@link #flush()}.
+ */
+abstract class LineFlushingOutputStream extends OutputStream {
+
+ static final int BUFFER_LENGTH = 8192;
+ protected static byte NEWLINE = '\n';
+
+ /**
+ * The buffer containing the characters that have not been flushed yet.
+ */
+ protected final byte[] buffer = new byte[BUFFER_LENGTH];
+
+ /**
+ * The length of the buffer that's actually used.
+ */
+ protected int len = 0;
+
+ @Override
+ public synchronized void write(byte[] b, int off, int inlen)
+ throws IOException {
+ if (len == BUFFER_LENGTH) {
+ flush();
+ }
+ int charsInLine = 0;
+ while(inlen > charsInLine) {
+ boolean sawNewline = (b[off + charsInLine] == NEWLINE);
+ charsInLine++;
+ if (sawNewline || len + charsInLine == BUFFER_LENGTH) {
+ System.arraycopy(b, off, buffer, len, charsInLine);
+ len += charsInLine;
+ off += charsInLine;
+ inlen -= charsInLine;
+ flush();
+ charsInLine = 0;
+ }
+ }
+ System.arraycopy(b, off, buffer, len, charsInLine);
+ len += charsInLine;
+ }
+
+ @Override
+ public void write(int byteAsInt) throws IOException {
+ byte b = (byte) byteAsInt; // make sure we work with bytes in comparisons
+ write(new byte[] {b}, 0, 1);
+ }
+
+ /**
+ * Close is implemented as {@link #flush()}. Client code must close the
+ * underlying output stream itself in case that's desired.
+ */
+ @Override
+ public synchronized void close() throws IOException {
+ flush();
+ }
+
+ @Override
+ public final synchronized void flush() throws IOException {
+ flushingHook(); // The point of using a hook is to make it synchronized.
+ }
+
+ /**
+ * The implementing class must define this method, which must at least flush
+ * the bytes in {@code buffer[0] - buffer[len - 1]}, and reset {@code len=0}.
+ *
+ * Don't forget to synchronized the implementation of this method on whatever
+ * underlying object it writes to!
+ */
+ protected abstract void flushingHook() throws IOException;
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStream.java
new file mode 100644
index 0000000..23d6cd7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStream.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A stream that writes to another one, emittig a prefix before every line
+ * it emits. This stream will also add a newline for every flush; so it's not
+ * useful for anything other than simple text data (e.g. log files). Here's
+ * an example which demonstrates how an explicit flush or a flush caused by
+ * a full buffer causes a newline to be added to the output.
+ *
+ * <code>
+ * foo bar
+ * baz ba[flush]ng
+ * boo
+ * </code>
+ *
+ * This results in this output being emitted:
+ *
+ * <code>
+ * my prefix: foo bar
+ * my prefix: ba
+ * my prefix: ng
+ * my prefix: boo
+ * </code>
+ */
+public final class LinePrefixingOutputStream extends LineFlushingOutputStream {
+
+ private byte[] linePrefix;
+ private final OutputStream sink;
+
+ public LinePrefixingOutputStream(String linePrefix, OutputStream sink) {
+ this.linePrefix = linePrefix.getBytes(UTF_8);
+ this.sink = sink;
+ }
+
+ @Override
+ protected void flushingHook() throws IOException {
+ synchronized (sink) {
+ if (len == 0) {
+ sink.flush();
+ return;
+ }
+ byte lastByte = buffer[len - 1];
+ boolean lineIsIncomplete = lastByte != NEWLINE;
+ sink.write(linePrefix);
+ sink.write(buffer, 0, len);
+ if (lineIsIncomplete) {
+ sink.write(NEWLINE);
+ }
+ sink.flush();
+ len = 0;
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/OutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/OutErr.java
new file mode 100644
index 0000000..c4700ea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/OutErr.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+
+/**
+ * A pair of output streams to be used for redirecting the output and error
+ * streams of a subprocess.
+ */
+public class OutErr {
+
+ private final OutputStream out;
+ private final OutputStream err;
+
+ public static final OutErr SYSTEM_OUT_ERR = create(System.out, System.err);
+
+ /**
+ * Creates a new OutErr instance from the specified output and error streams.
+ */
+ public static OutErr create(OutputStream out, OutputStream err) {
+ return new OutErr(out, err);
+ }
+
+ protected OutErr(OutputStream out, OutputStream err) {
+ this.out = out;
+ this.err = err;
+ }
+
+ /**
+ * This method redirects {@link System#out} / {@link System#err} into
+ * {@code this} object. After calling this method, writing to
+ * {@link System#out} or {@link System#err} will result in
+ * {@code "System.out: " + message} or {@code "System.err: " + message}
+ * being written to the OutputStreams of {@code this} instance.
+ *
+ * Note: This method affects global variables.
+ */
+ public void addSystemOutErrAsSource() {
+ System.setOut(new PrintStream(new LinePrefixingOutputStream("System.out: ", getOutputStream()),
+ /*autoflush=*/false));
+ System.setErr(new PrintStream(new LinePrefixingOutputStream("System.err: ", getErrorStream()),
+ /*autoflush=*/false));
+ }
+
+ /**
+ * Creates a new OutErr instance from the specified stream.
+ * Writes to either the output or err of the new OutErr are written
+ * to outputStream, synchronized.
+ */
+ public static OutErr createSynchronizedFunnel(final OutputStream outputStream) {
+ OutputStream syncOut = new OutputStream() {
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ outputStream.write(b);
+ }
+
+ @Override
+ public synchronized void write(byte b[]) throws IOException {
+ outputStream.write(b);
+ }
+
+ @Override
+ public synchronized void write(byte b[], int off, int len) throws IOException {
+ outputStream.write(b, off, len);
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ outputStream.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ outputStream.close();
+ }
+ };
+
+ return create(syncOut, syncOut);
+ }
+
+ public OutputStream getOutputStream() {
+ return out;
+ }
+
+ public OutputStream getErrorStream() {
+ return err;
+ }
+
+ /**
+ * Writes the specified string to the output stream, and flushes.
+ */
+ public void printOut(String s) {
+ PrintWriter writer = new PrintWriter(out, true);
+ writer.print(s);
+ writer.flush();
+ }
+
+ public void printOutLn(String s) {
+ printOut(s + "\n");
+ }
+
+ /**
+ * Writes the specified string to the error stream, and flushes.
+ */
+ public void printErr(String s) {
+ PrintWriter writer = new PrintWriter(err, true);
+ writer.print(s);
+ writer.flush();
+ }
+
+ public void printErrLn(String s) {
+ printErr(s + "\n");
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/RecordingOutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/RecordingOutErr.java
new file mode 100644
index 0000000..d276afc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/RecordingOutErr.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+
+/**
+ * An implementation of {@link OutErr} that captures all out / err output and
+ * makes it available as ISO-8859-1 strings. Useful for implementing test
+ * cases that assert particular output.
+ */
+public class RecordingOutErr extends OutErr {
+
+ public RecordingOutErr() {
+ super(new ByteArrayOutputStream(), new ByteArrayOutputStream());
+ }
+
+ public RecordingOutErr(ByteArrayOutputStream out, ByteArrayOutputStream err) {
+ super(out, err);
+ }
+
+ /**
+ * Reset the captured content; that is, reset the out / err buffers.
+ */
+ public void reset() {
+ getOutputStream().reset();
+ getErrorStream().reset();
+ }
+
+ /**
+ * Interprets the captured out content as an {@code ISO-8859-1} encoded
+ * string.
+ */
+ public String outAsLatin1() {
+ try {
+ return getOutputStream().toString("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * Interprets the captured err content as an {@code ISO-8859-1} encoded
+ * string.
+ */
+ public String errAsLatin1() {
+ try {
+ return getErrorStream().toString("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * Returns true if any output was recorded.
+ */
+ public boolean hasRecordedOutput() {
+ return getOutputStream().size() > 0 || getErrorStream().size() > 0;
+ }
+
+ @Override
+ public String toString() {
+ String out = outAsLatin1();
+ String err = errAsLatin1();
+ return "" + ((out.length() > 0) ? ("stdout: " + out + "\n") : "")
+ + ((err.length() > 0) ? ("stderr: " + err) : "");
+ }
+
+ @Override
+ public ByteArrayOutputStream getOutputStream() {
+ return (ByteArrayOutputStream) super.getOutputStream();
+ }
+
+ @Override
+ public ByteArrayOutputStream getErrorStream() {
+ return (ByteArrayOutputStream) super.getErrorStream();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/StreamDemultiplexer.java b/src/main/java/com/google/devtools/build/lib/util/io/StreamDemultiplexer.java
new file mode 100644
index 0000000..ffe0c19
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/StreamDemultiplexer.java
@@ -0,0 +1,186 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * The dual of {@link StreamMultiplexer}: This is an output stream into which
+ * you can dump the multiplexed stream, and it delegates the de-multiplexed
+ * content back into separate channels (instances of {@link OutputStream}).
+ *
+ * The format of the tagged output stream is as follows:
+ *
+ * <pre>
+ * combined :: = [ control_line payload ... ]+
+ * control_line :: = '@' marker '@'? '\n'
+ * payload :: = r'^[^\n]*\n'
+ * </pre>
+ *
+ * For more details, please see {@link StreamMultiplexer}.
+ */
+@ThreadCompatible
+public final class StreamDemultiplexer extends OutputStream {
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (selectedStream != null) {
+ selectedStream.flush();
+ }
+ }
+
+ private static final byte AT = '@';
+ private static final byte NEWLINE = '\n';
+
+ /**
+ * The output streams, conveniently in an array indexed by the marker byte.
+ * Some of these will be null, most likely.
+ */
+ private final OutputStream[] outputStreams =
+ new OutputStream[Byte.MAX_VALUE + 1];
+
+ /**
+ * Each state in this FSM corresponds to a position in the grammar, which is
+ * simple enough that we can just move through it from beginning to end as we
+ * parse things.
+ */
+ private enum State {
+ EXPECT_CONTROL_STARTING_AT,
+ EXPECT_MARKER_BYTE,
+ EXPECT_AT_OR_NEWLINE,
+ EXPECT_PAYLOAD_OR_NEWLINE
+ }
+
+ private State state = State.EXPECT_CONTROL_STARTING_AT;
+ private boolean addNewlineToPayload;
+ private OutputStream selectedStream;
+
+ /**
+ * Construct a new demultiplexer. The {@code smallestMarkerByte} indicates
+ * the marker byte we would expect for {@code outputStreams[0]} to be used.
+ * So, if this first stream is your stdout and you're using the
+ * {@link StreamMultiplexer}, then you will need to set this to
+ * {@code '1'}. Because {@link StreamDemultiplexer} extends
+ * {@link OutputStream}, this constructor effectively creates an
+ * {@link OutputStream} instance which demultiplexes the tagged data client
+ * code writes to it into {@code outputStreams}.
+ */
+ public StreamDemultiplexer(byte smallestMarkerByte,
+ OutputStream... outputStreams) {
+ for (int i = 0; i < outputStreams.length; i++) {
+ this.outputStreams[smallestMarkerByte + i] = outputStreams[i];
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ // This dispatch traverses the finite state machine / grammar.
+ switch (state) {
+ case EXPECT_CONTROL_STARTING_AT:
+ parseControlStartingAt((byte) b);
+ resetFields();
+ break;
+ case EXPECT_MARKER_BYTE:
+ parseMarkerByte((byte) b);
+ break;
+ case EXPECT_AT_OR_NEWLINE:
+ parseAtOrNewline((byte) b);
+ break;
+ case EXPECT_PAYLOAD_OR_NEWLINE:
+ parsePayloadOrNewline((byte) b);
+ break;
+ }
+ }
+
+ /**
+ * Handles {@link State#EXPECT_PAYLOAD_OR_NEWLINE}, which is the payload
+ * we are actually transporting over the wire. At this point we can rely
+ * on a stream having been preselected into {@link #selectedStream}, and
+ * also we will add a newline if {@link #addNewlineToPayload} is set.
+ * Flushes at the end of every payload segment.
+ */
+ private void parsePayloadOrNewline(byte b) throws IOException {
+ if (b == NEWLINE) {
+ if (addNewlineToPayload) {
+ selectedStream.write(NEWLINE);
+ }
+ selectedStream.flush();
+ state = State.EXPECT_CONTROL_STARTING_AT;
+ } else {
+ selectedStream.write(b);
+ selectedStream.flush(); // slow?
+ }
+ }
+
+ /**
+ * Handles {@link State#EXPECT_AT_OR_NEWLINE}, which is either the
+ * suppress newline indicator (at) at the end of a control line, or the end
+ * of a control line.
+ */
+ private void parseAtOrNewline(byte b) throws IOException {
+ if (b == NEWLINE) {
+ state = State.EXPECT_PAYLOAD_OR_NEWLINE;
+ } else if (b == AT) {
+ addNewlineToPayload = false;
+ } else {
+ throw new IOException("Expected @ or \\n. (" + b + ")");
+ }
+ }
+
+ /**
+ * Reset the fields that are affected by our state.
+ */
+ private void resetFields() {
+ selectedStream = null;
+ addNewlineToPayload = true;
+ }
+
+ /**
+ * Handles {@link State#EXPECT_MARKER_BYTE}. The byte determines which stream
+ * we will be using, and will set {@link #selectedStream}.
+ */
+ private void parseMarkerByte(byte markerByte) throws IOException {
+ if (markerByte < 0 || markerByte > Byte.MAX_VALUE) {
+ String msg = "Illegal marker byte (" + markerByte + ")";
+ throw new IllegalArgumentException(msg);
+ }
+ if (markerByte > outputStreams.length
+ || outputStreams[markerByte] == null) {
+ throw new IOException("stream " + markerByte + " not registered.");
+ }
+ selectedStream = outputStreams[markerByte];
+ state = State.EXPECT_AT_OR_NEWLINE;
+ }
+
+ /**
+ * Handles {@link State#EXPECT_CONTROL_STARTING_AT}, the very first '@' with
+ * which each message starts.
+ */
+ private void parseControlStartingAt(byte b) throws IOException {
+ if (b != AT) {
+ throw new IOException("Expected control starting @. (" + b + ", "
+ + (char) b + ")");
+ }
+ state = State.EXPECT_MARKER_BYTE;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/StreamMultiplexer.java b/src/main/java/com/google/devtools/build/lib/util/io/StreamMultiplexer.java
new file mode 100644
index 0000000..c214aa5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/StreamMultiplexer.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Instances of this class are multiplexers, which redirect multiple
+ * output streams into a single output stream with tagging so it can be
+ * de-multiplexed into multiple streams as needed. This allows us to
+ * use one connection for multiple streams, but more importantly it avoids
+ * multiple threads or select etc. on the receiving side: A client on the other
+ * end of a networking connection can simply read the tagged lines and then act
+ * on them within a sigle thread.
+ *
+ * The format of the tagged output stream is as follows:
+ *
+ * <pre>
+ * combined :: = [ control_line payload ... ]+
+ * control_line :: = '@' marker '@'? '\n'
+ * payload :: = r'^[^\n]*\n'
+ * </pre>
+ *
+ * So basically:
+ * <ul>
+ * <li>Control lines alternate with payload lines</li>
+ * <li>Both types of lines end with a newline, and never have a newline in
+ * them.</li>
+ * <li>The marker indicates which stream we mean.
+ * For now, '1'=stdout, '2'=stderr.</li>
+ * <li>The optional second '@' indicates that the following line is
+ * incomplete.</li>
+ * </ul>
+ *
+ * This format is optimized for easy interpretation by a Python client, but it's
+ * also a compromise in that it's still easy to interpret by a human (let's say
+ * you have to read the traffic over a wire for some reason).
+ */
+@ThreadSafe
+public final class StreamMultiplexer {
+
+ public static final byte STDOUT_MARKER = '1';
+ public static final byte STDERR_MARKER = '2';
+ public static final byte CONTROL_MARKER = '3';
+
+ private static final byte AT = '@';
+
+ private final Object mutex = new Object();
+ private final OutputStream multiplexed;
+
+ public StreamMultiplexer(OutputStream multiplexed) {
+ this.multiplexed = multiplexed;
+ }
+
+ private class MarkingStream extends LineFlushingOutputStream {
+
+ private final byte markerByte;
+
+ MarkingStream(byte markerByte) {
+ this.markerByte = markerByte;
+ }
+
+ @Override
+ protected void flushingHook() throws IOException {
+ synchronized (mutex) {
+ if (len == 0) {
+ multiplexed.flush();
+ return;
+ }
+ byte lastByte = buffer[len - 1];
+ boolean lineIsIncomplete = lastByte != NEWLINE;
+
+ multiplexed.write(AT);
+ multiplexed.write(markerByte);
+ if (lineIsIncomplete) {
+ multiplexed.write(AT);
+ }
+ multiplexed.write(NEWLINE);
+ multiplexed.write(buffer, 0, len);
+ if (lineIsIncomplete) {
+ multiplexed.write(NEWLINE);
+ }
+ multiplexed.flush();
+ }
+ len = 0;
+ }
+
+ }
+
+ /**
+ * Create a stream that will tag its contributions into the multiplexed stream
+ * with the marker '1', which means 'stdout'. Each newline byte leads
+ * to a forced automatic flush. Also, this stream never closes the underlying
+ * stream it delegates to - calling its {@code close()} method is equivalent
+ * to calling {@code flush}.
+ */
+ public OutputStream createStdout() {
+ return new MarkingStream(STDOUT_MARKER);
+ }
+
+ /**
+ * Like {@link #createStdout()}, except it tags with the marker '2' to
+ * indicate 'stderr'.
+ */
+ public OutputStream createStderr() {
+ return new MarkingStream(STDERR_MARKER);
+ }
+
+ /**
+ * Like {@link #createStdout()}, except it tags with the marker '3' to
+ * indicate control flow..
+ */
+ public OutputStream createControl() {
+ return new MarkingStream(CONTROL_MARKER);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/TimestampGranularityMonitor.java b/src/main/java/com/google/devtools/build/lib/util/io/TimestampGranularityMonitor.java
new file mode 100644
index 0000000..64575ae
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/TimestampGranularityMonitor.java
@@ -0,0 +1,194 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.Clock;
+
+/**
+ * A utility class for dealing with filesystem timestamp granularity issues.
+ *
+ * <p>
+ * Consider a sequence of commands such as
+ * <pre>
+ * echo ... > foo/bar
+ * blaze query ...
+ * echo ... > foo/bar
+ * blaze query ...
+ * </pre>
+ *
+ * If these commands all run very fast, it is possible that the timestamp
+ * on foo/bar is not changed by the second command, even though some time has
+ * passed, because the times are the same when rounded to the file system
+ * timestamp granularity (often 1 second).
+ * For performance, we assume that files
+ * timestamps haven't changed can safely be cached without reexamining their contents.
+ * But this assumption would be violated in the above scenario.
+ *
+ * <p>
+ * To address this, we record the current time at the start of executing
+ * a Blaze command, and whenever we check the timestamp of a source file
+ * or BUILD file, we check if the timestamp of that source file matches
+ * the current time. If so, we set a flag. At the end of the command,
+ * if the flag was set, then we wait until the clock has advanced, so
+ * that any file modifications performed after the command exits will
+ * result in a different file timestamp.
+ *
+ * <p>
+ * This class implicitly assumes that each filesystem's clock
+ * is the same as either System.currentTimeMillis() or
+ * System.currentTimeMillis() rounded down to the nearest second.
+ * That is not strictly correct; there might be clock skew between
+ * the cpu clock and the file system clocks (e.g. for NFS file systems),
+ * and some file systems might have different granularity (e.g. the old
+ * DOS FAT filesystem has TWO-second granularity timestamps).
+ * Clock skew can be addressed using NTP.
+ * Other granularities could be addressed by small changes to this class,
+ * if it turns out to be needed.
+ *
+ * <p>
+ * Another alternative design that we considered was to write to a file and
+ * read its timestamp. But doing that is a little tricky because we don't have
+ * a FileSystem or Path handy. Also, if we were going to do this, the stamp
+ * file that is used should be in the same file system as the input files.
+ * But the input file system(s) might not be writeable, and even if it is,
+ * it's hard for Blaze to find a writable file on the same filesystem as the
+ * input files.
+ */
+@ThreadCompatible
+public class TimestampGranularityMonitor {
+
+ /**
+ * The time of the start of the current Blaze command,
+ * in milliseconds.
+ */
+ private long commandStartTimeMillis;
+
+ /**
+ * The time of the start of the current Blaze command,
+ * in milliseconds, rounded to one second granularity.
+ */
+ private long commandStartTimeMillisRounded;
+
+ /**
+ * True iff we detected a source file or BUILD file whose (unrounded)
+ * timestamp matched the time at the start of the current Blaze command
+ * rounded to the nearest second.
+ */
+ private volatile boolean waitASecond;
+
+ /**
+ * True iff we detected a source file or BUILD file whose timestamp
+ * exactly matched the time at the start of the current Blaze command
+ * (measuring both in integral numbers of milliseconds).
+ */
+ private volatile boolean waitAMillisecond;
+
+ private final Clock clock;
+
+ public TimestampGranularityMonitor(Clock clock) {
+ this.clock = clock;
+ }
+
+ /**
+ * Record the time at which the Blaze command started.
+ * This is needed for use by waitForTimestampGranularity().
+ */
+ public void setCommandStartTime() {
+ this.commandStartTimeMillis = clock.currentTimeMillis();
+ this.commandStartTimeMillisRounded = roundDown(this.commandStartTimeMillis);
+ this.waitASecond = false;
+ this.waitAMillisecond = false;
+ }
+
+ /**
+ * Record that the output of this Blaze command depended on the contents
+ * of a build file or source file with the specified time stamp.
+ */
+ @ThreadSafe
+ public void notifyDependenceOnFileTime(long mtime) {
+ if (mtime == this.commandStartTimeMillis) {
+ this.waitAMillisecond = true;
+ }
+ if (mtime == this.commandStartTimeMillisRounded) {
+ this.waitASecond = true;
+ }
+ }
+
+ /**
+ * If needed, wait until the next "tick" of the filesystem timestamp clock.
+ * This is done to ensure that files created after the current Blaze command
+ * finishes will have timestamps different than files created before the
+ * current Blaze command started. Otherwise a sequence of commands
+ * such as
+ * <pre>
+ * echo ... > foo/BUILD
+ * blaze query ...
+ * echo ... > foo/BUILD
+ * blaze query ...
+ * </pre>
+ * could return wrong results, due to the contents of package foo
+ * being cached even though foo/BUILD changed.
+ */
+ public void waitForTimestampGranularity(OutErr outErr) {
+ if (this.waitASecond || this.waitAMillisecond) {
+ long startedWaiting = Profiler.nanoTimeMaybe();
+ boolean interrupted = false;
+
+ if (waitASecond) {
+ // 50ms slack after the whole-second boundary
+ while (clock.currentTimeMillis() < commandStartTimeMillisRounded + 1050) {
+ try {
+ Thread.sleep(50 /* milliseconds */);
+ } catch (InterruptedException e) {
+ if (!interrupted) {
+ outErr.printErrLn("INFO: Hang on a second...");
+ interrupted = true;
+ }
+ }
+ }
+ } else {
+ while (clock.currentTimeMillis() == commandStartTimeMillis) {
+ try {
+ Thread.sleep(1 /* milliseconds */);
+ } catch (InterruptedException e) {
+ if (!interrupted) {
+ outErr.printErrLn("INFO: Hang on a millisecond...");
+ interrupted = true;
+ }
+ }
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+
+ Profiler.instance().logSimpleTask(startedWaiting, ProfilerTask.WAIT,
+ "Timestamp granularity");
+ }
+ }
+
+ /**
+ * Rounds the specified time, in milliseconds, down to the nearest second,
+ * and returns the result in milliseconds.
+ */
+ private static long roundDown(long millis) {
+ return millis / 1000 * 1000;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/AbstractFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/AbstractFileSystem.java
new file mode 100644
index 0000000..dd4375c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/AbstractFileSystem.java
@@ -0,0 +1,136 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.unix.FileAccessException;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This class implements the FileSystem interface using direct calls to the
+ * UNIX filesystem.
+ */
+@ThreadSafe
+abstract class AbstractFileSystem extends FileSystem {
+
+ protected static final String ERR_PERMISSION_DENIED = " (Permission denied)";
+ protected static final Profiler profiler = Profiler.instance();
+
+ @Override
+ protected InputStream getInputStream(Path path) throws FileNotFoundException {
+ // This loop is a workaround for an apparent bug in FileInputStrean.open, which delegates
+ // ultimately to JVM_Open in the Hotspot JVM. This call is not EINTR-safe, so we must do the
+ // retry here.
+ for (;;) {
+ try {
+ return createFileInputStream(path);
+ } catch (FileNotFoundException e) {
+ if (e.getMessage().endsWith("(Interrupted system call)")) {
+ continue;
+ } else {
+ throw e;
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns either normal or profiled FileInputStream.
+ */
+ private InputStream createFileInputStream(Path path) throws FileNotFoundException {
+ final String name = path.toString();
+ if (profiler.isActive() && (profiler.isProfiling(ProfilerTask.VFS_READ) ||
+ profiler.isProfiling(ProfilerTask.VFS_OPEN))) {
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ // Replace default FileInputStream instance with the custom one that does profiling.
+ return new FileInputStream(name) {
+ @Override public int read(byte b[]) throws IOException {
+ return read(b, 0, b.length);
+ }
+ @Override public int read(byte b[], int off, int len) throws IOException {
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return super.read(b, off, len);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_READ, name);
+ }
+ }
+ };
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_OPEN, name);
+ }
+ } else {
+ // Use normal FileInputStream instance if profiler is not enabled.
+ return new FileInputStream(path.toString());
+ }
+ }
+
+ /**
+ * Returns either normal or profiled FileOutputStream. Should be used by subclasses
+ * to create default OutputStream instance.
+ */
+ protected OutputStream createFileOutputStream(Path path, boolean append)
+ throws FileNotFoundException {
+ final String name = path.toString();
+ if (profiler.isActive() && (profiler.isProfiling(ProfilerTask.VFS_WRITE) ||
+ profiler.isProfiling(ProfilerTask.VFS_OPEN))) {
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return new FileOutputStream(name, append) {
+ @Override public void write(byte b[]) throws IOException {
+ write(b, 0, b.length);
+ }
+ @Override public void write(byte b[], int off, int len) throws IOException {
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ super.write(b, off, len);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_WRITE, name);
+ }
+ }
+ };
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_OPEN, name);
+ }
+ } else {
+ return new FileOutputStream(name, append);
+ }
+ }
+
+ @Override
+ protected OutputStream getOutputStream(Path path, boolean append) throws IOException {
+ synchronized (path) {
+ try {
+ return createFileOutputStream(path, append);
+ } catch (FileNotFoundException e) {
+ // Why does it throw a *FileNotFoundException* if it can't write?
+ // That does not make any sense! And its in a completely different
+ // format than in other situations, no less!
+ if (e.getMessage().equals(path + ERR_PERMISSION_DENIED)) {
+ throw new FileAccessException(e.getMessage());
+ }
+ throw e;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/BatchStat.java b/src/main/java/com/google/devtools/build/lib/vfs/BatchStat.java
new file mode 100644
index 0000000..5144f31
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/BatchStat.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * An interface for doing a batch of stat() calls.
+ */
+public interface BatchStat {
+
+ /**
+ *
+ * @param includeDigest whether to include a file digest in the return values.
+ * @param includeLinks whether to include a symlink stat in the return values.
+ * @param paths The input paths to stat(), relative to the exec root.
+ * @return an array list of FileStatusWithDigest in the same order as the input. May
+ * contain null values.
+ * @throws IOException on unexpected failure.
+ * @throws InterruptedException on interrupt.
+ */
+ public List<FileStatusWithDigest> batchStat(boolean includeDigest,
+ boolean includeLinks,
+ Iterable<PathFragment> paths)
+ throws IOException, InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Canonicalizer.java b/src/main/java/com/google/devtools/build/lib/vfs/Canonicalizer.java
new file mode 100644
index 0000000..294a066
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/Canonicalizer.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+
+/**
+ * Static singleton holder for certain interning pools.
+ */
+public final class Canonicalizer<E> {
+
+ private static final Interner<PathFragment> FRAGMENT_INTERNER =
+ Interners.newWeakInterner();
+
+ /**
+ * Creates an instance of Canonicalizer tracking path fragments.
+ */
+ public static Interner<PathFragment> fragments() {
+ return FRAGMENT_INTERNER;
+ }
+
+ private Canonicalizer() {
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Dirent.java b/src/main/java/com/google/devtools/build/lib/vfs/Dirent.java
new file mode 100644
index 0000000..a2ee203
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/Dirent.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Preconditions;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Directory entry representation returned by {@link Path#readdir}.
+ */
+public class Dirent implements Serializable {
+
+ /** Type of the directory entry */
+ public enum Type {
+ FILE,
+ DIRECTORY,
+ SYMLINK,
+ UNKNOWN;
+ }
+
+ private final String name;
+ private final Type type;
+
+ /** Creates a new dirent with the given name and type, both of which must be non-null. */
+ public Dirent(String name, Type type) {
+ this.name = Preconditions.checkNotNull(name);
+ this.type = Preconditions.checkNotNull(type);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, type);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Dirent)) {
+ return false;
+ }
+ if (this == other) {
+ return true;
+ }
+ Dirent otherDirent = (Dirent) other;
+ return name.equals(otherDirent.name) && type.equals(otherDirent.type);
+ }
+
+ @Override
+ public String toString() {
+ return name + "[" + type.toString().toLowerCase() + "]";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/FileStatus.java
new file mode 100644
index 0000000..c57b223
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileStatus.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import java.io.IOException;
+
+/**
+ * File status: mode, mtime, size, etc.
+ *
+ * <p>The result of calling any {@code FileStatus} instance method is not
+ * guaranteed to result in I/O to the file system at the moment of the call.
+ * The I/O providing the result (and hence the throwing of an I/O exception,
+ * where applicable) may occur at any moment between the call to {@link
+ * FileSystem#stat} and the call of the {@code FileStatus} instance method.
+ *
+ * <p>Callers therefore cannot assume that all the values are populated
+ * atomically, or that the results of any two {@code FileStatus} methods
+ * correspond to state of the file system at a single moment in time. Nor may
+ * they assume that repeated successful calls to any method of the same
+ * instance will return the same value.
+ *
+ * <p>(This permits conforming implementations to use an atomic {@code stat(2)}
+ * call on file systems where it is available, and individual accessor methods
+ * on those where it is not. Caching is possible but not required.)
+ */
+public interface FileStatus {
+
+ /**
+ * Returns true iff this file is a regular or special file (e.g. socket,
+ * fifo or device).
+ */
+ boolean isFile();
+
+ /**
+ * Returns true iff this file is a directory.
+ */
+ boolean isDirectory();
+
+ /**
+ * Returns true iff this file is a symbolic link.
+ */
+ boolean isSymbolicLink();
+
+ /**
+ * Returns the total size, in bytes, of this file.
+ */
+ long getSize() throws IOException;
+
+ /**
+ * Returns the last modified time of this file's data (milliseconds since
+ * UNIX epoch).
+ */
+ long getLastModifiedTime() throws IOException;
+
+ /**
+ * Returns the last change time of this file, where change means any change
+ * to the file, including metadata changes (milliseconds since UNIX epoch).
+ *
+ * Note: UNIX uses seconds!
+ */
+ long getLastChangeTime() throws IOException;
+
+ /**
+ * Returns the unique file node id. Usually it is computed using both device
+ * and inode numbers.
+ *
+ * <p>Think of this value as a reference to the underlying inode. "mv"ing file a to file b
+ * ought to cause the node ID of b to change, but appending / modifying b should not.
+ */
+ public long getNodeId() throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigest.java b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigest.java
new file mode 100644
index 0000000..3dd62a1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigest.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import java.io.IOException;
+
+import javax.annotation.Nullable;
+
+/**
+ * A FileStatus that also optionally returns a Digest.
+ */
+public interface FileStatusWithDigest extends FileStatus {
+ /**
+ * @return the digest of the file - optional.
+ */
+ @Nullable
+ byte[] getDigest() throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigestAdapter.java b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigestAdapter.java
new file mode 100644
index 0000000..3f608ce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigestAdapter.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+
+import javax.annotation.Nullable;
+
+/**
+ * An adapter from FileStatus to FileStatusWithDigest.
+ */
+public class FileStatusWithDigestAdapter implements FileStatusWithDigest {
+ private final FileStatus stat;
+
+ public static FileStatusWithDigest adapt(FileStatus stat) {
+ return stat == null ? null : new FileStatusWithDigestAdapter(stat);
+ }
+
+ private FileStatusWithDigestAdapter(FileStatus stat) {
+ this.stat = Preconditions.checkNotNull(stat);
+ }
+
+ @Nullable
+ @Override
+ public byte[] getDigest() {
+ return null;
+ }
+
+ @Override
+ public boolean isFile() {
+ return stat.isFile();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return stat.isDirectory();
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return stat.isSymbolicLink();
+ }
+
+ @Override
+ public long getSize() throws IOException {
+ return stat.getSize();
+ }
+
+ @Override
+ public long getLastModifiedTime() throws IOException {
+ return stat.getLastModifiedTime();
+ }
+
+ @Override
+ public long getLastChangeTime() throws IOException {
+ return stat.getLastChangeTime();
+ }
+
+ @Override
+ public long getNodeId() throws IOException {
+ return stat.getNodeId();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
new file mode 100644
index 0000000..9d416098
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
@@ -0,0 +1,632 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.vfs;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.collect.Lists;
+import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
+import com.google.common.io.CharStreams;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.Dirent.Type;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * This interface models a file system using UNIX the naming scheme.
+ */
+@ThreadSafe
+public abstract class FileSystem {
+
+ /**
+ * An exception thrown when attempting to resolve an ordinary file as a symlink.
+ */
+ protected static final class NotASymlinkException extends IOException {
+ public NotASymlinkException(Path path) {
+ super(path.toString());
+ }
+ }
+
+ protected final Path rootPath;
+
+ protected FileSystem() {
+ this.rootPath = createRootPath();
+ }
+
+ /**
+ * Creates the root of all paths used by this filesystem. This is a hook
+ * allowing subclasses to define their own root path class. All other paths
+ * are created via the root path's {@link Path#createChildPath(String)} method.
+ * <p>
+ * Beware: this is called during the FileSystem constructor which may occur
+ * before subclasses are completely initialized.
+ */
+ protected Path createRootPath() {
+ return new Path(this);
+ }
+
+ /**
+ * Returns an absolute path instance, given an absolute path name, without
+ * double slashes, .., or . segments. While this method will normalize the
+ * path representation by creating a structured/parsed representation, it will
+ * not cause any IO. (e.g., it will not resolve symbolic links if it's a Unix
+ * file system.
+ */
+ public Path getPath(String pathName) {
+ return getPath(new PathFragment(pathName));
+ }
+
+ /**
+ * Returns an absolute path instance, given an absolute path name, without
+ * double slashes, .., or . segments. While this method will normalize the
+ * path representation by creating a structured/parsed representation, it will
+ * not cause any IO. (e.g., it will not resolve symbolic links if it's a Unix
+ * file system.
+ */
+ public Path getPath(PathFragment pathName) {
+ if (!pathName.isAbsolute()) {
+ throw new IllegalArgumentException(pathName.getPathString() + " (not an absolute path)");
+ }
+ return rootPath.getRelative(pathName);
+ }
+
+ /**
+ * Returns a path representing the root directory of the current file system.
+ */
+ public final Path getRootDirectory() {
+ return rootPath;
+ }
+
+ /**
+ * Returns whether or not the FileSystem supports modifications of files and
+ * file entries.
+ *
+ * <p>Returns true if FileSystem supports the following:
+ * <ul>
+ * <li>{@link #setWritable(Path, boolean)}</li>
+ * <li>{@link #setExecutable(Path, boolean)}</li>
+ * </ul>
+ *
+ * The above calls will result in an {@link UnsupportedOperationException} on
+ * a FileSystem where this method returns {@code false}.
+ */
+ public abstract boolean supportsModifications();
+
+ /**
+ * Returns whether or not the FileSystem supports symbolic links.
+ *
+ * <p>Returns true if FileSystem supports the following:
+ * <ul>
+ * <li>{@link #createSymbolicLink(Path, PathFragment)}</li>
+ * <li>{@link #getFileSize(Path, boolean)} where {@code followSymlinks=false}</li>
+ * <li>{@link #getLastModifiedTime(Path, boolean)} where {@code followSymlinks=false}</li>
+ * <li>{@link #readSymbolicLink(Path)} where the link points to a non-existent file</li>
+ * </ul>
+ *
+ * The above calls will result in an {@link UnsupportedOperationException} on
+ * a FileSystem where this method returns {@code false}.
+ */
+ public abstract boolean supportsSymbolicLinks();
+
+ /**
+ * Returns the type of the file system path belongs to.
+ *
+ * <p>The string returned is obtained directly from the operating system, so
+ * it's a best guess in absence of a guaranteed api.
+ *
+ * <p>This implementation uses <code>/proc/mounts</code> to determine the
+ * file system type.
+ */
+ public String getFileSystemType(Path path) {
+ String fileSystem = "unknown";
+ int bestMountPointSegmentCount = -1;
+ try {
+ Path canonicalPath = path.resolveSymbolicLinks();
+ Path mountTable = path.getRelative("/proc/mounts");
+ for (String line : CharStreams.readLines(new InputStreamReader(mountTable.getInputStream(),
+ ISO_8859_1))) {
+ String[] words = line.split("\\s+");
+ if (words.length >= 3) {
+ if (!words[1].startsWith("/")) {
+ continue;
+ }
+ Path mountPoint = path.getFileSystem().getPath(words[1]);
+ int segmentCount = mountPoint.asFragment().segmentCount();
+ if (canonicalPath.startsWith(mountPoint) && segmentCount > bestMountPointSegmentCount) {
+ bestMountPointSegmentCount = segmentCount;
+ fileSystem = words[2];
+ }
+ }
+ }
+ } catch (IOException e) {
+ // pass
+ }
+ return fileSystem;
+ }
+
+
+ /**
+ * Creates a directory with the name of the current path. See
+ * {@link Path#createDirectory} for specification.
+ */
+ protected abstract boolean createDirectory(Path path) throws IOException;
+
+ /**
+ * Returns the size in bytes of the file denoted by {@code path}. See
+ * {@link Path#getFileSize(Symlinks)} for specification.
+ *
+ * <p>Note: for <@link FileSystem>s where {@link #supportsSymbolicLinks()}
+ * returns false, this method will throw an
+ * {@link UnsupportedOperationException} if {@code followSymLinks=false}.
+ */
+ protected abstract long getFileSize(Path path, boolean followSymlinks) throws IOException;
+
+ /**
+ * Deletes the file denoted by {@code path}. See {@link Path#delete} for
+ * specification.
+ */
+ protected abstract boolean delete(Path path) throws IOException;
+
+ /**
+ * Returns the last modification time of the file denoted by {@code path}.
+ * See {@link Path#getLastModifiedTime(Symlinks)} for specification.
+ *
+ * Note: for {@link FileSystem}s where {@link #supportsSymbolicLinks()} returns
+ * false, this method will throw an {@link UnsupportedOperationException} if
+ * {@code followSymLinks=false}.
+ */
+ protected abstract long getLastModifiedTime(Path path,
+ boolean followSymlinks)
+ throws IOException;
+
+ /**
+ * Sets the last modification time of the file denoted by {@code path}. See
+ * {@link Path#setLastModifiedTime} for specification.
+ */
+ protected abstract void setLastModifiedTime(Path path, long newTime) throws IOException;
+
+ /**
+ * Returns value of the given extended attribute name or null if attribute
+ * does not exist or file system does not support extended attributes.
+ * <p>Default implementation assumes that file system does not support
+ * extended attributes and always returns null. Specific file system
+ * implementations should override this method if they do provide support
+ * for extended attributes.
+ *
+ * @param path the file whose extended attribute is to be returned.
+ * @param name the name of the extended attribute key.
+ * @return the value of the extended attribute associated with 'path', if
+ * any, or null if no such attribute is defined (ENODATA) or file
+ * system does not support extended attributes at all.
+ * @throws IOException if the call failed for any other reason.
+ */
+ protected byte[] getxattr(Path path, String name, boolean followSymlinks) throws IOException {
+ return null;
+ }
+
+ /**
+ * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null}
+ * if the filesystem doesn't support them.
+ */
+ protected String getFastDigestFunctionType(Path path) {
+ return null;
+ }
+
+ /**
+ * Gets a fast digest for the given path, or {@code null} if there isn't one available or the
+ * filesystem doesn't support them. This digest should be suitable for detecting changes to the
+ * file.
+ */
+ protected byte[] getFastDigest(Path path) throws IOException {
+ return null;
+ }
+
+ /**
+ * Returns the MD5 digest of the file denoted by {@code path}. See
+ * {@link Path#getMD5Digest} for specification.
+ */
+ protected byte[] getMD5Digest(final Path path) throws IOException {
+ // Naive I/O implementation. Subclasses may (and do) optimize.
+ // This code is only used by the InMemory or Zip or other weird FSs.
+ return new ByteSource() {
+ @Override
+ public InputStream openStream() throws IOException {
+ return getInputStream(path);
+ }
+ }.hash(Hashing.md5()).asBytes();
+ }
+
+ /**
+ * Returns true if "path" denotes an existing symbolic link. See
+ * {@link Path#isSymbolicLink} for specification.
+ */
+ protected abstract boolean isSymbolicLink(Path path);
+
+ /**
+ * Appends a single regular path segment 'child' to 'dir', recursively
+ * resolving symbolic links in 'child'. 'dir' must be canonical. 'maxLinks' is
+ * the maximum number of symbolic links that may be traversed before it gives
+ * up (the Linux kernel uses 32).
+ *
+ * <p>(This method does not need to be synchronized; but the result may be
+ * stale in the case of concurrent modification.)
+ *
+ * @throws IOException if 'dir' is not an existing directory; or if
+ * stat(child) fails for any reason, or if 'child' is a symlink and
+ * readlink(child) fails for any reason (e.g. ENOENT, EACCES), or if
+ * the chain of symbolic links exceeds 'maxLinks'.
+ */
+ private Path appendSegment(Path dir, String child, int maxLinks) throws IOException {
+ Path naive = dir.getChild(child);
+
+ PathFragment linkTarget = resolveOneLink(naive);
+ if (linkTarget == null) {
+ return naive; // regular file or directory
+ }
+
+ if (maxLinks-- == 0) {
+ throw new IOException(naive + " (Too many levels of symbolic links)");
+ }
+ if (linkTarget.isAbsolute()) { dir = rootPath; }
+ for (String name : linkTarget.segments()) {
+ if (name.equals(".") || name.equals("")) {
+ // no-op
+ } else if (name.equals("..")) {
+ Path parent = dir.getParentDirectory();
+ // root's parent is root, when canonicalizing, so this is a no-op.
+ if (parent != null) { dir = parent; }
+ } else {
+ dir = appendSegment(dir, name, maxLinks);
+ }
+ }
+ return dir;
+ }
+
+ /**
+ * Helper method of {@link #resolveSymbolicLinks(Path)}. This method
+ * encapsulates the I/O component of a full canonicalization operation.
+ * Subclasses can (and do) provide more efficient implementations.
+ *
+ * <p>(This method does not need to be synchronized; but the result may be
+ * stale in the case of concurrent modification.)
+ *
+ * @param path a path, of which all but the last segment is guaranteed to be
+ * canonical
+ * @return {@link #readSymbolicLink} iff path is a symlink or null iff
+ * path exists but is not a symlink
+ * @throws IOException if the file did not exist, or a parent directory could
+ * not be searched
+ */
+ protected PathFragment resolveOneLink(Path path) throws IOException {
+ try {
+ return readSymbolicLink(path);
+ } catch (NotASymlinkException e) {
+ // Not a symbolic link. Check it exists.
+
+ // (A simple call to lstat would replace all of this.)
+ if (!exists(path, false)) {
+ throw new FileNotFoundException(path + " (No such file or directory)");
+ }
+
+ // TODO(bazel-team): (2009) ideally, throw ENOTDIR if dir is not a dir, but that
+ // would require twice as many stats, or a much more convoluted
+ // implementation (like glibc's canonicalize.c).
+
+ return null; // exists.
+ }
+ }
+
+ /**
+ * Returns the canonical path for the given path. See
+ * {@link Path#resolveSymbolicLinks} for specification.
+ */
+ protected final Path resolveSymbolicLinks(Path path)
+ throws IOException {
+ Path parentNode = path.getParentDirectory();
+ return parentNode == null
+ ? path // (root)
+ : appendSegment(resolveSymbolicLinks(parentNode), path.getBaseName(), 32);
+ }
+
+ /**
+ * Returns the status of a file. See {@link Path#stat(Symlinks)} for
+ * specification.
+ *
+ * <p>The default implementation of this method is a "lazy" one, based on
+ * other accessor methods such as {@link #isFile}, etc. Subclasses may provide
+ * more efficient specializations. However, we still try to follow Unix-like
+ * semantics of failing fast in case of non-existent files (or in case of
+ * permission issues).
+ */
+ protected FileStatus stat(final Path path, final boolean followSymlinks) throws IOException {
+ FileStatus status = new FileStatus() {
+ volatile Boolean isFile;
+ volatile Boolean isDirectory;
+ volatile Boolean isSymbolicLink;
+ volatile long size = -1;
+ volatile long mtime = -1;
+
+ @Override
+ public boolean isFile() {
+ if (isFile == null) { isFile = FileSystem.this.isFile(path, followSymlinks); }
+ return isFile;
+ }
+
+ @Override
+ public boolean isDirectory() {
+ if (isDirectory == null) {
+ isDirectory = FileSystem.this.isDirectory(path, followSymlinks);
+ }
+ return isDirectory;
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ if (isSymbolicLink == null) { isSymbolicLink = FileSystem.this.isSymbolicLink(path); }
+ return isSymbolicLink;
+ }
+
+ @Override
+ public long getSize() throws IOException {
+ if (size == -1) { size = getFileSize(path, followSymlinks); }
+ return size;
+ }
+
+ @Override
+ public long getLastModifiedTime() throws IOException {
+ if (mtime == -1) { mtime = FileSystem.this.getLastModifiedTime(path, followSymlinks); }
+ return mtime;
+ }
+
+ @Override
+ public long getLastChangeTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getNodeId() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ // Fail fast in case if some operations will actually fail, since stat() call sometimes used
+ // to verify file existence as well. We will use getLastModifiedTime() method for that purpose.
+ status.getLastModifiedTime();
+
+ return status;
+ }
+
+ /**
+ * Like stat(), but returns null on failures instead of throwing.
+ */
+ protected FileStatus statNullable(Path path, boolean followSymlinks) {
+ try {
+ return stat(path, followSymlinks);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Like {@link #stat}, but returns null if the file is not found (corresponding to
+ * {@code ENOENT} or {@code ENOTDIR} in Unix's stat(2) function) instead of throwing. Note that
+ * this implementation does <i>not</i> successfully catch {@code ENOTDIR} exceptions. If the
+ * instantiated filesystem can catch such errors, it should override this method to do so.
+ */
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
+ try {
+ return stat(path, followSymlinks);
+ } catch (FileNotFoundException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Returns true iff {@code path} denotes an existing directory. See
+ * {@link Path#isDirectory(Symlinks)} for specification.
+ */
+ protected abstract boolean isDirectory(Path path, boolean followSymlinks);
+
+ /**
+ * Returns true iff {@code path} denotes an existing regular or special file.
+ * See {@link Path#isFile(Symlinks)} for specification.
+ */
+ protected abstract boolean isFile(Path path, boolean followSymlinks);
+
+ /**
+ * Creates a symbolic link. See {@link Path#createSymbolicLink(Path)} for
+ * specification.
+ *
+ * <p>Note: for {@link FileSystem}s where {@link #supportsSymbolicLinks()}
+ * returns false, this method will throw an
+ * {@link UnsupportedOperationException}
+ */
+ protected abstract void createSymbolicLink(Path linkPath, PathFragment targetFragment)
+ throws IOException;
+
+ /**
+ * Returns the target of a symbolic link. See {@link Path#readSymbolicLink}
+ * for specification.
+ *
+ * <p>Note: for {@link FileSystem}s where {@link #supportsSymbolicLinks()}
+ * returns false, this method will throw an
+ * {@link UnsupportedOperationException} if the link points to a non-existent
+ * file.
+ *
+ * @throws NotASymlinkException if the current path is not a symbolic link
+ * @throws IOException if the contents of the link could not be read for any reason.
+ */
+ protected abstract PathFragment readSymbolicLink(Path path) throws IOException;
+
+ /**
+ * Returns true iff {@code path} denotes an existing file of any kind. See
+ * {@link Path#exists(Symlinks)} for specification.
+ */
+ protected abstract boolean exists(Path path, boolean followSymlinks);
+
+ /**
+ * Returns a collection containing the names of all entities within the
+ * directory denoted by the {@code path}.
+ *
+ * @throws IOException if there was an error reading the directory entries
+ */
+ protected abstract Collection<Path> getDirectoryEntries(Path path) throws IOException;
+
+ /**
+ * Returns a Dirents structure, listing the names of all entries within the
+ * directory {@code path}, plus their types (file, directory, other).
+ *
+ * @param followSymlinks whether to follow symlinks when determining the file types of
+ * individual directory entries. No matter the value of this parameter, symlinks are
+ * followed when resolving the directory whose entries are to be read.
+ * @throws IOException if there was an error reading the directory entries
+ */
+ protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
+ Collection<Path> children = getDirectoryEntries(path);
+ List<Dirent> dirents = Lists.newArrayListWithCapacity(children.size());
+ for (Path child : children) {
+ FileStatus stat = statNullable(child, followSymlinks);
+ Dirent.Type type;
+ if (stat == null) {
+ type = Type.UNKNOWN;
+ } else if (stat.isFile()) {
+ type = Type.FILE;
+ } else if (stat.isDirectory()) {
+ type = Type.DIRECTORY;
+ } else if (stat.isSymbolicLink()) {
+ type = Type.SYMLINK;
+ } else {
+ type = Type.UNKNOWN;
+ }
+ dirents.add(new Dirent(child.getBaseName(), type));
+ }
+ return dirents;
+ }
+
+ /**
+ * Returns true iff the file represented by {@code path} is readable.
+ *
+ * @throws IOException if there was an error reading the file's metadata
+ */
+ protected abstract boolean isReadable(Path path) throws IOException;
+
+ /**
+ * Sets the file to readable (if the argument is true) or non-readable (if the
+ * argument is false)
+ *
+ * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()}
+ * returns false or which do not support unreadable files, this method will
+ * throw an {@link UnsupportedOperationException}.
+ *
+ * @throws IOException if there was an error reading or writing the file's metadata
+ */
+ protected abstract void setReadable(Path path, boolean readable)
+ throws IOException;
+
+ /**
+ * Returns true iff the file represented by {@code path} is writable.
+ *
+ * @throws IOException if there was an error reading the file's metadata
+ */
+ protected abstract boolean isWritable(Path path) throws IOException;
+
+ /**
+ * Sets the file to writable (if the argument is true) or non-writable (if the
+ * argument is false)
+ *
+ * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()}
+ * returns false, this method will throw an
+ * {@link UnsupportedOperationException}.
+ *
+ * @throws IOException if there was an error reading or writing the file's metadata
+ */
+ protected abstract void setWritable(Path path, boolean writable)
+ throws IOException;
+
+ /**
+ * Returns true iff the file represented by the path is executable.
+ *
+ * @throws IOException if there was an error reading the file's metadata
+ */
+ protected abstract boolean isExecutable(Path path) throws IOException;
+
+ /**
+ * Sets the file to executable, if the argument is true. It is currently not
+ * supported to unset the executable status of a file, so {code
+ * executable=false} yields an {@link UnsupportedOperationException}.
+ *
+ * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()}
+ * returns false, this method will throw an
+ * {@link UnsupportedOperationException}.
+ *
+ * @throws IOException if there was an error reading or writing the file's metadata
+ */
+ protected abstract void setExecutable(Path path, boolean executable) throws IOException;
+
+ /**
+ * Sets the file permissions. If permission changes on this {@link FileSystem}
+ * are slow (e.g. one syscall per change), this method should aim to be faster
+ * than setting each permission individually. If this {@link FileSystem} does
+ * not support group or others permissions, those bits will be ignored.
+ *
+ * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()}
+ * returns false, this method will throw an
+ * {@link UnsupportedOperationException}.
+ *
+ * @throws IOException if there was an error reading or writing the file's metadata
+ */
+ protected void chmod(Path path, int mode) throws IOException {
+ setReadable(path, (mode & 0400) != 0);
+ setWritable(path, (mode & 0200) != 0);
+ setExecutable(path, (mode & 0100) != 0);
+ }
+
+ /**
+ * Creates an InputStream accessing the file denoted by the path.
+ *
+ * @throws IOException if there was an error opening the file for reading
+ */
+ protected abstract InputStream getInputStream(Path path) throws IOException;
+
+ /**
+ * Creates an OutputStream accessing the file denoted by path.
+ *
+ * @throws IOException if there was an error opening the file for writing
+ */
+ protected final OutputStream getOutputStream(Path path) throws IOException {
+ return getOutputStream(path, false);
+ }
+
+ /**
+ * Creates an OutputStream accessing the file denoted by path.
+ *
+ * @param append whether to open the output stream in append mode
+ * @throws IOException if there was an error opening the file for writing
+ */
+ protected abstract OutputStream getOutputStream(Path path, boolean append) throws IOException;
+
+ /**
+ * Renames the file denoted by "sourceNode" to the location "targetNode".
+ * See {@link Path#renameTo} for specification.
+ */
+ protected abstract void renameTo(Path sourcePath, Path targetPath) throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java
new file mode 100644
index 0000000..bc55032
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java
@@ -0,0 +1,988 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteSink;
+import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Helper functions that implement often-used complex operations on file
+ * systems.
+ */
+@ConditionallyThreadSafe // ThreadSafe except for deleteTree.
+public class FileSystemUtils {
+
+ static final Logger LOG = Logger.getLogger(FileSystemUtils.class.getName());
+ static final boolean LOG_FINER = LOG.isLoggable(Level.FINER);
+
+ private FileSystemUtils() {}
+
+ /****************************************************************************
+ * Path and PathFragment functions.
+ */
+
+ /**
+ * Throws exceptions if {@code baseName} is not a valid base name. A valid
+ * base name:
+ * <ul>
+ * <li>Is not null
+ * <li>Is not an empty string
+ * <li>Is not "." or ".."
+ * <li>Does not contain a slash
+ * </ul>
+ */
+ @ThreadSafe
+ public static void checkBaseName(String baseName) {
+ if (baseName.length() == 0) {
+ throw new IllegalArgumentException("Child must not be empty string ('')");
+ }
+ if (baseName.equals(".") || baseName.equals("..")) {
+ throw new IllegalArgumentException("baseName must not be '" + baseName + "'");
+ }
+ if (baseName.indexOf('/') != -1) {
+ throw new IllegalArgumentException("baseName must not contain a slash: '" + baseName + "'");
+ }
+ }
+
+ /**
+ * Returns the common ancestor between two paths, or null if none (including
+ * if they are on different filesystems).
+ */
+ public static Path commonAncestor(Path a, Path b) {
+ while (a != null && !b.startsWith(a)) {
+ a = a.getParentDirectory(); // returns null at root
+ }
+ return a;
+ }
+
+ /**
+ * Returns the longest common ancestor of the two path fragments, or either "/" or "" (depending
+ * on whether {@code a} is absolute or relative) if there is none.
+ */
+ public static PathFragment commonAncestor(PathFragment a, PathFragment b) {
+ while (a != null && !b.startsWith(a)) {
+ a = a.getParentDirectory();
+ }
+
+ return a;
+ }
+ /**
+ * Returns a path fragment from a given from-dir to a given to-path. May be
+ * either a short relative path "foo/bar", an up'n'over relative path
+ * "../../foo/bar" or an absolute path.
+ */
+ public static PathFragment relativePath(Path fromDir, Path to) {
+ if (to.getFileSystem() != fromDir.getFileSystem()) {
+ throw new IllegalArgumentException("fromDir and to must be on the same FileSystem");
+ }
+
+ return relativePath(fromDir.asFragment(), to.asFragment());
+ }
+
+ /**
+ * Returns a path fragment from a given from-dir to a given to-path.
+ */
+ public static PathFragment relativePath(PathFragment fromDir, PathFragment to) {
+ if (to.equals(fromDir)) {
+ return new PathFragment("."); // same dir, just return '.'
+ }
+ if (to.startsWith(fromDir)) {
+ return to.relativeTo(fromDir); // easy case--it's a descendant
+ }
+ PathFragment ancestor = commonAncestor(fromDir, to);
+ if (ancestor == null) {
+ return to; // no common ancestor, use 'to'
+ }
+ int levels = fromDir.relativeTo(ancestor).segmentCount();
+ StringBuilder dotdots = new StringBuilder();
+ for (int i = 0; i < levels; i++) {
+ dotdots.append("../");
+ }
+ return new PathFragment(dotdots.toString()).getRelative(to.relativeTo(ancestor));
+ }
+
+ /**
+ * Returns the longest prefix from a given set of 'prefixes' that are
+ * contained in 'path'. I.e the closest ancestor directory containing path.
+ * Returns null if none found.
+ */
+ public static PathFragment longestPathPrefix(PathFragment path, Set<PathFragment> prefixes) {
+ for (int i = path.segmentCount(); i >= 1; i--) {
+ PathFragment prefix = path.subFragment(0, i);
+ if (prefixes.contains(prefix)) {
+ return prefix;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Removes the shortest suffix beginning with '.' from the basename of the
+ * filename string. If the basename contains no '.', the filename is returned
+ * unchanged.
+ *
+ * e.g. "foo/bar.x" -> "foo/bar"
+ */
+ @ThreadSafe
+ public static String removeExtension(String filename) {
+ int lastDotIndex = filename.lastIndexOf('.');
+ if (lastDotIndex == -1) { return filename; }
+ int lastSlashIndex = filename.lastIndexOf('/');
+ if (lastSlashIndex > lastDotIndex) {
+ return filename;
+ }
+ return filename.substring(0, lastDotIndex);
+ }
+
+ /**
+ * Removes the shortest suffix beginning with '.' from the basename of the
+ * PathFragment. If the basename contains no '.', the filename is returned
+ * unchanged.
+ *
+ * <p>e.g. "foo/bar.x" -> "foo/bar"
+ */
+ @ThreadSafe
+ public static PathFragment removeExtension(PathFragment path) {
+ return path.replaceName(removeExtension(path.getBaseName()));
+ }
+
+ /**
+ * Removes the shortest suffix beginning with '.' from the basename of the
+ * Path. If the basename contains no '.', the filename is returned
+ * unchanged.
+ *
+ * <p>e.g. "foo/bar.x" -> "foo/bar"
+ */
+ @ThreadSafe
+ public static Path removeExtension(Path path) {
+ return path.getFileSystem().getPath(removeExtension(path.asFragment()));
+ }
+
+ /**
+ * Returns a new {@code PathFragment} formed by replacing the extension of the
+ * last path segment of {@code path} with {@code newExtension}. Null is
+ * returned iff {@code path} has zero segments.
+ */
+ public static PathFragment replaceExtension(PathFragment path, String newExtension) {
+ return path.replaceName(removeExtension(path.getBaseName()) + newExtension);
+ }
+
+ /**
+ * Returns a new {@code PathFragment} formed by replacing the extension of the
+ * last path segment of {@code path} with {@code newExtension}. Null is
+ * returned iff {@code path} has zero segments or it doesn't end with {@code oldExtension}.
+ */
+ public static PathFragment replaceExtension(PathFragment path, String newExtension,
+ String oldExtension) {
+ String base = path.getBaseName();
+ if (!base.endsWith(oldExtension)) {
+ return null;
+ }
+ String newBase = base.substring(0, base.length() - oldExtension.length()) + newExtension;
+ return path.replaceName(newBase);
+ }
+
+ /**
+ * Returns a new {@code Path} formed by replacing the extension of the
+ * last path segment of {@code path} with {@code newExtension}. Null is
+ * returned iff {@code path} has zero segments.
+ */
+ public static Path replaceExtension(Path path, String newExtension) {
+ PathFragment fragment = replaceExtension(path.asFragment(), newExtension);
+ return fragment == null ? null : path.getFileSystem().getPath(fragment);
+ }
+
+ /**
+ * Returns a new {@code PathFragment} formed by adding the extension to the last path segment of
+ * {@code path}. Null is returned if {@code path} has zero segments.
+ */
+ public static PathFragment appendExtension(PathFragment path, String newExtension) {
+ return path.replaceName(path.getBaseName() + newExtension);
+ }
+
+ /**
+ * Returns a new {@code PathFragment} formed by replacing the first, or all if
+ * {@code replaceAll} is true, {@code oldSegment} of {@code path} with {@code
+ * newSegment}.
+ */
+ public static PathFragment replaceSegments(PathFragment path,
+ String oldSegment, String newSegment, boolean replaceAll) {
+ int count = path.segmentCount();
+ for (int i = 0; i < count; i++) {
+ if (path.getSegment(i).equals(oldSegment)) {
+ path = new PathFragment(path.subFragment(0, i),
+ new PathFragment(newSegment),
+ path.subFragment(i+1, count));
+ if (!replaceAll) {
+ return path;
+ }
+ }
+ }
+ return path;
+ }
+
+ /**
+ * Returns a new {@code PathFragment} formed by appending the given string to the last path
+ * segment of {@code path} without removing the extension. Returns null if {@code path}
+ * has zero segments.
+ */
+ public static PathFragment appendWithoutExtension(PathFragment path, String toAppend) {
+ return path.replaceName(appendWithoutExtension(path.getBaseName(), toAppend));
+ }
+
+ /**
+ * Given a string that represents a file with an extension separated by a '.' and a string
+ * to append, return a string in which {@code toAppend} has been appended to {@code name}
+ * before the last '.' character. If {@code name} does not include a '.', appends {@code
+ * toAppend} at the end.
+ *
+ * <p>For example,
+ * ("libfoo.jar", "-src") ==> "libfoo-src.jar"
+ * ("libfoo", "-src") ==> "libfoo-src"
+ */
+ private static String appendWithoutExtension(String name, String toAppend) {
+ int dotIndex = name.lastIndexOf(".");
+ if (dotIndex > 0) {
+ String baseName = name.substring(0, dotIndex);
+ String extension = name.substring(dotIndex);
+ return baseName + toAppend + extension;
+ } else {
+ return name + toAppend;
+ }
+ }
+
+ /****************************************************************************
+ * FileSystem property functions.
+ */
+
+ /**
+ * Return the current working directory as expressed by the System property
+ * 'user.dir'.
+ */
+ public static Path getWorkingDirectory(FileSystem fs) {
+ return fs.getPath(getWorkingDirectory());
+ }
+
+ /**
+ * Returns the current working directory as expressed by the System property
+ * 'user.dir'. This version does not require a {@link FileSystem}.
+ */
+ public static PathFragment getWorkingDirectory() {
+ return new PathFragment(System.getProperty("user.dir", "/"));
+ }
+
+ /****************************************************************************
+ * Path FileSystem mutating operations.
+ */
+
+ /**
+ * "Touches" the file or directory specified by the path, following symbolic
+ * links. If it does not exist, it is created as an empty file; otherwise, the
+ * time of last access is updated to the current time.
+ *
+ * @throws IOException if there was an error while touching the file
+ */
+ @ThreadSafe
+ public static void touchFile(Path path) throws IOException {
+ if (path.exists()) {
+ // -1L means "use the current time", and is ultimately implemented by
+ // utime(path, null), thereby using the kernel's clock, not the JVM's.
+ // (A previous implementation based on the JVM clock was found to be
+ // skewy.)
+ path.setLastModifiedTime(-1L);
+ } else {
+ createEmptyFile(path);
+ }
+ }
+
+ /**
+ * Creates an empty regular file with the name of the current path, following
+ * symbolic links.
+ *
+ * @throws IOException if the file could not be created for any reason
+ * (including that there was already a file at that location)
+ */
+ public static void createEmptyFile(Path path) throws IOException {
+ path.getOutputStream().close();
+ }
+
+ /**
+ * Creates or updates a symbolic link from 'link' to 'target'. Replaces
+ * existing symbolic links with target, and skips the link creation if it is
+ * already present. Will also create any missing ancestor directories of the
+ * link. This method is non-atomic
+ *
+ * <p>Note: this method will throw an IOException if there is an unequal
+ * non-symlink at link.
+ *
+ * @throws IOException if the creation of the symbolic link was unsuccessful
+ * for any reason.
+ */
+ @ThreadSafe // but not atomic
+ public static void ensureSymbolicLink(Path link, Path target) throws IOException {
+ ensureSymbolicLink(link, target.asFragment());
+ }
+
+ /**
+ * Creates or updates a symbolic link from 'link' to 'target'. Replaces
+ * existing symbolic links with target, and skips the link creation if it is
+ * already present. Will also create any missing ancestor directories of the
+ * link. This method is non-atomic
+ *
+ * <p>Note: this method will throw an IOException if there is an unequal
+ * non-symlink at link.
+ *
+ * @throws IOException if the creation of the symbolic link was unsuccessful
+ * for any reason.
+ */
+ @ThreadSafe // but not atomic
+ public static void ensureSymbolicLink(Path link, String target) throws IOException {
+ ensureSymbolicLink(link, new PathFragment(target));
+ }
+
+ /**
+ * Creates or updates a symbolic link from 'link' to 'target'. Replaces
+ * existing symbolic links with target, and skips the link creation if it is
+ * already present. Will also create any missing ancestor directories of the
+ * link. This method is non-atomic
+ *
+ * <p>Note: this method will throw an IOException if there is an unequal
+ * non-symlink at link.
+ *
+ * @throws IOException if the creation of the symbolic link was unsuccessful
+ * for any reason.
+ */
+ @ThreadSafe // but not atomic
+ public static void ensureSymbolicLink(Path link, PathFragment target) throws IOException {
+ // TODO(bazel-team): (2009) consider adding the logic for recovering from the case when
+ // we have already created a parent directory symlink earlier.
+ try {
+ if (link.readSymbolicLink().equals(target)) {
+ return; // Do nothing if the link is already there.
+ }
+ } catch (IOException e) { // link missing or broken
+ /* fallthru and do the work below */
+ }
+ if (link.isSymbolicLink()) {
+ link.delete(); // Remove the symlink since it is pointing somewhere else.
+ } else {
+ createDirectoryAndParents(link.getParentDirectory());
+ }
+ try {
+ link.createSymbolicLink(target);
+ } catch (IOException e) {
+ // Only pass on exceptions caused by a true link creation failure.
+ if (!link.isSymbolicLink() ||
+ !link.resolveSymbolicLinks().equals(link.getRelative(target))) {
+ throw e;
+ }
+ }
+ }
+
+ private static ByteSource asByteSource(final Path path) {
+ return new ByteSource() {
+ @Override public InputStream openStream() throws IOException {
+ return path.getInputStream();
+ }
+ };
+ }
+
+ private static ByteSink asByteSink(final Path path, final boolean append) {
+ return new ByteSink() {
+ @Override public OutputStream openStream() throws IOException {
+ return path.getOutputStream(append);
+ }
+ };
+ }
+
+ private static ByteSink asByteSink(final Path path) {
+ return asByteSink(path, false);
+ }
+
+ /**
+ * Copies the file from location "from" to location "to", while overwriting a
+ * potentially existing "to". File's last modified time, executable and
+ * writable bits are also preserved.
+ *
+ * <p>If no error occurs, the method returns normally. If a parent directory does
+ * not exist, a FileNotFoundException is thrown. An IOException is thrown when
+ * other erroneous situations occur. (e.g. read errors)
+ */
+ @ThreadSafe // but not atomic
+ public static void copyFile(Path from, Path to) throws IOException {
+ try {
+ to.delete();
+ } catch (IOException e) {
+ throw new IOException("error copying file: "
+ + "couldn't delete destination: " + e.getMessage());
+ }
+ asByteSource(from).copyTo(asByteSink(to));
+ to.setLastModifiedTime(from.getLastModifiedTime()); // Preserve mtime.
+ if (!from.isWritable()) {
+ to.setWritable(false); // Make file read-only if original was read-only.
+ }
+ to.setExecutable(from.isExecutable()); // Copy executable bit.
+ }
+
+ /**
+ * Copies a tool binary from one path to another, returning the target path.
+ * The directory of the target path must already exist. The target copy's time
+ * is set to match, as well as its read-only and executable flags. The
+ * operation is skipped if the target file has the same time and size as the
+ * source.
+ */
+ public static Path copyTool(Path source, Path target) throws IOException {
+ FileStatus sourceStat = null;
+ FileStatus targetStat = target.statNullable();
+ if (targetStat != null) {
+ // stat the source file only if we'll need the stat.
+ sourceStat = source.stat(Symlinks.FOLLOW);
+ }
+ if (targetStat == null ||
+ targetStat.getLastModifiedTime() != sourceStat.getLastModifiedTime() ||
+ targetStat.getSize() != sourceStat.getSize()) {
+ copyFile(source, target);
+ target.setWritable(source.isWritable());
+ target.setExecutable(source.isExecutable());
+ target.setLastModifiedTime(source.getLastModifiedTime());
+ }
+ return target;
+ }
+
+ /****************************************************************************
+ * Directory tree operations.
+ */
+
+ /**
+ * Returns a new collection containing all of the paths below a given root
+ * path, for which the given predicate is true. Symbolic links are not
+ * followed, and may appear in the result.
+ *
+ * @throws IOException If the root does not denote a directory
+ */
+ @ThreadSafe
+ public static Collection<Path> traverseTree(Path root, Predicate<? super Path> predicate)
+ throws IOException {
+ List<Path> paths = new ArrayList<>();
+ traverseTree(paths, root, predicate);
+ return paths;
+ }
+
+ /**
+ * Populates an existing Path List, adding all of the paths below a given root
+ * path for which the given predicate is true. Symbolic links are not
+ * followed, and may appear in the result.
+ *
+ * @throws IOException If the root does not denote a directory
+ */
+ @ThreadSafe
+ public static void traverseTree(Collection<Path> paths, Path root,
+ Predicate<? super Path> predicate) throws IOException {
+ for (Path p : root.getDirectoryEntries()) {
+ if (predicate.apply(p)) {
+ paths.add(p);
+ }
+ if (p.isDirectory(Symlinks.NOFOLLOW)) {
+ traverseTree(paths, p, predicate);
+ }
+ }
+ }
+
+ /**
+ * Deletes 'p', and everything recursively beneath it if it's a directory.
+ * Does not follow any symbolic links.
+ *
+ * @throws IOException if any file could not be removed.
+ */
+ @ThreadSafe
+ public static void deleteTree(Path p) throws IOException {
+ deleteTreesBelow(p);
+ p.delete();
+ }
+
+ /**
+ * Deletes all dir trees recursively beneath 'dir' if it's a directory,
+ * nothing otherwise. Does not follow any symbolic links.
+ *
+ * @throws IOException if any file could not be removed.
+ */
+ @ThreadSafe
+ public static void deleteTreesBelow(Path dir) throws IOException {
+ if (dir.isDirectory(Symlinks.NOFOLLOW)) { // real directories (not symlinks)
+ dir.setReadable(true);
+ dir.setWritable(true);
+ dir.setExecutable(true);
+ for (Path child : dir.getDirectoryEntries()) {
+ deleteTree(child);
+ }
+ }
+ }
+
+ /**
+ * Delete all dir trees under a given 'dir' that don't start with one of a set
+ * of given 'prefixes'. Does not follow any symbolic links.
+ */
+ @ThreadSafe
+ public static void deleteTreesBelowNotPrefixed(Path dir, String[] prefixes) throws IOException {
+ dirloop:
+ for (Path p : dir.getDirectoryEntries()) {
+ String name = p.getBaseName();
+ for (int i = 0; i < prefixes.length; i++) {
+ if (name.startsWith(prefixes[i])) {
+ continue dirloop;
+ }
+ }
+ deleteTree(p);
+ }
+ }
+
+ /**
+ * Copies all dir trees under a given 'from' dir to location 'to', while overwriting
+ * all files in the potentially existing 'to'. Does not follow any symbolic links,
+ * but copies them instead.
+ *
+ * <p>The source and the destination must be non-overlapping, otherwise an
+ * IllegalArgumentException will be thrown. This method cannot be used to copy
+ * a dir tree to a sub tree of itself.
+ *
+ * <p>If no error occurs, the method returns normally. If the given 'from' does
+ * not exist, a FileNotFoundException is thrown. An IOException is thrown when
+ * other erroneous situations occur. (e.g. read errors)
+ */
+ @ThreadSafe
+ public static void copyTreesBelow(Path from , Path to) throws IOException {
+ if (to.startsWith(from)) {
+ throw new IllegalArgumentException(to + " is a subdirectory of " + from);
+ }
+
+ Collection<Path> entries = from.getDirectoryEntries();
+ for (Path entry : entries) {
+ if (entry.isDirectory(Symlinks.NOFOLLOW)) {
+ Path subDir = to.getChild(entry.getBaseName());
+ subDir.createDirectory();
+ copyTreesBelow(entry, subDir);
+ } else if (entry.isSymbolicLink()) {
+ Path newLink = to.getChild(entry.getBaseName());
+ newLink.createSymbolicLink(entry.readSymbolicLink());
+ } else {
+ Path newEntry = to.getChild(entry.getBaseName());
+ copyFile(entry, newEntry);
+ }
+ }
+ }
+
+ /**
+ * Attempts to create a directory with the name of the given path, creating
+ * ancestors as necessary.
+ *
+ * <p>Postcondition: completes normally iff {@code dir} denotes an existing
+ * directory (not necessarily canonical); completes abruptly otherwise.
+ *
+ * @return true if the directory was successfully created anew, false if it
+ * already existed (including the case where {@code dir} denotes a symlink
+ * to an existing directory)
+ * @throws IOException if the directory could not be created
+ */
+ @ThreadSafe
+ public static boolean createDirectoryAndParents(Path dir) throws IOException {
+ // Optimised for minimal number of I/O calls.
+
+ // Don't attempt to create the root directory.
+ if (dir.getParentDirectory() == null) { return false; }
+
+ FileSystem filesystem = dir.getFileSystem();
+ if (filesystem instanceof UnionFileSystem) {
+ // If using UnionFS, make sure that we do not traverse filesystem boundaries when creating
+ // parent directories by rehoming the path on the most specific filesystem.
+ FileSystem delegate = ((UnionFileSystem) filesystem).getDelegate(dir);
+ dir = delegate.getPath(dir.asFragment());
+ }
+
+ try {
+ return dir.createDirectory();
+ } catch (IOException e) {
+ if (e.getMessage().endsWith(" (No such file or directory)")) { // ENOENT
+ createDirectoryAndParents(dir.getParentDirectory());
+ return dir.createDirectory();
+ } else if (e.getMessage().endsWith(" (File exists)") && dir.isDirectory()) { // EEXIST
+ return false;
+ } else {
+ throw e; // some other error (e.g. ENOTDIR, EACCES, etc.)
+ }
+ }
+ }
+
+ /**
+ * Attempts to remove a relative chain of directories under a given base.
+ * Returns {@code true} if the removal was successful, and returns {@code
+ * false} if the removal fails because a directory was not empty. An
+ * {@link IOException} is thrown for any other errors.
+ */
+ @ThreadSafe
+ public static boolean removeDirectoryAndParents(Path base, PathFragment toRemove) {
+ if (toRemove.isAbsolute()) {
+ return false;
+ }
+ try {
+ for (; toRemove.segmentCount() > 0; toRemove = toRemove.getParentDirectory()) {
+ Path p = base.getRelative(toRemove);
+ if (p.exists()) {
+ p.delete();
+ }
+ }
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Takes a map of directory fragments to root paths, and creates a symlink
+ * forest under an existing linkRoot to the corresponding source dirs or
+ * files. Symlink are made at the highest dir possible, linking files directly
+ * only when needed with nested packages.
+ */
+ public static void plantLinkForest(ImmutableMap<PathFragment, Path> packageRootMap, Path linkRoot)
+ throws IOException {
+ // Create a sorted map of all dirs (packages and their ancestors) to sets of their roots.
+ // Packages come from exactly one root, but their shared ancestors may come from more.
+ // The map is maintained sorted lexicographically, so parents are before their children.
+ Map<PathFragment, Set<Path>> dirRootsMap = Maps.newTreeMap();
+ for (Map.Entry<PathFragment, Path> entry : packageRootMap.entrySet()) {
+ PathFragment pkgDir = entry.getKey();
+ Path pkgRoot = entry.getValue();
+ for (int i = 1; i <= pkgDir.segmentCount(); i++) {
+ PathFragment dir = pkgDir.subFragment(0, i);
+ Set<Path> roots = dirRootsMap.get(dir);
+ if (roots == null) {
+ roots = Sets.newHashSet();
+ dirRootsMap.put(dir, roots);
+ }
+ roots.add(pkgRoot);
+ }
+ }
+ // Now add in roots for all non-pkg dirs that are in between two packages, and missed above.
+ for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) {
+ PathFragment dir = entry.getKey();
+ if (!packageRootMap.containsKey(dir)) {
+ PathFragment pkgDir = longestPathPrefix(dir, packageRootMap.keySet());
+ if (pkgDir != null) {
+ entry.getValue().add(packageRootMap.get(pkgDir));
+ }
+ }
+ }
+ // Create output dirs for all dirs that have more than one root and need to be split.
+ for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) {
+ PathFragment dir = entry.getKey();
+ if (entry.getValue().size() > 1) {
+ if (LOG_FINER) {
+ LOG.finer("mkdir " + linkRoot.getRelative(dir));
+ }
+ createDirectoryAndParents(linkRoot.getRelative(dir));
+ }
+ }
+ // Make dir links for single rooted dirs.
+ for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) {
+ PathFragment dir = entry.getKey();
+ Set<Path> roots = entry.getValue();
+ // Simple case of one root for this dir.
+ if (roots.size() == 1) {
+ if (dir.segmentCount() > 1 && dirRootsMap.get(dir.getParentDirectory()).size() == 1) {
+ continue; // skip--an ancestor will link this one in from above
+ }
+ // This is the top-most dir that can be linked to a single root. Make it so.
+ Path root = roots.iterator().next(); // lone root in set
+ if (LOG_FINER) {
+ LOG.finer("ln -s " + root.getRelative(dir) + " " + linkRoot.getRelative(dir));
+ }
+ linkRoot.getRelative(dir).createSymbolicLink(root.getRelative(dir));
+ }
+ }
+ // Make links for dirs within packages, skip parent-only dirs.
+ for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) {
+ PathFragment dir = entry.getKey();
+ if (entry.getValue().size() > 1) {
+ // If this dir is at or below a package dir, link in its contents.
+ PathFragment pkgDir = longestPathPrefix(dir, packageRootMap.keySet());
+ if (pkgDir != null) {
+ Path root = packageRootMap.get(pkgDir);
+ try {
+ Path absdir = root.getRelative(dir);
+ if (absdir.isDirectory()) {
+ if (LOG_FINER) {
+ LOG.finer("ln -s " + absdir + "/* " + linkRoot.getRelative(dir) + "/");
+ }
+ for (Path target : absdir.getDirectoryEntries()) {
+ PathFragment p = target.relativeTo(root);
+ if (!dirRootsMap.containsKey(p)) {
+ //LOG.finest("ln -s " + target + " " + linkRoot.getRelative(p));
+ linkRoot.getRelative(p).createSymbolicLink(target);
+ }
+ }
+ } else {
+ LOG.fine("Symlink planting skipping dir '" + absdir + "'");
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ // Otherwise its just an otherwise empty common parent dir.
+ }
+ }
+ }
+ }
+
+ /****************************************************************************
+ * Whole-file I/O utilities for characters and bytes. These convenience
+ * methods are not efficient and should not be used for large amounts of data!
+ */
+
+ private static char[] convertFromLatin1(byte[] content) {
+ char[] latin1 = new char[content.length];
+ for (int i = 0; i < latin1.length; i++) { // yeah, latin1 is this easy! :-)
+ latin1[i] = (char) (0xff & content[i]);
+ }
+ return latin1;
+ }
+
+ /**
+ * Writes lines to file using ISO-8859-1 encoding (isolatin1).
+ */
+ @ThreadSafe // but not atomic
+ public static void writeIsoLatin1(Path file, String... lines) throws IOException {
+ writeLinesAs(file, ISO_8859_1, lines);
+ }
+
+ /**
+ * Append lines to file using ISO-8859-1 encoding (isolatin1).
+ */
+ @ThreadSafe // but not atomic
+ public static void appendIsoLatin1(Path file, String... lines) throws IOException {
+ appendLinesAs(file, ISO_8859_1, lines);
+ }
+
+ /**
+ * Writes the specified String as ISO-8859-1 (latin1) encoded bytes to the
+ * file. Follows symbolic links.
+ *
+ * @throws IOException if there was an error
+ */
+ public static void writeContentAsLatin1(Path outputFile, String content) throws IOException {
+ writeContent(outputFile, ISO_8859_1, content);
+ }
+
+ /**
+ * Writes the specified String using the specified encoding to the file.
+ * Follows symbolic links.
+ *
+ * @throws IOException if there was an error
+ */
+ public static void writeContent(Path outputFile, Charset charset, String content)
+ throws IOException {
+ asByteSink(outputFile).asCharSink(charset).write(content);
+ }
+
+ /**
+ * Writes lines to file using the given encoding, ending every line with a
+ * line break '\n' character.
+ */
+ @ThreadSafe // but not atomic
+ public static void writeLinesAs(Path file, Charset charset, String... lines)
+ throws IOException {
+ writeLinesAs(file, charset, Arrays.asList(lines));
+ }
+
+ /**
+ * Appends lines to file using the given encoding, ending every line with a
+ * line break '\n' character.
+ */
+ @ThreadSafe // but not atomic
+ public static void appendLinesAs(Path file, Charset charset, String... lines)
+ throws IOException {
+ appendLinesAs(file, charset, Arrays.asList(lines));
+ }
+
+ /**
+ * Writes lines to file using the given encoding, ending every line with a
+ * line break '\n' character.
+ */
+ @ThreadSafe // but not atomic
+ public static void writeLinesAs(Path file, Charset charset, Iterable<String> lines)
+ throws IOException {
+ createDirectoryAndParents(file.getParentDirectory());
+ asByteSink(file).asCharSink(charset).writeLines(lines);
+ }
+
+ /**
+ * Appends lines to file using the given encoding, ending every line with a
+ * line break '\n' character.
+ */
+ @ThreadSafe // but not atomic
+ public static void appendLinesAs(Path file, Charset charset, Iterable<String> lines)
+ throws IOException {
+ createDirectoryAndParents(file.getParentDirectory());
+ asByteSink(file, true).asCharSink(charset).writeLines(lines);
+ }
+
+ /**
+ * Writes the specified byte array to the output file. Follows symbolic links.
+ *
+ * @throws IOException if there was an error
+ */
+ public static void writeContent(Path outputFile, byte[] content) throws IOException {
+ asByteSink(outputFile).write(content);
+ }
+
+ /**
+ * Returns the entirety of the specified input stream and returns it as a char
+ * array, decoding characters using ISO-8859-1 (Latin1).
+ *
+ * @throws IOException if there was an error
+ */
+ public static char[] readContentAsLatin1(InputStream in) throws IOException {
+ return convertFromLatin1(ByteStreams.toByteArray(in));
+ }
+
+ /**
+ * Returns the entirety of the specified file and returns it as a char array,
+ * decoding characters using ISO-8859-1 (Latin1).
+ *
+ * @throws IOException if there was an error
+ */
+ public static char[] readContentAsLatin1(Path inputFile) throws IOException {
+ return convertFromLatin1(readContent(inputFile));
+ }
+
+ /**
+ * Returns an iterable that allows iterating over ISO-8859-1 (Latin1) text
+ * file contents line by line. If the file ends in a line break, the iterator
+ * will return an empty string as the last element.
+ *
+ * @throws IOException if there was an error
+ */
+ public static Iterable<String> iterateLinesAsLatin1(Path inputFile) throws IOException {
+ return asByteSource(inputFile).asCharSource(ISO_8859_1).readLines();
+ }
+
+ /**
+ * Returns the entirety of the specified file and returns it as a byte array.
+ *
+ * @throws IOException if there was an error
+ */
+ public static byte[] readContent(Path inputFile) throws IOException {
+ return asByteSource(inputFile).read();
+ }
+
+ /**
+ * Reads at most {@code limit} bytes from {@code inputFile} and returns it as a byte array.
+ *
+ * @throws IOException if there was an error.
+ */
+ public static byte[] readContentWithLimit(Path inputFile, int limit) throws IOException {
+ Preconditions.checkArgument(limit >= 0, "limit needs to be >=0, but it is %s", limit);
+ ByteSource byteSource = asByteSource(inputFile);
+ byte[] buffer = new byte[limit];
+ try (InputStream inputStream = byteSource.openBufferedStream()) {
+ int read = ByteStreams.read(inputStream, buffer, 0, limit);
+ return Arrays.copyOf(buffer, read);
+ }
+ }
+
+ /**
+ * Dumps diagnostic information about the specified filesystem to {@code out}.
+ * This is the implementation of the filesystem part of the 'blaze dump'
+ * command. It lives here, rather than in DumpCommand, because it requires
+ * privileged access to members of this package.
+ *
+ * <p>Its results are unspecified and MUST NOT be interpreted programmatically.
+ */
+ public static void dump(FileSystem fs, final PrintStream out) {
+ if (!(fs instanceof UnixFileSystem)) {
+ out.println(" Not a UnixFileSystem.");
+ return;
+ }
+
+ // Unfortunately there's no "letrec" for anonymous functions so we have to
+ // (a) name the function, (b) put it in a box and (c) use List not array
+ // because of the generic type. *sigh*.
+ final List<Predicate<Path>> dumpFunction = new ArrayList<>();
+ dumpFunction.add(new Predicate<Path>() {
+ @Override
+ public boolean apply(Path child) {
+ Path path = child;
+ out.println(" " + path + " (" + path.toDebugString() + ")");
+ path.applyToChildren(dumpFunction.get(0));
+ return false;
+ }
+ });
+
+ fs.getRootDirectory().applyToChildren(dumpFunction.get(0));
+ }
+
+ /**
+ * Returns the type of the file system path belongs to.
+ */
+ public static String getFileSystem(Path path) {
+ return path.getFileSystem().getFileSystemType(path);
+ }
+
+ /**
+ * Returns whether the given path starts with any of the paths in the given
+ * list of prefixes.
+ */
+ public static boolean startsWithAny(Path path, Iterable<Path> prefixes) {
+ for (Path prefix : prefixes) {
+ if (path.startsWith(prefix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether the given path starts with any of the paths in the given
+ * list of prefixes.
+ */
+ public static boolean startsWithAny(PathFragment path, Iterable<PathFragment> prefixes) {
+ for (PathFragment prefix : prefixes) {
+ if (path.startsWith(prefix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystems.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystems.java
new file mode 100644
index 0000000..1e7aaae
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystems.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.vfs;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+/**
+ * This static file system singleton manages access to a single default
+ * {@link FileSystem} instance created within the methods of this class.
+ */
+@ThreadSafe
+@Deprecated // Instantiate and inject FileSystem instances directly, or use
+ // com.google.devtools.build.lib.vfs.util.FileSystems in tests.
+public final class FileSystems {
+
+ private FileSystems() {}
+
+ private static FileSystem defaultFileSystem;
+
+ /**
+ * Initializes the default {@link FileSystem} instance as a platform native
+ * (Unix) file system, creating one iff needed, and returns the instance.
+ *
+ * <p>This method is idempotent as long as the initialization is of the same
+ * type (Native/JavaIo/Union).
+ */
+ public static synchronized FileSystem initDefaultAsNative() {
+ if (!(defaultFileSystem instanceof UnixFileSystem)) {
+ defaultFileSystem = new UnixFileSystem();
+ }
+ return defaultFileSystem;
+ }
+
+ /**
+ * Initializes the default {@link FileSystem} instance as a java.io.File
+ * file system, creating one iff needed, and returns the instance.
+ *
+ * <p>This method is idempotent as long as the initialization is of the same
+ * type (Native/JavaIo/Union).
+ */
+ public static synchronized FileSystem initDefaultAsJavaIo() {
+ if (!(defaultFileSystem instanceof JavaIoFileSystem)) {
+ defaultFileSystem = new JavaIoFileSystem();
+ }
+ return defaultFileSystem;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/IORuntimeException.java b/src/main/java/com/google/devtools/build/lib/vfs/IORuntimeException.java
new file mode 100644
index 0000000..2769fe9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/IORuntimeException.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// All Rights Reserved.
+
+package com.google.devtools.build.lib.vfs;
+
+import java.io.IOException;
+
+/**
+ * Signals that an I/O exception of some sort has occurred. Contrary to
+ * <code>java.io.IOException</code>, this class is a subclass of
+ * <code>RuntimeException</code>, which allows you to signal an I/O problem
+ * without polluting the callers. For details on why checked exceptions is bad,
+ * try searching for "java checked exception mistake" on Google.
+ */
+public class IORuntimeException extends RuntimeException {
+ /**
+ * Constructs a new IORuntimeException with null as its detail message.
+ */
+ public IORuntimeException() {
+ super();
+ }
+
+ /**
+ * Constructs a new IORuntimeException with the specified detail message.
+ */
+ public IORuntimeException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs a new IORuntimeException with the specified detail message and
+ * cause.
+ *
+ * @param message the detail message, which is saved for later retrieval by
+ * the <code>Throwable.getMessage()</code> method.
+ * @param cause the cause (which is saved for later retrieval by the
+ * <code>Throwable.getCause()</code> method). (A null value is
+ * permitted, and indicates that the cause is nonexistent or unknown.)
+ */
+ public IORuntimeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a new IORuntimeException as a wrapper on a root cause
+ */
+ public IORuntimeException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * @return the actual IOException that caused this exception, or null if it
+ * was not caused by an IOException. Call <code>getCause()</code>
+ * instead if it was caused by other types of exceptions.
+ */
+ public IOException getCauseIOException() {
+ Throwable cause = getCause();
+ if (cause instanceof IOException) {
+ return (IOException) cause;
+ } else {
+ return null;
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/JavaIoFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/JavaIoFileSystem.java
new file mode 100644
index 0000000..08e67f7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/JavaIoFileSystem.java
@@ -0,0 +1,486 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.unix.FileAccessException;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * A FileSystem that does not use any JNI and hence, does not require a shared library be present at
+ * execution.
+ *
+ * <p>Note: Blaze profiler tasks are defined on the system call level - thus we do not distinguish
+ * (from profiling perspective) between different methods on this class that end up doing stat()
+ * system call - they all are associated with the VFS_STAT task.
+ */
+@ThreadSafe
+public class JavaIoFileSystem extends AbstractFileSystem {
+ private static final LinkOption[] NO_LINK_OPTION = new LinkOption[0];
+ // This isn't generally safe; we rely on the file system APIs not modifying the array.
+ private static final LinkOption[] NOFOLLOW_LINKS_OPTION =
+ new LinkOption[] { LinkOption.NOFOLLOW_LINKS };
+
+ protected static final String ERR_IS_DIRECTORY = " (Is a directory)";
+ protected static final String ERR_DIRECTORY_NOT_EMPTY = " (Directory not empty)";
+ protected static final String ERR_FILE_EXISTS = " (File exists)";
+ protected static final String ERR_NO_SUCH_FILE_OR_DIR = " (No such file or directory)";
+ protected static final String ERR_NOT_A_DIRECTORY = " (Not a directory)";
+
+ protected File getIoFile(Path path) {
+ return new File(path.toString());
+ }
+
+ private LinkOption[] linkOpts(boolean followSymlinks) {
+ return followSymlinks ? NO_LINK_OPTION : NOFOLLOW_LINKS_OPTION;
+ }
+
+ @Override
+ protected Collection<Path> getDirectoryEntries(Path path) throws IOException {
+ File file = getIoFile(path);
+ String[] entries = null;
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ entries = file.list();
+ if (entries == null) {
+ if (file.exists()) {
+ throw new IOException(path + ERR_NOT_A_DIRECTORY);
+ } else {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ }
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, file.getPath());
+ }
+ Collection<Path> result = new ArrayList<>(entries.length);
+ for (String entry : entries) {
+ if (!entry.equals(".") && !entry.equals("..")) {
+ result.add(path.getChild(entry));
+ }
+ }
+ return result;
+ }
+
+ @Override
+ protected boolean exists(Path path, boolean followSymlinks) {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return Files.exists(file.toPath(), linkOpts(followSymlinks));
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path.toString());
+ }
+ }
+
+ @Override
+ protected boolean isDirectory(Path path, boolean followSymlinks) {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ if (!followSymlinks && fileIsSymbolicLink(file)) {
+ return false;
+ }
+ return file.isDirectory();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path.toString());
+ }
+ }
+
+ @Override
+ protected boolean isFile(Path path, boolean followSymlinks) {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ if (!followSymlinks && fileIsSymbolicLink(file)) {
+ return false;
+ }
+ return file.isFile();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path.toString());
+ }
+ }
+
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ if (!file.exists()) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ return file.canRead();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath());
+ }
+ }
+
+ @Override
+ protected boolean isWritable(Path path) throws IOException {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ if (!file.exists()) {
+ if (linkExists(file)) {
+ throw new IOException(path + ERR_PERMISSION_DENIED);
+ } else {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ }
+ return file.canWrite();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath());
+ }
+ }
+
+ @Override
+ protected boolean isExecutable(Path path) throws IOException {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ if (!file.exists()) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ return file.canExecute();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath());
+ }
+ }
+
+ @Override
+ protected void setReadable(Path path, boolean readable) throws IOException {
+ File file = getIoFile(path);
+ if (!file.exists()) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ file.setReadable(readable);
+ }
+
+ @Override
+ protected void setWritable(Path path, boolean writable) throws IOException {
+ File file = getIoFile(path);
+ if (!file.exists()) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ file.setWritable(writable);
+ }
+
+ @Override
+ protected void setExecutable(Path path, boolean executable) throws IOException {
+ File file = getIoFile(path);
+ if (!file.exists()) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ file.setExecutable(executable);
+ }
+
+ @Override
+ public boolean supportsModifications() {
+ return true;
+ }
+
+ @Override
+ public boolean supportsSymbolicLinks() {
+ return true;
+ }
+
+ @Override
+ protected boolean createDirectory(Path path) throws IOException {
+
+ // We always synchronize on the current path before doing it on the parent path and file system
+ // path structure ensures that this locking order will never be reversed.
+ // When refactoring, check that subclasses still work as expected and there can be no
+ // deadlocks.
+ synchronized (path) {
+ File file = getIoFile(path);
+ if (file.mkdir()) {
+ return true;
+ }
+
+ // We will be checking the state of the parent path as well. Synchronize on it before
+ // attempting anything.
+ Path parentDirectory = path.getParentDirectory();
+ synchronized (parentDirectory) {
+ if (fileIsSymbolicLink(file)) {
+ throw new IOException(path + ERR_FILE_EXISTS);
+ }
+ if (file.isDirectory()) {
+ return false; // directory already existed
+ } else if (file.exists()) {
+ throw new IOException(path + ERR_FILE_EXISTS);
+ } else if (!file.getParentFile().exists()) {
+ throw new FileNotFoundException(path.getParentDirectory() + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ // Parent directory apparently exists - try to create our directory again - protecting
+ // against the case where parent directory would be created right before us obtaining
+ // synchronization lock.
+ if (file.mkdir()) {
+ return true; // Everything is fine finally.
+ } else if (!file.getParentFile().canWrite()) {
+ throw new FileAccessException(path + ERR_PERMISSION_DENIED);
+ } else {
+ // Parent exists, is writable, yet we can't create our directory.
+ throw new FileNotFoundException(path.getParentDirectory() + ERR_NOT_A_DIRECTORY);
+ }
+ }
+ }
+ }
+
+ private boolean linkExists(File file) {
+ String shortName = file.getName();
+ File parentFile = file.getParentFile();
+ if (parentFile == null) {
+ return false;
+ }
+ String[] filenames = parentFile.list();
+ if (filenames == null) {
+ return false;
+ }
+ for (String name : filenames) {
+ if (name.equals(shortName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ protected void createSymbolicLink(Path linkPath, PathFragment targetFragment)
+ throws IOException {
+ File file = getIoFile(linkPath);
+ try {
+ Files.createSymbolicLink(file.toPath(), new File(targetFragment.getPathString()).toPath());
+ } catch (java.nio.file.FileAlreadyExistsException e) {
+ throw new IOException(linkPath + ERR_FILE_EXISTS);
+ } catch (java.nio.file.AccessDeniedException e) {
+ throw new IOException(linkPath + ERR_PERMISSION_DENIED);
+ } catch (java.nio.file.NoSuchFileException e) {
+ throw new FileNotFoundException(linkPath + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ }
+
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ String link = Files.readSymbolicLink(file.toPath()).toString();
+ return new PathFragment(link);
+ } catch (java.nio.file.NotLinkException e) {
+ throw new NotASymlinkException(path);
+ } catch (java.nio.file.NoSuchFileException e) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_LINK, file.getPath());
+ }
+ }
+
+ @Override
+ protected void renameTo(Path sourcePath, Path targetPath) throws IOException {
+ synchronized (sourcePath) {
+ File sourceFile = getIoFile(sourcePath);
+ File targetFile = getIoFile(targetPath);
+ if (!sourceFile.renameTo(targetFile)) {
+ if (!sourceFile.exists()) {
+ throw new FileNotFoundException(sourcePath + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ if (targetFile.exists()) {
+ if (targetFile.isDirectory() && targetFile.list().length > 0) {
+ throw new IOException(targetPath + ERR_DIRECTORY_NOT_EMPTY);
+ } else if (sourceFile.isDirectory() && targetFile.isFile()) {
+ throw new IOException(sourcePath + " -> " + targetPath + ERR_NOT_A_DIRECTORY);
+ } else if (sourceFile.isFile() && targetFile.isDirectory()) {
+ throw new IOException(sourcePath + " -> " + targetPath + ERR_IS_DIRECTORY);
+ } else {
+ throw new IOException(sourcePath + " -> " + targetPath + ERR_PERMISSION_DENIED);
+ }
+ } else {
+ throw new FileAccessException(sourcePath + " -> " + targetPath + ERR_PERMISSION_DENIED);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected long getFileSize(Path path, boolean followSymlinks) throws IOException {
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return stat(path, followSymlinks).getSize();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path);
+ }
+ }
+
+ @Override
+ protected boolean delete(Path path) throws IOException {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ synchronized (path) {
+ try {
+ if (file.delete()) {
+ return true;
+ }
+ if (file.exists()) {
+ if (file.isDirectory() && file.list().length > 0) {
+ throw new IOException(path + ERR_DIRECTORY_NOT_EMPTY);
+ } else {
+ throw new IOException(path + ERR_PERMISSION_DENIED);
+ }
+ }
+ return false;
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_DELETE, file.getPath());
+ }
+ }
+ }
+
+ @Override
+ protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return stat(path, followSymlinks).getLastModifiedTime();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath());
+ }
+ }
+
+ @Override
+ protected boolean isSymbolicLink(Path path) {
+ File file = getIoFile(path);
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return fileIsSymbolicLink(file);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath());
+ }
+ }
+
+ private boolean fileIsSymbolicLink(File file) {
+ return Files.isSymbolicLink(file.toPath());
+ }
+
+ @Override
+ protected void setLastModifiedTime(Path path, long newTime) throws IOException {
+ File file = getIoFile(path);
+ if (!file.setLastModified(newTime)) {
+ if (!file.exists()) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ } else if (!file.getParentFile().canWrite()) {
+ throw new FileAccessException(path.getParentDirectory() + ERR_PERMISSION_DENIED);
+ } else {
+ throw new FileAccessException(path + ERR_PERMISSION_DENIED);
+ }
+ }
+ }
+
+ @Override
+ protected byte[] getMD5Digest(Path path) throws IOException {
+ String name = path.toString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return super.getMD5Digest(path);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_MD5, name);
+ }
+ }
+
+ /**
+ * Returns the status of a file. See {@link Path#stat(Symlinks)} for
+ * specification.
+ *
+ * <p>The default implementation of this method is a "lazy" one, based on
+ * other accessor methods such as {@link #isFile}, etc. Subclasses may provide
+ * more efficient specializations. However, we still try to follow Unix-like
+ * semantics of failing fast in case of non-existent files (or in case of
+ * permission issues).
+ */
+ @Override
+ protected FileStatus stat(final Path path, final boolean followSymlinks) throws IOException {
+ File file = getIoFile(path);
+ final BasicFileAttributes attributes;
+ try {
+ attributes = Files.readAttributes(
+ file.toPath(), BasicFileAttributes.class, linkOpts(followSymlinks));
+ } catch (java.nio.file.FileSystemException e) {
+ throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR);
+ }
+ FileStatus status = new FileStatus() {
+ @Override
+ public boolean isFile() {
+ return attributes.isRegularFile();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return attributes.isDirectory();
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return attributes.isSymbolicLink();
+ }
+
+ @Override
+ public long getSize() throws IOException {
+ return attributes.size();
+ }
+
+ @Override
+ public long getLastModifiedTime() throws IOException {
+ return attributes.lastModifiedTime().toMillis();
+ }
+
+ @Override
+ public long getLastChangeTime() {
+ // This is the best we can do with Java NIO...
+ return attributes.lastModifiedTime().toMillis();
+ }
+
+ @Override
+ public long getNodeId() {
+ // TODO(bazel-team): Consider making use of attributes.fileKey().
+ return -1;
+ }
+ };
+
+ return status;
+ }
+
+ @Override
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) {
+ try {
+ return stat(path, followSymlinks);
+ } catch (FileNotFoundException e) {
+ // JavaIoFileSystem#stat (incorrectly) only throws FileNotFoundException (because it calls
+ // #getLastModifiedTime, which can only throw a FileNotFoundException), so we always hit this
+ // codepath. Thus, this method will incorrectly not throw an exception for some filesystem
+ // errors.
+ return null;
+ } catch (IOException e) {
+ // If this codepath is ever hit, then this method should be rewritten to properly distinguish
+ // between not-found exceptions and others.
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ModifiedFileSet.java b/src/main/java/com/google/devtools/build/lib/vfs/ModifiedFileSet.java
new file mode 100644
index 0000000..3d6a638
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/ModifiedFileSet.java
@@ -0,0 +1,126 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * An immutable set of modified source files. The scope of these files is context-dependent; in some
+ * uses this may mean information about all files in the client, while in other uses this may mean
+ * information about some specific subset of files. {@link #EVERYTHING_MODIFIED} can be used to
+ * indicate that all files of interest have been modified.
+ */
+public final class ModifiedFileSet {
+
+ public static final ModifiedFileSet EVERYTHING_MODIFIED = new ModifiedFileSet(null);
+ public static final ModifiedFileSet NOTHING_MODIFIED = new ModifiedFileSet(
+ ImmutableSet.<PathFragment>of());
+
+ @Nullable private final ImmutableSet<PathFragment> modified;
+
+ /**
+ * Whether all files of interest should be treated as potentially modified.
+ */
+ public boolean treatEverythingAsModified() {
+ return modified == null;
+ }
+
+ /**
+ * The set of files of interest that were modified.
+ *
+ * @throws IllegalStateException if {@link #treatEverythingAsModified} returns true.
+ */
+ public ImmutableSet<PathFragment> modifiedSourceFiles() {
+ if (treatEverythingAsModified()) {
+ throw new IllegalStateException();
+ }
+ return modified;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ModifiedFileSet)) {
+ return false;
+ }
+ ModifiedFileSet other = (ModifiedFileSet) o;
+ return Objects.equals(modified, other.modified);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(modified);
+ }
+
+ @Override
+ public String toString() {
+ if (this == EVERYTHING_MODIFIED) {
+ return "EVERYTHING_MODIFIED";
+ } else if (this == NOTHING_MODIFIED) {
+ return "NOTHING_MODIFIED";
+ } else {
+ return modified.toString();
+ }
+ }
+
+ private ModifiedFileSet(ImmutableSet<PathFragment> modified) {
+ this.modified = modified;
+ }
+
+ /**
+ * The builder for {@link ModifiedFileSet}.
+ */
+ public static class Builder {
+ private final ImmutableSet.Builder<PathFragment> setBuilder =
+ ImmutableSet.<PathFragment>builder();
+
+ public ModifiedFileSet build() {
+ ImmutableSet<PathFragment> modified = setBuilder.build();
+ return modified.isEmpty() ? NOTHING_MODIFIED : new ModifiedFileSet(modified);
+ }
+
+ public Builder modify(PathFragment pathFragment) {
+ setBuilder.add(pathFragment);
+ return this;
+ }
+
+ public Builder modifyAll(Iterable<PathFragment> pathFragments) {
+ setBuilder.addAll(pathFragments);
+ return this;
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static ModifiedFileSet union(ModifiedFileSet mfs1, ModifiedFileSet mfs2) {
+ if (mfs1.treatEverythingAsModified() || mfs2.treatEverythingAsModified()) {
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+ if (mfs1.equals(ModifiedFileSet.NOTHING_MODIFIED)) {
+ return mfs2;
+ }
+ if (mfs2.equals(ModifiedFileSet.NOTHING_MODIFIED)) {
+ return mfs1;
+ }
+ return ModifiedFileSet.builder()
+ .modifyAll(mfs1.modifiedSourceFiles())
+ .modifyAll(mfs2.modifiedSourceFiles())
+ .build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
new file mode 100644
index 0000000..de222fe
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
@@ -0,0 +1,1099 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.lang.ref.Reference;
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.WeakReference;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.Objects;
+
+/**
+ * <p>Instances of this class represent pathnames, forming a tree
+ * structure to implement sharing of common prefixes (parent directory names).
+ * A node in these trees is something like foo, bar, .., ., or /. If the
+ * instance is not a root path, it will have a parent path. A path can also
+ * have children, which are indexed by name in a map.
+ *
+ * <p>There is some limited support for Windows-style paths. Most importantly, drive identifiers
+ * in front of a path (c:/abc) are supported. However, Windows-style backslash separators
+ * (C:\\foo\\bar) and drive-relative paths ("C:foo") are explicitly not supported, same with
+ * advanced features like \\\\network\\paths and \\\\?\\unc\\paths.
+ *
+ * <p>{@link FileSystem} implementations maintain pointers into this graph.
+ */
+@ThreadSafe
+public class Path implements Comparable<Path>, Serializable {
+
+ private static FileSystem fileSystemForSerialization;
+
+ /**
+ * We need to specify used FileSystem. In this case we can save memory during the serialization.
+ */
+ public static void setFileSystemForSerialization(FileSystem fileSystem) {
+ fileSystemForSerialization = fileSystem;
+ }
+
+ /**
+ * Returns FileSystem that we are using.
+ */
+ public static FileSystem getFileSystemForSerialization() {
+ return fileSystemForSerialization;
+ }
+
+ // These are basically final, but can't be marked as such in order to support serialization.
+ private FileSystem fileSystem;
+ private String name;
+ private Path parent;
+ private int depth;
+ private int hashCode;
+
+ private static final ReferenceQueue<Path> REFERENCE_QUEUE = new ReferenceQueue<>();
+
+ private static class PathWeakReferenceForCleanup extends WeakReference<Path> {
+ final Path parent;
+ final String baseName;
+
+ PathWeakReferenceForCleanup(Path referent, ReferenceQueue<Path> referenceQueue) {
+ super(referent, referenceQueue);
+ parent = referent.getParentDirectory();
+ baseName = referent.getBaseName();
+ }
+ }
+
+ private static final Thread PATH_CHILD_CACHE_CLEANUP_THREAD = new Thread("Path cache cleanup") {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ PathWeakReferenceForCleanup ref = (PathWeakReferenceForCleanup) REFERENCE_QUEUE.remove();
+ Path parent = ref.parent;
+ synchronized (parent) {
+ // It's possible that since this reference was enqueued for deletion, the Path was
+ // recreated with a new entry in the map. We definitely shouldn't delete that entry, so
+ // check to make sure they're the same.
+ Reference<Path> currentRef = ref.parent.children.get(ref.baseName);
+ if (currentRef == ref) {
+ ref.parent.children.remove(ref.baseName);
+ }
+ }
+ } catch (InterruptedException e) {
+ // Ignored.
+ }
+ }
+ }
+ };
+
+ static {
+ PATH_CHILD_CACHE_CLEANUP_THREAD.setDaemon(true);
+ PATH_CHILD_CACHE_CLEANUP_THREAD.start();
+ }
+
+ /**
+ * A mapping from a child file name to the {@link Path} representing it.
+ *
+ * <p>File names must be a single path segment. The strings must be
+ * canonical. We use IdentityHashMap instead of HashMap for reasons of space
+ * efficiency: instances are smaller by a single word. Also, since all path
+ * segments are interned, the universe of Paths holds a minimal number of
+ * references to strings. (It's doubtful that there's any time gain from use
+ * of an IdentityHashMap, since the time saved by avoiding string equality
+ * tests during hash lookups is probably equal to the time spent eagerly
+ * interning strings, unless the collision rate is high.)
+ *
+ * <p>The Paths are stored as weak references to ensure that a live
+ * Path for a directory does not hold a strong reference to all of its
+ * descendants, which would prevent collection of paths we never intend to
+ * use again. Stale references in the map must be treated as absent.
+ *
+ * <p>A Path may be recycled once there is no Path that refers to it or
+ * to one of its descendants. This means that any data stored in the
+ * Path instance by one of its subclasses must be recoverable: it's ok to
+ * store data in Paths as an optimization, but there must be another
+ * source for that data in case the Path is recycled.
+ *
+ * <p>We intentionally avoid using the existing library classes for reasons of
+ * space efficiency: while ConcurrentHashMap would reduce our locking
+ * overhead, and ReferenceMap would simplify the code a little, both of those
+ * classes have much higher per-instance overheads than IdentityHashMap.
+ *
+ * <p>The Path object must be synchronized while children is being
+ * accessed.
+ */
+ private IdentityHashMap<String, Reference<Path>> children;
+
+ /**
+ * Create a path instance. Should only be called by {@link #createChildPath}.
+ *
+ * @param name the name of this path; it must be canonicalized with {@link
+ * StringCanonicalizer#intern}
+ * @param parent this path's parent
+ */
+ protected Path(FileSystem fileSystem, String name, Path parent) {
+ this.fileSystem = fileSystem;
+ this.name = name;
+ this.parent = parent;
+ this.depth = parent == null ? 0 : parent.depth + 1;
+ this.hashCode = Objects.hash(parent, name);
+ }
+
+ /**
+ * Create the root path. Should only be called by
+ * {@link FileSystem#createRootPath()}.
+ */
+ protected Path(FileSystem fileSystem) {
+ this(fileSystem, StringCanonicalizer.intern("/"), null);
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ Preconditions.checkState(fileSystem == fileSystemForSerialization, fileSystem);
+ out.writeUTF(getPathString());
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException {
+ fileSystem = fileSystemForSerialization;
+ String p = in.readUTF();
+ PathFragment pf = new PathFragment(p);
+ PathFragment parentDir = pf.getParentDirectory();
+ if (parentDir == null) {
+ this.name = "/";
+ this.parent = null;
+ this.depth = 0;
+ } else {
+ this.name = pf.getBaseName();
+ this.parent = fileSystem.getPath(parentDir);
+ this.depth = this.parent.depth + 1;
+ }
+ this.hashCode = Objects.hash(parent, name);
+ }
+
+ /**
+ * Returns the filesystem instance to which this path belongs.
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ public boolean isRootDirectory() {
+ return parent == null;
+ }
+
+ protected Path createChildPath(String childName) {
+ return new Path(fileSystem, childName, this);
+ }
+
+ /**
+ * Returns the child path named name, or creates such a path (and caches it)
+ * if it doesn't already exist.
+ */
+ private Path getCachedChildPath(String childName) {
+ // Don't hold the lock for the interning operation. It increases lock contention.
+ childName = StringCanonicalizer.intern(childName);
+ synchronized(this) {
+ if (children == null) {
+ // 66% of Paths have size == 1, 80% <= 2
+ children = new IdentityHashMap<String, Reference<Path>>(1);
+ }
+ Reference<Path> childRef = children.get(childName);
+ Path child;
+ if (childRef == null || (child = childRef.get()) == null) {
+ child = createChildPath(childName);
+ children.put(childName, new PathWeakReferenceForCleanup(child, REFERENCE_QUEUE));
+ }
+ return child;
+ }
+ }
+
+ /**
+ * Applies the specified function to each {@link Path} that is an existing direct
+ * descendant of this one. The Predicate is evaluated only for its
+ * side-effects.
+ *
+ * <p>This function exists to hide the "children" field, whose complex
+ * synchronization and identity requirements are too unsafe to be exposed to
+ * subclasses. For example, the "children" field must be synchronized for
+ * the duration of any iteration over it; it may be null; and references
+ * within it may be stale, and must be ignored.
+ */
+ protected synchronized void applyToChildren(Predicate<Path> function) {
+ if (children != null) {
+ for (Reference<Path> childRef : children.values()) {
+ Path child = childRef.get();
+ if (child != null) {
+ function.apply(child);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns whether this path is recursively "under" {@code prefix} - that is,
+ * whether {@code path} is a prefix of this path.
+ *
+ * <p>This method returns {@code true} when called with this path itself. This
+ * method acts independently of the existence of files or folders.
+ *
+ * @param prefix a path which may or may not be a prefix of this path
+ */
+ public boolean startsWith(Path prefix) {
+ Path n = this;
+ for (int i = 0, len = depth - prefix.depth; i < len; i++) {
+ n = n.getParentDirectory();
+ }
+ return prefix.equals(n);
+ }
+
+ /**
+ * Computes a string representation of this path, and writes it to the
+ * given string builder. Only called locally with a new instance.
+ */
+ private void buildPathString(StringBuilder result) {
+ if (isRootDirectory()) {
+ result.append('/');
+ } else {
+ if (parent.isWindowsVolumeName()) {
+ result.append(parent.name);
+ } else {
+ parent.buildPathString(result);
+ }
+ if (!parent.isRootDirectory()) {
+ result.append('/');
+ }
+ result.append(name);
+ }
+ }
+
+ /**
+ * Returns true if the current path represents a Windows volume name (such as "c:" or "d:").
+ *
+ * <p>Paths such as '\\\\vol\\foo' are not supported.
+ */
+ private boolean isWindowsVolumeName() {
+ return OS.getCurrent() == OS.WINDOWS
+ && parent != null && parent.isRootDirectory() && name.length() == 2
+ && PathFragment.getWindowsDriveLetter(name) != '\0';
+ }
+
+ /**
+ * Returns the path as a string.
+ */
+ public String getPathString() {
+ // Profile driven optimization:
+ // Preallocate a size determined by the depth, so that
+ // we do not have to expand the capacity of the StringBuilder
+ StringBuilder builder = new StringBuilder(depth * 20);
+ buildPathString(builder);
+ return builder.toString();
+ }
+
+ /**
+ * Returns the path as a string.
+ */
+ @Override
+ public String toString() {
+ return getPathString();
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof Path)) {
+ return false;
+ }
+ Path otherPath = (Path) other;
+ return fileSystem.equals(otherPath.fileSystem) && name.equals(otherPath.name)
+ && Objects.equals(parent, otherPath.parent);
+ }
+
+ /**
+ * Returns a string of debugging information associated with this path.
+ * The results are unspecified and MUST NOT be interpreted programmatically.
+ */
+ protected String toDebugString() {
+ return "";
+ }
+
+ /**
+ * Returns a path representing the parent directory of this path,
+ * or null iff this Path represents the root of the filesystem.
+ *
+ * <p>Note: This method normalises ".." and "." path segments by string
+ * processing, not by directory lookups.
+ */
+ public Path getParentDirectory() {
+ return parent;
+ }
+
+ /**
+ * Returns true iff this path denotes an existing file of any kind. Follows
+ * symbolic links.
+ */
+ public boolean exists() {
+ return fileSystem.exists(this, true);
+ }
+
+ /**
+ * Returns true iff this path denotes an existing file of any kind.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is dereferenced until a file other than a
+ * symbolic link is found
+ */
+ public boolean exists(Symlinks followSymlinks) {
+ return fileSystem.exists(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Returns a new, immutable collection containing the names of all entities
+ * within the directory denoted by the current path. Follows symbolic links.
+ *
+ * @throws FileNotFoundException If the directory is not found
+ * @throws IOException If the path does not denote a directory
+ */
+ public Collection<Path> getDirectoryEntries() throws IOException, FileNotFoundException {
+ return fileSystem.getDirectoryEntries(this);
+ }
+
+ /**
+ * Returns a collection of the names and types of all entries within the directory
+ * denoted by the current path. Follows symbolic links if {@code followSymlinks} is true.
+ * Note that the order of the returned entries is not guaranteed.
+ *
+ * @param followSymlinks whether to follow symlinks or not
+ *
+ * @throws FileNotFoundException If the directory is not found
+ * @throws IOException If the path does not denote a directory
+ */
+ public Collection<Dirent> readdir(Symlinks followSymlinks) throws IOException {
+ return fileSystem.readdir(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Returns a new, immutable collection containing the names of all entities
+ * within the directory denoted by the current path, for which the given
+ * predicate is true.
+ *
+ * @throws FileNotFoundException If the directory is not found
+ * @throws IOException If the path does not denote a directory
+ */
+ public Collection<Path> getDirectoryEntries(Predicate<? super Path> predicate)
+ throws IOException, FileNotFoundException {
+ return ImmutableList.<Path>copyOf(Iterables.filter(getDirectoryEntries(), predicate));
+ }
+
+ /**
+ * Returns the status of a file, following symbolic links.
+ *
+ * @throws IOException if there was an error obtaining the file status. Note,
+ * some implementations may defer the I/O, and hence the throwing of
+ * the exception, until the accessor methods of {@code FileStatus} are
+ * called.
+ */
+ public FileStatus stat() throws IOException {
+ return fileSystem.stat(this, true);
+ }
+
+ /**
+ * Like stat(), but returns null on file-nonexistence instead of throwing.
+ */
+ public FileStatus statNullable() {
+ return statNullable(Symlinks.FOLLOW);
+ }
+
+ /**
+ * Like stat(), but returns null on file-nonexistence instead of throwing.
+ */
+ public FileStatus statNullable(Symlinks symlinks) {
+ return fileSystem.statNullable(this, symlinks.toBoolean());
+ }
+
+ /**
+ * Returns the status of a file, optionally following symbolic links.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is dereferenced until a file other than a
+ * symbolic link is found
+ * @throws IOException if there was an error obtaining the file status. Note,
+ * some implementations may defer the I/O, and hence the throwing of
+ * the exception, until the accessor methods of {@code FileStatus} are
+ * called
+ */
+ public FileStatus stat(Symlinks followSymlinks) throws IOException {
+ return fileSystem.stat(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Like {@link #stat}, but may return null if the file is not found (corresponding to
+ * {@code ENOENT} and {@code ENOTDIR} in Unix's stat(2) function) instead of throwing. Follows
+ * symbolic links.
+ */
+ public FileStatus statIfFound() throws IOException {
+ return fileSystem.statIfFound(this, true);
+ }
+
+ /**
+ * Like {@link #stat}, but may return null if the file is not found (corresponding to
+ * {@code ENOENT} and {@code ENOTDIR} in Unix's stat(2) function) instead of throwing.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is dereferenced until a file other than a
+ * symbolic link is found
+ */
+ public FileStatus statIfFound(Symlinks followSymlinks) throws IOException {
+ return fileSystem.statIfFound(this, followSymlinks.toBoolean());
+ }
+
+
+ /**
+ * Returns true iff this path denotes an existing directory. Follows symbolic
+ * links.
+ */
+ public boolean isDirectory() {
+ return fileSystem.isDirectory(this, true);
+ }
+
+ /**
+ * Returns true iff this path denotes an existing directory.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is dereferenced until a file other than a
+ * symbolic link is found
+ */
+ public boolean isDirectory(Symlinks followSymlinks) {
+ return fileSystem.isDirectory(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Returns true iff this path denotes an existing regular or special file.
+ * Follows symbolic links.
+ *
+ * <p>For our purposes, "file" includes special files (socket, fifo, block or
+ * char devices) too; it excludes symbolic links and directories.
+ */
+ public boolean isFile() {
+ return fileSystem.isFile(this, true);
+ }
+
+ /**
+ * Returns true iff this path denotes an existing regular or special file.
+ *
+ * <p>For our purposes, a "file" includes special files (socket, fifo, block
+ * or char devices) too; it excludes symbolic links and directories.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is dereferenced until a file other than a
+ * symbolic link is found.
+ */
+ public boolean isFile(Symlinks followSymlinks) {
+ return fileSystem.isFile(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Returns true iff this path denotes an existing symbolic link. Does not
+ * follow symbolic links.
+ */
+ public boolean isSymbolicLink() {
+ return fileSystem.isSymbolicLink(this);
+ }
+
+ /**
+ * Returns the last segment of this path, or "/" for the root directory.
+ */
+ public String getBaseName() {
+ return name;
+ }
+
+ /**
+ * Interprets the name of a path segment relative to the current path and
+ * returns the result.
+ *
+ * <p>This is a purely syntactic operation, i.e. it does no I/O, it does not
+ * validate the existence of any path, nor resolve symbolic links. If 'prefix'
+ * is not canonical, then a 'name' of '..' will be interpreted incorrectly.
+ *
+ * @precondition segment contains no slashes.
+ */
+ private Path getCanonicalPath(String segment) {
+ if (segment.equals(".") || segment.equals("")) {
+ return this; // that's a noop
+ } else if (segment.equals("..")) {
+ // root's parent is root, when canonicalising:
+ return parent == null || isWindowsVolumeName() ? this : parent;
+ } else {
+ return getCachedChildPath(segment);
+ }
+ }
+
+ /**
+ * Returns the path formed by appending the single non-special segment
+ * "baseName" to this path.
+ *
+ * <p>You should almost always use {@link #getRelative} instead, which has
+ * the same performance characteristics if the given name is a valid base
+ * name, and which also works for '.', '..', and strings containing '/'.
+ *
+ * @throws IllegalArgumentException if {@code baseName} is not a valid base
+ * name according to {@link FileSystemUtils#checkBaseName}
+ */
+ public Path getChild(String baseName) {
+ FileSystemUtils.checkBaseName(baseName);
+ return getCachedChildPath(baseName);
+ }
+
+ /**
+ * Returns the path formed by appending the relative or absolute path fragment
+ * {@code suffix} to this path.
+ *
+ * <p>If suffix is absolute, the current path will be ignored; otherwise, they
+ * will be combined. Up-level references ("..") cause the preceding path
+ * segment to be elided; this interpretation is only correct if the base path
+ * is canonical.
+ */
+ public Path getRelative(PathFragment suffix) {
+ Path result = suffix.isAbsolute() ? fileSystem.getRootDirectory() : this;
+ if (!suffix.windowsVolume().isEmpty()) {
+ result = result.getCanonicalPath(suffix.windowsVolume());
+ }
+ for (String segment : suffix.segments()) {
+ result = result.getCanonicalPath(segment);
+ }
+ return result;
+ }
+
+ /**
+ * Returns the path formed by appending the relative or absolute string
+ * {@code path} to this path.
+ *
+ * <p>If the given path string is absolute, the current path will be ignored;
+ * otherwise, they will be combined. Up-level references ("..") cause the
+ * preceding path segment to be elided.
+ *
+ * <p>This is a purely syntactic operation, i.e. it does no I/O, it does not
+ * validate the existence of any path, nor resolve symbolic links.
+ */
+ public Path getRelative(String path) {
+ // Fast path for valid base names.
+ if ((path.length() == 0) || (path.equals("."))) {
+ return this;
+ } else if (path.equals("..")) {
+ return parent == null ? this : parent;
+ } else if ((path.indexOf('/') != -1)) {
+ return getRelative(new PathFragment(path));
+ } else {
+ return getCachedChildPath(path);
+ }
+ }
+
+ /**
+ * Returns an absolute PathFragment representing this path.
+ */
+ public PathFragment asFragment() {
+ String[] resultSegments = new String[depth];
+ Path currentPath = this;
+ for (int pos = depth - 1; pos >= 0; pos--) {
+ resultSegments[pos] = currentPath.getBaseName();
+ currentPath = currentPath.getParentDirectory();
+ }
+
+ char driveLetter = '\0';
+ if (resultSegments.length > 0) {
+ driveLetter = PathFragment.getWindowsDriveLetter(resultSegments[0]);
+ if (driveLetter != '\0') {
+ // Strip off the first segment that contains the volume name.
+ resultSegments = Arrays.copyOfRange(resultSegments, 1, resultSegments.length);
+ }
+ }
+
+ return new PathFragment(driveLetter, true, resultSegments);
+ }
+
+
+ /**
+ * Returns a relative path fragment to this path, relative to {@code
+ * ancestorDirectory}. {@code ancestorDirectory} must be on the same
+ * filesystem as this path. (Currently, both this path and "ancestorDirectory"
+ * must be absolute, though this restriction could be loosened.)
+ * <p>
+ * <code>x.relativeTo(z) == y</code> implies
+ * <code>z.getRelative(y.getPathString()) == x</code>.
+ * <p>
+ * For example, <code>"/foo/bar/wiz".relativeTo("/foo")</code> returns
+ * <code>"bar/wiz"</code>.
+ *
+ * @throws IllegalArgumentException if this path is not beneath {@code
+ * ancestorDirectory} or if they are not part of the same filesystem
+ */
+ public PathFragment relativeTo(Path ancestorPath) {
+ checkSameFilesystem(ancestorPath);
+
+ // Fast path: when otherPath is the ancestor of this path
+ int resultSegmentCount = depth - ancestorPath.depth;
+ if (resultSegmentCount >= 0) {
+ String[] resultSegments = new String[resultSegmentCount];
+ Path currentPath = this;
+ for (int pos = resultSegmentCount - 1; pos >= 0; pos--) {
+ resultSegments[pos] = currentPath.getBaseName();
+ currentPath = currentPath.getParentDirectory();
+ }
+ if (ancestorPath.equals(currentPath)) {
+ return new PathFragment('\0', false, resultSegments);
+ }
+ }
+
+ throw new IllegalArgumentException("Path " + this + " is not beneath " + ancestorPath);
+ }
+
+ /**
+ * Checks that "this" and "that" are paths on the same filesystem.
+ */
+ protected void checkSameFilesystem(Path that) {
+ if (this.fileSystem != that.fileSystem) {
+ throw new IllegalArgumentException("Files are on different filesystems: "
+ + this + ", " + that);
+ }
+ }
+
+ /**
+ * Returns an output stream to the file denoted by the current path, creating
+ * it and truncating it if necessary. The stream is opened for writing.
+ *
+ * @throws FileNotFoundException If the file cannot be found or created.
+ * @throws IOException If a different error occurs.
+ */
+ public OutputStream getOutputStream() throws IOException, FileNotFoundException {
+ return getOutputStream(false);
+ }
+
+ /**
+ * Returns an output stream to the file denoted by the current path, creating
+ * it and truncating it if necessary. The stream is opened for writing.
+ *
+ * @param append whether to open the file in append mode.
+ * @throws FileNotFoundException If the file cannot be found or created.
+ * @throws IOException If a different error occurs.
+ */
+ public OutputStream getOutputStream(boolean append) throws IOException, FileNotFoundException {
+ return fileSystem.getOutputStream(this, append);
+ }
+
+ /**
+ * Creates a directory with the name of the current path, not following
+ * symbolic links. Returns normally iff the directory exists after the call:
+ * true if the directory was created by this call, false if the directory was
+ * already in existence. Throws an exception if the directory could not be
+ * created for any reason.
+ *
+ * @throws IOException if the directory creation failed for any reason
+ */
+ public boolean createDirectory() throws IOException {
+ return fileSystem.createDirectory(this);
+ }
+
+ /**
+ * Creates a symbolic link with the name of the current path, following
+ * symbolic links. The referent of the created symlink is is the absolute path
+ * "target"; it is not possible to create relative symbolic links via this
+ * method.
+ *
+ * @throws IOException if the creation of the symbolic link was unsuccessful
+ * for any reason
+ */
+ public void createSymbolicLink(Path target) throws IOException {
+ checkSameFilesystem(target);
+ fileSystem.createSymbolicLink(this, target.asFragment());
+ }
+
+ /**
+ * Creates a symbolic link with the name of the current path, following
+ * symbolic links. The referent of the created symlink is is the path fragment
+ * "target", which may be absolute or relative.
+ *
+ * @throws IOException if the creation of the symbolic link was unsuccessful
+ * for any reason
+ */
+ public void createSymbolicLink(PathFragment target) throws IOException {
+ fileSystem.createSymbolicLink(this, target);
+ }
+
+ /**
+ * Returns the target of the current path, which must be a symbolic link. The
+ * link contents are returned exactly, and may contain an absolute or relative
+ * path. Analogous to readlink(2).
+ *
+ * @return the content (i.e. target) of the symbolic link
+ * @throws IOException if the current path is not a symbolic link, or the
+ * contents of the link could not be read for any reason
+ */
+ public PathFragment readSymbolicLink() throws IOException {
+ return fileSystem.readSymbolicLink(this);
+ }
+
+ /**
+ * Returns the canonical path for this path, by repeatedly replacing symbolic
+ * links with their referents. Analogous to realpath(3).
+ *
+ * @return the canonical path for this path
+ * @throws IOException if any symbolic link could not be resolved, or other
+ * error occurred (for example, the path does not exist)
+ */
+ public Path resolveSymbolicLinks() throws IOException {
+ return fileSystem.resolveSymbolicLinks(this);
+ }
+
+ /**
+ * Renames the file denoted by the current path to the location "target", not
+ * following symbolic links.
+ *
+ * <p>Files cannot be atomically renamed across devices; copying is required.
+ * Use {@link FileSystemUtils#copyFile} followed by {@link Path#delete}.
+ *
+ * @throws IOException if the rename failed for any reason
+ */
+ public void renameTo(Path target) throws IOException {
+ checkSameFilesystem(target);
+ fileSystem.renameTo(this, target);
+ }
+
+ /**
+ * Returns the size in bytes of the file denoted by the current path,
+ * following symbolic links.
+ *
+ * <p>The size of directory or special file is undefined.
+ *
+ * @throws FileNotFoundException if the file denoted by the current path does
+ * not exist
+ * @throws IOException if the file's metadata could not be read, or some other
+ * error occurred
+ */
+ public long getFileSize() throws IOException, FileNotFoundException {
+ return fileSystem.getFileSize(this, true);
+ }
+
+ /**
+ * Returns the size in bytes of the file denoted by the current path.
+ *
+ * <p>The size of directory or special file is undefined. The size of a symbolic
+ * link is the length of the name of its referent.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is deferenced until a file other than a
+ * symbol link is found
+ * @throws FileNotFoundException if the file denoted by the current path does
+ * not exist
+ * @throws IOException if the file's metadata could not be read, or some other
+ * error occurred
+ */
+ public long getFileSize(Symlinks followSymlinks) throws IOException, FileNotFoundException {
+ return fileSystem.getFileSize(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Deletes the file denoted by this path, not following symbolic links.
+ * Returns normally iff the file doesn't exist after the call: true if this
+ * call deleted the file, false if the file already didn't exist. Throws an
+ * exception if the file could not be deleted for any reason.
+ *
+ * @return true iff the file was actually deleted by this call
+ * @throws IOException if the deletion failed but the file was present prior
+ * to the call
+ */
+ public boolean delete() throws IOException {
+ return fileSystem.delete(this);
+ }
+
+ /**
+ * Returns the last modification time of the file, in milliseconds since the
+ * UNIX epoch, of the file denoted by the current path, following symbolic
+ * links.
+ *
+ * <p>Caveat: many filesystems store file times in seconds, so do not rely on
+ * the millisecond precision.
+ *
+ * @throws IOException if the operation failed for any reason
+ */
+ public long getLastModifiedTime() throws IOException {
+ return fileSystem.getLastModifiedTime(this, true);
+ }
+
+ /**
+ * Returns the last modification time of the file, in milliseconds since the
+ * UNIX epoch, of the file denoted by the current path.
+ *
+ * <p>Caveat: many filesystems store file times in seconds, so do not rely on
+ * the millisecond precision.
+ *
+ * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a
+ * symbolic link, the link is dereferenced until a file other than a
+ * symbolic link is found
+ * @throws IOException if the modification time for the file could not be
+ * obtained for any reason
+ */
+ public long getLastModifiedTime(Symlinks followSymlinks) throws IOException {
+ return fileSystem.getLastModifiedTime(this, followSymlinks.toBoolean());
+ }
+
+ /**
+ * Sets the modification time of the file denoted by the current path. Follows
+ * symbolic links. If newTime is -1, the current time according to the kernel
+ * is used; this may differ from the JVM's clock.
+ *
+ * <p>Caveat: many filesystems store file times in seconds, so do not rely on
+ * the millisecond precision.
+ *
+ * @param newTime time, in milliseconds since the UNIX epoch, or -1L, meaning
+ * use the kernel's current time
+ * @throws IOException if the modification time for the file could not be set
+ * for any reason
+ */
+ public void setLastModifiedTime(long newTime) throws IOException {
+ fileSystem.setLastModifiedTime(this, newTime);
+ }
+
+ /**
+ * Returns value of the given extended attribute name or null if attribute does not exist or
+ * file system does not support extended attributes. Follows symlinks.
+ */
+ public byte[] getxattr(String name) throws IOException {
+ return fileSystem.getxattr(this, name, true);
+ }
+
+ /**
+ * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null}
+ * if the filesystem doesn't support them.
+ */
+ public String getFastDigestFunctionType() {
+ return fileSystem.getFastDigestFunctionType(this);
+ }
+
+ /**
+ * Gets a fast digest for the given path, or {@code null} if there isn't one available. The
+ * digest should be suitable for detecting changes to the file.
+ */
+ public byte[] getFastDigest() throws IOException {
+ return fileSystem.getFastDigest(this);
+ }
+
+ /**
+ * Returns the MD5 digest of the file denoted by the current path, following
+ * symbolic links.
+ *
+ * <p>This method runs in O(n) time where n is the length of the file, but
+ * certain implementations may be much faster than the worst case.
+ *
+ * @return a new 16-byte array containing the file's MD5 digest
+ * @throws IOException if the MD5 digest could not be computed for any reason
+ */
+ public byte[] getMD5Digest() throws IOException {
+ return fileSystem.getMD5Digest(this);
+ }
+
+ /**
+ * Opens the file denoted by this path, following symbolic links, for reading,
+ * and returns an input stream to it.
+ *
+ * @throws IOException if the file was not found or could not be opened for
+ * reading
+ */
+ public InputStream getInputStream() throws IOException {
+ return fileSystem.getInputStream(this);
+ }
+
+ /**
+ * Returns a java.io.File representation of this path.
+ *
+ * <p>Caveat: the result may be useless if this path's getFileSystem() is not
+ * the UNIX filesystem.
+ */
+ public File getPathFile() {
+ return new File(getPathString());
+ }
+
+ /**
+ * Returns true if the file denoted by the current path, following symbolic
+ * links, is writable for the current user.
+ *
+ * @throws FileNotFoundException if the file does not exist, a dangling
+ * symbolic link was encountered, or the file's metadata could not be
+ * read
+ */
+ public boolean isWritable() throws IOException, FileNotFoundException {
+ return fileSystem.isWritable(this);
+ }
+
+ /**
+ * Sets the read permissions of the file denoted by the current path,
+ * following symbolic links. Permissions apply to the current user.
+ *
+ * @param readable if true, the file is set to readable; otherwise the file is
+ * made non-readable
+ * @throws FileNotFoundException if the file does not exist
+ * @throws IOException If the action cannot be taken (ie. permissions)
+ */
+ public void setReadable(boolean readable) throws IOException, FileNotFoundException {
+ fileSystem.setReadable(this, readable);
+ }
+
+ /**
+ * Sets the write permissions of the file denoted by the current path,
+ * following symbolic links. Permissions apply to the current user.
+ *
+ * <p>TODO(bazel-team): (2009) what about owner/group/others?
+ *
+ * @param writable if true, the file is set to writable; otherwise the file is
+ * made non-writable
+ * @throws FileNotFoundException if the file does not exist
+ * @throws IOException If the action cannot be taken (ie. permissions)
+ */
+ public void setWritable(boolean writable) throws IOException, FileNotFoundException {
+ fileSystem.setWritable(this, writable);
+ }
+
+ /**
+ * Returns true iff the file specified by the current path, following symbolic
+ * links, is executable by the current user.
+ *
+ * @throws FileNotFoundException if the file does not exist or a dangling
+ * symbolic link was encountered
+ * @throws IOException if some other I/O error occurred
+ */
+ public boolean isExecutable() throws IOException, FileNotFoundException {
+ return fileSystem.isExecutable(this);
+ }
+
+ /**
+ * Returns true iff the file specified by the current path, following symbolic
+ * links, is readable by the current user.
+ *
+ * @throws FileNotFoundException if the file does not exist or a dangling
+ * symbolic link was encountered
+ * @throws IOException if some other I/O error occurred
+ */
+ public boolean isReadable() throws IOException, FileNotFoundException {
+ return fileSystem.isReadable(this);
+ }
+
+ /**
+ * Sets the execute permission on the file specified by the current path,
+ * following symbolic links. Permissions apply to the current user.
+ *
+ * @throws FileNotFoundException if the file does not exist or a dangling
+ * symbolic link was encountered
+ * @throws IOException if the metadata change failed, for example because of
+ * permissions
+ */
+ public void setExecutable(boolean executable) throws IOException, FileNotFoundException {
+ fileSystem.setExecutable(this, executable);
+ }
+
+ /**
+ * Sets the permissions on the file specified by the current path, following
+ * symbolic links. If permission changes on this path's {@link FileSystem} are
+ * slow (e.g. one syscall per change), this method should aim to be faster
+ * than setting each permission individually. If this path's
+ * {@link FileSystem} does not support group and others permissions, those
+ * bits will be ignored.
+ *
+ * @throws FileNotFoundException if the file does not exist or a dangling
+ * symbolic link was encountered
+ * @throws IOException if the metadata change failed, for example because of
+ * permissions
+ */
+ public void chmod(int mode) throws IOException {
+ fileSystem.chmod(this, mode);
+ }
+
+ /**
+ * Compare Paths of the same file system using their PathFragments.
+ *
+ * <p>Paths from different filesystems will be compared using the identity
+ * hash code of their respective filesystems.
+ */
+ @Override
+ public int compareTo(Path o) {
+ // Fast-path.
+ if (equals(o)) {
+ return 0;
+ }
+
+ // If they are on different file systems, the file system decides the ordering.
+ FileSystem otherFs = o.getFileSystem();
+ if (!fileSystem.equals(otherFs)) {
+ int thisFileSystemHash = System.identityHashCode(fileSystem);
+ int otherFileSystemHash = System.identityHashCode(otherFs);
+ if (thisFileSystemHash < otherFileSystemHash) {
+ return -1;
+ } else if (thisFileSystemHash > otherFileSystemHash) {
+ return 1;
+ } else {
+ // TODO(bazel-team): Add a name to every file system to be used here.
+ return 0;
+ }
+ }
+
+ // Equal file system, but different paths, because of the canonicalization.
+ // We expect to often compare Paths that are very similar, for example for files in the same
+ // directory. This can be done efficiently by going up segment by segment until we get the
+ // identical path (canonicalization again), and then just compare the immediate child segments.
+ // Overall this is much faster than creating PathFragment instances, and comparing those, which
+ // requires us to always go up to the top-level directory and copy all segments into a new
+ // string array.
+ // This was previously showing up as a hotspot in a profile of globbing a large directory.
+ Path a = this, b = o;
+ int maxDepth = Math.min(a.depth, b.depth);
+ while (a.depth > maxDepth) {
+ a = a.getParentDirectory();
+ }
+ while (b.depth > maxDepth) {
+ b = b.getParentDirectory();
+ }
+ // One is the child of the other.
+ if (a.equals(b)) {
+ // If a is the same as this, this.depth must be less than o.depth.
+ return equals(a) ? -1 : 1;
+ }
+ Path previousa, previousb;
+ do {
+ previousa = a;
+ previousb = b;
+ a = a.getParentDirectory();
+ b = b.getParentDirectory();
+ } while (a != b); // This has to happen eventually.
+ return previousa.name.compareTo(previousb.name);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java b/src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java
new file mode 100644
index 0000000..1ee65a8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java
@@ -0,0 +1,655 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+
+import java.io.File;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Set;
+
+/**
+ * This class represents an immutable UNIX filesystem path, which may be absolute or relative. The
+ * path is maintained as a simple ordered list of path segment strings.
+ *
+ * <p>This class is independent from other VFS classes, especially anything requiring native code.
+ * It is safe to use in places that need simple segmented string path functionality.
+ *
+ * <p>There is some limited support for Windows-style paths. Most importantly, drive identifiers
+ * in front of a path (c:/abc) are supported and such paths are correctly recognized as absolute.
+ * However, Windows-style backslash separators (C:\\foo\\bar) are explicitly not supported, same
+ * with advanced features like \\\\network\\paths and \\\\?\\unc\\paths.
+ */
+@Immutable @ThreadSafe
+public final class PathFragment implements Comparable<PathFragment>, Serializable {
+
+ public static final int INVALID_SEGMENT = -1;
+
+ public static final char SEPARATOR_CHAR = '/';
+
+ public static final char EXTRA_SEPARATOR_CHAR =
+ (OS.getCurrent() == OS.WINDOWS) ? '\\' : '/';
+
+ public static final String ROOT_DIR = "/";
+
+ /** An empty path fragment. */
+ public static final PathFragment EMPTY_FRAGMENT = new PathFragment("");
+
+ public static final Function<String, PathFragment> TO_PATH_FRAGMENT =
+ new Function<String, PathFragment>() {
+ @Override
+ public PathFragment apply(String str) {
+ return new PathFragment(str);
+ }
+ };
+
+ public static final Predicate<PathFragment> IS_ABSOLUTE =
+ new Predicate<PathFragment>() {
+ @Override
+ public boolean apply(PathFragment input) {
+ return input.isAbsolute();
+ }
+ };
+
+ private static final Function<PathFragment, String> TO_SAFE_PATH_STRING =
+ new Function<PathFragment, String>() {
+ @Override
+ public String apply(PathFragment path) {
+ return path.getSafePathString();
+ }
+ };
+
+ // We have 3 word-sized fields (segments, hashCode and path), and 2
+ // byte-sized ones, which fits in 16 bytes. Object sizes are rounded
+ // to 16 bytes. Medium sized builds can easily hold millions of
+ // live PathFragments, so do not add further fields on a whim.
+
+ // The individual path components.
+ private final String[] segments;
+
+ // True both for UNIX-style absolute paths ("/foo") and Windows-style ("C:/foo").
+ private final boolean isAbsolute;
+
+ // Upper case windows drive letter, or '\0' if none. While a volumeName string is more
+ // general, we create a lot of these objects, so space is at a premium.
+ private final char driveLetter;
+
+ // hashCode and path are lazily initialized but semantically immutable.
+ private int hashCode;
+ private String path;
+
+ /**
+ * Construct a PathFragment from a string, which is an absolute or relative UNIX or Windows path.
+ */
+ public PathFragment(String path) {
+ this.driveLetter = getWindowsDriveLetter(path);
+ if (driveLetter != '\0') {
+ path = path.substring(2);
+ // TODO(bazel-team): Decide what to do about non-absolute paths with a volume name, e.g. C:x.
+ }
+ this.isAbsolute = path.length() > 0 && isSeparator(path.charAt(0));
+ this.segments = segment(path, isAbsolute ? 1 : 0);
+ }
+
+ private static boolean isSeparator(char c) {
+ return c == SEPARATOR_CHAR || c == EXTRA_SEPARATOR_CHAR;
+ }
+
+ /**
+ * Construct a PathFragment from a java.io.File, which is an absolute or
+ * relative UNIX path. Does not support Windows-style Files.
+ */
+ public PathFragment(File path) {
+ this(path.getPath());
+ }
+
+ /**
+ * Constructs a PathFragment, taking ownership of segments. Package-private,
+ * because it does not perform a defensive clone of the segments array. Used
+ * here in PathFragment, and by Path.asFragment() and Path.relativeTo().
+ */
+ PathFragment(char driveLetter, boolean isAbsolute, String[] segments) {
+ this.driveLetter = driveLetter;
+ this.isAbsolute = isAbsolute;
+ this.segments = segments;
+ }
+
+ /**
+ * Construct a PathFragment from a sequence of other PathFragments. The new
+ * fragment will be absolute iff the first fragment was absolute.
+ */
+ public PathFragment(PathFragment first, PathFragment second, PathFragment... more) {
+ // TODO(bazel-team): The handling of absolute path fragments in this constructor is unexpected.
+ this.segments = new String[sumLengths(first, second, more)];
+ int offset = 0;
+ offset += addSegments(offset, first);
+ offset += addSegments(offset, second);
+ for (PathFragment fragment : more) {
+ offset += addSegments(offset, fragment);
+ }
+ this.isAbsolute = first.isAbsolute;
+ this.driveLetter = first.driveLetter;
+ }
+
+ private int addSegments(int offset, PathFragment fragment) {
+ int count = fragment.segmentCount();
+ System.arraycopy(fragment.segments, 0, this.segments, offset, count);
+ return count;
+ }
+
+ private static int sumLengths(PathFragment first, PathFragment second, PathFragment[] more) {
+ int total = first.segmentCount() + second.segmentCount();
+ for (PathFragment fragment : more) {
+ total += fragment.segmentCount();
+ }
+ return total;
+ }
+
+ /**
+ * Segments the string passed in as argument and returns an array of strings.
+ * The split is performed along occurrences of (sequences of) the slash
+ * character.
+ *
+ * @param toSegment the string to segment
+ * @param offset how many characters from the start of the string to ignore.
+ */
+ private static String[] segment(String toSegment, int offset) {
+ char[] chars = toSegment.toCharArray();
+ int length = chars.length;
+
+ // Handle "/" and "" quickly.
+ if (length == offset) {
+ return new String[0];
+ }
+
+ // We make two passes through the array of characters: count & alloc,
+ // because simply using ArrayList was a bottleneck showing up during profiling.
+ int seg = 0;
+ int start = offset;
+ for (int i = offset; i < length; i++) {
+ if (isSeparator(chars[i])) {
+ if (i > start) { // to skip repeated separators
+ seg++;
+ }
+ start = i + 1;
+ }
+ }
+ if (start < length) {
+ seg++;
+ }
+ String[] result = new String[seg];
+ seg = 0;
+ start = offset;
+ for (int i = offset; i < length; i++) {
+ if (isSeparator(chars[i])) {
+ if (i > start) { // to skip repeated separators
+ // Make a copy of the String here to allow the interning to save memory. String.substring
+ // does not make a copy, but refers to the original char array, preventing garbage
+ // collection of the parts that are unnecessary.
+ result[seg] = StringCanonicalizer.intern(new String(chars, start, i - start));
+ seg++;
+ }
+ start = i + 1;
+ }
+ }
+ if (start < length) {
+ result[seg] = StringCanonicalizer.intern(new String(chars, start, length - start));
+ seg++;
+ }
+ return result;
+ }
+
+ private Object writeReplace() {
+ return new PathFragmentSerializationProxy(toString());
+ }
+
+ private void readObject(ObjectInputStream stream) throws InvalidObjectException {
+ throw new InvalidObjectException("Serialization is allowed only by proxy");
+ }
+
+ /**
+ * Returns the path string using '/' as the name-separator character. Returns "" if the path
+ * is both relative and empty.
+ */
+ public String getPathString() {
+ // Double-checked locking works, even without volatile, because path is a String, according to:
+ // http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html
+ if (path == null) {
+ synchronized (this) {
+ if (path == null) {
+ path = StringCanonicalizer.intern(joinSegments(SEPARATOR_CHAR));
+ }
+ }
+ }
+ return path;
+ }
+
+ /**
+ * Returns "." if the path fragment is both relative and empty, or {@link
+ * #getPathString} otherwise.
+ */
+ // TODO(bazel-team): Change getPathString to do this - this behavior makes more sense.
+ public String getSafePathString() {
+ return (!isAbsolute && (segmentCount() == 0)) ? "." : getPathString();
+ }
+
+ /**
+ * Returns a sequence consisting of the {@link #getSafePathString()} return of each item in
+ * {@code fragments}.
+ */
+ public static Iterable<String> safePathStrings(Iterable<PathFragment> fragments) {
+ return Iterables.transform(fragments, TO_SAFE_PATH_STRING);
+ }
+
+ private String joinSegments(char separatorChar) {
+ if (segments.length == 0 && isAbsolute) {
+ return windowsVolume() + ROOT_DIR;
+ }
+
+ // Profile driven optimization:
+ // Preallocate a size determined by the number of segments, so that
+ // we do not have to expand the capacity of the StringBuilder.
+ // Heuristically, this estimate is right for about 99% of the time.
+ int estimateSize =
+ ((driveLetter != '\0') ? 2 : 0)
+ + ((segments.length == 0) ? 0 : (segments.length + 1) * 20);
+ StringBuilder result = new StringBuilder(estimateSize);
+ result.append(windowsVolume());
+ boolean initialSegment = true;
+ for (String segment : segments) {
+ if (!initialSegment || isAbsolute) {
+ result.append(separatorChar);
+ }
+ initialSegment = false;
+ result.append(segment);
+ }
+ return result.toString();
+ }
+
+ /**
+ * Return true iff none of the segments are either "." or "..".
+ */
+ public boolean isNormalized() {
+ for (String segment : segments) {
+ if (segment.equals(".") || segment.equals("..")) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Normalizes the path fragment: removes "." and ".." segments if possible
+ * (if there are too many ".." segments, the resulting PathFragment will still
+ * start with "..").
+ */
+ public PathFragment normalize() {
+ String[] scratchSegments = new String[segments.length];
+ int segmentCount = 0;
+
+ for (String segment : segments) {
+ if (segment.equals(".")) {
+ // Just discard it
+ } else if (segment.equals("..")) {
+ if (segmentCount > 0 && !scratchSegments[segmentCount - 1].equals("..")) {
+ // Remove the last segment, if there is one and it is not "..". This
+ // means that the resulting PathFragment can still contain ".."
+ // segments at the beginning.
+ segmentCount--;
+ } else {
+ scratchSegments[segmentCount++] = segment;
+ }
+ } else {
+ scratchSegments[segmentCount++] = segment;
+ }
+ }
+
+ if (segmentCount == segments.length) {
+ // Optimization, no new PathFragment needs to be created.
+ return this;
+ }
+
+ return new PathFragment(driveLetter, isAbsolute,
+ subarray(scratchSegments, 0, segmentCount));
+ }
+
+ /**
+ * Returns the path formed by appending the relative or absolute path fragment
+ * {@code suffix} to this path.
+ *
+ * <p>If suffix is absolute, the current path will be ignored; otherwise, they
+ * will be concatenated. This is a purely syntactic operation, with no path
+ * normalization or I/O performed.
+ */
+ public PathFragment getRelative(PathFragment otherFragment) {
+ return otherFragment.isAbsolute()
+ ? otherFragment
+ : new PathFragment(this, otherFragment);
+ }
+
+ /**
+ * Returns the path formed by appending the relative or absolute string
+ * {@code path} to this path.
+ *
+ * <p>If the given path string is absolute, the current path will be ignored;
+ * otherwise, they will be concatenated. This is a purely syntactic operation,
+ * with no path normalization or I/O performed.
+ */
+ public PathFragment getRelative(String path) {
+ return getRelative(new PathFragment(path));
+ }
+
+ /**
+ * Returns the path formed by appending the single non-special segment
+ * "baseName" to this path.
+ *
+ * <p>You should almost always use {@link #getRelative} instead, which has
+ * the same performance characteristics if the given name is a valid base
+ * name, and which also works for '.', '..', and strings containing '/'.
+ *
+ * @throws IllegalArgumentException if {@code baseName} is not a valid base
+ * name according to {@link FileSystemUtils#checkBaseName}
+ */
+ public PathFragment getChild(String baseName) {
+ FileSystemUtils.checkBaseName(baseName);
+ baseName = StringCanonicalizer.intern(baseName);
+ String[] newSegments = new String[segments.length + 1];
+ System.arraycopy(segments, 0, newSegments, 0, segments.length);
+ newSegments[newSegments.length - 1] = baseName;
+ return new PathFragment(driveLetter, isAbsolute, newSegments);
+ }
+
+ /**
+ * Returns the last segment of this path, or "" for the empty fragment.
+ */
+ public String getBaseName() {
+ return (segments.length == 0) ? "" : segments[segments.length - 1];
+ }
+
+ /**
+ * Returns a relative path fragment to this path, relative to
+ * {@code ancestorDirectory}.
+ * <p>
+ * <code>x.relativeTo(z) == y</code> implies
+ * <code>z.getRelative(y) == x</code>.
+ * <p>
+ * For example, <code>"foo/bar/wiz".relativeTo("foo")</code>
+ * returns <code>"bar/wiz"</code>.
+ */
+ public PathFragment relativeTo(PathFragment ancestorDirectory) {
+ String[] ancestorSegments = ancestorDirectory.segments();
+ int ancestorLength = ancestorSegments.length;
+
+ if (isAbsolute != ancestorDirectory.isAbsolute()
+ || segments.length < ancestorLength) {
+ throw new IllegalArgumentException("PathFragment " + this
+ + " is not beneath " + ancestorDirectory);
+ }
+
+ for (int index = 0; index < ancestorLength; index++) {
+ if (!segments[index].equals(ancestorSegments[index])) {
+ throw new IllegalArgumentException("PathFragment " + this
+ + " is not beneath " + ancestorDirectory);
+ }
+ }
+
+ int length = segments.length - ancestorLength;
+ String[] resultSegments = subarray(segments, ancestorLength, length);
+ return new PathFragment('\0', false, resultSegments);
+ }
+
+ /**
+ * Returns a relative path fragment to this path, relative to {@code path}.
+ */
+ public PathFragment relativeTo(String path) {
+ return relativeTo(new PathFragment(path));
+ }
+
+ /**
+ * Returns a new PathFragment formed by appending {@code newName} to the
+ * parent directory. Null is returned iff this method is called on a
+ * PathFragment with zero segments. If {@code newName} designates an absolute path,
+ * the value of {@code this} will be ignored and a PathFragment corresponding to
+ * {@code newName} will be returned. This behavior is consistent with the behavior of
+ * {@link #getRelative(String)}.
+ */
+ public PathFragment replaceName(String newName) {
+ return segments.length == 0 ? null : getParentDirectory().getRelative(newName);
+ }
+
+ /**
+ * Returns a path representing the parent directory of this path,
+ * or null iff this Path represents the root of the filesystem.
+ *
+ * <p>Note: This method DOES NOT normalize ".." and "." path segments.
+ */
+ public PathFragment getParentDirectory() {
+ return segments.length == 0 ? null : subFragment(0, segments.length - 1);
+ }
+
+ /**
+ * Returns true iff {@code prefix}, considered as a list of path segments, is
+ * a prefix of {@code this}, and that they are both relative or both
+ * absolute.
+ *
+ * This is a reflexive, transitive, anti-symmetric relation (i.e. a partial
+ * order)
+ */
+ public boolean startsWith(PathFragment prefix) {
+ if (this.isAbsolute != prefix.isAbsolute ||
+ this.segments.length < prefix.segments.length ||
+ this.driveLetter != prefix.driveLetter) {
+ return false;
+ }
+ for (int i = 0, len = prefix.segments.length; i < len; i++) {
+ if (!this.segments[i].equals(prefix.segments[i])) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns true iff {@code suffix}, considered as a list of path segments, is
+ * relative and a suffix of {@code this}, or both are absolute and equal.
+ *
+ * This is a reflexive, transitive, anti-symmetric relation (i.e. a partial
+ * order)
+ */
+ public boolean endsWith(PathFragment suffix) {
+ if ((suffix.isAbsolute && !suffix.equals(this)) ||
+ this.segments.length < suffix.segments.length) {
+ return false;
+ }
+ int offset = this.segments.length - suffix.segments.length;
+ for (int i = 0; i < suffix.segments.length; i++) {
+ if (!this.segments[offset + i].equals(suffix.segments[i])) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String[] subarray(String[] array, int start, int length) {
+ String[] subarray = new String[length];
+ System.arraycopy(array, start, subarray, 0, length);
+ return subarray;
+ }
+
+ /**
+ * Returns a new path fragment that is a sub fragment of this one.
+ * The sub fragment begins at the specified <code>beginIndex</code> segment
+ * and ends at the segment at index <code>endIndex - 1</code>. Thus the number
+ * of segments in the new PathFragment is <code>endIndex - beginIndex</code>.
+ *
+ * @param beginIndex the beginning index, inclusive.
+ * @param endIndex the ending index, exclusive.
+ * @return the specified sub fragment, never null.
+ * @exception IndexOutOfBoundsException if the
+ * <code>beginIndex</code> is negative, or
+ * <code>endIndex</code> is larger than the length of
+ * this <code>String</code> object, or
+ * <code>beginIndex</code> is larger than
+ * <code>endIndex</code>.
+ */
+ public PathFragment subFragment(int beginIndex, int endIndex) {
+ int count = segments.length;
+ if ((beginIndex < 0) || (beginIndex > endIndex) || (endIndex > count)) {
+ throw new IndexOutOfBoundsException(String.format("path: %s, beginIndex: %d endIndex: %d",
+ toString(), beginIndex, endIndex));
+ }
+ boolean isAbsolute = (beginIndex == 0) && this.isAbsolute;
+ return ((beginIndex == 0) && (endIndex == count)) ? this :
+ new PathFragment(driveLetter, isAbsolute,
+ subarray(segments, beginIndex, endIndex - beginIndex));
+ }
+
+ /**
+ * Returns true iff the path represented by this object is absolute.
+ */
+ public boolean isAbsolute() {
+ return isAbsolute;
+ }
+
+ /**
+ * Returns the segments of this path fragment. This array should not be
+ * modified.
+ */
+ String[] segments() {
+ return segments;
+ }
+
+ public String windowsVolume() {
+ if (OS.getCurrent() != OS.WINDOWS) {
+ return "";
+ }
+ return (driveLetter != '\0') ? driveLetter + ":" : "";
+ }
+
+ /**
+ * Returns the number of segments in this path.
+ */
+ public int segmentCount() {
+ return segments.length;
+ }
+
+ /**
+ * Returns the specified segment of this path; index must be positive and
+ * less than numSegments().
+ */
+ public String getSegment(int index) {
+ return segments[index];
+ }
+
+ /**
+ * Returns the index of the first segment which equals one of the input values
+ * or {@link PathFragment#INVALID_SEGMENT} if none of the segments match.
+ */
+ public int getFirstSegment(Set<String> values) {
+ for (int i = 0; i < segments.length; i++) {
+ if (values.contains(segments[i])) {
+ return i;
+ }
+ }
+ return INVALID_SEGMENT;
+ }
+
+ /**
+ * Returns true iff this path contains uplevel references "..".
+ */
+ public boolean containsUplevelReferences() {
+ for (String segment : segments) {
+ if (segment.equals("..")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Given a path, returns the Windows drive letter ('X'), or an null character if no volume
+ * name was specified.
+ */
+ static char getWindowsDriveLetter(String path) {
+ if (OS.getCurrent() == OS.WINDOWS
+ && path.length() >= 2 && path.charAt(1) == ':' && Character.isLetter(path.charAt(0))) {
+ return Character.toUpperCase(path.charAt(0));
+ }
+ return '\0';
+ }
+
+ @Override
+ public int hashCode() {
+ int h = hashCode;
+ if (h == 0) {
+ h = isAbsolute ? 1 : 0;
+ for (String segment : segments) {
+ h = h * 31 + segment.hashCode();
+ }
+ hashCode = h;
+ }
+ return h;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof PathFragment)) {
+ return false;
+ }
+ PathFragment otherPath = (PathFragment) other;
+ return isAbsolute == otherPath.isAbsolute &&
+ Arrays.equals(otherPath.segments, segments);
+ }
+
+ /**
+ * Compares two PathFragments using the lexicographical order.
+ */
+ @Override
+ public int compareTo(PathFragment p2) {
+ if (isAbsolute != p2.isAbsolute) {
+ return isAbsolute ? -1 : 1;
+ }
+ PathFragment p1 = this;
+ String[] segments1 = p1.segments;
+ String[] segments2 = p2.segments;
+ int len1 = segments1.length;
+ int len2 = segments2.length;
+ int n = Math.min(len1, len2);
+ for (int i = 0; i < n; i++) {
+ String segment1 = segments1[i];
+ String segment2 = segments2[i];
+ if (!segment1.equals(segment2)) {
+ return segment1.compareTo(segment2);
+ }
+ }
+ return len1 - len2;
+ }
+
+ @Override
+ public String toString() {
+ return getPathString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/PathFragmentSerializationProxy.java b/src/main/java/com/google/devtools/build/lib/vfs/PathFragmentSerializationProxy.java
new file mode 100644
index 0000000..6e1b04d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/PathFragmentSerializationProxy.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectOutput;
+
+
+/**
+ * A helper proxy for serializing immutable {@link PathFragment} objects.
+ */
+public final class PathFragmentSerializationProxy implements Externalizable {
+ private String pathFragmentString;
+
+ public PathFragmentSerializationProxy(String pathFragmentString) {
+ this.pathFragmentString = pathFragmentString;
+ }
+
+ // For deserialization machinery.
+ public PathFragmentSerializationProxy() {
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ // Manual serialization gives us about a 30% reduction in size.
+ out.writeUTF(pathFragmentString);
+ }
+
+ @Override
+ public void readExternal(java.io.ObjectInput in) throws IOException {
+ this.pathFragmentString = in.readUTF();
+ }
+
+ private Object readResolve() {
+ return new PathFragment(pathFragmentString);
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ReadonlyFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/ReadonlyFileSystem.java
new file mode 100644
index 0000000..fc668db
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/ReadonlyFileSystem.java
@@ -0,0 +1,103 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An abstract partial implementation of FileSystem for read-only
+ * implementations.
+ *
+ * <p>Any ReadonlyFileSystem does not support the following:
+ * <ul>
+ * <li>{@link #createDirectory(Path)}</li>
+ * <li>{@link #createSymbolicLink(Path, PathFragment)}</li>
+ * <li>{@link #delete(Path)}</li>
+ * <li>{@link #getOutputStream(Path)}</li>
+ * <li>{@link #renameTo(Path, Path)}</li>
+ * <li>{@link #setExecutable(Path, boolean)}</li>
+ * <li>{@link #setLastModifiedTime(Path, long)}</li>
+ * <li>{@link #setWritable(Path, boolean)}</li>
+ * </ul>
+ * The above calls will always result in an {@link IOException}.
+ */
+public abstract class ReadonlyFileSystem extends FileSystem {
+
+ protected ReadonlyFileSystem() {
+ }
+
+ protected IOException modificationException() {
+ String longname = this.getClass().getName();
+ String shortname = longname.substring(longname.lastIndexOf(".") + 1);
+ return new IOException(
+ shortname + " does not support mutating operations");
+ }
+
+ @Override
+ protected OutputStream getOutputStream(Path path, boolean append) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected void setReadable(Path path, boolean readable) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected void setWritable(Path path, boolean writable) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected void setExecutable(Path path, boolean executable) {
+ throw new UnsupportedOperationException("setExecutable");
+ }
+
+ @Override
+ public boolean supportsModifications() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsSymbolicLinks() {
+ return false;
+ }
+
+ @Override
+ protected boolean createDirectory(Path path) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected void renameTo(Path sourcePath, Path targetPath) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected boolean delete(Path path) throws IOException {
+ throw modificationException();
+ }
+
+ @Override
+ protected void setLastModifiedTime(Path path, long newTime) throws IOException {
+ throw modificationException();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/RootedPath.java b/src/main/java/com/google/devtools/build/lib/vfs/RootedPath.java
new file mode 100644
index 0000000..c753aa6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/RootedPath.java
@@ -0,0 +1,116 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Preconditions;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * A {@link PathFragment} relative to a root, which is an absolute {@link Path}. Typically the root
+ * will be a package path entry.
+ *
+ * Two {@link RootedPath}s are considered equal iff they have equal roots and equal relative paths.
+ *
+ * TODO(bazel-team): refactor Artifact to use this instead of Root.
+ * TODO(bazel-team): use an opaque root representation so as to not expose the absolute path to
+ * clients via #asPath or #getRoot.
+ */
+public class RootedPath implements Serializable {
+
+ private final Path root;
+ private final PathFragment relativePath;
+ private final Path path;
+
+ /**
+ * Constructs a {@link RootedPath} from an absolute root path and a non-absolute relative path.
+ */
+ private RootedPath(Path root, PathFragment relativePath) {
+ Preconditions.checkState(!relativePath.isAbsolute(), "relativePath: %s root: %s", relativePath,
+ root);
+ this.root = root;
+ this.relativePath = relativePath.normalize();
+ this.path = root.getRelative(this.relativePath);
+ }
+
+ /**
+ * Returns a rooted path representing {@code relativePath} relative to {@code root}.
+ */
+ public static RootedPath toRootedPath(Path root, PathFragment relativePath) {
+ return new RootedPath(root, relativePath);
+ }
+
+ /**
+ * Returns a rooted path representing {@code path} under the root {@code root}.
+ */
+ public static RootedPath toRootedPath(Path root, Path path) {
+ Preconditions.checkState(path.startsWith(root), "path: %s root: %s", path, root);
+ return new RootedPath(root, path.relativeTo(root));
+ }
+
+ /**
+ * Returns a rooted path representing {@code path} under one of the package roots, or under the
+ * filesystem root if it's not under any package root.
+ */
+ public static RootedPath toRootedPathMaybeUnderRoot(Path path, Iterable<Path> packagePathRoots) {
+ for (Path root : packagePathRoots) {
+ if (path.startsWith(root)) {
+ return toRootedPath(root, path);
+ }
+ }
+ return toRootedPath(path.getFileSystem().getRootDirectory(), path);
+ }
+
+ public Path asPath() {
+ // Ideally, this helper method would not be needed. But Skyframe's FileFunction and
+ // DirectoryListingFunction need to do filesystem operations on the absolute path and
+ // Path#getRelative(relPath) is O(relPath.segmentCount()). Therefore we precompute the absolute
+ // path represented by this relative path.
+ return path;
+ }
+
+ public Path getRoot() {
+ return root;
+ }
+
+ /**
+ * Returns the (normalized) path relative to {@code #getRoot}.
+ */
+ public PathFragment getRelativePath() {
+ return relativePath;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof RootedPath)) {
+ return false;
+ }
+ RootedPath other = (RootedPath) obj;
+ return Objects.equals(root, other.root) && Objects.equals(relativePath, other.relativePath);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(root, relativePath);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + root.toString() + "]/[" + relativePath.toString() + "]";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystem.java
new file mode 100644
index 0000000..429de18
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystem.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile;
+
+import java.io.IOException;
+
+/**
+ * A file system that's capable of identifying paths residing outside its scope
+ * and using a delegator (such as {@link UnionFileSystem}) to re-route them
+ * to appropriate alternative file systems.
+ *
+ * <p>This is most useful for symlinks, which may ostensibly fall beneath some
+ * file system but resolve to paths outside that file system.
+ *
+ * <p>Note that we don't protect against cross-filesystem circular references.
+ * Therefore, care should be taken not to mix two scopable file systems that
+ * can reference each other. This theoretical safety cost is balanced by
+ * decreased code complexity requirements in implementations.
+ */
+public abstract class ScopeEscapableFileSystem extends FileSystem {
+
+ private FileSystem delegator;
+ protected final PathFragment scopeRoot;
+ private boolean enableScopeChecking = true; // Used for testing.
+
+ /**
+ * Instantiates a new ScopeEscapableFileSystem.
+ *
+ * @param scopeRoot the root path for the file system's scope. Any path
+ * that isn't beneath this one is considered out of scope according
+ * to {@link #inScope}. If null, scope checking is disabled. Note
+ * this is not the same thing as {@link FileSystem#rootPath}, which
+ * generally resolves to "/".
+ */
+ protected ScopeEscapableFileSystem(PathFragment scopeRoot) {
+ this.scopeRoot = scopeRoot;
+ }
+
+ @VisibleForTesting
+ void enableScopeChecking(boolean enable) {
+ this.enableScopeChecking = enable;
+ }
+
+ /**
+ * Sets the delegator used to resolve paths that fall outside this file
+ * system's scope.
+ *
+ * <p>This method is not thread safe. It's intended to be called during
+ * instance initialization, not during active usage. The only reason this
+ * isn't set as immutable state within the constructor is that the delegator
+ * may need a reference to this instance for its own constructor.
+ */
+ @ThreadHostile
+ public void setDelegator(FileSystem delegator) {
+ this.delegator = delegator;
+ }
+
+ /**
+ * Uses the delegator to convert a path fragment to a path that's bound
+ * to the file system that manages that path.
+ */
+ protected Path getDelegatedPath(PathFragment path) {
+ Preconditions.checkState(delegator != null);
+ return delegator.getPath(path);
+ }
+
+ /**
+ * Proxy for {@link FileSystem#resolveOneLink} that sends the input path
+ * through the delegator.
+ */
+ protected PathFragment resolveOneLinkWithDelegator(final PathFragment path) throws IOException {
+ Preconditions.checkState(delegator != null);
+ return delegator.resolveOneLink(getDelegatedPath(path));
+ }
+
+ /**
+ * Proxy for {@link FileSystem#stat} that sends the input path through
+ * the delegator.
+ */
+ protected FileStatus statWithDelegator(final PathFragment path, final boolean followSymlinks)
+ throws IOException {
+ Preconditions.checkState(delegator != null);
+ return delegator.stat(getDelegatedPath(path), followSymlinks);
+ }
+
+ /**
+ * Returns true if the given path is within this file system's scope, false
+ * otherwise.
+ *
+ * @param parentDepth the number of segments in the path's parent directory
+ * (only meaningful for paths that begin with ".."). The parent directory
+ * itself is assumed to be in scope.
+ * @param normalizedPath input path, expected to be normalized such that all
+ * ".." and "." segments are removed (with the exception of a possible
+ * prefix sequence of contiguous ".." segments)
+ */
+ protected boolean inScope(int parentDepth, PathFragment normalizedPath) {
+ if (scopeRoot == null || !enableScopeChecking) {
+ return true;
+ } else if (normalizedPath.isAbsolute()) {
+ return normalizedPath.startsWith(scopeRoot);
+ } else {
+ // Efficiency note: we're not accounting for "/scope/root/../root" paths here, i.e. paths
+ // that appear to go out of scope but ultimately stay within scope. This may result in
+ // unnecessary re-delegation back into the same FS. we're choosing to forgo that
+ // optimization under the assumption that such scenarios are rare and unimportant to
+ // overall performance. We can always enhance this if needed.
+ return parentDepth - leadingParentReferences(normalizedPath) >= scopeRoot.segmentCount();
+ }
+ }
+
+ /**
+ * Given a path that's normalized (no ".." or "." segments), except for a possible
+ * prefix sequence of contiguous ".." segments, returns the size of that prefix
+ * sequence.
+ *
+ * <p>Example allowed inputs: "/absolute/path", "relative/path", "../../relative/path".
+ * Example disallowed inputs: "/absolute/path/../path2", "relative/../path", "../relative/../p".
+ */
+ protected int leadingParentReferences(PathFragment normalizedPath) {
+ int leadingParentReferences = 0;
+ for (int i = 0; i < normalizedPath.segmentCount() &&
+ normalizedPath.getSegment(i).equals(".."); i++) {
+ leadingParentReferences++;
+ }
+ return leadingParentReferences;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Symlinks.java b/src/main/java/com/google/devtools/build/lib/vfs/Symlinks.java
new file mode 100644
index 0000000..ceb353a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/Symlinks.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+/**
+ * An enumeration for selecting between {@code stat}- and {@code lstat}-like
+ * behavior in various {@link Path} operations.
+ */
+public enum Symlinks {
+
+ /** Follow symbolic links; stat(2)-like behaviour. */
+ FOLLOW,
+
+ /** Do not follow symbolic links; lstat(2)-like behaviour. */
+ NOFOLLOW;
+
+ boolean toBoolean() { return this == FOLLOW; }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java
new file mode 100644
index 0000000..b349b53
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java
@@ -0,0 +1,419 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.util.StringTrie;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Presents a unified view of multiple virtual {@link FileSystem} instances, to which requests are
+ * delegated based on a {@link PathFragment} prefix mapping.
+ * If multiple prefixes apply to a given path, the *longest* (i.e. most specific) match is used.
+ * The order in which the delegates are specified does not influence the mapping.
+ *
+ * <p>Paths are preserved absolutely, contrary to how "mount" works, e.g.:
+ * /foo/bar maps to /foo/bar on the delegate, even if it is mounted at /foo.
+ *
+ * <p>For example:
+ * "/in" maps to InFileSystem, "/" maps to OtherFileSystem.
+ * Reading from "/in/base/BUILD" through the UnionFileSystem will delegate the read operation to
+ * InFileSystem, which will read "/in/base/BUILD" relative to its root.
+ * ("mount" behavior would remap it to "/base/BUILD" on the delegate).
+ *
+ * <p>Intra-filesystem symbolic links are resolved to their ultimate targets.
+ * Cross-filesystem links are not currently supported.
+ */
+@ThreadSafety.ThreadSafe
+public class UnionFileSystem extends FileSystem {
+
+ // Prefix trie index, allowing children to easily inherit prefix mappings
+ // of their parents.
+ // This does not currently handle unicode filenames.
+ private StringTrie<FileSystem> pathDelegate;
+
+ // True iff the filesystem can be modified. If false, mutating operations
+ // will throw UnsupportedOperationExceptions.
+ private final boolean readOnly;
+
+ /**
+ * Creates a new modifiable UnionFileSystem with prefix mappings
+ * specified by a map.
+ *
+ * @param prefixMapping map of path prefixes to {@link FileSystem}s
+ */
+ public UnionFileSystem(Map<PathFragment, FileSystem> prefixMapping,
+ FileSystem rootFileSystem) {
+ this(prefixMapping, rootFileSystem, /* readOnly */ false);
+ }
+
+ /**
+ * Creates a new modifiable or read-only UnionFileSystem with prefix mappings
+ * specified by a map.
+ *
+ * @param prefixMapping map of path prefixes to delegate {@link FileSystem}s
+ * @param rootFileSystem root for default requests; i.e. mapping of "/"
+ * @param readOnly if true, mutating operations will throw
+ */
+ public UnionFileSystem(Map<PathFragment, FileSystem> prefixMapping,
+ FileSystem rootFileSystem, boolean readOnly) {
+ super();
+ Preconditions.checkNotNull(prefixMapping);
+ Preconditions.checkNotNull(rootFileSystem);
+ Preconditions.checkArgument(rootFileSystem != this, "Circular root filesystem.");
+ Preconditions.checkArgument(
+ !prefixMapping.containsKey(PathFragment.EMPTY_FRAGMENT),
+ "Attempted to specify an explicit root prefix mapping; " +
+ "please use the rootFileSystem argument instead.");
+
+ this.readOnly = readOnly;
+ this.pathDelegate = new StringTrie<FileSystem>();
+
+ for (Map.Entry<PathFragment, FileSystem> prefix : prefixMapping.entrySet()) {
+ FileSystem delegate = prefix.getValue();
+ PathFragment prefixPath = prefix.getKey();
+
+ // Extra slash prevents within-directory mappings, which Path can't handle.
+ String path = prefixPath.getPathString();
+ pathDelegate.put(path, delegate);
+ }
+ pathDelegate.put(PathFragment.EMPTY_FRAGMENT.getPathString(), rootFileSystem);
+ }
+
+ /**
+ * Retrieves the filesystem delegate of a path mapping.
+ * Does not follow symlinks (but you can call on a path preprocessed with
+ * {@link #resolveSymbolicLinks} to support this use case).
+ *
+ * @param path the {@link Path} to map to a filesystem
+ * @throws IllegalArgumentException if no delegate exists for the path
+ */
+ protected FileSystem getDelegate(Path path) {
+ Preconditions.checkNotNull(path);
+
+ String pathString = path.getPathString();
+ FileSystem immediateDelegate = pathDelegate.get(pathString);
+
+ // Should never actually happen if the root delegate is present.
+ Preconditions.checkArgument(immediateDelegate != null, "No delegate filesystem exists for %s",
+ pathString);
+ return immediateDelegate;
+ }
+
+ // Associates the path with the root of the given delegate filesystem.
+ // Necessary to avoid null pointer problems inside of the delegates.
+ protected Path adjustPath(Path path, FileSystem delegate) {
+ return delegate.getPath(path.asFragment());
+ }
+
+ /**
+ * Follow a symbolic link once using the appropriate delegate filesystem, also
+ * resolving parent directory symlinks.
+ *
+ * @param path {@link Path} to the symbolic link
+ */
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ Preconditions.checkNotNull(path);
+ FileSystem delegate = getDelegate(path);
+ return delegate.readSymbolicLink(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected PathFragment resolveOneLink(Path path) throws IOException {
+ Preconditions.checkNotNull(path);
+ FileSystem delegate = getDelegate(path);
+ return delegate.resolveOneLink(adjustPath(path, delegate));
+ }
+
+ private void checkModifiable() {
+ if (!supportsModifications()) {
+ throw new UnsupportedOperationException(
+ "Modifications to this " + getClass().getSimpleName() + " are disabled.");
+ }
+ }
+
+ @Override
+ public boolean supportsModifications() {
+ return !readOnly;
+ }
+
+ @Override
+ public boolean supportsSymbolicLinks() {
+ return true;
+ }
+
+ @Override
+ public String getFileSystemType(Path path) {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getFileSystemType(path);
+ }
+
+ @Override
+ protected byte[] getMD5Digest(Path path) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getMD5Digest(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected boolean createDirectory(Path path) throws IOException {
+ checkModifiable();
+ // When creating the exact directory that is mapped,
+ // create it on both the parent's delegate and the path's delegate.
+ // This is necessary both for the parent to see the directory and for the
+ // delegate to use it.
+ // This is present to address this problematic case:
+ // / -> RootFs
+ // /foo -> FooFs
+ // mkdir /foo
+ // ls / ("foo" would be missing if not created on the parent)
+ // ls /foo (would fail if foo weren't also present on the child)
+ FileSystem delegate = getDelegate(path);
+ Path parent = path.getParentDirectory();
+ if (parent != null) {
+ FileSystem parentDelegate = getDelegate(parent);
+ if (parentDelegate != delegate) {
+ // There's a possibility it already exists on the parent, so don't die
+ // if the directory can't be created there.
+ parentDelegate.createDirectory(adjustPath(path, parentDelegate));
+ }
+ }
+ return delegate.createDirectory(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected long getFileSize(Path path, boolean followSymlinks) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getFileSize(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ protected boolean delete(Path path) throws IOException {
+ checkModifiable();
+ FileSystem delegate = getDelegate(path);
+ return delegate.delete(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getLastModifiedTime(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ protected void setLastModifiedTime(Path path, long newTime) throws IOException {
+ checkModifiable();
+ FileSystem delegate = getDelegate(path);
+ delegate.setLastModifiedTime(adjustPath(path, delegate), newTime);
+ }
+
+ @Override
+ protected boolean isSymbolicLink(Path path) {
+ FileSystem delegate = getDelegate(path);
+ path = adjustPath(path, delegate);
+ return delegate.isSymbolicLink(path);
+ }
+
+ @Override
+ protected boolean isDirectory(Path path, boolean followSymlinks) {
+ FileSystem delegate = getDelegate(path);
+ return delegate.isDirectory(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ protected boolean isFile(Path path, boolean followSymlinks) {
+ FileSystem delegate = getDelegate(path);
+ return delegate.isFile(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) throws IOException {
+ checkModifiable();
+ if (!supportsSymbolicLinks()) {
+ throw new UnsupportedOperationException(
+ "Attempted to create a symlink, but symlink support is disabled.");
+ }
+
+ FileSystem delegate = getDelegate(linkPath);
+ delegate.createSymbolicLink(adjustPath(linkPath, delegate), targetFragment);
+ }
+
+ @Override
+ protected boolean exists(Path path, boolean followSymlinks) {
+ FileSystem delegate = getDelegate(path);
+ return delegate.exists(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ protected FileStatus stat(final Path path, final boolean followSymlinks) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.stat(adjustPath(path, delegate), followSymlinks);
+ }
+
+ // Needs to be overridden for the delegation logic, because the
+ // UnixFileSystem implements statNullable and stat as separate codepaths.
+ // More generally, we wish to delegate all filesystem operations.
+ @Override
+ protected FileStatus statNullable(Path path, boolean followSymlinks) {
+ FileSystem delegate = getDelegate(path);
+ return delegate.statNullable(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ @Nullable
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.statIfFound(adjustPath(path, delegate), followSymlinks);
+ }
+
+ /**
+ * Retrieves the directory entries for the specified path under the assumption
+ * that {@code resolvedPath} is the resolved path of {@code path} in one of the
+ * underlying file systems.
+ *
+ * @param path the {@link Path} whose children are to be retrieved
+ */
+ @Override
+ protected Collection<Path> getDirectoryEntries(Path path) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ Path resolvedPath = adjustPath(path, delegate);
+ Collection<Path> entries = resolvedPath.getDirectoryEntries();
+ Collection<Path> result = Lists.newArrayListWithCapacity(entries.size());
+ for (Path entry : entries) {
+ result.add(path.getChild(entry.getBaseName()));
+ }
+ return result;
+ }
+
+ // No need for the more complex logic of getDirectoryEntries; it calls it implicitly.
+ @Override
+ protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.readdir(adjustPath(path, delegate), followSymlinks);
+ }
+
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.isReadable(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected void setReadable(Path path, boolean readable) throws IOException {
+ checkModifiable();
+ FileSystem delegate = getDelegate(path);
+ delegate.setReadable(adjustPath(path, delegate), readable);
+ }
+
+ @Override
+ protected boolean isWritable(Path path) throws IOException {
+ if (!supportsModifications()) {
+ return false;
+ }
+ FileSystem delegate = getDelegate(path);
+ return delegate.isWritable(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected void setWritable(Path path, boolean writable) throws IOException {
+ checkModifiable();
+ FileSystem delegate = getDelegate(path);
+ delegate.setWritable(adjustPath(path, delegate), writable);
+ }
+
+ @Override
+ protected boolean isExecutable(Path path) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.isExecutable(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected void setExecutable(Path path, boolean executable) throws IOException {
+ checkModifiable();
+ FileSystem delegate = getDelegate(path);
+ delegate.setExecutable(adjustPath(path, delegate), executable);
+ }
+
+ @Override
+ protected String getFastDigestFunctionType(Path path) {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getFastDigestFunctionType(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected byte[] getFastDigest(Path path) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getFastDigest(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected byte[] getxattr(Path path, String name, boolean followSymlinks) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getxattr(adjustPath(path, delegate), name, followSymlinks);
+ }
+
+ @Override
+ protected InputStream getInputStream(Path path) throws IOException {
+ FileSystem delegate = getDelegate(path);
+ return delegate.getInputStream(adjustPath(path, delegate));
+ }
+
+ @Override
+ protected OutputStream getOutputStream(Path path, boolean append) throws IOException {
+ checkModifiable();
+ FileSystem delegate = getDelegate(path);
+ return delegate.getOutputStream(adjustPath(path, delegate), append);
+ }
+
+ @Override
+ protected void renameTo(Path sourcePath, Path targetPath) throws IOException {
+ checkModifiable();
+ FileSystem sourceDelegate = getDelegate(sourcePath);
+ if (!sourceDelegate.supportsModifications()) {
+ throw new UnsupportedOperationException(
+ "The filesystem for the source path "
+ + sourcePath.getPathString() + " does not support modifications.");
+ }
+ sourcePath = adjustPath(sourcePath, sourceDelegate);
+
+ FileSystem targetDelegate = getDelegate(targetPath);
+ if (!targetDelegate.supportsModifications()) {
+ throw new UnsupportedOperationException(
+ "The filesystem for the target path "
+ + targetPath.getPathString() + " does not support modifications.");
+ }
+ targetPath = adjustPath(targetPath, targetDelegate);
+
+ if (sourceDelegate == targetDelegate) {
+ // Easy, same filesystem.
+ sourceDelegate.renameTo(sourcePath, targetPath);
+ return;
+ } else {
+ // Copy across filesystems, then delete.
+ // copyFile throws on failure, so delete will never be reached if it fails.
+ FileSystemUtils.copyFile(sourcePath, targetPath);
+ sourceDelegate.delete(sourcePath);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnixFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/UnixFileSystem.java
new file mode 100644
index 0000000..c7dd3a8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/UnixFileSystem.java
@@ -0,0 +1,414 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.unix.ErrnoFileStatus;
+import com.google.devtools.build.lib.unix.FilesystemUtils;
+import com.google.devtools.build.lib.unix.FilesystemUtils.Dirents;
+import com.google.devtools.build.lib.unix.FilesystemUtils.ReadTypes;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * This class implements the FileSystem interface using direct calls to the
+ * UNIX filesystem.
+ */
+// Not final only for testing.
+@ThreadSafe
+public class UnixFileSystem extends AbstractFileSystem {
+
+ public static final UnixFileSystem INSTANCE = new UnixFileSystem();
+ /**
+ * Eager implementation of FileStatus for file systems that have an atomic
+ * stat(2) syscall. A proxy for {@link com.google.devtools.build.lib.unix.FileStatus}.
+ * Note that isFile and getLastModifiedTime have slightly different meanings
+ * between UNIX and VFS.
+ */
+ @VisibleForTesting
+ protected static class UnixFileStatus implements FileStatus {
+
+ private final com.google.devtools.build.lib.unix.FileStatus status;
+
+ UnixFileStatus(com.google.devtools.build.lib.unix.FileStatus status) {
+ this.status = status;
+ }
+
+ @Override
+ public boolean isFile() { return !isDirectory() && !isSymbolicLink(); }
+
+ @Override
+ public boolean isDirectory() { return status.isDirectory(); }
+
+ @Override
+ public boolean isSymbolicLink() { return status.isSymbolicLink(); }
+
+ @Override
+ public long getSize() { return status.getSize(); }
+
+ @Override
+ public long getLastModifiedTime() {
+ return (status.getLastModifiedTime() * 1000)
+ + (status.getFractionalLastModifiedTime() / 1000000);
+ }
+
+ @Override
+ public long getLastChangeTime() {
+ return (status.getLastChangeTime() * 1000)
+ + (status.getFractionalLastChangeTime() / 1000000);
+ }
+
+ @Override
+ public long getNodeId() {
+ // Note that we may want to include more information in this id number going forward,
+ // especially the device number.
+ return status.getInodeNumber();
+ }
+
+ int getPermissions() { return status.getPermissions(); }
+
+ @Override
+ public String toString() { return status.toString(); }
+ }
+
+ @Override
+ protected Collection<Path> getDirectoryEntries(Path path) throws IOException {
+ String name = path.getPathString();
+ String[] entries;
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ entries = FilesystemUtils.readdir(name);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, name);
+ }
+ Collection<Path> result = new ArrayList<>(entries.length);
+ for (String entry : entries) {
+ result.add(path.getChild(entry));
+ }
+ return result;
+ }
+
+ @Override
+ protected PathFragment resolveOneLink(Path path) throws IOException {
+ // Beware, this seemingly simple code belies the complex specification of
+ // FileSystem.resolveOneLink().
+ return stat(path, false).isSymbolicLink()
+ ? readSymbolicLink(path)
+ : null;
+ }
+
+ /**
+ * Converts from {@link com.google.devtools.build.lib.unix.FilesystemUtils.Dirents.Type} to
+ * {@link com.google.devtools.build.lib.vfs.Dirent.Type}.
+ */
+ private static Dirent.Type convertToDirentType(Dirents.Type type) {
+ switch (type) {
+ case FILE:
+ return Dirent.Type.FILE;
+ case DIRECTORY:
+ return Dirent.Type.DIRECTORY;
+ case SYMLINK:
+ return Dirent.Type.SYMLINK;
+ case UNKNOWN:
+ return Dirent.Type.UNKNOWN;
+ default:
+ throw new IllegalArgumentException("Unknown type " + type);
+ }
+ }
+
+ @Override
+ protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
+ String name = path.getPathString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ Dirents unixDirents = FilesystemUtils.readdir(name,
+ followSymlinks ? ReadTypes.FOLLOW : ReadTypes.NOFOLLOW);
+ Preconditions.checkState(unixDirents.hasTypes());
+ List<Dirent> dirents = Lists.newArrayListWithCapacity(unixDirents.size());
+ for (int i = 0; i < unixDirents.size(); i++) {
+ dirents.add(new Dirent(unixDirents.getName(i),
+ convertToDirentType(unixDirents.getType(i))));
+ }
+ return dirents;
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, name);
+ }
+ }
+
+ @Override
+ protected UnixFileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ String name = path.getPathString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return new UnixFileStatus(followSymlinks
+ ? FilesystemUtils.stat(name)
+ : FilesystemUtils.lstat(name));
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name);
+ }
+ }
+
+ // Like stat(), but returns null instead of throwing.
+ // This is a performance optimization in the case where clients
+ // catch and don't re-throw.
+ @Override
+ protected UnixFileStatus statNullable(Path path, boolean followSymlinks) {
+ String name = path.getPathString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ ErrnoFileStatus stat = followSymlinks
+ ? FilesystemUtils.errnoStat(name)
+ : FilesystemUtils.errnoLstat(name);
+ return stat.hasError() ? null : new UnixFileStatus(stat);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name);
+ }
+ }
+
+ @Override
+ protected boolean exists(Path path, boolean followSymlinks) {
+ return statNullable(path, followSymlinks) != null;
+ }
+
+ /**
+ * Return true iff the {@code stat} of {@code path} resulted in an {@code ENOENT}
+ * or {@code ENOTDIR} error.
+ */
+ @Override
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
+ String name = path.getPathString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ ErrnoFileStatus stat = followSymlinks
+ ? FilesystemUtils.errnoStat(name)
+ : FilesystemUtils.errnoLstat(name);
+ if (!stat.hasError()) {
+ return new UnixFileStatus(stat);
+ }
+ int errno = stat.getErrno();
+ if (errno == ErrnoFileStatus.ENOENT || errno == ErrnoFileStatus.ENOTDIR) {
+ return null;
+ }
+ // This should not return -- we are calling stat here just to throw the proper exception.
+ // However, since there may be transient IO errors, we cannot guarantee that an exception will
+ // be thrown.
+ // TODO(bazel-team): Extract the exception-construction code and make it visible separately in
+ // FilesystemUtils to avoid having to do a duplicate stat call.
+ return stat(path, followSymlinks);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name);
+ }
+ }
+
+ @Override
+ protected boolean isDirectory(Path path, boolean followSymlinks) {
+ UnixFileStatus stat = statNullable(path, followSymlinks);
+ return stat != null && stat.isDirectory();
+ }
+
+ @Override
+ protected boolean isFile(Path path, boolean followSymlinks) {
+ // Note, FileStatus.isFile means *regular* file whereas Path.isFile may
+ // mean special file too, so we don't return FileStatus.isFile here.
+ UnixFileStatus status = statNullable(path, followSymlinks);
+ return status != null && !(status.isSymbolicLink() || status.isDirectory());
+ }
+
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ return (stat(path, true).getPermissions() & 0400) != 0;
+ }
+
+ @Override
+ protected boolean isWritable(Path path) throws IOException {
+ return (stat(path, true).getPermissions() & 0200) != 0;
+ }
+
+ @Override
+ protected boolean isExecutable(Path path) throws IOException {
+ return (stat(path, true).getPermissions() & 0100) != 0;
+ }
+
+ /**
+ * Adds or remove the bits specified in "permissionBits" to the permission
+ * mask of the file specified by {@code path}. If the argument {@code add} is
+ * true, the specified permissions are added, otherwise they are removed.
+ *
+ * @throws IOException if there was an error writing the file's metadata
+ */
+ private void modifyPermissionBits(Path path, int permissionBits, boolean add)
+ throws IOException {
+ synchronized (path) {
+ int oldMode = stat(path, true).getPermissions();
+ int newMode = add ? (oldMode | permissionBits) : (oldMode & ~permissionBits);
+ FilesystemUtils.chmod(path.toString(), newMode);
+ }
+ }
+
+ @Override
+ protected void setReadable(Path path, boolean readable) throws IOException {
+ modifyPermissionBits(path, 0400, readable);
+ }
+
+ @Override
+ protected void setWritable(Path path, boolean writable) throws IOException {
+ modifyPermissionBits(path, 0200, writable);
+ }
+
+ @Override
+ protected void setExecutable(Path path, boolean executable) throws IOException {
+ modifyPermissionBits(path, 0111, executable);
+ }
+
+ @Override
+ protected void chmod(Path path, int mode) throws IOException {
+ synchronized (path) {
+ FilesystemUtils.chmod(path.toString(), mode);
+ }
+ }
+
+ @Override
+ public boolean supportsModifications() {
+ return true;
+ }
+
+ @Override
+ public boolean supportsSymbolicLinks() {
+ return true;
+ }
+
+ @Override
+ protected boolean createDirectory(Path path) throws IOException {
+ synchronized (path) {
+ // Note: UNIX mkdir(2), FilesystemUtils.mkdir() and createDirectory all
+ // have different ways of representing failure!
+ if (FilesystemUtils.mkdir(path.toString(), 0777)) {
+ return true; // successfully created
+ }
+
+ // false => EEXIST: something is already in the way (file/dir/symlink)
+ if (isDirectory(path, false)) {
+ return false; // directory already existed
+ } else {
+ throw new IOException(path + " (File exists)");
+ }
+ }
+ }
+
+ @Override
+ protected void createSymbolicLink(Path linkPath, PathFragment targetFragment)
+ throws IOException {
+ synchronized (linkPath) {
+ FilesystemUtils.symlink(targetFragment.toString(), linkPath.toString());
+ }
+ }
+
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ String name = path.toString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return new PathFragment(FilesystemUtils.readlink(name));
+ } catch (IOException e) {
+ // EINVAL => not a symbolic link. Anything else is a real error.
+ throw e.getMessage().endsWith("(Invalid argument)") ? new NotASymlinkException(path) : e;
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_LINK, name);
+ }
+ }
+
+ @Override
+ protected void renameTo(Path sourcePath, Path targetPath) throws IOException {
+ synchronized (sourcePath) {
+ FilesystemUtils.rename(sourcePath.toString(), targetPath.toString());
+ }
+ }
+
+ @Override
+ protected long getFileSize(Path path, boolean followSymlinks) throws IOException {
+ return stat(path, followSymlinks).getSize();
+ }
+
+ @Override
+ protected boolean delete(Path path) throws IOException {
+ String name = path.toString();
+ long startTime = Profiler.nanoTimeMaybe();
+ synchronized (path) {
+ try {
+ return FilesystemUtils.remove(name);
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_DELETE, name);
+ }
+ }
+ }
+
+ @Override
+ protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException {
+ return stat(path, followSymlinks).getLastModifiedTime();
+ }
+
+ @Override
+ protected boolean isSymbolicLink(Path path) {
+ UnixFileStatus stat = statNullable(path, false);
+ return stat != null && stat.isSymbolicLink();
+ }
+
+ @Override
+ protected void setLastModifiedTime(Path path, long newTime) throws IOException {
+ synchronized (path) {
+ if (newTime == -1L) { // "now"
+ FilesystemUtils.utime(path.toString(), true, 0, 0);
+ } else {
+ // newTime > MAX_INT => -ve unixTime
+ int unixTime = (int) (newTime / 1000);
+ FilesystemUtils.utime(path.toString(), false, unixTime, unixTime);
+ }
+ }
+ }
+
+ @Override
+ protected byte[] getxattr(Path path, String name, boolean followSymlinks) throws IOException {
+ String pathName = path.toString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return followSymlinks
+ ? FilesystemUtils.getxattr(pathName, name) : FilesystemUtils.lgetxattr(pathName, name);
+ } catch (UnsupportedOperationException e) {
+ // getxattr() syscall is not supported by the underlying filesystem (it returned ENOTSUP).
+ // Per method contract, treat this as ENODATA.
+ return null;
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_XATTR, pathName);
+ }
+ }
+
+ @Override
+ protected byte[] getMD5Digest(Path path) throws IOException {
+ String name = path.toString();
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ return FilesystemUtils.md5sum(name).asBytes();
+ } finally {
+ profiler.logSimpleTask(startTime, ProfilerTask.VFS_MD5, name);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
new file mode 100644
index 0000000..d512abc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
@@ -0,0 +1,785 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+
+/**
+ * Implementation of a subset of UNIX-style file globbing, expanding "*" and "?" as wildcards, but
+ * not [a-z] ranges.
+ *
+ * <p><code>**</code> gets special treatment in include patterns. If it is used as a complete path
+ * segment it matches the filenames in subdirectories recursively.
+ */
+public final class UnixGlob {
+ private UnixGlob() {}
+
+ private static List<Path> globInternal(Path base, Collection<String> patterns,
+ Collection<String> excludePatterns,
+ boolean excludeDirectories,
+ Predicate<Path> dirPred,
+ boolean checkForInterruption,
+ FilesystemCalls syscalls,
+ ThreadPoolExecutor threadPool)
+ throws IOException, InterruptedException {
+ GlobVisitor visitor = (threadPool == null)
+ ? new GlobVisitor(checkForInterruption)
+ : new GlobVisitor(threadPool, checkForInterruption);
+ return visitor.glob(base, patterns, excludePatterns, excludeDirectories, dirPred, syscalls);
+ }
+
+ private static Future<List<Path>> globAsyncInternal(Path base, Collection<String> patterns,
+ Collection<String> excludePatterns,
+ boolean excludeDirectories,
+ Predicate<Path> dirPred,
+ FilesystemCalls syscalls,
+ boolean checkForInterruption,
+ ThreadPoolExecutor threadPool) {
+ GlobVisitor visitor = (threadPool == null)
+ ? new GlobVisitor(checkForInterruption)
+ : new GlobVisitor(threadPool, checkForInterruption);
+ return visitor.globAsync(base, patterns, excludePatterns, excludeDirectories, dirPred,
+ syscalls);
+ }
+
+ /**
+ * Checks that each pattern is valid, splits it into segments and checks
+ * that each segment contains only valid wildcards.
+ *
+ * @return list of segment arrays
+ */
+ private static List<String[]> checkAndSplitPatterns(Collection<String> patterns) {
+ List<String[]> list = Lists.newArrayListWithCapacity(patterns.size());
+ for (String pattern : patterns) {
+ String error = checkPatternForError(pattern);
+ if (error != null) {
+ throw new IllegalArgumentException(error + " (in glob pattern '" + pattern + "')");
+ }
+ Iterable<String> segments = Splitter.on('/').split(pattern);
+ list.add(Iterables.toArray(segments, String.class));
+ }
+ return list;
+ }
+
+ /**
+ * @return whether or not {@code pattern} contains illegal characters
+ */
+ public static String checkPatternForError(String pattern) {
+ if (pattern.isEmpty()) {
+ return "pattern cannot be empty";
+ }
+ if (pattern.charAt(0) == '/') {
+ return "pattern cannot be absolute";
+ }
+ for (int i = 0; i < pattern.length(); i++) {
+ char c = pattern.charAt(i);
+ switch (c) {
+ case '(': case ')':
+ case '{': case '}':
+ case '[': case ']':
+ return "illegal character '" + c + "'";
+ }
+ }
+ Iterable<String> segments = Splitter.on('/').split(pattern);
+ for (String segment : segments) {
+ if (segment.isEmpty()) {
+ return "empty segment not permitted";
+ }
+ if (segment.equals(".") || segment.equals("..")) {
+ return "segment '" + segment + "' not permitted";
+ }
+ if (segment.contains("**") && !segment.equals("**")) {
+ return "recursive wildcard must be its own segment";
+ }
+ }
+ return null;
+ }
+
+ private static boolean excludedOnMatch(Path path, List<String[]> excludePatterns,
+ int idx, Cache<String, Pattern> cache,
+ Predicate<Path> dirPred) {
+ for (String[] excludePattern : excludePatterns) {
+ String text = path.getBaseName();
+ if (idx == excludePattern.length
+ && matches(excludePattern[idx - 1], text, cache)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the exclude patterns in {@code excludePatterns} which could
+ * apply to the children of {@code base}
+ *
+ * @param idx index into {@code excludePatterns} for the part of the pattern
+ * which might match {@code base}
+ */
+ private static List<String[]> getRelevantExcludes(
+ final Path base, List<String[]> excludePatterns, final int idx,
+ final Cache<String, Pattern> cache) {
+ if (excludePatterns.isEmpty()) {
+ return excludePatterns;
+ }
+ List<String[]> list = new ArrayList<>();
+ for (String[] patterns : excludePatterns) {
+ if (excludePatternMatches(patterns, idx, base, cache)) {
+ list.add(patterns);
+ }
+ }
+ return list;
+ }
+
+ /**
+ * @param patterns a list of patterns
+ * @param idx index into {@code patterns}
+ */
+ private static boolean excludePatternMatches(String[] patterns, int idx,
+ Path base,
+ Cache<String, Pattern> cache) {
+ if (idx == 0) {
+ return true;
+ }
+ String text = base.getBaseName();
+ return patterns.length > idx && matches(patterns[idx - 1], text, cache);
+ }
+
+ /**
+ * Calls {@link #matches(String, String, Cache) matches(pattern, str, null)}
+ */
+ public static boolean matches(String pattern, String str) {
+ return matches(pattern, str, null);
+ }
+
+ /**
+ * Returns whether {@code str} matches the glob pattern {@code pattern}. This
+ * method may use the {@code patternCache} to speed up the matching process.
+ *
+ * @param pattern a glob pattern
+ * @param str the string to match
+ * @param patternCache a cache from patterns to compiled Pattern objects, or
+ * {@code null} to skip caching
+ */
+ public static boolean matches(String pattern, String str,
+ Cache<String, Pattern> patternCache) {
+ if (pattern.length() == 0 || str.length() == 0) {
+ return false;
+ }
+
+ // Common case: **
+ if (pattern.equals("**")) {
+ return true;
+ }
+
+ // Common case: *
+ if (pattern.equals("*")) {
+ return true;
+ }
+
+ // If a filename starts with '.', this char must be matched explicitly.
+ if (str.charAt(0) == '.' && pattern.charAt(0) != '.') {
+ return false;
+ }
+
+ // Common case: *.xyz
+ if (pattern.charAt(0) == '*' && pattern.lastIndexOf('*') == 0) {
+ return str.endsWith(pattern.substring(1));
+ }
+ // Common case: xyz*
+ int lastIndex = pattern.length() - 1;
+ // The first clause of this if statement is unnecessary, but is an
+ // optimization--charAt runs faster than indexOf.
+ if (pattern.charAt(lastIndex) == '*' && pattern.indexOf('*') == lastIndex) {
+ return str.startsWith(pattern.substring(0, lastIndex));
+ }
+
+ Pattern regex = patternCache == null ? null : patternCache.getIfPresent(pattern);
+ if (regex == null) {
+ regex = makePatternFromWildcard(pattern);
+ if (patternCache != null) {
+ patternCache.put(pattern, regex);
+ }
+ }
+ return regex.matcher(str).matches();
+ }
+
+ /**
+ * Returns a regular expression implementing a matcher for "pattern", in which
+ * "*" and "?" are wildcards.
+ *
+ * <p>e.g. "foo*bar?.java" -> "foo.*bar.\\.java"
+ */
+ private static Pattern makePatternFromWildcard(String pattern) {
+ StringBuilder regexp = new StringBuilder();
+ for(int i = 0, len = pattern.length(); i < len; i++) {
+ char c = pattern.charAt(i);
+ switch(c) {
+ case '*':
+ regexp.append(".*");
+ break;
+ case '?':
+ regexp.append('.');
+ break;
+ //escape the regexp special characters that are allowed in wildcards
+ case '^': case '$': case '|': case '+':
+ case '{': case '}': case '[': case ']':
+ case '\\': case '.':
+ regexp.append('\\');
+ regexp.append(c);
+ break;
+ default:
+ regexp.append(c);
+ break;
+ }
+ }
+ return Pattern.compile(regexp.toString());
+ }
+
+ /**
+ * Filesystem calls required for glob().
+ */
+ public static interface FilesystemCalls {
+ /**
+ * Get directory entries and their types.
+ */
+ Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException;
+
+ /**
+ * Return the stat() for the given path, or null.
+ */
+ FileStatus statNullable(Path path, Symlinks symlinks);
+ }
+
+ public static FilesystemCalls DEFAULT_SYSCALLS = new FilesystemCalls() {
+ @Override
+ public Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException {
+ return path.readdir(symlinks);
+ }
+
+ @Override
+ public FileStatus statNullable(Path path, Symlinks symlinks) {
+ return path.statNullable(symlinks);
+ }
+ };
+
+ public static final AtomicReference<FilesystemCalls> DEFAULT_SYSCALLS_REF =
+ new AtomicReference<FilesystemCalls>(DEFAULT_SYSCALLS);
+
+ public static Builder forPath(Path path) {
+ return new Builder(path);
+ }
+
+ /**
+ * Builder class for UnixGlob.
+ *
+ *
+ */
+ public static class Builder {
+ private Path base;
+ private List<String> patterns;
+ private List<String> excludes;
+ private boolean excludeDirectories;
+ private Predicate<Path> pathFilter;
+ private ThreadPoolExecutor threadPool;
+ private AtomicReference<? extends FilesystemCalls> syscalls =
+ new AtomicReference<>(DEFAULT_SYSCALLS);
+
+ /**
+ * Creates a glob builder with the given base path.
+ */
+ public Builder(Path base) {
+ this.base = base;
+ this.patterns = Lists.newArrayList();
+ this.excludes = Lists.newArrayList();
+ this.excludeDirectories = false;
+ this.pathFilter = Predicates.alwaysTrue();
+ }
+
+ /**
+ * Adds a pattern to include to the glob builder.
+ *
+ * <p>For a description of the syntax of the patterns, see {@link UnixGlob}.
+ */
+ public Builder addPattern(String pattern) {
+ this.patterns.add(pattern);
+ return this;
+ }
+
+ /**
+ * Adds a pattern to include to the glob builder.
+ *
+ * <p>For a description of the syntax of the patterns, see {@link UnixGlob}.
+ */
+ public Builder addPatterns(String... patterns) {
+ for (String pattern : patterns) {
+ this.patterns.add(pattern);
+ }
+ return this;
+ }
+
+ /**
+ * Adds a pattern to include to the glob builder.
+ *
+ * <p>For a description of the syntax of the patterns, see {@link UnixGlob}.
+ */
+ public Builder addPatterns(Collection<String> patterns) {
+ this.patterns.addAll(patterns);
+ return this;
+ }
+
+ /**
+ * Sets the FilesystemCalls interface to use on this glob().
+ */
+ public Builder setFilesystemCalls(AtomicReference<? extends FilesystemCalls> syscalls) {
+ this.syscalls = (syscalls == null)
+ ? new AtomicReference<FilesystemCalls>(DEFAULT_SYSCALLS)
+ : syscalls;
+ return this;
+ }
+
+ /**
+ * Adds patterns to exclude from the results to the glob builder.
+ *
+ * <p>For a description of the syntax of the patterns, see {@link UnixGlob}.
+ */
+ public Builder addExcludes(String... excludes) {
+ this.excludes.addAll(Arrays.asList(excludes));
+ return this;
+ }
+
+ /**
+ * Adds patterns to exclude from the results to the glob builder.
+ *
+ * <p>For a description of the syntax of the patterns, see {@link UnixGlob}.
+ */
+ public Builder addExcludes(Collection<String> excludes) {
+ this.excludes.addAll(excludes);
+ return this;
+ }
+
+ /**
+ * If set to true, directories are not returned in the glob result.
+ */
+ public Builder setExcludeDirectories(boolean excludeDirectories) {
+ this.excludeDirectories = excludeDirectories;
+ return this;
+ }
+
+
+ /**
+ * Sets the threadpool to use for parallel glob evaluation.
+ * If unset, evaluation is done in-thread.
+ */
+ public Builder setThreadPool(ThreadPoolExecutor pool) {
+ this.threadPool = pool;
+ return this;
+ }
+
+
+ /**
+ * If set, the given predicate is called for every directory
+ * encountered. If it returns false, the corresponding item is not
+ * returned in the output and directories are not traversed either.
+ */
+ public Builder setDirectoryFilter(Predicate<Path> pathFilter) {
+ this.pathFilter = pathFilter;
+ return this;
+ }
+
+ /**
+ * Executes the glob.
+ */
+ public List<Path> glob() throws IOException {
+ try {
+ return globInternal(base, patterns, excludes, excludeDirectories, pathFilter, false,
+ syscalls.get(), threadPool);
+ } catch (InterruptedException e) {
+ // cannot happen, since we told globInternal not to throw
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Executes the glob.
+ *
+ * @throws InterruptedException if the thread is interrupted.
+ */
+ public List<Path> globInterruptible() throws IOException, InterruptedException {
+ return globInternal(base, patterns, excludes, excludeDirectories, pathFilter, true,
+ syscalls.get(), threadPool);
+ }
+
+ /**
+ * Executes the glob asynchronously.
+ *
+ * @param checkForInterrupt if the returned future may throw
+ * InterruptedException.
+ */
+ public Future<List<Path>> globAsync(boolean checkForInterrupt) {
+ return globAsyncInternal(base, patterns, excludes, excludeDirectories, pathFilter,
+ syscalls.get(), checkForInterrupt, threadPool);
+ }
+ }
+
+ /**
+ * Adapts the result of the glob visitation as a Future.
+ */
+ private static class GlobFuture extends AbstractFuture<List<Path>> {
+ private final GlobVisitor visitor;
+ private final boolean checkForInterrupt;
+ private final Object completionLock = new Object();
+
+ public GlobFuture(GlobVisitor visitor, boolean checkForInterrupt) {
+ this.visitor = visitor;
+ this.checkForInterrupt = checkForInterrupt;
+ }
+
+ private List<Path> getSafe() throws InterruptedException, ExecutionException {
+ boolean interrupted = false;
+ try {
+ while (true) {
+ try {
+ return super.get();
+ } catch (InterruptedException e) {
+ if (checkForInterrupt) {
+ throw e;
+ }
+ interrupted = true;
+ } catch (ExecutionException e) {
+ // The checkForInterrupt logic is already handled in
+ // GlobVisitor#waitForCompletion().
+ Throwables.propagateIfInstanceOf(e.getCause(), InterruptedException.class);
+ throw e;
+ }
+ }
+ } finally {
+ if (!checkForInterrupt && interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ @Override
+ public List<Path> get() throws InterruptedException, ExecutionException {
+ synchronized (completionLock) {
+ if (isDone()) {
+ return getSafe();
+ }
+
+ try {
+ visitor.waitForCompletion();
+ super.set(Lists.newArrayList(visitor.results));
+ } catch (Throwable t) {
+ super.setException(t);
+ }
+ List<Path> result = getSafe();
+ return result;
+ }
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ synchronized (completionLock) {
+ if (isDone()) {
+ return false;
+ }
+
+ visitor.interrupt();
+ return true;
+ }
+ }
+ }
+
+ /**
+ * GlobVisitor executes a glob using parallelism, which is useful when
+ * the glob() requires many readdir() calls on high latency filesystems.
+ */
+ private static final class GlobVisitor extends AbstractQueueVisitor {
+ // These collections are used across workers and must therefore be
+ // thread-safe.
+
+ private final static String THREAD_NAME = "GlobVisitor";
+
+ private final Collection<Path> results =
+ Collections.synchronizedSet(Sets.<Path>newTreeSet());
+ private final Cache<String, Pattern> cache = CacheBuilder.newBuilder().build(
+ new CacheLoader<String, Pattern>() {
+ @Override
+ public Pattern load(String wildcard) {
+ return makePatternFromWildcard(wildcard);
+ }
+ });
+
+ private final GlobFuture result;
+ private final boolean failFastOnInterrupt;
+
+ public GlobVisitor(ThreadPoolExecutor executor, boolean failFastOnInterrupt) {
+ super(executor, /*shutdownOnCompletion=*/false, /*failFastOnException=*/true,
+ /*failFastOnInterrupt=*/failFastOnInterrupt);
+ this.result = new GlobFuture(this, failFastOnInterrupt);
+ this.failFastOnInterrupt = failFastOnInterrupt;
+ }
+
+ public GlobVisitor(boolean failFastOnInterrupt) {
+ super(/*concurrent=*/false, 0, 0, 0, null, /*failFastOnException=*/true,
+ /*failFastOnInterrupt=*/failFastOnInterrupt, THREAD_NAME);
+ this.result = new GlobFuture(this, failFastOnInterrupt);
+ this.failFastOnInterrupt = failFastOnInterrupt;
+ }
+
+ /**
+ * Performs wildcard globbing: returns the sorted list of filenames that match any of
+ * {@code patterns} relative to {@code base}, but which do not match {@code excludePatterns}.
+ * Directories are traversed if and only if they match {@code dirPred}. The predicate is also
+ * called for the root of the traversal.
+ *
+ * <p>Patterns may include "*" and "?", but not "[a-z]".
+ *
+ * <p><code>**</code> gets special treatment in include patterns. If it is
+ * used as a complete path segment it matches the filenames in
+ * subdirectories recursively.
+ *
+ * @throws IllegalArgumentException if any glob or exclude pattern
+ * {@linkplain #checkPatternForError(String) contains errors} or if
+ * any exclude pattern segment contains <code>**</code> or if any
+ * include pattern segment contains <code>**</code> but not equal to
+ * it.
+ */
+ public List<Path> glob(Path base, Collection<String> patterns,
+ Collection<String> excludePatterns, boolean excludeDirectories,
+ Predicate<Path> dirPred, FilesystemCalls syscalls)
+ throws IOException, InterruptedException {
+ try {
+ return globAsync(base, patterns, excludePatterns, excludeDirectories,
+ dirPred, syscalls).get();
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Throwables.propagateIfPossible(cause, IOException.class);
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Future<List<Path>> globAsync(Path base, Collection<String> patterns,
+ Collection<String> excludePatterns, boolean excludeDirectories,
+ Predicate<Path> dirPred, FilesystemCalls syscalls) {
+
+ FileStatus baseStat = syscalls.statNullable(base, Symlinks.FOLLOW);
+ if (baseStat == null || patterns.isEmpty()) {
+ return Futures.immediateFuture(Collections.<Path>emptyList());
+ }
+
+ List<String[]> splitPatterns = checkAndSplitPatterns(patterns);
+ List<String[]> splitExcludes = checkAndSplitPatterns(excludePatterns);
+
+ // We do a dumb loop, even though it will likely duplicate work
+ // (e.g., readdir calls). In order to optimize, we would need
+ // to keep track of which patterns shared sub-patterns and which did not
+ // (for example consider the glob [*/*.java, sub/*.java, */*.txt]).
+ for (String[] splitPattern : splitPatterns) {
+ queueGlob(base, baseStat.isDirectory(), splitPattern, 0, excludeDirectories,
+ splitExcludes, 0, results, cache, dirPred, syscalls);
+ }
+
+ return result;
+ }
+
+ protected void waitForCompletion() throws IOException, InterruptedException {
+ try {
+ super.work(failFastOnInterrupt);
+ } catch (InterruptedException e) {
+ if (failFastOnInterrupt) {
+ throw e;
+ } else {
+ Thread.currentThread().interrupt();
+ }
+ } catch (IORuntimeException e) {
+ if (Thread.interrupted()) {
+ // As per the contract of AbstractQueueVisitor#work, if an unchecked exception is thrown
+ // and the build is interrupted, the thrown exception is what will be rethrown. Since the
+ // user presumably wanted to interrupt the build, we ignore the thrown IORuntimeException
+ // (which doesn't indicate a programming bug) and throw an InterruptedException.
+ if (failFastOnInterrupt) {
+ throw new InterruptedException();
+ }
+ Thread.currentThread().interrupt();
+ }
+ throw e.getCauseIOException();
+ }
+ }
+
+ private void queueGlob(final Path base, final boolean baseIsDir,
+ final String[] patternParts, final int idx,
+ final boolean excludeDirectories,
+ final List<String[]> excludePatterns,
+ final int excludeIdx,
+ final Collection<Path> results, final Cache<String, Pattern> cache,
+ final Predicate<Path> dirPred, final FilesystemCalls syscalls) {
+ enqueue(new Runnable() {
+ @Override
+ public void run() {
+ Profiler.instance().startTask(ProfilerTask.VFS_GLOB, this);
+ try {
+ reallyGlob(base, baseIsDir, patternParts, idx, excludeDirectories,
+ excludePatterns, excludeIdx, results, cache, dirPred, syscalls);
+ } catch (IOException e) {
+ throw new IORuntimeException(e);
+ } catch (InterruptedException e) {
+ // When we get to this point, the main thread already knows that the
+ // globbing has been interrupted, so we do not need to report the
+ // error condition.
+ } finally {
+ Profiler.instance().completeTask(ProfilerTask.VFS_GLOB);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "%s glob(include=[%s], exclude=[%s], exclude_directories=%s)",
+ base.getPathString(),
+ "\"" + Joiner.on("\", \"").join(patternParts) + "\"",
+ "\"" + Joiner.on("\", \"").join(excludePatterns) + "\"",
+ excludeDirectories);
+ }
+ });
+
+ }
+
+ /**
+ * Expressed in Haskell:
+ * <pre>
+ * reallyGlob base [] = { base }
+ * reallyGlob base [x:xs] = union { reallyGlob(f, xs) | f results "base/x" }
+ * </pre>
+ */
+ private void reallyGlob(Path base, boolean baseIsDir, String[] patternParts, int idx,
+ boolean excludeDirectories,
+ List<String[]> excludePatterns,
+ int excludeIdx,
+ Collection<Path> results, Cache<String, Pattern> cache,
+ Predicate<Path> dirPred,
+ FilesystemCalls syscalls) throws IOException, InterruptedException {
+ if (failFastOnInterrupt && Thread.interrupted()) {
+ throw new InterruptedException();
+ }
+
+ if (baseIsDir && !dirPred.apply(base)) {
+ return;
+ }
+
+ if (idx == patternParts.length) { // Base case.
+ if (!(excludeDirectories && baseIsDir) &&
+ !excludedOnMatch(base, excludePatterns, excludeIdx, cache, dirPred)) {
+ results.add(base);
+ }
+
+ return;
+ }
+
+ if (!baseIsDir) {
+ // Nothing to find here.
+ return;
+ }
+
+ List<String[]> relevantExcludes
+ = getRelevantExcludes(base, excludePatterns, excludeIdx, cache);
+ final String pattern = patternParts[idx];
+
+ // ** is special: it can match nothing at all.
+ // For example, x/** matches x, **/y matches y, and x/**/y matches x/y.
+ if ("**".equals(pattern)) {
+ queueGlob(base, baseIsDir, patternParts, idx + 1, excludeDirectories,
+ excludePatterns, excludeIdx, results, cache, dirPred, syscalls);
+ }
+
+ if (!pattern.contains("*") && !pattern.contains("?")) {
+ // We do not need to do a readdir in this case, just a stat.
+ Path child = base.getChild(pattern);
+ FileStatus status = syscalls.statNullable(child, Symlinks.FOLLOW);
+ if (status == null || (!status.isDirectory() && !status.isFile())) {
+ // The file is a dangling symlink, fifo, does not exist, etc.
+ return;
+ }
+
+ boolean childIsDir = status.isDirectory();
+
+ queueGlob(child, childIsDir, patternParts, idx + 1, excludeDirectories,
+ relevantExcludes, excludeIdx + 1, results, cache, dirPred, syscalls);
+ return;
+ }
+
+ Collection<Dirent> dents = syscalls.readdir(base, Symlinks.FOLLOW);
+
+ for (Dirent dent : dents) {
+ Dirent.Type type = dent.getType();
+ if (type == Dirent.Type.UNKNOWN) {
+ // The file is a dangling symlink, fifo, etc.
+ continue;
+ }
+ boolean childIsDir = (type == Dirent.Type.DIRECTORY);
+ String text = dent.getName();
+ Path child = base.getChild(text);
+
+ if ("**".equals(pattern)) {
+ // Recurse without shifting the pattern.
+ if (childIsDir) {
+ queueGlob(child, childIsDir, patternParts, idx, excludeDirectories,
+ relevantExcludes, excludeIdx + 1, results, cache, dirPred, syscalls);
+ }
+ }
+ if (matches(pattern, text, cache)) {
+ // Recurse and consume one segment of the pattern.
+ if (childIsDir) {
+ queueGlob(child, childIsDir, patternParts, idx + 1, excludeDirectories,
+ relevantExcludes, excludeIdx + 1, results, cache, dirPred, syscalls);
+ } else {
+ // Instead of using an async call, just repeat the base case above.
+ if (idx + 1 == patternParts.length &&
+ !excludedOnMatch(child, relevantExcludes, excludeIdx + 1, cache, dirPred)) {
+ results.add(child);
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ZipFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/ZipFileSystem.java
new file mode 100644
index 0000000..558263d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/ZipFileSystem.java
@@ -0,0 +1,253 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.base.Predicate;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+/**
+ * A FileSystem that provides a read-only filesystem view on a zip file.
+ * Inherits the constraints imposed by ReadonlyFileSystem.
+ */
+@ThreadSafe
+public class ZipFileSystem extends ReadonlyFileSystem {
+
+ private final ZipFile zipFile;
+
+ /**
+ * The sole purpose of this field is to hold a strong reference to all leaf
+ * {@link Path}s which have a non-null "entry" field, preventing them from
+ * being garbage-collected. (The leaf paths hold string references to their
+ * parents, so we don't need to include them here.)
+ *
+ * <p>This is necessary because {@link Path}s may be recycled when they
+ * become unreachable, but the ZipFileSystem uses them to hold the {@link
+ * ZipEntry} for that path, if any. Without this additional strong
+ * reference, ZipEntries would seem to "disappear" during garbage collection.
+ */
+ @SuppressWarnings("unused")
+ private final Object paths;
+
+ /**
+ * Constructs a ZipFileSystem from a zip file identified with a given path.
+ */
+ public ZipFileSystem(Path zipPath) throws IOException {
+ // Throw some more specific exceptions than ZipFile does.
+ // We do this using File instead of Path, in case zipPath points to an
+ // InMemoryFileSystem. This case is not really supported but
+ // can occur in tests.
+ File file = zipPath.getPathFile();
+ if (!file.exists()) {
+ throw new FileNotFoundException(String.format("File '%s' does not exist", zipPath));
+ }
+ if (!file.isFile()) {
+ throw new IOException(String.format("'%s' is not a file", zipPath));
+ }
+ if (!file.canRead()) {
+ throw new IOException(String.format("File '%s' is not readable", zipPath));
+ }
+
+ this.zipFile = new ZipFile(file);
+ this.paths = populatePathTree();
+ }
+
+ // ZipPath extends Path with a set-once ZipEntry field.
+ // TODO(bazel-team): (2009) Delete class ZipPath, and perform the
+ // Path-to-ZipEntry lookup in {@link #zipEntry} and {@link
+ // #getDirectoryEntries}. Then this field becomes redundant.
+ @ThreadSafe
+ private static class ZipPath extends Path {
+ /**
+ * Non-null iff this file/directory exists. Set by setZipEntry for files
+ * explicitly mentioned in the zipfile's table of contents, or implicitly
+ * an ancestor of them.
+ */
+ ZipEntry entry = null;
+
+ // Root path.
+ ZipPath(ZipFileSystem fileSystem) {
+ super(fileSystem);
+ }
+
+ // Non-root paths.
+ ZipPath(ZipFileSystem fileSystem, String name, ZipPath parent) {
+ super(fileSystem, name, parent);
+ }
+
+ void setZipEntry(ZipEntry entry) {
+ if (this.entry != null) {
+ throw new IllegalStateException("setZipEntry(" + entry
+ + ") called twice!");
+ }
+ this.entry = entry;
+
+ // Ensure all parents of this path have a directory ZipEntry:
+ for (ZipPath path = (ZipPath) getParentDirectory();
+ path != null && path.entry == null;
+ path = (ZipPath) path.getParentDirectory()) {
+ // Note, the ZipEntry for the root path is called "//", but that's ok.
+ path.setZipEntry(new ZipEntry(path + "/")); // trailing "/" => isDir
+ }
+ }
+
+ @Override
+ protected ZipPath createChildPath(String childName) {
+ return new ZipPath((ZipFileSystem) getFileSystem(), childName, this);
+ }
+ }
+
+ /**
+ * Scans the Zip file and associates a ZipEntry with each filename
+ * (ZipPath) that is mentioned in the table of contents. Returns a
+ * collection of all corresponding Paths.
+ */
+ private Collection<Path> populatePathTree() {
+ Collection<Path> paths = new ArrayList<>();
+ for (ZipEntry entry : Collections.list(zipFile.entries())) {
+ PathFragment frag = new PathFragment(entry.getName());
+ Path path = rootPath.getRelative(frag);
+ paths.add(path);
+ ((ZipPath) path).setZipEntry(entry);
+ }
+ return paths;
+ }
+
+ @Override
+ public String getFileSystemType(Path path) {
+ return "zipfs";
+ }
+
+ @Override
+ protected Path createRootPath() {
+ return new ZipPath(this);
+ }
+
+ /** Returns the ZipEntry associated with a given path name, if any. */
+ private static ZipEntry zipEntry(Path path) {
+ return ((ZipPath) path).entry;
+ }
+
+ /** Like zipEntry, but throws FileNotFoundException unless path exists. */
+ private static ZipEntry zipEntryNonNull(Path path)
+ throws FileNotFoundException {
+ ZipEntry zipEntry = zipEntry(path);
+ if (zipEntry == null) {
+ throw new FileNotFoundException(path + " (No such file or directory)");
+ }
+ return zipEntry;
+ }
+
+ @Override
+ protected InputStream getInputStream(Path path) throws IOException {
+ return zipFile.getInputStream(zipEntryNonNull(path));
+ }
+
+ @Override
+ protected Collection<Path> getDirectoryEntries(Path path)
+ throws IOException {
+ zipEntryNonNull(path);
+ final Collection<Path> result = new ArrayList<>();
+ ((ZipPath) path).applyToChildren(new Predicate<Path>() {
+ @Override
+ public boolean apply(Path child) {
+ if (zipEntry(child) != null) {
+ result.add(child);
+ }
+ return true;
+ }
+ });
+ return result;
+ }
+
+ @Override
+ protected boolean exists(Path path, boolean followSymlinks) {
+ return zipEntry(path) != null;
+ }
+
+ @Override
+ protected boolean isDirectory(Path path, boolean followSymlinks) {
+ ZipEntry entry = zipEntry(path);
+ return entry != null && entry.isDirectory();
+ }
+
+ @Override
+ protected boolean isFile(Path path, boolean followSymlinks) {
+ ZipEntry entry = zipEntry(path);
+ return entry != null && !entry.isDirectory();
+ }
+
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ zipEntryNonNull(path);
+ return true;
+ }
+
+ @Override
+ protected boolean isWritable(Path path) throws IOException {
+ zipEntryNonNull(path);
+ return false;
+ }
+
+ @Override
+ protected boolean isExecutable(Path path) throws IOException {
+ zipEntryNonNull(path);
+ return false;
+ }
+
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ zipEntryNonNull(path);
+ throw new NotASymlinkException(path);
+ }
+
+ @Override
+ protected long getFileSize(Path path, boolean followSymlinks)
+ throws IOException {
+ return zipEntryNonNull(path).getSize();
+ }
+
+ @Override
+ protected long getLastModifiedTime(Path path, boolean followSymlinks)
+ throws FileNotFoundException {
+ return zipEntryNonNull(path).getTime();
+ }
+
+ @Override
+ protected boolean isSymbolicLink(Path path) {
+ return false;
+ }
+
+ @Override
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) {
+ try {
+ return stat(path, followSymlinks);
+ } catch (FileNotFoundException e) {
+ return null;
+ } catch (IOException e) {
+ // getLastModifiedTime can only throw FileNotFoundException, which is what stat uses.
+ throw new IllegalStateException (e);
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java
new file mode 100644
index 0000000..fff562f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * This interface represents a mutable file stored in an InMemoryFileSystem.
+ */
+@ThreadSafe
+public abstract class FileInfo extends InMemoryContentInfo {
+ protected FileInfo(Clock clock) {
+ super(clock);
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return false;
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return false;
+ }
+
+ @Override
+ public boolean isFile() {
+ return true;
+ }
+
+ protected abstract byte[] readContent() throws IOException;
+
+ protected abstract OutputStream getOutputStream(boolean append) throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java
new file mode 100644
index 0000000..0e7de71
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java
@@ -0,0 +1,212 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+
+/**
+ * This interface defines the function directly supported by the "files" stored
+ * in a InMemoryFileSystem. This corresponds to a file or inode in UNIX: it
+ * doesn't have a path (it could have many paths due to hard links, or none if
+ * it's unlinked, i.e. garbage).
+ *
+ * <p>This class is thread-safe: instances may be accessed and modified from
+ * concurrent threads. Subclasses must preserve this property.
+ */
+@ThreadSafe
+public abstract class InMemoryContentInfo implements ScopeEscapableStatus {
+
+ private final Clock clock;
+
+ /**
+ * Stores the time when the file was last modified. This is atomically updated
+ * whenever the file changes, so all accesses must be synchronized.
+ */
+ private long lastModifiedTime;
+
+ /**
+ * Stores the time when the file information was changed. This is atomically updated
+ * whenever the file changes, so all accesses must be synchronized.
+ */
+ private long lastChangeTime;
+
+ /**
+ * Modifications to the isWritable field do not update the lastModifiedTime,
+ * so we don't need to synchronize; using volatile is enough.
+ */
+ private volatile boolean isWritable = true;
+ private volatile boolean isExecutable = false;
+ private volatile boolean isReadable = true;
+
+ protected InMemoryContentInfo(Clock clock) {
+ this(clock, true);
+ }
+
+ protected InMemoryContentInfo(Clock clock, boolean isMutable) {
+ this.clock = clock;
+ // When we create the file, it is modified.
+ if (isMutable) {
+ markModificationTime();
+ }
+ }
+
+ /**
+ * Returns true if the current object is a directory.
+ */
+ @Override
+ public abstract boolean isDirectory();
+
+ /**
+ * Returns true if the current object is a symbolic link.
+ */
+ @Override
+ public abstract boolean isSymbolicLink();
+
+ /**
+ * Returns true if the current object is a regular file.
+ */
+ @Override
+ public abstract boolean isFile();
+
+ /**
+ * Returns the size of the entity denoted by the current object. For files,
+ * this is the length in bytes, for directories the number of children. The
+ * size of links is unspecified.
+ */
+ @Override
+ public abstract long getSize() throws IOException;
+
+ /**
+ * Returns the time when the entity denoted by the current object was last
+ * modified.
+ */
+ @Override
+ public synchronized long getLastModifiedTime() {
+ return lastModifiedTime;
+ }
+
+ /**
+ * Returns the time when the entity denoted by the current object was last
+ * changed.
+ */
+ @Override
+ public synchronized long getLastChangeTime() {
+ return lastChangeTime;
+ }
+
+ /**
+ * Returns the file node id for the given instance, emulated by the
+ * identity hash code.
+ */
+ @Override
+ public long getNodeId() {
+ return System.identityHashCode(this);
+ }
+
+ /**
+ * Sets the time that denotes when the entity denoted by this object was last
+ * modified.
+ */
+ synchronized void setLastModifiedTime(long newTime) {
+ lastModifiedTime = newTime;
+ markChangeTime();
+ }
+
+ /**
+ * Sets the last modification and change times to the current time.
+ */
+ protected synchronized void markModificationTime() {
+ Preconditions.checkState(clock != null);
+ lastModifiedTime = clock.currentTimeMillis();
+ lastChangeTime = lastModifiedTime;
+ }
+
+ /**
+ * Sets the last change time to the current time.
+ */
+ protected synchronized void markChangeTime() {
+ Preconditions.checkState(clock != null);
+ lastChangeTime = clock.currentTimeMillis();
+ }
+
+ /**
+ * Sets whether the current file is readable.
+ */
+ boolean isReadable() {
+ return isReadable;
+ }
+
+ /**
+ * Returns whether the current file is readable.
+ */
+ void setReadable(boolean readable) {
+ isReadable = readable;
+ }
+
+
+ /**
+ * Sets whether the current file is writable.
+ */
+ void setWritable(boolean writable) {
+ isWritable = writable;
+ markChangeTime();
+ }
+
+ /**
+ * Returns whether the current file is writable.
+ */
+ boolean isWritable() {
+ return isWritable;
+ }
+
+ /**
+ * Sets whether the current file is executable.
+ */
+ void setExecutable(boolean executable) {
+ isExecutable = executable;
+ markChangeTime();
+ }
+
+ /**
+ * Returns whether the current file is executable.
+ */
+ boolean isExecutable() {
+ return isExecutable;
+ }
+
+ @Override
+ public boolean outOfScope() {
+ return false;
+ }
+
+ @Override
+ public PathFragment getEscapingPath() {
+ return null;
+ }
+
+ /**
+ * Called just before this inode is moved.
+ *
+ * @param targetPath where the inode is relocated.
+ * @throws IOException
+ */
+ protected void movedTo(Path targetPath) throws IOException {
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryDirectoryInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryDirectoryInfo.java
new file mode 100644
index 0000000..400490b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryDirectoryInfo.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.common.collect.MapMaker;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * This class represents a directory stored in an {@link InMemoryFileSystem}.
+ */
+@ThreadSafe
+class InMemoryDirectoryInfo extends InMemoryContentInfo {
+
+ private final ConcurrentMap<String, InMemoryContentInfo> directoryContent =
+ new MapMaker().makeMap();
+
+ InMemoryDirectoryInfo(Clock clock) {
+ this(clock, true);
+ }
+
+ protected InMemoryDirectoryInfo(Clock clock, boolean isMutable) {
+ super(clock, isMutable);
+ if (isMutable) {
+ setExecutable(true);
+ }
+ }
+
+ /**
+ * Adds a new child to this directory under the name "name". Callers must
+ * ensure that no entry of that name exists already.
+ */
+ synchronized void addChild(String name, InMemoryContentInfo inode) {
+ if (name == null) { throw new NullPointerException(); }
+ if (inode == null) { throw new NullPointerException(); }
+ if (directoryContent.put(name, inode) != null) {
+ throw new IllegalArgumentException("File already exists: " + name);
+ }
+ markModificationTime();
+ }
+
+ /**
+ * Does a directory lookup, and returns the "inode" for the specified name.
+ * Returns null if the child is not found.
+ */
+ synchronized InMemoryContentInfo getChild(String name) {
+ return directoryContent.get(name);
+ }
+
+ /**
+ * Removes a previously existing child from the directory specified by this
+ * object.
+ */
+ synchronized void removeChild(String name) {
+ if (directoryContent.remove(name) == null) {
+ throw new IllegalArgumentException(name + " is not a member of this directory");
+ }
+ markModificationTime();
+ }
+
+ /**
+ * This function returns the content of a directory. For now, it returns a set
+ * to reflect the semantics of the value returned (ie. unordered, no
+ * duplicates). If thats too slow, it should be changed later.
+ */
+ Set<String> getAllChildren() {
+ return directoryContent.keySet();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return true;
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return false;
+ }
+
+ @Override
+ public boolean isFile() {
+ return false;
+ }
+
+ /**
+ * In the InMemory hierarchy, the getSize on a directory always returns the
+ * number of children in the directory.
+ */
+ @Override
+ public long getSize() {
+ return directoryContent.size();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java
new file mode 100644
index 0000000..f88285d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * InMemoryFileInfo manages file contents by storing them entirely in memory.
+ */
+@ThreadSafe
+public class InMemoryFileInfo extends FileInfo {
+
+ /**
+ * Updates to the content must atomically update the lastModifiedTime. So all
+ * accesses to this field must be synchronized.
+ */
+ protected byte[] content;
+
+ protected InMemoryFileInfo(Clock clock) {
+ super(clock);
+ content = new byte[0]; // New files start out empty.
+ }
+
+ @Override
+ public synchronized long getSize() {
+ return content.length;
+ }
+
+ @Override
+ public synchronized byte[] readContent() {
+ return content.clone();
+ }
+
+ private synchronized void setContent(byte[] newContent) {
+ content = newContent;
+ markModificationTime();
+ }
+
+ @Override
+ protected synchronized OutputStream getOutputStream(boolean append)
+ throws IOException {
+ OutputStream out = new ByteArrayOutputStream() {
+ private boolean closed = false;
+
+ @Override
+ public void write(byte[] data) throws IOException {
+ Preconditions.checkState(!closed);
+ super.write(data);
+ }
+
+ @Override
+ public void write(int dataByte) {
+ Preconditions.checkState(!closed);
+ super.write(dataByte);
+ }
+
+ @Override
+ public void write(byte[] data, int offset, int length) {
+ Preconditions.checkState(!closed);
+ super.write(data, offset, length);
+ }
+
+ @Override
+ public void close() {
+ flush();
+ closed = true;
+ }
+
+ @Override
+ public void flush() {
+ setContent(toByteArray().clone());
+ }
+ };
+
+ if (append) {
+ out.write(readContent());
+ }
+ return out;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java
new file mode 100644
index 0000000..8a3b823
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java
@@ -0,0 +1,920 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.unix.FileAccessException;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.JavaClock;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.ScopeEscapableFileSystem;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.Stack;
+
+import javax.annotation.Nullable;
+
+/**
+ * This class provides a complete in-memory file system.
+ *
+ * <p>Naming convention: we use "path" for all {@link Path} variables, since these
+ * represent *names* and we use "node" or "inode" for InMemoryContentInfo
+ * variables, since these correspond to inodes in the UNIX file system.
+ *
+ * <p>The code is structured to be as similar to the implementation of UNIX "namei"
+ * as is reasonably possibly. This provides a firm reference point for many
+ * concepts and makes compatibility easier to achieve.
+ *
+ * <p>As a scope-escapable file system, this class supports re-delegation of symbolic links
+ * that escape its root. This is done through the use of {@link OutOfScopeFileStatus}
+ * and {@link OutOfScopeDirectoryStatus} objects, which may be returned by
+ * getDirectory, pathWalk, and scopeLimitedStat. Any code that calls one of these
+ * methods (either directly or indirectly) is obligated to check the possibility
+ * that its info represents an out-of-scope path. Lack of such a check will result
+ * in unchecked runtime exceptions upon any request for status data (as well as
+ * possible logical errors).
+ */
+@ThreadSafe
+public class InMemoryFileSystem extends ScopeEscapableFileSystem {
+
+ private final Clock clock;
+
+ // The root inode (a directory).
+ private final InMemoryDirectoryInfo rootInode;
+
+ // Maximum number of traversals before ELOOP is thrown.
+ private static final int MAX_TRAVERSALS = 256;
+
+ /**
+ * Creates a new InMemoryFileSystem with scope checking disabled (all paths are considered to be
+ * within scope) and a default clock.
+ */
+ public InMemoryFileSystem() {
+ this(new JavaClock());
+ }
+
+ /**
+ * Creates a new InMemoryFileSystem with scope checking disabled (all
+ * paths are considered to be within scope).
+ */
+ public InMemoryFileSystem(Clock clock) {
+ this(clock, null);
+ }
+
+ /**
+ * Creates a new InMemoryFileSystem with scope checking bound to
+ * scopeRoot, i.e. any path that's not below scopeRoot is considered
+ * to be out of scope.
+ */
+ protected InMemoryFileSystem(Clock clock, PathFragment scopeRoot) {
+ super(scopeRoot);
+ this.clock = clock;
+ this.rootInode = new InMemoryDirectoryInfo(clock);
+ rootInode.addChild(".", rootInode);
+ rootInode.addChild("..", rootInode);
+ }
+
+ /**
+ * The errors that {@link InMemoryFileSystem} might issue for different sorts of IO failures.
+ */
+ public enum Error {
+ ENOENT("No such file or directory"),
+ EACCES("Permission denied"),
+ ENOTDIR("Not a directory"),
+ EEXIST("File exists"),
+ EBUSY("Device or resource busy"),
+ ENOTEMPTY("Directory not empty"),
+ EISDIR("Is a directory"),
+ ELOOP("Too many levels of symbolic links");
+
+ private final String message;
+
+ private Error(String message) {
+ this.message = message;
+ }
+
+ @Override
+ public String toString() {
+ return message;
+ }
+
+ /** Implemented by exceptions that contain the extra info of which Error caused them. */
+ private static interface WithError {
+ Error getError();
+ }
+
+ /**
+ * The exceptions below extend their parent classes in order to additionally store the error
+ * that caused them. However, they must impersonate their parents to any outside callers,
+ * including in their toString() method, which prints the class name followed by the exception
+ * method. This method returns the same value as the toString() method of a {@link Throwable}'s
+ * parent would, so that the child class can have the same toString() value.
+ */
+ private static String parentThrowableToString(Throwable obj) {
+ String s = obj.getClass().getSuperclass().getName();
+ String message = obj.getLocalizedMessage();
+ return (message != null) ? (s + ": " + message) : s;
+ }
+
+ private static class IOExceptionWithError extends IOException implements WithError {
+ private final Error errorCode;
+
+ private IOExceptionWithError(String message, Error errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ @Override
+ public Error getError() {
+ return errorCode;
+ }
+
+ @Override
+ public String toString() {
+ return parentThrowableToString(this);
+ }
+ }
+
+
+ private static class FileNotFoundExceptionWithError
+ extends FileNotFoundException implements WithError {
+ private final Error errorCode;
+
+ private FileNotFoundExceptionWithError(String message, Error errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ @Override
+ public Error getError() {
+ return errorCode;
+ }
+
+ @Override
+ public String toString() {
+ return parentThrowableToString(this);
+ }
+ }
+
+
+ private static class FileAccessExceptionWithError
+ extends FileAccessException implements WithError {
+ private final Error errorCode;
+
+ private FileAccessExceptionWithError(String message, Error errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ @Override
+ public Error getError() {
+ return errorCode;
+ }
+
+ @Override
+ public String toString() {
+ return parentThrowableToString(this);
+ }
+ }
+
+ /**
+ * Returns a new IOException for the error. The exception message
+ * contains 'path', and is consistent with the messages returned by
+ * c.g.common.unix.FilesystemUtils.
+ */
+ public IOException exception(Path path) throws IOException {
+ String m = path + " (" + message + ")";
+ if (this == EACCES) {
+ throw new FileAccessExceptionWithError(m, this);
+ } else if (this == ENOENT) {
+ throw new FileNotFoundExceptionWithError(m, this);
+ } else {
+ throw new IOExceptionWithError(m, this);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>If <code>/proc/mounts</code> does not exist return {@code "inmemoryfs"}.
+ */
+ @Override
+ public String getFileSystemType(Path path) {
+ return path.getRelative("/proc/mounts").exists() ? super.getFileSystemType(path) : "inmemoryfs";
+ }
+
+ /****************************************************************************
+ * "Kernel" primitives: basic directory lookup primitives, in topological
+ * order.
+ */
+
+ /**
+ * Unlinks the entry 'child' from its existing parent directory 'dir'. Dual to
+ * insert. This succeeds even if 'child' names a non-empty directory; we need
+ * that for renameTo. 'child' must be a member of its parent directory,
+ * however. Fails if the directory was read-only.
+ */
+ private void unlink(InMemoryDirectoryInfo dir, String child, Path errorPath)
+ throws IOException {
+ if (!dir.isWritable()) { throw Error.EACCES.exception(errorPath); }
+ dir.removeChild(child);
+ }
+
+ /**
+ * Inserts inode 'childInode' into the existing directory 'dir' under the
+ * specified 'name'. Dual to unlink. Fails if the directory was read-only.
+ */
+ private void insert(InMemoryDirectoryInfo dir, String child,
+ InMemoryContentInfo childInode, Path errorPath)
+ throws IOException {
+ if (!dir.isWritable()) { throw Error.EACCES.exception(errorPath); }
+ dir.addChild(child, childInode);
+ }
+
+ /**
+ * Given an existing directory 'dir', looks up 'name' within it and returns
+ * its inode. Assumes the file exists, unless 'create', in which case it will
+ * try to create it. May fail with ENOTDIR, EACCES, ENOENT. Error messages
+ * will be reported against file 'path'.
+ */
+ private InMemoryContentInfo directoryLookup(InMemoryContentInfo dir,
+ String name,
+ boolean create,
+ Path path) throws IOException {
+ if (!dir.isDirectory()) { throw Error.ENOTDIR.exception(path); }
+ InMemoryDirectoryInfo imdi = (InMemoryDirectoryInfo) dir;
+ if (!imdi.isExecutable()) { throw Error.EACCES.exception(path); }
+ InMemoryContentInfo child = imdi.getChild(name);
+ if (child == null) {
+ if (!create) {
+ throw Error.ENOENT.exception(path);
+ } else {
+ child = makeFileInfo(clock, path.asFragment());
+ insert(imdi, name, child, path);
+ }
+ }
+ return child;
+ }
+
+ /**
+ * Low-level path-to-inode lookup routine. Analogous to path_walk() in many
+ * UNIX kernels. Given 'path', walks the directory tree from the root,
+ * resolving all symbolic links, and returns the designated inode.
+ *
+ * <p>If 'create' is false, the inode must exist; otherwise, it will be created
+ * and added to its parent directory, which must exist.
+ *
+ * <p>Iff the given path escapes this file system's scope, the returned value
+ * is an {@link OutOfScopeFileStatus} instance. Any code that calls this method
+ * needs to check for that possibility (via {@link ScopeEscapableStatus#outOfScope}).
+ *
+ * <p>May fail with ENOTDIR, ENOENT, EACCES, ELOOP.
+ */
+ private synchronized InMemoryContentInfo pathWalk(Path path, boolean create)
+ throws IOException {
+ // Implementation note: This is where we check for out-of-scope symlinks and
+ // trigger re-delegation to another file system accordingly. This code handles
+ // both absolute and relative symlinks. Some assumptions we make: First, only
+ // symlink targets as read from getNormalizedLinkContent() can escape our scope.
+ // This is because Path objects are all canonicalized (see {@link Path#getRelative},
+ // etc.) and symlink target segments that get added to the stack are in-scope by
+ // definition. Second, symlink targets with relative segments must have the form
+ // [".."]*[standard segment]+, i.e. only the ".." non-standard segment is allowed
+ // and it may only appear as part of a contiguous prefix sequence.
+
+ Stack<String> stack = new Stack<>();
+ PathFragment rootPathFragment = rootPath.asFragment();
+ for (Path p = path; !p.asFragment().equals(rootPathFragment); p = p.getParentDirectory()) {
+ stack.push(p.getBaseName());
+ }
+
+ InMemoryContentInfo inode = rootInode;
+ int parentDepth = -1;
+ int traversals = 0;
+
+ while (!stack.isEmpty()) {
+ traversals++;
+
+ String name = stack.pop();
+ parentDepth += name.equals("..") ? -1 : 1;
+
+ // ENOENT on last segment with 'create' => create a new file.
+ InMemoryContentInfo child = directoryLookup(inode, name, create && stack.isEmpty(), path);
+ if (child.isSymbolicLink()) {
+ PathFragment linkTarget = ((InMemoryLinkInfo) child).getNormalizedLinkContent();
+ if (!inScope(parentDepth, linkTarget)) {
+ return outOfScopeStatus(linkTarget, parentDepth, stack);
+ }
+ if (linkTarget.isAbsolute()) {
+ inode = rootInode;
+ parentDepth = -1;
+ }
+ if (traversals > MAX_TRAVERSALS) {
+ throw Error.ELOOP.exception(path);
+ }
+ for (int ii = linkTarget.segmentCount() - 1; ii >= 0; --ii) {
+ stack.push(linkTarget.getSegment(ii)); // Note this may include ".." segments.
+ }
+ } else {
+ inode = child;
+ }
+ }
+ return inode;
+ }
+
+ /**
+ * Helper routine for pathWalk: given a symlink target known to escape this file system's
+ * scope (and that has the form [".."]*[standard segment]+), the number of segments
+ * in the directory containing the symlink, and the remaining path segments following
+ * the symlink in the original input to pathWalk, returns an OutofScopeFileStatus
+ * initialized with an appropriate out-of-scope reformulation of pathWalk's original
+ * input.
+ */
+ private OutOfScopeFileStatus outOfScopeStatus(PathFragment linkTarget, int parentDepth,
+ Stack<String> descendantSegments) {
+
+ PathFragment escapingPath;
+ if (linkTarget.isAbsolute()) {
+ escapingPath = linkTarget;
+ } else {
+ // Relative out-of-scope paths must look like "../../../a/b/c". Find the target's
+ // parent path depth by subtracting one from parentDepth for each ".." reference.
+ // Then use that to retrieve a prefix of the scope root, which is the target's
+ // canonicalized parent path.
+ int leadingParentRefs = leadingParentReferences(linkTarget);
+ int baseDepth = parentDepth - leadingParentRefs;
+ Preconditions.checkState(baseDepth < scopeRoot.segmentCount());
+ escapingPath = baseDepth > 0
+ ? scopeRoot.subFragment(0, baseDepth)
+ : scopeRoot.subFragment(0, 0);
+ // Now add in everything that comes after the ".." sequence.
+ for (int i = leadingParentRefs; i < linkTarget.segmentCount(); i++) {
+ escapingPath = escapingPath.getRelative(linkTarget.getSegment(i));
+ }
+ }
+
+ // We've now converted the symlink to its target in canonicalized absolute path
+ // form. Since the symlink wasn't necessarily the final segment in the original
+ // input sent to pathWalk, now add in every segment that came after.
+ while (!descendantSegments.empty()) {
+ escapingPath = escapingPath.getRelative(descendantSegments.pop());
+ }
+
+ return new OutOfScopeFileStatus(escapingPath);
+ }
+
+ /**
+ * Given 'path', returns the existing directory inode it designates,
+ * following symbolic links.
+ *
+ * <p>May fail with ENOTDIR, or any exception from pathWalk.
+ *
+ * <p>Iff the given path escapes this file system's scope, this method skips
+ * ENOTDIR checking and returns an OutOfScopeDirectoryStatus instance. Any
+ * code that calls this method needs to check for that possibility
+ * (via {@link ScopeEscapableStatus#outOfScope}).
+ */
+ private InMemoryDirectoryInfo getDirectory(Path path) throws IOException {
+ InMemoryContentInfo dirInfo = pathWalk(path, false);
+ if (dirInfo.outOfScope()) {
+ return new OutOfScopeDirectoryStatus(dirInfo.getEscapingPath());
+ } else if (!dirInfo.isDirectory()) {
+ throw Error.ENOTDIR.exception(path);
+ } else {
+ return (InMemoryDirectoryInfo) dirInfo;
+ }
+ }
+
+ /**
+ * Helper method for stat, scopeLimitedStat: lock the internal state and return the
+ * path's (no symlink-followed) stat if the path's parent directory is within scope,
+ * else return an "out of scope" reference to the path's parent directory (which will
+ * presumably be re-delegated to another FS).
+ */
+ private synchronized InMemoryContentInfo getNoFollowStatOrOutOfScopeParent(Path path)
+ throws IOException {
+ InMemoryDirectoryInfo dirInfo = getDirectory(path.getParentDirectory());
+ return dirInfo.outOfScope()
+ ? dirInfo
+ : directoryLookup(dirInfo, path.getBaseName(), /*create=*/false, path);
+ }
+
+ /**
+ * Given 'path', returns the existing inode it designates, optionally
+ * following symbolic links. Analogous to UNIX stat(2)/lstat(2), except that
+ * it returns a mutable inode we can modify directly.
+ */
+ @Override
+ public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (followSymlinks) {
+ InMemoryContentInfo status = scopeLimitedStat(path, true);
+ return status.outOfScope()
+ ? statWithDelegator(status.getEscapingPath(), true)
+ : status;
+ } else {
+ if (path.equals(rootPath)) {
+ return rootInode;
+ } else {
+ InMemoryContentInfo status = getNoFollowStatOrOutOfScopeParent(path);
+ // If out of scope, status references the path's parent directory. Else it references the
+ // path itself.
+ return status.outOfScope()
+ ? getDelegatedPath(status.getEscapingPath().getRelative(
+ path.getBaseName())).stat(Symlinks.NOFOLLOW)
+ : status;
+ }
+ }
+ }
+
+ @Override
+ @Nullable
+ public FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
+ try {
+ return stat(path, followSymlinks);
+ } catch (IOException e) {
+ if (e instanceof Error.WithError) {
+ Error errorCode = ((Error.WithError) e).getError();
+ if (errorCode == Error.ENOENT || errorCode == Error.ENOTDIR) {
+ return null;
+ }
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Version of stat that returns an inode if the input path stays entirely within
+ * this file system's scope, otherwise an {@link OutOfScopeFileStatus}.
+ *
+ * <p>Any code that calls this method needs to check for either possibility via
+ * {@link ScopeEscapableStatus#outOfScope}.
+ */
+ protected InMemoryContentInfo scopeLimitedStat(Path path, boolean followSymlinks)
+ throws IOException {
+ if (followSymlinks) {
+ return pathWalk(path, false);
+ } else {
+ if (path.equals(rootPath)) {
+ return rootInode;
+ } else {
+ InMemoryContentInfo status = getNoFollowStatOrOutOfScopeParent(path);
+ // If out of scope, status references the path's parent directory. Else it references the
+ // path itself.
+ return status.outOfScope()
+ ? new OutOfScopeFileStatus(status.getEscapingPath().getRelative(path.getBaseName()))
+ : status;
+ }
+ }
+ }
+
+ /****************************************************************************
+ * FileSystem methods
+ */
+
+ /**
+ * This is a helper routing for {@link #resolveSymbolicLinks(Path)}, i.e.
+ * the "user-mode" routing for canonicalising paths. It is analogous to the
+ * code in glibc's realpath(3).
+ *
+ * <p>Just like realpath, resolveSymbolicLinks requires a quadratic number of
+ * directory lookups: n path segments are statted, and each stat requires a
+ * linear amount of work in the "kernel" routine.
+ */
+ @Override
+ protected PathFragment resolveOneLink(Path path) throws IOException {
+ // Beware, this seemingly simple code belies the complex specification of
+ // FileSystem.resolveOneLink().
+ InMemoryContentInfo status = scopeLimitedStat(path, false);
+ if (status.outOfScope()) {
+ return resolveOneLinkWithDelegator(status.getEscapingPath());
+ } else {
+ return status.isSymbolicLink()
+ ? ((InMemoryLinkInfo) status).getLinkContent()
+ : null;
+ }
+ }
+
+ @Override
+ protected boolean isDirectory(Path path, boolean followSymlinks) {
+ try {
+ return stat(path, followSymlinks).isDirectory();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean isFile(Path path, boolean followSymlinks) {
+ try {
+ return stat(path, followSymlinks).isFile();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean isSymbolicLink(Path path) {
+ try {
+ return stat(path, false).isSymbolicLink();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean exists(Path path, boolean followSymlinks) {
+ try {
+ stat(path, followSymlinks);
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Like {@link #exists}, but checks for existence within this filesystem's scope.
+ */
+ protected boolean scopeLimitedExists(Path path, boolean followSymlinks) {
+ try {
+ // Path#asFragment() always returns an absolute path, so inScope() is called with
+ // parentDepth = 0.
+ return inScope(0, path.asFragment()) && !scopeLimitedStat(path, followSymlinks).outOfScope();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ InMemoryContentInfo status = scopeLimitedStat(path, true);
+ return status.outOfScope()
+ ? getDelegatedPath(status.getEscapingPath()).isReadable()
+ : status.isReadable();
+ }
+
+ @Override
+ protected void setReadable(Path path, boolean readable) throws IOException {
+ InMemoryContentInfo status;
+ synchronized (this) {
+ status = scopeLimitedStat(path, true);
+ if (!status.outOfScope()) {
+ status.setReadable(readable);
+ return;
+ }
+ }
+ // If we get here, we're out of scope.
+ getDelegatedPath(status.getEscapingPath()).setReadable(readable);
+ }
+
+ @Override
+ protected boolean isWritable(Path path) throws IOException {
+ InMemoryContentInfo status = scopeLimitedStat(path, true);
+ return status.outOfScope()
+ ? getDelegatedPath(status.getEscapingPath()).isWritable()
+ : status.isWritable();
+ }
+
+ @Override
+ protected void setWritable(Path path, boolean writable) throws IOException {
+ InMemoryContentInfo status;
+ synchronized (this) {
+ status = scopeLimitedStat(path, true);
+ if (!status.outOfScope()) {
+ status.setWritable(writable);
+ return;
+ }
+ }
+ // If we get here, we're out of scope.
+ getDelegatedPath(status.getEscapingPath()).setWritable(writable);
+ }
+
+ @Override
+ protected boolean isExecutable(Path path) throws IOException {
+ InMemoryContentInfo status = scopeLimitedStat(path, true);
+ return status.outOfScope()
+ ? getDelegatedPath(status.getEscapingPath()).isExecutable()
+ : status.isExecutable();
+ }
+
+ @Override
+ protected void setExecutable(Path path, boolean executable)
+ throws IOException {
+ InMemoryContentInfo status;
+ synchronized (this) {
+ status = scopeLimitedStat(path, true);
+ if (!status.outOfScope()) {
+ status.setExecutable(executable);
+ return;
+ }
+ }
+ // If we get here, we're out of scope.
+ getDelegatedPath(status.getEscapingPath()).setExecutable(executable);
+ }
+
+ @Override
+ public boolean supportsModifications() {
+ return true;
+ }
+
+ @Override
+ public boolean supportsSymbolicLinks() {
+ return true;
+ }
+
+ /**
+ * Constructs a new inode. Provided so that subclasses of InMemoryFileSystem
+ * can inject subclasses of FileInfo properly.
+ */
+ protected FileInfo makeFileInfo(Clock clock, PathFragment frag) {
+ return new InMemoryFileInfo(clock);
+ }
+
+ /**
+ * Returns a new path constructed by appending the child's base name to the
+ * escaped parent path. For example, assume our file system root is /foo
+ * and /foo/link1 -> /bar. This method can be used on child = /foo/link1/link2/name
+ * and parent = /bar/link2 to return /bar/link2/name, which is a semi-resolved
+ * path bound to a different file system.
+ */
+ private Path getDelegatedPath(PathFragment escapedParent, Path child) {
+ return getDelegatedPath(escapedParent.getRelative(child.getBaseName()));
+ }
+
+ @Override
+ protected boolean createDirectory(Path path) throws IOException {
+ if (path.equals(rootPath)) { throw Error.EACCES.exception(path); }
+
+ InMemoryDirectoryInfo parent;
+ synchronized (this) {
+ parent = getDirectory(path.getParentDirectory());
+ if (!parent.outOfScope()) {
+ InMemoryContentInfo child = parent.getChild(path.getBaseName());
+ if (child != null) { // already exists
+ if (child.isDirectory()) {
+ return false;
+ } else {
+ throw Error.EEXIST.exception(path);
+ }
+ }
+
+ InMemoryDirectoryInfo newDir = new InMemoryDirectoryInfo(clock);
+ newDir.addChild(".", newDir);
+ newDir.addChild("..", parent);
+ insert(parent, path.getBaseName(), newDir, path);
+
+ return true;
+ }
+ }
+
+ // If we get here, we're out of scope.
+ return getDelegatedPath(parent.getEscapingPath(), path).createDirectory();
+ }
+
+ @Override
+ protected void createSymbolicLink(Path path, PathFragment targetFragment)
+ throws IOException {
+ if (path.equals(rootPath)) { throw Error.EACCES.exception(path); }
+
+ InMemoryDirectoryInfo parent;
+ synchronized (this) {
+ parent = getDirectory(path.getParentDirectory());
+ if (!parent.outOfScope()) {
+ if (parent.getChild(path.getBaseName()) != null) { throw Error.EEXIST.exception(path); }
+ insert(parent, path.getBaseName(), new InMemoryLinkInfo(clock, targetFragment), path);
+ return;
+ }
+ }
+
+ // If we get here, we're out of scope.
+ getDelegatedPath(parent.getEscapingPath(), path).createSymbolicLink(targetFragment);
+ }
+
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ InMemoryContentInfo status = scopeLimitedStat(path, false);
+ if (status.outOfScope()) {
+ return getDelegatedPath(status.getEscapingPath()).readSymbolicLink();
+ } else if (status.isSymbolicLink()) {
+ Preconditions.checkState(status instanceof InMemoryLinkInfo);
+ return ((InMemoryLinkInfo) status).getLinkContent();
+ } else {
+ throw new NotASymlinkException(path);
+ }
+ }
+
+ @Override
+ protected long getFileSize(Path path, boolean followSymlinks)
+ throws IOException {
+ return stat(path, followSymlinks).getSize();
+ }
+
+ @Override
+ protected Collection<Path> getDirectoryEntries(Path path) throws IOException {
+ InMemoryDirectoryInfo dirInfo;
+ synchronized (this) {
+ dirInfo = getDirectory(path);
+ if (!dirInfo.outOfScope()) {
+ FileStatus status = stat(path, false);
+ Preconditions.checkState(status instanceof InMemoryContentInfo);
+ if (!((InMemoryContentInfo) status).isReadable()) {
+ throw new IOException("Directory is not readable");
+ }
+
+ Set<String> allChildren = dirInfo.getAllChildren();
+ List<Path> result = new ArrayList<>(allChildren.size());
+ for (String child : allChildren) {
+ if (!(child.equals(".") || child.equals(".."))) {
+ result.add(path.getChild(child));
+ }
+ }
+ return result;
+ }
+ }
+
+ // If we get here, we're out of scope.
+ return getDelegatedPath(dirInfo.getEscapingPath()).getDirectoryEntries();
+ }
+
+ @Override
+ protected boolean delete(Path path) throws IOException {
+ if (path.equals(rootPath)) { throw Error.EBUSY.exception(path); }
+ if (!exists(path, false)) { return false; }
+
+ InMemoryDirectoryInfo parent;
+ synchronized (this) {
+ parent = getDirectory(path.getParentDirectory());
+ if (!parent.outOfScope()) {
+ InMemoryContentInfo child = parent.getChild(path.getBaseName());
+ if (child.isDirectory() && child.getSize() > 2) { throw Error.ENOTEMPTY.exception(path); }
+ unlink(parent, path.getBaseName(), path);
+ return true;
+ }
+ }
+
+ // If we get here, we're out of scope.
+ return getDelegatedPath(parent.getEscapingPath(), path).delete();
+ }
+
+ @Override
+ protected long getLastModifiedTime(Path path, boolean followSymlinks)
+ throws IOException {
+ return stat(path, followSymlinks).getLastModifiedTime();
+ }
+
+ @Override
+ protected void setLastModifiedTime(Path path, long newTime) throws IOException {
+ InMemoryContentInfo status;
+ synchronized (this) {
+ status = scopeLimitedStat(path, true);
+ if (!status.outOfScope()) {
+ status.setLastModifiedTime(newTime == -1L
+ ? clock.currentTimeMillis()
+ : newTime);
+ return;
+ }
+ }
+
+ // If we get here, we're out of scope.
+ getDelegatedPath(status.getEscapingPath()).setLastModifiedTime(newTime);
+ }
+
+ @Override
+ protected InputStream getInputStream(Path path) throws IOException {
+ InMemoryContentInfo status;
+ synchronized (this) {
+ status = scopeLimitedStat(path, true);
+ if (!status.outOfScope()) {
+ if (status.isDirectory()) { throw Error.EISDIR.exception(path); }
+ if (!path.isReadable()) { throw Error.EACCES.exception(path); }
+ Preconditions.checkState(status instanceof FileInfo);
+ return new ByteArrayInputStream(((FileInfo) status).readContent());
+ }
+ }
+
+ // If we get here, we're out of scope.
+ return getDelegatedPath(status.getEscapingPath()).getInputStream();
+ }
+
+ /**
+ * Creates a new file at the given path and returns its inode. If the path
+ * escapes this file system's scope, trivially returns an "out of scope" status.
+ * Calling code should check for both possibilities via
+ * {@link ScopeEscapableStatus#outOfScope}.
+ */
+ protected InMemoryContentInfo getOrCreateWritableInode(Path path)
+ throws IOException {
+ // open(WR_ONLY) of a dangling link writes through the link. That means
+ // that the usual path lookup operations have to behave differently when
+ // resolving a path with the intent to create it: instead of failing with
+ // ENOENT they have to return an open file. This is exactly how UNIX
+ // kernels do it, which is what we're trying to emulate.
+ InMemoryContentInfo child = pathWalk(path, /*create=*/true);
+ Preconditions.checkNotNull(child);
+ if (child.outOfScope()) {
+ return child;
+ } else if (child.isDirectory()) {
+ throw Error.EISDIR.exception(path);
+ } else { // existing or newly-created file
+ if (!child.isWritable()) { throw Error.EACCES.exception(path); }
+ return child;
+ }
+ }
+
+ @Override
+ protected OutputStream getOutputStream(Path path, boolean append)
+ throws IOException {
+ InMemoryContentInfo status;
+ synchronized (this) {
+ status = getOrCreateWritableInode(path);
+ if (!status.outOfScope()) {
+ return ((FileInfo) getOrCreateWritableInode(path)).getOutputStream(append);
+ }
+ }
+ // If we get here, we're out of scope.
+ return getDelegatedPath(status.getEscapingPath()).getOutputStream(append);
+ }
+
+ @Override
+ protected void renameTo(Path sourcePath, Path targetPath)
+ throws IOException {
+ if (sourcePath.equals(rootPath)) { throw Error.EACCES.exception(sourcePath); }
+ if (targetPath.equals(rootPath)) { throw Error.EACCES.exception(targetPath); }
+
+ InMemoryDirectoryInfo sourceParent;
+ InMemoryDirectoryInfo targetParent;
+
+ synchronized (this) {
+ sourceParent = getDirectory(sourcePath.getParentDirectory());
+ targetParent = getDirectory(targetPath.getParentDirectory());
+
+ // Handle the rename if both paths are within our scope.
+ if (!sourceParent.outOfScope() && !targetParent.outOfScope()) {
+ InMemoryContentInfo sourceInode = sourceParent.getChild(sourcePath.getBaseName());
+ if (sourceInode == null) { throw Error.ENOENT.exception(sourcePath); }
+ InMemoryContentInfo targetInode = targetParent.getChild(targetPath.getBaseName());
+
+ unlink(sourceParent, sourcePath.getBaseName(), sourcePath);
+ try {
+ // TODO(bazel-team): (2009) test with symbolic links.
+
+ // Precondition checks:
+ if (targetInode != null) { // already exists
+ if (targetInode.isDirectory()) {
+ if (!sourceInode.isDirectory()) {
+ throw new IOException(sourcePath + " -> " + targetPath + " (" + Error.EISDIR + ")");
+ }
+ if (targetInode.getSize() > 2) {
+ throw Error.ENOTEMPTY.exception(targetPath);
+ }
+ } else if (sourceInode.isDirectory()) {
+ throw new IOException(sourcePath + " -> " + targetPath + " (" + Error.ENOTDIR + ")");
+ }
+ unlink(targetParent, targetPath.getBaseName(), targetPath);
+ }
+ sourceInode.movedTo(targetPath);
+ insert(targetParent, targetPath.getBaseName(), sourceInode, targetPath);
+ return;
+
+ } catch (IOException e) {
+ sourceInode.movedTo(sourcePath);
+ insert(sourceParent, sourcePath.getBaseName(), sourceInode, sourcePath); // restore source
+ throw e;
+ }
+ }
+ }
+
+ // If we get here, either one or both paths is out of scope.
+ if (sourceParent.outOfScope() && targetParent.outOfScope()) {
+ Path delegatedSource = getDelegatedPath(sourceParent.getEscapingPath(), sourcePath);
+ Path delegatedTarget = getDelegatedPath(targetParent.getEscapingPath(), targetPath);
+ delegatedSource.renameTo(delegatedTarget);
+ } else {
+ // We don't support cross-file system renaming.
+ throw Error.EACCES.exception(targetPath);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryLinkInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryLinkInfo.java
new file mode 100644
index 0000000..f8837ee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryLinkInfo.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * This interface represents a symbolic link to an absolute or relative path,
+ * stored in an InMemoryFileSystem.
+ */
+@ThreadSafe @Immutable
+class InMemoryLinkInfo extends InMemoryContentInfo {
+
+ private final PathFragment linkContent;
+ private final PathFragment normalizedLinkContent;
+
+ InMemoryLinkInfo(Clock clock, PathFragment linkContent) {
+ super(clock);
+ this.linkContent = linkContent;
+ this.normalizedLinkContent = linkContent.normalize();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return false;
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return true;
+ }
+
+ @Override
+ public boolean isFile() {
+ return false;
+ }
+
+ @Override
+ public long getSize() {
+ return linkContent.toString().length();
+ }
+
+ /**
+ * Returns the content of the symbolic link.
+ */
+ PathFragment getLinkContent() {
+ return linkContent;
+ }
+
+ /**
+ * Returns the content of the symbolic link, with ".." and "." removed
+ * (except for the possibility of necessary ".." segments at the beginning).
+ */
+ PathFragment getNormalizedLinkContent() {
+ return normalizedLinkContent;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " -> " + linkContent;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeDirectoryStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeDirectoryStatus.java
new file mode 100644
index 0000000..b757acd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeDirectoryStatus.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Set;
+
+/**
+ * A directory status that signifies a path has left this file system's
+ * scope. All methods beside {@link #outOfScope} and {@link #getEscapingPath}
+ * are disabled.
+ */
+final class OutOfScopeDirectoryStatus extends InMemoryDirectoryInfo {
+ /**
+ * Contains the requested path resolved up to the point where it
+ * first escapes the scope. See
+ * {@link ScopeEscapableStatus#getEscapingPath} for an example.
+ */
+ private final PathFragment escapingPath;
+
+ public OutOfScopeDirectoryStatus(PathFragment escapingPath) {
+ super(null, false);
+ this.escapingPath = escapingPath;
+ }
+
+ @Override
+ public boolean outOfScope() {
+ return true;
+ }
+
+ @Override
+ public PathFragment getEscapingPath() {
+ return escapingPath;
+ }
+
+ private static UnsupportedOperationException failure() {
+ return new UnsupportedOperationException();
+ }
+
+ @Override public boolean isDirectory() { throw failure(); }
+ @Override public boolean isSymbolicLink() { throw failure(); }
+ @Override public boolean isFile() { throw failure(); }
+ @Override public long getSize() { throw failure(); }
+ @Override protected void markModificationTime() { throw failure(); }
+ @Override public synchronized long getLastModifiedTime() { throw failure(); }
+ @Override void setLastModifiedTime(long newTime) { throw failure(); }
+ @Override public synchronized long getLastChangeTime() { throw failure(); }
+ @Override boolean isReadable() { throw failure(); }
+ @Override void setReadable(boolean readable) { throw failure(); }
+ @Override void setWritable(boolean writable) { throw failure(); }
+ @Override void setExecutable(boolean executable) { throw failure(); }
+ @Override boolean isWritable() { throw failure(); }
+ @Override boolean isExecutable() { throw failure(); }
+ @Override void addChild(String name, InMemoryContentInfo inode) { throw failure(); }
+ @Override InMemoryContentInfo getChild(String name) { throw failure(); }
+ @Override void removeChild(String name) { throw failure(); }
+ @Override Set<String> getAllChildren() { throw failure(); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeFileStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeFileStatus.java
new file mode 100644
index 0000000..177ac11
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeFileStatus.java
@@ -0,0 +1,65 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A file status that signifies a path has left this file system's
+ * scope. All methods beside {@link #outOfScope} and {@link #getEscapingPath}
+ * are disabled.
+ */
+final class OutOfScopeFileStatus extends InMemoryContentInfo {
+
+ /**
+ * Contains the requested path resolved up to the point where it
+ * first escapes the scope. See
+ * {@link ScopeEscapableStatus#getEscapingPath} for an example.
+ */
+ private final PathFragment escapingPath;
+
+ public OutOfScopeFileStatus(PathFragment escapingPath) {
+ super(null, false);
+ this.escapingPath = escapingPath;
+ }
+
+ @Override
+ public boolean outOfScope() {
+ return true;
+ }
+
+ @Override
+ public PathFragment getEscapingPath() {
+ return escapingPath;
+ }
+
+ private static UnsupportedOperationException failure() {
+ return new UnsupportedOperationException();
+ }
+
+ @Override public boolean isDirectory() { throw failure(); }
+ @Override public boolean isSymbolicLink() { throw failure(); }
+ @Override public boolean isFile() { throw failure(); }
+ @Override public long getSize() { throw failure(); }
+ @Override protected void markModificationTime() { throw failure(); }
+ @Override public synchronized long getLastModifiedTime() { throw failure(); }
+ @Override void setLastModifiedTime(long newTime) { throw failure(); }
+ @Override public synchronized long getLastChangeTime() { throw failure(); }
+ @Override boolean isReadable() { throw failure(); }
+ @Override void setReadable(boolean readable) { throw failure(); }
+ @Override void setWritable(boolean writable) { throw failure(); }
+ @Override boolean isWritable() { throw failure(); }
+ @Override void setExecutable(boolean executable) { throw failure(); }
+ @Override boolean isExecutable() { throw failure(); }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/ScopeEscapableStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/ScopeEscapableStatus.java
new file mode 100644
index 0000000..4afec78
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/ScopeEscapableStatus.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.ScopeEscapableFileSystem;
+
+/**
+ * Interface definition for a file status that may signify that the
+ * referenced path falls outside the scope of the file system (see
+ * {@link ScopeEscapableFileSystem}) and can provide the "escaped"
+ * version of that path suitable for re-delegation to another file
+ * system.
+ */
+interface ScopeEscapableStatus extends FileStatus {
+
+ /**
+ * Returns true if this status corresponds to a path that leaves
+ * the file system's scope, false otherwise.
+ */
+ boolean outOfScope();
+
+ /**
+ * If this status represents a path that leaves the file system's scope,
+ * returns the requested path resolved up to the point where it first
+ * escapes the file system. For example: if the file system is mapped to
+ * /foo, the requested path is /foo/link1/link2/link3, and link1 -> /bar,
+ * this returns /bar/link2/link3.
+ *
+ * <p>If this status doesn't represent a scope-escaping path, returns
+ * null.
+ */
+ PathFragment getEscapingPath();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/IndexPageHandler.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/IndexPageHandler.java
new file mode 100644
index 0000000..c9eb3ed
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/IndexPageHandler.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.webstatusserver;
+
+import com.google.common.collect.ImmutableList;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * Handlers for displaying the index page of server.
+ *
+ */
+public class IndexPageHandler {
+ private List<TestStatusHandler> testHandlers = new ArrayList<>();
+ private IndexPageJsonData dataHandler;
+ private StaticResourceHandler frontendHandler;
+
+ public IndexPageHandler(HttpServer server, List<TestStatusHandler> testHandlers) {
+ this.testHandlers = testHandlers;
+ this.dataHandler = new IndexPageJsonData(this);
+ this.frontendHandler =
+ StaticResourceHandler.createFromRelativePath("static/index.html", "text/html");
+ server.createContext("/", frontendHandler);
+ server.createContext("/tests/list", dataHandler);
+ }
+
+ /**
+ * Puts data from the build log into json suitable for frontend.
+ *
+ */
+ private class IndexPageJsonData implements HttpHandler {
+ private IndexPageHandler pageHandler;
+ private Gson gson = new Gson();
+ public IndexPageJsonData(IndexPageHandler indexPageHandler) {
+ this.pageHandler = indexPageHandler;
+ }
+
+ @Override
+ public void handle(HttpExchange exchange) throws IOException {
+ exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("application/json"));
+ JsonArray response = new JsonArray();
+ for (TestStatusHandler handler : this.pageHandler.testHandlers) {
+ WebStatusBuildLog buildLog = handler.getBuildLog();
+ JsonObject test = new JsonObject();
+ test.add("targets", gson.toJsonTree(buildLog.getTargetList()));
+ test.addProperty("startTime", buildLog.getStartTime());
+ test.addProperty("finished", buildLog.finished());
+ test.addProperty("uuid", buildLog.getCommandId().toString());
+ response.add(test);
+ }
+ String serializedResponse = response.toString();
+ exchange.sendResponseHeaders(200, serializedResponse.length());
+ OutputStream os = exchange.getResponseBody();
+ os.write(serializedResponse.getBytes());
+ os.close();
+ }
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/StaticResourceHandler.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/StaticResourceHandler.java
new file mode 100644
index 0000000..cd9eb5f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/StaticResourceHandler.java
@@ -0,0 +1,80 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.webstatusserver;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.CharStreams;
+import com.google.devtools.build.lib.util.ResourceFileLoader;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.List;
+
+/**
+ * Handler for static resources (JS, html, css...)
+ */
+public class StaticResourceHandler implements HttpHandler {
+ private String response;
+ private List<String> contentType;
+ private int httpCode;
+
+ public static StaticResourceHandler createFromAbsolutePath(String path, String contentType) {
+ return new StaticResourceHandler(path, contentType, true);
+ }
+
+ public static StaticResourceHandler createFromRelativePath(String path, String contentType) {
+ return new StaticResourceHandler(path, contentType, false);
+ }
+
+ private StaticResourceHandler(String path, String contentType, boolean absolutePath) {
+ try {
+ if (absolutePath) {
+ InputStream resourceStream = loadFromAbsolutePath(WebStatusServerModule.class, path);
+ response = CharStreams.toString(new InputStreamReader(resourceStream));
+
+ } else {
+ response = ResourceFileLoader.loadResource(WebStatusServerModule.class, path);
+ }
+ httpCode = 200;
+ } catch (IOException e) {
+ throw new IllegalArgumentException("resource " + path + " not found");
+ }
+ this.contentType = ImmutableList.of(contentType);
+ }
+
+ @Override
+ public void handle(HttpExchange exchange) throws IOException {
+ exchange.getResponseHeaders().put("Content-Type", contentType);
+ exchange.sendResponseHeaders(httpCode, response.length());
+ OutputStream os = exchange.getResponseBody();
+ os.write(response.getBytes());
+ os.close();
+ }
+
+ public static InputStream loadFromAbsolutePath(Class<?> loadingClass, String path)
+ throws IOException {
+ URL resourceUrl = loadingClass.getClassLoader().getResource(path);
+ if (resourceUrl == null) {
+ throw new IllegalArgumentException("resource " + path + " not found");
+ }
+ return resourceUrl.openStream();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/TestStatusHandler.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/TestStatusHandler.java
new file mode 100644
index 0000000..41cb06d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/TestStatusHandler.java
@@ -0,0 +1,148 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.webstatusserver;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.reflect.TypeToken;
+
+import com.sun.net.httpserver.HttpContext;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Type;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Collection of handlers for displaying the test data.
+ */
+class TestStatusHandler {
+ private StaticResourceHandler frontendHandler;
+ private WebStatusBuildLog buildLog;
+ private HttpHandler detailsHandler;
+ private HttpServer server;
+ private ImmutableList<HttpContext> contexts;
+ private CommandJsonData commandHandler;
+ private Gson gson = new Gson();
+
+ public TestStatusHandler(HttpServer server, WebStatusBuildLog buildLog) {
+ Builder<HttpContext> builder = ImmutableList.builder();
+ this.buildLog = buildLog;
+ this.server = server;
+ detailsHandler = new TestStatusResultJsonData(this);
+ commandHandler = new CommandJsonData(this);
+ frontendHandler = StaticResourceHandler.createFromRelativePath("static/test.html", "text/html");
+ builder.add(
+ server.createContext("/tests/" + buildLog.getCommandId() + "/details", detailsHandler));
+ builder.add(
+ server.createContext("/tests/" + buildLog.getCommandId() + "/info", commandHandler));
+ builder.add(server.createContext("/tests/" + buildLog.getCommandId(), frontendHandler));
+ contexts = builder.build();
+ }
+
+ public WebStatusBuildLog getBuildLog() {
+ return buildLog;
+ }
+
+
+ /**
+ * Serves JSON objects containing command info, which will be rendered by frontend.
+ */
+ private class CommandJsonData implements HttpHandler {
+ private TestStatusHandler testStatusHandler;
+
+ public CommandJsonData(TestStatusHandler testStatusHandler) {
+ this.testStatusHandler = testStatusHandler;
+ }
+
+ @Override
+ public void handle(HttpExchange exchange) throws IOException {
+ exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("application/json"));
+ Type commandInfoType = new TypeToken<Map<String, JsonElement>>() {}.getType();
+ JsonObject response = gson.toJsonTree(testStatusHandler.buildLog.getCommandInfo(),
+ commandInfoType).getAsJsonObject();
+ response.addProperty("startTime", testStatusHandler.buildLog.getStartTime());
+ response.addProperty("finished", testStatusHandler.buildLog.finished());
+
+ String serializedResponse = response.toString();
+ exchange.sendResponseHeaders(200, serializedResponse.length());
+ OutputStream os = exchange.getResponseBody();
+ os.write(serializedResponse.getBytes());
+ os.close();
+ }
+ }
+
+ /**
+ * Serves JSON objects containing test cases, which will be rendered by frontend.
+ */
+ private class TestStatusResultJsonData implements HttpHandler {
+ private TestStatusHandler testStatusHandler;
+
+ public TestStatusResultJsonData(TestStatusHandler testStatusHandler) {
+ this.testStatusHandler = testStatusHandler;
+ }
+
+ @Override
+ public void handle(HttpExchange exchange) throws IOException {
+ Map<String, JsonObject> testInfo = testStatusHandler.buildLog.getTestCases();
+ exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("application/json"));
+ JsonObject response = new JsonObject();
+ for (Entry<String, JsonObject> testCase : testInfo.entrySet()) {
+ response.add(testCase.getKey(), testCase.getValue());
+ }
+
+ String serializedResponse = response.toString();
+ exchange.sendResponseHeaders(200, serializedResponse.length());
+ OutputStream os = exchange.getResponseBody();
+ os.write(serializedResponse.getBytes());
+ os.close();
+ }
+ }
+
+ /**
+ * Adds another URI for existing test data. If specified URI is already used by some other
+ * handler, the previous handler will be removed.
+ */
+ public void overrideURI(String uri) {
+ String detailsPath = uri + "/details";
+ String commandPath = uri + "/info";
+ try {
+ this.server.removeContext(detailsPath);
+ this.server.removeContext(commandPath);
+ } catch (IllegalArgumentException e) {
+ // There was nothing to remove, so proceed with creation (unfortunately the server api doesn't
+ // have "hasContext" method)
+ }
+ this.server.createContext(detailsPath, this.detailsHandler);
+ this.server.createContext(commandPath, this.commandHandler);
+ }
+
+ /**
+ * Deregisters all the handlers associated with the test.
+ */
+ public void deregister() {
+ for (HttpContext c : this.contexts) {
+ this.server.removeContext(c);
+ }
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusBuildLog.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusBuildLog.java
new file mode 100644
index 0000000..86eed88
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusBuildLog.java
@@ -0,0 +1,200 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.webstatusserver;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.logging.Logger;
+
+/**
+ * Stores information about one build command. The data is stored in JSON so that it can be
+ * can be easily fed to frontend.
+ *
+ * <p> The information is grouped into following structures:
+ * <ul>
+ * <li> {@link #commandInfo} contain information about the build known when it starts but before
+ * anything is actually compiled/run
+ * <li> {@link #testCases} contain detailed information about each test case ran, for now they're
+ *
+ * </ul>
+ */
+public class WebStatusBuildLog {
+ private Gson gson = new Gson();
+ private boolean complete = false;
+ private static final Logger LOG =
+ Logger.getLogger(WebStatusEventCollector.class.getCanonicalName());
+ private Map<String, JsonElement> commandInfo = new HashMap<String, JsonElement>();
+ private Map<String, JsonObject> testCases = new HashMap<String, JsonObject>();
+ private long startTime;
+ private ImmutableList<String> targetList;
+ private UUID commandId;
+
+ public WebStatusBuildLog(UUID commandId) {
+ this.commandId = commandId;
+ }
+
+ public WebStatusBuildLog addInfo(String key, Object value) {
+ commandInfo.put(key, gson.toJsonTree(value));
+ return this;
+ }
+
+ public void addStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public void addTargetList(List<String> targets) {
+ this.targetList = ImmutableList.copyOf(targets);
+ }
+
+ public void finish() {
+ commandInfo = ImmutableMap.copyOf(commandInfo);
+ complete = true;
+ }
+
+ public Map<String, JsonElement> getCommandInfo() {
+ return commandInfo;
+ }
+
+ public ImmutableMap<String, JsonObject> getTestCases() {
+ // TODO(bazel-team): not really immutable, since one can do addProperty on
+ // values (unfortunately gson doesn't support immutable JsonObjects)
+ return ImmutableMap.copyOf(testCases);
+ }
+
+ public boolean finished() {
+ return complete;
+ }
+
+ public List<String> getTargetList() {
+ return targetList;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void addTestTarget(Label label) {
+ String targetName = label.toShorthandString();
+ if (!testCases.containsKey(targetName)) {
+ JsonObject summary = createTestCaseEmptyJsonNode(targetName);
+ summary.addProperty("finished", false);
+ summary.addProperty("status", "started");
+ testCases.put(targetName, summary);
+ } else {
+ // TODO(bazel-team): figure out if there are any situations it can happen
+ }
+ }
+
+ public void addTestSummary(Label label, BlazeTestStatus status, List<Long> testTimes,
+ boolean isCached) {
+ JsonObject testCase = testCases.get(label.toShorthandString());
+ testCase.addProperty("status", status.toString());
+ testCase.add("times", gson.toJsonTree(testTimes));
+ testCase.addProperty("cached", isCached);
+ testCase.addProperty("finished", true);
+ }
+
+ public void addTargetBuilt(Label label, boolean success) {
+ if (testCases.containsKey(label.toShorthandString())) {
+ if (success) {
+ testCases.get(label.toShorthandString()).addProperty("status", "built");
+ } else {
+ testCases.get(label.toShorthandString()).addProperty("status", "build failure");
+ }
+ } else {
+ LOG.info("Unhandled target: " + label);
+ }
+ }
+
+ @VisibleForTesting
+ static JsonObject createTestCaseEmptyJsonNode(String fullName) {
+ JsonObject currentNode = new JsonObject();
+ currentNode.addProperty("fullName", fullName);
+ currentNode.addProperty("name", "");
+ currentNode.addProperty("className", "");
+ currentNode.add("results", new JsonObject());
+ currentNode.add("times", new JsonObject());
+ currentNode.add("children", new JsonObject());
+ currentNode.add("failures", new JsonObject());
+ currentNode.add("errors", new JsonObject());
+ return currentNode;
+ }
+
+ private static JsonObject createTestCaseEmptyJsonNode(String fullName, TestCase testCase) {
+ JsonObject currentNode = createTestCaseEmptyJsonNode(fullName);
+ currentNode.addProperty("name", testCase.getName());
+ currentNode.addProperty("className", testCase.getClassName());
+ return currentNode;
+ }
+
+ private JsonObject mergeTestCases(JsonObject currentNode, String fullName, TestCase testCase,
+ int shardNumber) {
+ if (currentNode == null) {
+ currentNode = createTestCaseEmptyJsonNode(fullName, testCase);
+ }
+
+ if (testCase.getRun()) {
+ JsonObject results = (JsonObject) currentNode.get("results");
+ JsonObject times = (JsonObject) currentNode.get("times");
+
+ if (testCase.hasResult()) {
+ results.addProperty(Integer.toString(shardNumber), testCase.getResult());
+ }
+
+ if (testCase.hasStatus()) {
+ results.addProperty(Integer.toString(shardNumber), testCase.getStatus().toString());
+ }
+
+ if (testCase.hasRunDurationMillis()) {
+ times.addProperty(Integer.toString(shardNumber), testCase.getRunDurationMillis());
+ }
+ }
+ JsonObject children = (JsonObject) currentNode.get("children");
+
+ for (TestCase child : testCase.getChildList()) {
+ String fullChildName = child.getClassName() + "." + child.getName();
+ JsonObject childNode = mergeTestCases((JsonObject) children.get(fullChildName), fullChildName,
+ child, shardNumber);
+ if (!children.has(fullChildName)) {
+ children.add(fullChildName, childNode);
+ }
+ }
+ return currentNode;
+ }
+
+ public void addTestResult(Label label, TestCase testCase, int shardNumber) {
+ String testResultFullName = label.toShorthandString();
+ if (!testCases.containsKey(testResultFullName)) {
+ testCases.put(testResultFullName, createTestCaseEmptyJsonNode(testResultFullName, testCase));
+ }
+ mergeTestCases(testCases.get(testResultFullName), testResultFullName, testCase, shardNumber);
+ }
+
+ public UUID getCommandId() {
+ return commandId;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusEventCollector.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusEventCollector.java
new file mode 100644
index 0000000..40b0908
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusEventCollector.java
@@ -0,0 +1,135 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.webstatusserver;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.runtime.CommandCompleteEvent;
+import com.google.devtools.build.lib.runtime.CommandStartEvent;
+import com.google.devtools.build.lib.runtime.TestSummary;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.logging.Logger;
+
+/**
+ * This class monitors the build progress, collects events and preprocesses them for use by
+ * frontend.
+ *
+ */
+public class WebStatusEventCollector {
+ private static final Logger LOG =
+ Logger.getLogger(WebStatusEventCollector.class.getCanonicalName());
+ private final EventBus eventBus;
+ private final Reporter reporter;
+ private final int port;
+ private WebStatusBuildLog currentBuild;
+ private WebStatusServerModule serverModule;
+
+ public WebStatusEventCollector(EventBus eventBus, Reporter reporter,
+ WebStatusServerModule webStatusServerModule) {
+ this.eventBus = eventBus;
+ this.eventBus.register(this);
+ this.reporter = reporter;
+ this.port = webStatusServerModule.getPort();
+ this.serverModule = webStatusServerModule;
+ LOG.info("Created new status collector");
+ }
+
+ @Subscribe
+ public void buildStarted(BuildStartingEvent startingEvent) {
+ BuildRequest request = startingEvent.getRequest();
+ BlazeVersionInfo versionInfo = BlazeVersionInfo.instance();
+ currentBuild.addStartTime(request.getStartTime());
+ currentBuild.addTargetList(request.getTargets());
+ currentBuild
+ .addInfo("version", versionInfo)
+ .addInfo("commandName", request.getCommandName())
+ .addInfo("outputFs", startingEvent.getOutputFileSystem())
+ .addInfo("symlinkPrefix", request.getSymlinkPrefix())
+ .addInfo("optionsDescription", request.getOptionsDescription())
+ .addInfo("targets", request.getTargets())
+ .addInfo("viewOptions", request.getViewOptions());
+ }
+
+ @Subscribe
+ @SuppressWarnings("unused")
+ public void commandComplete(CommandCompleteEvent completeEvent) {
+ currentBuild.addInfo("endTime", completeEvent.getEventTimeInEpochTime());
+ currentBuild.finish();
+ }
+
+ @Subscribe
+ @SuppressWarnings("unused")
+ public void commandStarted(CommandStartEvent event) {
+ this.currentBuild = new WebStatusBuildLog(event.getCommandId());
+ this.serverModule.commandStarted();
+ String webStatusServerUrl = "http://localhost:" + port;
+ this.reporter.handle(Event.info("Status page: " + webStatusServerUrl + "/tests/"
+ + this.currentBuild.getCommandId() + " (alternative link: " + webStatusServerUrl
+ + WebStatusServerModule.LAST_TEST_URI + " )"));
+ }
+
+ @Subscribe
+ public void doneTestFiltering(TestFilteringCompleteEvent event) {
+ if (event.getTestTargets() != null) {
+ Builder<Label> builder = ImmutableList.builder();
+ for (ConfiguredTarget target : event.getTestTargets()) {
+ builder.add(target.getLabel());
+ }
+ doneTestFiltering(builder.build());
+ }
+ }
+
+ @VisibleForTesting
+ public void doneTestFiltering(Iterable<Label> testLabels) {
+ for (Label label : testLabels) {
+ currentBuild.addTestTarget(label);
+ }
+ }
+
+ @Subscribe
+ public void testTargetComplete(TestSummary summary) {
+ currentBuild.addTestSummary(summary.getTarget().getLabel(), summary.getStatus(),
+ summary.getTestTimes(), summary.isCached());
+ }
+
+ @Subscribe
+ public void testTargetResult(TestResult result) {
+ currentBuild.addTestResult(result.getTestAction().getOwner().getLabel(),
+ result.getData().getTestCase(), result.getShardNum());
+ }
+
+ @Subscribe
+ public void targetComplete(TargetCompleteEvent event) {
+ // TODO(bazel-team): would getting more details about failure be useful?
+ currentBuild.addTargetBuilt(event.getTarget().getTarget().getLabel(), !event.failed());
+ }
+
+ public WebStatusBuildLog getBuildLog() {
+ return this.currentBuild;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusServerModule.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusServerModule.java
new file mode 100644
index 0000000..13d4c8b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusServerModule.java
@@ -0,0 +1,159 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.webstatusserver;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.util.LinkedList;
+import java.util.UUID;
+import java.util.logging.Logger;
+
+/**
+ * Web server for monitoring blaze commands status.
+ */
+public class WebStatusServerModule extends BlazeModule {
+ static final String LAST_TEST_URI = "/tests/last";
+ // 100 is an arbitrary limit; it seems like a reasonable size for history and it's okay to change
+ // it
+ private static final int MAX_TESTS_STORED = 100;
+
+ private HttpServer server;
+ private boolean running = false;
+ private BlazeServerStartupOptions serverOptions;
+ private static final Logger LOG =
+ Logger.getLogger(WebStatusServerModule.class.getCanonicalName());
+ private int port;
+ private LinkedList<TestStatusHandler> testsRan = new LinkedList<>();
+ @SuppressWarnings("unused")
+ private WebStatusEventCollector collector;
+ @SuppressWarnings("unused")
+ private IndexPageHandler indexHandler;
+
+ @Override
+ public Iterable<Class<? extends OptionsBase>> getStartupOptions() {
+ return ImmutableList.<Class<? extends OptionsBase>>of(BlazeServerStartupOptions.class);
+ }
+
+ @Override
+ public void blazeStartup(OptionsProvider startupOptions, BlazeVersionInfo versionInfo,
+ UUID instanceId, BlazeDirectories directories, Clock clock) throws AbruptExitException {
+ serverOptions = startupOptions.getOptions(BlazeServerStartupOptions.class);
+ if (serverOptions.useWebStatusServer <= 0) {
+ LOG.info("web status server disabled");
+ return;
+ }
+ port = serverOptions.useWebStatusServer;
+ try {
+ server = HttpServer.create(new InetSocketAddress(port), 0);
+ serveStaticContent();
+ TextHandler lastCommandHandler = new TextHandler("No commands ran yet.");
+ server.createContext("/last", lastCommandHandler);
+ server.setExecutor(null);
+ server.start();
+ indexHandler = new IndexPageHandler(server, this.testsRan);
+ running = true;
+ LOG.info("Running web status server on port " + port);
+ } catch (IOException e) {
+ // TODO(bazel-team): Display information about why it failed
+ running = false;
+ LOG.warning("Unable to run web status server on port " + port);
+ }
+ }
+
+ @Override
+ public void beforeCommand(BlazeRuntime blazeRuntime, Command command) throws AbruptExitException {
+ if (!running) {
+ return;
+ }
+ collector =
+ new WebStatusEventCollector(blazeRuntime.getEventBus(), blazeRuntime.getReporter(), this);
+ }
+
+ public void commandStarted() {
+ WebStatusBuildLog currentBuild = collector.getBuildLog();
+
+ if (testsRan.size() == MAX_TESTS_STORED) {
+ TestStatusHandler oldestTest = testsRan.removeLast();
+ oldestTest.deregister();
+ }
+
+ TestStatusHandler lastTest = new TestStatusHandler(server, currentBuild);
+ testsRan.add(lastTest);
+
+ lastTest.overrideURI(LAST_TEST_URI);
+ }
+
+ private void serveStaticContent() {
+ StaticResourceHandler testjs =
+ StaticResourceHandler.createFromRelativePath("static/test.js", "application/javascript");
+ StaticResourceHandler indexjs =
+ StaticResourceHandler.createFromRelativePath("static/index.js", "application/javascript");
+ StaticResourceHandler style =
+ StaticResourceHandler.createFromRelativePath("static/style.css", "text/css");
+ StaticResourceHandler d3 = StaticResourceHandler.createFromAbsolutePath(
+ "third_party/javascript/d3/d3-js.js", "application/javascript");
+ StaticResourceHandler jquery = StaticResourceHandler.createFromAbsolutePath(
+ "third_party/javascript/jquery/v2_0_3/jquery_uncompressed.jslib",
+ "application/javascript");
+ StaticResourceHandler testFrontend =
+ StaticResourceHandler.createFromRelativePath("static/test.html", "text/html");
+
+ server.createContext("/css/style.css", style);
+ server.createContext("/js/test.js", testjs);
+ server.createContext("/js/index.js", indexjs);
+ server.createContext("/js/lib/d3.js", d3);
+ server.createContext("/js/lib/jquery.js", jquery);
+ server.createContext(LAST_TEST_URI, testFrontend);
+ }
+
+ private static class TextHandler implements HttpHandler {
+ private String response;
+
+ private TextHandler(String response) {
+ this.response = response;
+ }
+
+ @Override
+ public void handle(HttpExchange exchange) throws IOException {
+ exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("text/plain"));
+ exchange.sendResponseHeaders(200, response.length());
+ OutputStream os = exchange.getResponseBody();
+ os.write(response.getBytes());
+ os.close();
+ }
+ }
+
+ public int getPort() {
+ return port;
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.html b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.html
new file mode 100644
index 0000000..f57bc30
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.html
@@ -0,0 +1,14 @@
+<html>
+<head>
+ <title> Bazel web server </title>
+ <link rel="stylesheet" type="text/css" href="/css/style.css"></link>
+ <script src="/js/lib/d3.js" type="application/javascript"></script>
+ <script src="/js/lib/jquery.js" type="application/javascript"></script>
+ <script src="/js/index.js" type="application/javascript"></script>
+</head>
+<body onload="showData()">
+ <h1> Bazel web server status page </h1>
+ <div id="testsList">
+ </div>
+</body>
+</html>
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.js b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.js
new file mode 100644
index 0000000..4ef9671
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.js
@@ -0,0 +1,76 @@
+var icons = {
+ running: '\u25B6',
+ finished: '\u2611'
+};
+
+function showData() {
+ renderTestList(getTestsData());
+}
+
+function getTestsData() {
+ // TODO(bazel-team): change it to async callback retrieving data in background
+ // (for simplicity this is synchronous now)
+ return $.ajax({
+ type: 'GET',
+ url: document.URL + 'tests/list',
+ async: false
+ }).responseJSON;
+}
+
+function renderTestList(tests) {
+ var rows = d3.select('#testsList')
+ .selectAll()
+ .data(tests)
+ .enter().append('div')
+ .classed('info-cell', true);
+
+ // status
+ rows.append('div').classed('info-detail', true).text(function(j) {
+ return j.finished ? icons.finished : icons.running;
+ });
+
+ // target(s) name(s)
+ rows.append('div').classed('info-detail', true).text(function(j) {
+ if (j.targets.length == 1) {
+ return j.targets[0];
+ }
+ if (j.targets.length == 0) {
+ return 'Unknown target.';
+ }
+ return j.targets;
+ });
+
+ // start time
+ rows.append('div').classed('info-detail', true).text(function(j) {
+ // Pad value with 2 zeroes
+ function pad(value) {
+ return value < 10 ? '0' + value : value;
+ }
+
+ var
+ date = new Date(j.startTime),
+ today = new Date(Date.now()),
+ h = pad(date.getHours()),
+ m = pad(date.getMinutes()),
+ dd = pad(date.getDay()),
+ mm = pad(date.getMonth()),
+ yy = date.getYear(),
+ day;
+
+ // don't show date if ran today
+ if (dd != today.getDay() && mm != today.getMonth() &&
+ yy != today.getYear()) {
+ day = ' on ' + yy + '-' + mm + '-' + dd;
+ } else {
+ day = '';
+ }
+ return h + ':' + m;
+ });
+
+ // link
+ rows.append('div').classed('info-detail', true).classed('button', true)
+ .append('a').attr('href', function(datum, index) {
+ return '/tests/' + datum.uuid;
+ })
+ .text('link');
+}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.html b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.html
new file mode 100644
index 0000000..04a6fb7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.html
@@ -0,0 +1,28 @@
+<html>
+<head><title>Tests Result Page</title>
+ <link rel="stylesheet" type="text/css" href="/css/style.css"></link>
+ <script src="/js/lib/d3.js" type="application/javascript"></script>
+ <script src="/js/lib/jquery.js" type="application/javascript"></script>
+ <script src="/js/test.js" type="application/javascript"></script>
+</head>
+<body>
+<h1> Bazel web status server </h1>
+<div id="testInfo">
+ No test info to display.
+</div>
+<br>
+<div id="testFilters">
+ <div class="info-cell">
+ <input placeholder="Filter by name" type=text id="search"></input>
+ <!-- TODO(bazel-team) this is very simplistic view of tests,
+ we probably need more filters -->
+ <input type=checkbox checked=true id="boxPassed">passed</input>
+ <input type=checkbox checked=true id="boxFailed">failed</input>
+ <button id="clearFilters"> clear filters </button>
+ </div>
+</div>
+<div id="testDetails">
+ No test details to display.
+</div>
+</body>
+</html>
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.js b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.js
new file mode 100644
index 0000000..406dcab
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.js
@@ -0,0 +1,384 @@
+var icons = {
+ running: '?',
+ passed: '\u2705',
+ errors: '\u274c'
+};
+
+
+function showData() {
+ renderDetails(getDetailsData(), false);
+ renderInfo(getCommandInfo());
+}
+
+function getCommandInfo() {
+ var url = document.URL;
+ if (url[url.length - 1] != '/') {
+ url += '/';
+ }
+ return $.ajax({
+ type: 'GET',
+ url: url + 'info',
+ async: false
+ }).responseJSON;
+}
+
+function getDetailsData() {
+ // TODO(bazel-team): auto refresh, async callback
+ var url = document.URL;
+ if (url[url.length - 1] != '/') {
+ url += '/';
+ }
+ return $.ajax({
+ type: 'GET',
+ url: url + 'details',
+ async: false
+ }).responseJSON;
+}
+
+
+function showDate(d) {
+ function pad(x) {
+ return x < 10 ? '0' + x : '' + x;
+ }
+ var today = new Date();
+ var result = pad(d.getHours()) + ':' + pad(d.getMinutes()) + ':' +
+ pad(d.getSeconds());
+ if (d.getDate() === today.getDate() && d.getMonth() === today.getMonth() &&
+ d.getYear() === today.getYear()) {
+ result += ' today';
+ } else {
+ result += pad(d.getDate()) + ' ' + pad(d.getMonth()) + ' ' + d.getYear();
+ }
+ return result;
+}
+
+function renderInfo(info) {
+ $('#testInfo').empty();
+ var data = [
+ ['Targets: ', info['targets']],
+ ['Started at: ', showDate(new Date(info['startTime']))],
+ ];
+ if (info['finished']) {
+ data.push(['Finished at: ', showDate(new Date(info['endTime']))]);
+ } else {
+ data.push(['Still running']);
+ }
+ var selection = d3.select('#testInfo').selectAll()
+ .data(data)
+ .enter().append('div')
+ .classed('info-cell', true);
+ selection
+ .append('div')
+ .classed('info-detail', true)
+ .text(function(d) { return d[0]; });
+ selection
+ .append('div')
+ .classed('info-detail', true)
+ .text(function(d) { return d[1]; });
+}
+
+// predicate is either a predicate function or null - in the latter case
+// everything is shown
+function renderDetails(tests, predicate) {
+ $('#testDetails').empty();
+ if (tests.length == 0) {
+ $('#testDetails').text('No test details to display.');
+ return;
+ }
+ // flatten object to array and set visibility
+ tests = $.map(tests, function(element) {
+ if (predicate) {
+ setVisibility(predicate, element);
+ }
+ return element;
+ });
+ var rows = d3.select('#testDetails').selectAll()
+ .data(tests)
+ .enter().append('div')
+ .classed('test-case', true);
+
+ function addTestDetail(selection, toplevel) {
+ function fullName() {
+ selection.append('div').classed('test-detail', true).text(function(j) {
+ return j.fullName;
+ });
+ }
+ function propagateStatus(j) {
+ var result = '';
+ var failures = [];
+ var errors = [];
+ $.each(j.results, function(key, value) {
+ if (value == 'FAILED') {
+ failures.push(key);
+ }
+ if (value == 'ERROR') {
+ errors.push(key);
+ }
+ });
+ if (failures.length > 0) {
+ var s = failures.length > 1 ? 's' : '';
+ result += 'Failed on ' + failures.length + ' shard' + s + ': ' +
+ failures.join();
+ }
+ if (errors.length > 0) {
+ var s = failures.length > 1 ? 's' : '';
+ result += 'Errors on ' + errors.length + ' shard' + s + ': ' +
+ errors.join();
+ }
+ if (result == '') {
+ return j.status;
+ }
+ return result;
+ }
+ function testCaseStatus() {
+ selection.append('div')
+ .classed('test-detail', true)
+ .text(propagateStatus);
+ }
+ function testTargetStatus() {
+ selection.append('div')
+ .classed('test-detail', true)
+ .text(function(target) {
+ var childStatus = propagateStatus(target);
+ if (target.finished = false) {
+ return target.status + ' ' + stillRunning;
+ } else {
+ if (childStatus == 'PASSED') {
+ return target.status;
+ } else {
+ return target.status + ' ' + childStatus;
+ }
+ }
+ });
+ }
+ function testTargetStatusIcon() {
+ selection.append('div')
+ .classed('test-detail', true)
+ .attr('color', function(target) {
+ var childStatus = propagateStatus(target);
+ if (target.finished == false) {
+ return 'running';
+ } else {
+ if (childStatus == 'PASSED') {
+ return 'passed';
+ } else {
+ return 'errors';
+ }
+ }})
+ .text(function(target) {
+ var childStatus = propagateStatus(target);
+ if (target.finished == false) {
+ return icons.running;
+ } else {
+ if (childStatus == 'PASSED') {
+ return icons.passed;
+ } else {
+ return icons.errors;
+ }
+ }
+ });
+ }
+ function testCaseTime() {
+ selection.append('div').classed('test-detail', true).text(function(j) {
+ var times = $.map(j.times, function(element, key) { return element });
+ if (times.length < 1) {
+ return '?';
+ } else {
+ return Math.max.apply(Math, times) / 1000 + ' s';
+ }
+ });
+ }
+
+ function visibilityFilter() {
+ selection.attr('show', function(datum) {
+ return ('show' in datum) ? datum['show'] : true;
+ });
+ }
+
+ // Toplevel nodes represent test targets, so they look a bit different
+ if (toplevel) {
+ testTargetStatusIcon();
+ fullName();
+ } else {
+ testTargetStatusIcon();
+ fullName();
+ testCaseStatus();
+ testCaseTime();
+ }
+ visibilityFilter();
+ }
+
+ function addNestedDetails(table, toplevel) {
+ table.sort(function(data1, data2) {
+ if (data1.fullName < data2.fullName) {
+ return -1;
+ }
+ if (data1.fullName > data2.fullName) {
+ return 1;
+ }
+ return 0;
+ });
+
+ addTestDetail(table, toplevel);
+
+ // Add children nodes + show/hide button
+ var nonLeafNodes = table.filter(function(data, index) {
+ return !($.isEmptyObject(data.children));
+ });
+ var nextLevelNodes = nonLeafNodes.selectAll().data(function(d) {
+ return $.map(d.children, function(element, key) { return element });
+ });
+
+ if (nextLevelNodes.enter().empty()) {
+ return;
+ }
+
+ nonLeafNodes
+ .append('div')
+ .classed('test-detail', true)
+ .classed('button', true)
+ .text(function(j) {
+ return 'Show details';
+ })
+ .attr('toggle', 'off')
+ .on('click', function(datum) {
+ if ($(this).attr('toggle') == 'on') {
+ $(this).siblings('.test-case').not('[show=false]').hide();
+ $(this).attr('toggle', 'off');
+ $(this).text('Show details');
+ } else {
+ $(this).siblings('.test-case').not('[show=false]').show();
+ $(this).attr('toggle', 'on');
+ $(this).text('Hide details');
+ }
+ });
+ nextLevelNodes.enter().append('div').classed('test-case', true);
+ addNestedDetails(nextLevelNodes, false);
+ }
+
+ addNestedDetails(rows, true);
+ $('.button').siblings('.test-case').hide();
+ if (predicate) {
+ toggleVisibility();
+ }
+}
+
+function toggleVisibility() {
+ $('#testDetails > [show=false]').hide();
+ $('#testDetails > [show=true]').show();
+ $('[toggle=on]').siblings('[show=false]').hide();
+ $('[toggle=on]').siblings('[show=true]').show();
+}
+
+function setVisibility(predicate, object) {
+ var show = predicate(object);
+ var childrenPredicate = predicate;
+ // It rarely makes sense to show a non-leaf node and hide its children, so
+ // we just show all children
+ if (show) {
+ childrenPredicate = function() { return true; };
+ }
+ if ('children' in object) {
+ for (var child in object.children) {
+ setVisibility(childrenPredicate, object.children[child]);
+ show = object.children[child]['show'] || show;
+ }
+ }
+ object['show'] = show;
+}
+
+// given a list of predicates, return a function
+function intersectFilters(filterList) {
+ var filters = filterList.filter(function(x) { return x });
+ return function(x) {
+ for (var i = 0; i < filters.length; i++) {
+ if (!filters[i](x)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
+
+function textFilterActive() {
+ return $('#search').val();
+}
+
+function getTestFilters() {
+ var statusFilter = null;
+ var textFilter = null;
+ var filters = [];
+ var passed = $('#boxPassed').prop('checked');
+ var failed = $('#boxFailed').prop('checked');
+ // add checkbox filters only when necessary (ie. something is unchecked - when
+ // everything is checked this means user wants to see everything).
+ if (!(passed && failed)) {
+ var checkBoxFilters = [];
+ if (passed) {
+ checkBoxFilters.push(function(object) {
+ return object.status == 'PASSED';
+ });
+ }
+ if (failed) {
+ checkBoxFilters.push(function(object) {
+ return 'status' in object && object.status != 'PASSED';
+ });
+ }
+ filters.push(function(object) {
+ return checkBoxFilters.some(function(f) { return f(object); });
+ });
+ }
+ if (textFilterActive()) {
+ filters.push(function(object) {
+ // TODO(bazel-team): would case insentive search make more sense?
+ return ('fullName' in object &&
+ object.fullName.indexOf($('#search').val()) != -1);
+ });
+ }
+ return filters;
+}
+
+function redraw() {
+ renderDetails(getDetailsData(), intersectFilters(getTestFilters()));
+}
+
+function updateVisibleCases() {
+ var predicate = intersectFilters(getTestFilters());
+ var parentCases = d3.selectAll('#testDetails > div').data();
+ parentCases.forEach(function(element, index) {
+ setVisibility(predicate, element);
+ });
+ d3.selectAll('.test-detail').attr('show', function(datum) {
+ return ('show' in datum) ? datum['show'] : true;
+ });
+ d3.selectAll('.test-case').attr('show', function(datum) {
+ return ('show' in datum) ? datum['show'] : true;
+ });
+ toggleVisibility();
+ if (textFilterActive()) {
+ // expand nodes to save some clicking - if user searched for something that
+ // is leaf of the tree, she definitely wants to see it
+ $('#testDetails > [show=true]').find('[toggle=off]').click();
+ }
+}
+
+function enableControls() {
+ var redrawTimeout = null;
+ $('#boxPassed').click(updateVisibleCases);
+ $('#boxFailed').click(updateVisibleCases);
+ $('#search').keyup(function() {
+ clearTimeout(redrawTimeout);
+ redrawTimeout = setTimeout(updateVisibleCases, 500);
+ });
+ $('#clearFilters').click(function() {
+ $('#boxPassed').prop('checked', true);
+ $('#boxFailed').prop('checked', true);
+ $('#search').val('');
+ updateVisibleCases();
+ });
+}
+
+$(function() {
+ showData();
+ enableControls();
+});
diff --git a/src/main/java/com/google/devtools/build/skyframe/BuildDriver.java b/src/main/java/com/google/devtools/build/skyframe/BuildDriver.java
new file mode 100644
index 0000000..938735b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/BuildDriver.java
@@ -0,0 +1,32 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.skyframe;
+
+import com.google.devtools.build.lib.events.EventHandler;
+
+/**
+ * A BuildDriver wraps a MemoizingEvaluator, passing along the proper Version.
+ */
+public interface BuildDriver {
+ /**
+ * See {@link MemoizingEvaluator#evaluate}, which has the same semantics except for the
+ * inclusion of a {@link Version} value.
+ */
+ <T extends SkyValue> EvaluationResult<T> evaluate(
+ Iterable<SkyKey> roots, boolean keepGoing, int numThreads, EventHandler reporter)
+ throws InterruptedException;
+
+ MemoizingEvaluator getGraphForTesting();
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/BuildingState.java b/src/main/java/com/google/devtools/build/skyframe/BuildingState.java
new file mode 100644
index 0000000..21deec1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/BuildingState.java
@@ -0,0 +1,437 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.util.GroupedList;
+import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Data the NodeEntry uses to maintain its state before it is done building. It allows the
+ * {@link NodeEntry} to keep the current state of the entry across invalidation and successive
+ * evaluations. A done node does not contain any of this data. However, if a node is marked dirty,
+ * its entry acquires a new {@code BuildingState} object, which persists until it is done again.
+ *
+ * <p>This class should be considered a private inner class of {@link NodeEntry} -- no other
+ * classes should instantiate a {@code BuildingState} object or call any of its methods directly.
+ * It is in a separate file solely to keep the {@link NodeEntry} class readable. In particular, the
+ * caller must synchronize access to this class.
+ */
+@ThreadCompatible
+final class BuildingState {
+ enum DirtyState {
+ /**
+ * The node's dependencies need to be checked to see if it needs to be rebuilt. The
+ * dependencies must be obtained through calls to {@link #getNextDirtyDirectDeps} and checked.
+ */
+ CHECK_DEPENDENCIES,
+ /**
+ * All of the node's dependencies are unchanged, and the value itself was not marked changed,
+ * so its current value is still valid -- it need not be rebuilt.
+ */
+ VERIFIED_CLEAN,
+ /**
+ * A rebuilding is required or in progress, because either the node itself changed or one of
+ * its dependencies did.
+ */
+ REBUILDING
+ }
+
+ /**
+ * During its life, a node can go through states as follows:
+ * <ol>
+ * <li>Non-existent
+ * <li>Just created ({@code evaluating} is false)
+ * <li>Evaluating ({@code evaluating} is true)
+ * <li>Done (meaning this buildingState object is null)
+ * <li>Just created (when it is dirtied during evaluation)
+ * <li>Reset (just before it is re-evaluated)
+ * <li>Evaluating
+ * <li>Done
+ * </ol>
+ *
+ * <p>The "just created" state is there to allow the {@link EvaluableGraph#createIfAbsent} and
+ * {@link NodeEntry#addReverseDepAndCheckIfDone} methods to be separate. All callers have to
+ * call both methods in that order if they want to create a node. The second method calls
+ * {@link #startEvaluating}, which transitions the current node to the "evaluating" state and
+ * returns true only the first time it was called. A caller that gets "true" back from that call
+ * must start the evaluation of this node, while any subsequent callers must not.
+ *
+ * <p>An entry is set to "evaluating" as soon as it is scheduled for evaluation. Thus, even a
+ * node that is never actually built (for instance, a dirty node that is verified as clean) is
+ * in the "evaluating" state until it is done.
+ */
+ private boolean evaluating = false;
+
+ /**
+ * The state of a dirty node. A node is marked dirty in the BuildingState constructor, and goes
+ * into either the state {@link DirtyState#CHECK_DEPENDENCIES} or {@link DirtyState#REBUILDING},
+ * depending on whether the caller specified that the node was itself changed or not. A non-null
+ * {@code dirtyState} indicates that the node {@link #isDirty} in some way.
+ */
+ private DirtyState dirtyState = null;
+
+ /**
+ * The number of dependencies that are known to be done in a {@link NodeEntry}. There is a
+ * potential check-then-act race here, so we need to make sure that when this is increased, we
+ * always check if the new value is equal to the number of required dependencies, and if so, we
+ * must re-schedule the node for evaluation.
+ *
+ * <p>There are two potential pitfalls here: 1) If multiple dependencies signal this node in
+ * close succession, this node should be scheduled exactly once. 2) If a thread is still working
+ * on this node, it should not be scheduled.
+ *
+ * <p>The first problem is solved by the {@link #signalDep} method, which also returns if the
+ * node needs to be re-scheduled, and ensures that only one thread gets a true return value.
+ *
+ * <p>The second problem is solved by first adding the newly discovered deps to a node's
+ * {@link #directDeps}, and then looping through the direct deps and registering this node as a
+ * reverse dependency. This ensures that the signaledDeps counter can only reach
+ * {@link #directDeps}.size() on the very last iteration of the loop, i.e., the thread is not
+ * working on the node anymore. Note that this requires that there is no code after the loop in
+ * {@code ParallelEvaluator.Evaluate#run}.
+ */
+ private int signaledDeps = 0;
+
+ /**
+ * Direct dependencies discovered during the build. They will be written to the immutable field
+ * {@code ValueEntry#directDeps} and the dependency group data to {@code ValueEntry#groupData}
+ * once the node is finished building. {@link SkyFunction}s can request deps in groups, and these
+ * groupings are preserved in this field.
+ */
+ private final GroupedList<SkyKey> directDeps = new GroupedList<>();
+
+ /**
+ * The set of reverse dependencies that are registered before the node has finished building.
+ * Upon building, these reverse deps will be signaled and then stored in the permanent
+ * {@code ValueEntry#reverseDeps}.
+ */
+ // TODO(bazel-team): Remove this field. With eager invalidation, all direct deps on this dirty
+ // node will be removed by the time evaluation starts, so reverse deps to signal can just be
+ // reverse deps in the main ValueEntry object.
+ private Object reverseDepsToSignal = ImmutableList.of();
+ private List<SkyKey> reverseDepsToRemove = null;
+ private boolean reverseDepIsSingleObject = false;
+
+ private static final ReverseDepsUtil<BuildingState> REVERSE_DEPS_UTIL =
+ new ReverseDepsUtil<BuildingState>() {
+ @Override
+ void setReverseDepsObject(BuildingState container, Object object) {
+ container.reverseDepsToSignal = object;
+ }
+
+ @Override
+ void setSingleReverseDep(BuildingState container, boolean singleObject) {
+ container.reverseDepIsSingleObject = singleObject;
+ }
+
+ @Override
+ void setReverseDepsToRemove(BuildingState container, List<SkyKey> object) {
+ container.reverseDepsToRemove = object;
+ }
+
+ @Override
+ Object getReverseDepsObject(BuildingState container) {
+ return container.reverseDepsToSignal;
+ }
+
+ @Override
+ boolean isSingleReverseDep(BuildingState container) {
+ return container.reverseDepIsSingleObject;
+ }
+
+ @Override
+ List<SkyKey> getReverseDepsToRemove(BuildingState container) {
+ return container.reverseDepsToRemove;
+ }
+ };
+
+ // Below are fields that are used for dirty nodes.
+
+ /**
+ * The dependencies requested (with group markers) last time the node was built (and below, the
+ * value last time the node was built). They will be compared to dependencies requested on this
+ * build to check whether this node has changed in {@link NodeEntry#setValue}. If they are null,
+ * it means that this node is being built for the first time. See {@link #directDeps} for more on
+ * dependency group storage.
+ */
+ private final GroupedList<SkyKey> lastBuildDirectDeps;
+ private final SkyValue lastBuildValue;
+
+ /**
+ * Which child should be re-evaluated next in the process of determining if this entry needs to
+ * be re-evaluated. Used by {@link #getNextDirtyDirectDeps} and {@link #signalDep(boolean)}.
+ */
+ private Iterator<Iterable<SkyKey>> dirtyDirectDepIterator = null;
+
+ BuildingState() {
+ lastBuildDirectDeps = null;
+ lastBuildValue = null;
+ }
+
+ private BuildingState(boolean isChanged, GroupedList<SkyKey> lastBuildDirectDeps,
+ SkyValue lastBuildValue) {
+ this.lastBuildDirectDeps = lastBuildDirectDeps;
+ this.lastBuildValue = Preconditions.checkNotNull(lastBuildValue);
+ Preconditions.checkState(isChanged || !this.lastBuildDirectDeps.isEmpty(),
+ "is being marked dirty, not changed, but has no children that could have dirtied it", this);
+ dirtyState = isChanged ? DirtyState.REBUILDING : DirtyState.CHECK_DEPENDENCIES;
+ if (dirtyState == DirtyState.CHECK_DEPENDENCIES) {
+ // We need to iterate through the deps to see if they have changed. Initialize the iterator.
+ dirtyDirectDepIterator = lastBuildDirectDeps.iterator();
+ }
+ }
+
+ static BuildingState newDirtyState(boolean isChanged,
+ GroupedList<SkyKey> lastBuildDirectDeps, SkyValue lastBuildValue) {
+ return new BuildingState(isChanged, lastBuildDirectDeps, lastBuildValue);
+ }
+
+ void markChanged() {
+ Preconditions.checkState(isDirty(), this);
+ Preconditions.checkState(!isChanged(), this);
+ Preconditions.checkState(!evaluating, this);
+ dirtyState = DirtyState.REBUILDING;
+ }
+
+ void forceChanged() {
+ Preconditions.checkState(isDirty(), this);
+ Preconditions.checkState(!isChanged(), this);
+ Preconditions.checkState(evaluating, this);
+ Preconditions.checkState(isReady(), this);
+ dirtyState = DirtyState.REBUILDING;
+ }
+
+ /**
+ * Returns whether all known children of this node have signaled that they are done.
+ */
+ boolean isReady() {
+ int directDepsSize = directDeps.size();
+ Preconditions.checkState(signaledDeps <= directDepsSize, "%s %s", directDepsSize, this);
+ return signaledDeps == directDepsSize;
+ }
+
+ /**
+ * Returns true if the entry is marked dirty, meaning that at least one of its transitive
+ * dependencies is marked changed.
+ *
+ * @see NodeEntry#isDirty()
+ */
+ boolean isDirty() {
+ return dirtyState != null;
+ }
+
+ /**
+ * Returns true if the entry is known to require re-evaluation.
+ *
+ * @see NodeEntry#isChanged()
+ */
+ boolean isChanged() {
+ return dirtyState == DirtyState.REBUILDING;
+ }
+
+ private boolean rebuilding() {
+ return dirtyState == DirtyState.REBUILDING;
+ }
+
+ /**
+ * Helper method to assert that node has finished building, as far as we can tell. We would
+ * actually like to check that the node has been evaluated, but that is not available in
+ * this context.
+ */
+ private void checkNotProcessing() {
+ Preconditions.checkState(evaluating, "not started building %s", this);
+ Preconditions.checkState(!isDirty() || dirtyState == DirtyState.VERIFIED_CLEAN
+ || rebuilding(), "not done building %s", this);
+ Preconditions.checkState(isReady(), "not done building %s", this);
+ }
+
+ /**
+ * Puts the node in the "evaluating" state if it is not already in it. Returns whether or not the
+ * node was already evaluating. Should only be called by
+ * {@link NodeEntry#addReverseDepAndCheckIfDone}.
+ */
+ boolean startEvaluating() {
+ boolean result = !evaluating;
+ evaluating = true;
+ return result;
+ }
+
+ /**
+ * Increments the number of children known to be finished. Returns true if the number of children
+ * finished is equal to the number of known children.
+ *
+ * <p>If the node is dirty and checking its deps for changes, this also updates {@link
+ * #dirtyState} as needed -- {@link DirtyState#REBUILDING} if the child has changed,
+ * and {@link DirtyState#VERIFIED_CLEAN} if the child has not changed and this was the last
+ * child to be checked (as determined by {@link #dirtyDirectDepIterator} == null, isReady(), and
+ * a flag set in {@link #getNextDirtyDirectDeps}).
+ *
+ * @see NodeEntry#signalDep(Version)
+ */
+ boolean signalDep(boolean childChanged) {
+ signaledDeps++;
+ if (isDirty() && !rebuilding()) {
+ // Synchronization isn't needed here because the only caller is ValueEntry, which does it
+ // through the synchronized method signalDep(long).
+ if (childChanged) {
+ dirtyState = DirtyState.REBUILDING;
+ } else if (dirtyState == DirtyState.CHECK_DEPENDENCIES && isReady()
+ && dirtyDirectDepIterator == null) {
+ // No other dep already marked this as REBUILDING, no deps outstanding, and this was
+ // the last block of deps to be checked.
+ dirtyState = DirtyState.VERIFIED_CLEAN;
+ }
+ }
+ return isReady();
+ }
+
+ /**
+ * Returns true if {@code newValue}.equals the value from the last time this node was built, and
+ * the deps requested during this evaluation are exactly those requested the last time this node
+ * was built, in the same order. Should only be used by {@link NodeEntry#setValue}.
+ */
+ boolean unchangedFromLastBuild(SkyValue newValue) {
+ checkNotProcessing();
+ return lastBuildValue.equals(newValue) && lastBuildDirectDeps.equals(directDeps);
+ }
+
+ boolean noDepsLastBuild() {
+ return lastBuildDirectDeps.isEmpty();
+ }
+
+ SkyValue getLastBuildValue() {
+ return Preconditions.checkNotNull(lastBuildValue, this);
+ }
+
+ /**
+ * Gets the current state of checking this dirty entry to see if it must be re-evaluated. Must be
+ * called each time evaluation of a dirty entry starts to find the proper action to perform next,
+ * as enumerated by {@link DirtyState}.
+ *
+ * @see NodeEntry#getDirtyState()
+ */
+ DirtyState getDirtyState() {
+ // Entry may not be ready if being built just for its errors.
+ Preconditions.checkState(isDirty(), "must be dirty to get dirty state %s", this);
+ Preconditions.checkState(evaluating, "must be evaluating to get dirty state %s", this);
+ return dirtyState;
+ }
+
+ /**
+ * Gets the next children to be re-evaluated to see if this dirty node needs to be re-evaluated.
+ *
+ * <p>If this is the last group of children to be checked, then sets {@link
+ * #dirtyDirectDepIterator} to null so that the final call to {@link #signalDep(boolean)} will
+ * know to mark this entry as {@link DirtyState#VERIFIED_CLEAN} if no deps have changed.
+ *
+ * See {@link NodeEntry#getNextDirtyDirectDeps}.
+ */
+ Collection<SkyKey> getNextDirtyDirectDeps() {
+ Preconditions.checkState(isDirty(), this);
+ Preconditions.checkState(dirtyState == DirtyState.CHECK_DEPENDENCIES, this);
+ Preconditions.checkState(evaluating, this);
+ List<SkyKey> nextDeps = ImmutableList.copyOf(dirtyDirectDepIterator.next());
+ if (!dirtyDirectDepIterator.hasNext()) {
+ // Done checking deps. If this last group is clean, the state will become VERIFIED_CLEAN.
+ dirtyDirectDepIterator = null;
+ }
+ return nextDeps;
+ }
+
+ void addDirectDeps(GroupedListHelper<SkyKey> depsThisRun) {
+ directDeps.append(depsThisRun);
+ }
+
+ /**
+ * Returns the direct deps found so far on this build. Should only be called before the node has
+ * finished building.
+ *
+ * @see NodeEntry#getTemporaryDirectDeps()
+ */
+ Set<SkyKey> getDirectDepsForBuild() {
+ return directDeps.toSet();
+ }
+
+ /**
+ * Returns the direct deps (in groups) found on this build. Should only be called when the node
+ * is done.
+ *
+ * @see NodeEntry#setStateFinishedAndReturnReverseDeps
+ */
+ GroupedList<SkyKey> getFinishedDirectDeps() {
+ return directDeps;
+ }
+
+ /**
+ * Returns reverse deps to signal that have been registered this build.
+ *
+ * @see NodeEntry#getReverseDeps()
+ */
+ ImmutableSet<SkyKey> getReverseDepsToSignal() {
+ return REVERSE_DEPS_UTIL.getReverseDeps(this);
+ }
+
+ /**
+ * Adds a reverse dependency that should be notified when this entry is done.
+ *
+ * @see NodeEntry#addReverseDepAndCheckIfDone(SkyKey)
+ */
+ void addReverseDepToSignal(SkyKey newReverseDep) {
+ REVERSE_DEPS_UTIL.consolidateReverseDepsRemovals(this);
+ REVERSE_DEPS_UTIL.addReverseDeps(this, Collections.singleton(newReverseDep));
+ }
+
+ /**
+ * @see NodeEntry#removeReverseDep(SkyKey)
+ */
+ void removeReverseDepToSignal(SkyKey reverseDep) {
+ REVERSE_DEPS_UTIL.removeReverseDep(this, reverseDep);
+ }
+
+ /**
+ * Removes a set of deps from the set of known direct deps. This is complicated by the need
+ * to maintain the group data. If we remove a dep that ended a group, then its predecessor's
+ * group data must be changed to indicate that it now ends the group.
+ *
+ * @see NodeEntry#removeUnfinishedDeps
+ */
+ void removeDirectDeps(Set<SkyKey> unfinishedDeps) {
+ directDeps.remove(unfinishedDeps);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public String toString() {
+ return Objects.toStringHelper(this) // MoreObjects is not in Guava
+ .add("evaluating", evaluating)
+ .add("dirtyState", dirtyState)
+ .add("signaledDeps", signaledDeps)
+ .add("directDeps", directDeps)
+ .add("reverseDepsToSignal", REVERSE_DEPS_UTIL.toString(this))
+ .add("lastBuildDirectDeps", lastBuildDirectDeps)
+ .add("lastBuildValue", lastBuildValue)
+ .add("dirtyDirectDepIterator", dirtyDirectDepIterator).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/CycleDeduper.java b/src/main/java/com/google/devtools/build/skyframe/CycleDeduper.java
new file mode 100644
index 0000000..f52333c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/CycleDeduper.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Dedupes C candidate cycles of size O(L) in O(CL) time and memory in the common case and
+ * O(C^2 * L) time and O(CL) memory in the extreme case.
+ *
+ * Two cycles are considered duplicates if they are exactly the same except for the entry point.
+ * For example, 'a' -> 'b' -> 'c' -> 'a' is the considered the same as 'b' -> 'c' -> 'a' -> 'b'.
+ */
+class CycleDeduper<T> {
+
+ private HashMultimap<ImmutableSet<T>, ImmutableList<T>> knownCyclesByMembers =
+ HashMultimap.create();
+
+ /**
+ * Marks a non-empty list representing a cycle of unique values as being seen and returns true
+ * iff the cycle hasn't been seen before, accounting for logical equivalence of cycles.
+ *
+ * For example, the cycle 'a' -> 'b' -> 'c' -> 'a' is represented by the list ['a', 'b', 'c']
+ * and is logically equivalent to the cycle represented by the list ['b', 'c', 'a'].
+ */
+ public boolean seen(ImmutableList<T> cycle) {
+ ImmutableSet<T> cycleMembers = ImmutableSet.copyOf(cycle);
+ Preconditions.checkState(!cycle.isEmpty());
+ Preconditions.checkState(cycle.size() == cycleMembers.size(),
+ "cycle doesn't have unique members: " + cycle);
+
+ if (knownCyclesByMembers.containsEntry(cycleMembers, cycle)) {
+ return false;
+ }
+
+ // Of the C cycles, suppose there are D cycles that have the same members (but are in an
+ // incompatible order). This code path takes O(D * L) time. The common case is that D is
+ // very small.
+ boolean found = false;
+ for (ImmutableList<T> candidateCycle : knownCyclesByMembers.get(cycleMembers)) {
+ int startPos = candidateCycle.indexOf(cycle.get(0));
+ // The use of a multimap keyed by cycle members guarantees that the first element of 'cycle'
+ // is present in 'candidateCycle'.
+ Preconditions.checkState(startPos >= 0);
+ if (equalsWithSingleLoopFrom(cycle, candidateCycle, startPos)) {
+ found = true;
+ break;
+ }
+ }
+ // We add the cycle even if it's a duplicate so that future exact copies of this can be
+ // processed in O(L) time. We are already using O(CL) memory, and this optimization doesn't
+ // change that.
+ knownCyclesByMembers.put(cycleMembers, cycle);
+ return !found;
+ }
+
+ /**
+ * Returns true iff
+ * listA[0], listA[1], ..., listA[listA.size()]
+ * is the same as
+ * listB[start], listB[start+1], ..., listB[listB.size()-1], listB[0], ..., listB[start-1]
+ */
+ private boolean equalsWithSingleLoopFrom(ImmutableList<T> listA, ImmutableList<T> listB,
+ int start) {
+ if (listA.size() != listB.size()) {
+ return false;
+ }
+ int length = listA.size();
+ for (int i = 0; i < length; i++) {
+ if (!listA.get(i).equals(listB.get((i + start) % length))) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/CycleInfo.java b/src/main/java/com/google/devtools/build/skyframe/CycleInfo.java
new file mode 100644
index 0000000..a44d2fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/CycleInfo.java
@@ -0,0 +1,144 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Data for a single cycle in the graph, together with the path to the cycle. For any value, the
+ * head of path to the cycle should be the value itself, or, if the value is actually in the cycle,
+ * the cycle should start with the value.
+ */
+public class CycleInfo implements Serializable {
+ private final ImmutableList<SkyKey> cycle;
+ private final ImmutableList<SkyKey> pathToCycle;
+
+ @VisibleForTesting
+ public CycleInfo(Iterable<SkyKey> cycle) {
+ this(ImmutableList.<SkyKey>of(), cycle);
+ }
+
+ CycleInfo(Iterable<SkyKey> pathToCycle, Iterable<SkyKey> cycle) {
+ this.pathToCycle = ImmutableList.copyOf(pathToCycle);
+ this.cycle = ImmutableList.copyOf(cycle);
+ }
+
+ // If a cycle is already known, but we are processing a value in the middle of the cycle, we need
+ // to shift the cycle so that the value is at the head.
+ private CycleInfo(Iterable<SkyKey> cycle, int cycleStart) {
+ Preconditions.checkState(cycleStart >= 0, cycleStart);
+ ImmutableList.Builder<SkyKey> cycleTail = ImmutableList.builder();
+ ImmutableList.Builder<SkyKey> cycleHead = ImmutableList.builder();
+ int index = 0;
+ for (SkyKey key : cycle) {
+ if (index >= cycleStart) {
+ cycleHead.add(key);
+ } else {
+ cycleTail.add(key);
+ }
+ index++;
+ }
+ Preconditions.checkState(cycleStart < index, "%s >= %s ??", cycleStart, index);
+ this.cycle = cycleHead.addAll(cycleTail.build()).build();
+ this.pathToCycle = ImmutableList.of();
+ }
+
+ public ImmutableList<SkyKey> getCycle() {
+ return cycle;
+ }
+
+ public ImmutableList<SkyKey> getPathToCycle() {
+ return pathToCycle;
+ }
+
+ // Given a cycle and a value, if the value is part of the cycle, shift the cycle. Otherwise,
+ // prepend the value to the head of pathToCycle.
+ private static CycleInfo normalizeCycle(final SkyKey value, CycleInfo cycle) {
+ int index = cycle.cycle.indexOf(value);
+ if (index > -1) {
+ if (!cycle.pathToCycle.isEmpty()) {
+ // The head value we are considering is already part of a cycle, but we have reached it by a
+ // roundabout way. Since we should have reached it directly as well, filter this roundabout
+ // way out. Example (c has a dependence on top):
+ // top
+ // / ^
+ // a |
+ // / \ /
+ // b-> c
+ // In the traversal, we start at top, visit a, then c, then top. This yields the
+ // cycle {top,a,c}. Then we visit b, getting (b, {top,a,c}). Then we construct the full
+ // error for a. The error should just be the cycle {top,a,c}, but we have an extra copy of
+ // it via the path through b.
+ return null;
+ }
+ return new CycleInfo(cycle.cycle, index);
+ }
+ return new CycleInfo(Iterables.concat(ImmutableList.of(value), cycle.pathToCycle),
+ cycle.cycle);
+ }
+
+ /**
+ * Normalize multiple cycles. This includes removing multiple paths to the same cycle, so that
+ * a value does not depend on the same cycle multiple ways through the same child value. Note that
+ * a value can still depend on the same cycle multiple ways, it's just that each way must be
+ * through a different child value (a path with a different first element).
+ */
+ static Iterable<CycleInfo> prepareCycles(final SkyKey value, Iterable<CycleInfo> cycles) {
+ final Set<ImmutableList<SkyKey>> alreadyDoneCycles = new HashSet<>();
+ return Iterables.filter(Iterables.transform(cycles,
+ new Function<CycleInfo, CycleInfo>() {
+ @Override
+ public CycleInfo apply(CycleInfo input) {
+ CycleInfo normalized = normalizeCycle(value, input);
+ if (normalized != null && alreadyDoneCycles.add(normalized.cycle)) {
+ return normalized;
+ }
+ return null;
+ }
+ }), Predicates.notNull());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(cycle, pathToCycle);
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ }
+ if (!(that instanceof CycleInfo)) {
+ return false;
+ }
+
+ CycleInfo thatCycle = (CycleInfo) that;
+ return thatCycle.cycle.equals(this.cycle) && thatCycle.pathToCycle.equals(this.pathToCycle);
+ }
+
+ @Override
+ public String toString() {
+ return Iterables.toString(pathToCycle) + " -> " + Iterables.toString(cycle);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/CyclesReporter.java b/src/main/java/com/google/devtools/build/skyframe/CyclesReporter.java
new file mode 100644
index 0000000..a9b0d8e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/CyclesReporter.java
@@ -0,0 +1,102 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.EventHandler;
+
+/**
+ * An utility for custom reporting of errors from cycles in the the Skyframe graph. This class is
+ * stateful in order to differentiate between new cycles and cycles that have already been
+ * reported (do not reuse the instances or cache the results as it could end up printing
+ * inconsistent information or leak memory). It treats two cycles as the same if they contain the
+ * same {@link SkyKey}s in the same order, but perhaps with different starting points. See
+ * {@link CycleDeduper} for more information.
+ */
+public class CyclesReporter {
+
+ /**
+ * Interface for reporting custom information about a single cycle.
+ */
+ public interface SingleCycleReporter {
+
+ /**
+ * Reports the given cycle and returns {@code true}, or return {@code false} if this
+ * {@link SingleCycleReporter} doesn't know how to report the cycle.
+ *
+ * @param topLevelKey the top level key that transitively depended on the cycle
+ * @param cycleInfo the cycle
+ * @param alreadyReported whether the cycle has already been reported to the
+ * {@link CyclesReporter}.
+ * @param eventHandler the eventHandler to which to report the error
+ */
+ boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, boolean alreadyReported,
+ EventHandler eventHandler);
+ }
+
+ private final ImmutableList<SingleCycleReporter> cycleReporters;
+ private final CycleDeduper<SkyKey> cycleDeduper = new CycleDeduper<>();
+
+ /**
+ * Constructs a {@link CyclesReporter} that delegates to the given {@link SingleCycleReporter}s,
+ * in the given order, to report custom information about cycles.
+ */
+ public CyclesReporter(SingleCycleReporter... cycleReporters) {
+ this.cycleReporters = ImmutableList.copyOf(cycleReporters);
+ }
+
+ /**
+ * Reports the given cycles, differentiating between cycles that have already been reported.
+ *
+ * @param cycles The {@code Iterable} of cycles.
+ * @param topLevelKey This key represents the top level value key that returned cycle errors.
+ * @param eventHandler the eventHandler to which to report the error
+ */
+ public void reportCycles(Iterable<CycleInfo> cycles, SkyKey topLevelKey,
+ EventHandler eventHandler) {
+ Preconditions.checkNotNull(eventHandler);
+ for (CycleInfo cycleInfo : cycles) {
+ boolean alreadyReported = false;
+ if (!cycleDeduper.seen(cycleInfo.getCycle())) {
+ alreadyReported = true;
+ }
+ boolean successfullyReported = false;
+ for (SingleCycleReporter cycleReporter : cycleReporters) {
+ if (cycleReporter.maybeReportCycle(topLevelKey, cycleInfo, alreadyReported, eventHandler)) {
+ successfullyReported = true;
+ break;
+ }
+ }
+ Preconditions.checkState(successfullyReported,
+ printArbitraryCycle(topLevelKey, cycleInfo, alreadyReported));
+ }
+ }
+
+ private String printArbitraryCycle(SkyKey topLevelKey, CycleInfo cycleInfo,
+ boolean alreadyReported) {
+ StringBuilder cycleMessage = new StringBuilder()
+ .append("topLevelKey: " + topLevelKey + "\n")
+ .append("alreadyReported: " + alreadyReported + "\n")
+ .append("path to cycle:\n");
+ for (SkyKey skyKey : cycleInfo.getPathToCycle()) {
+ cycleMessage.append(skyKey + "\n");
+ }
+ cycleMessage.append("cycle:\n");
+ for (SkyKey skyKey : cycleInfo.getCycle()) {
+ cycleMessage.append(skyKey + "\n");
+ }
+ return cycleMessage.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Differencer.java b/src/main/java/com/google/devtools/build/skyframe/Differencer.java
new file mode 100644
index 0000000..f6433ac
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/Differencer.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import java.util.Map;
+
+/**
+ * Calculate set of changed values in a graph.
+ */
+public interface Differencer {
+
+ /**
+ * Represents a set of changed values.
+ */
+ interface Diff {
+ /**
+ * Returns the value keys whose values have changed, but for which we don't have the new values.
+ */
+ Iterable<SkyKey> changedKeysWithoutNewValues();
+
+ /**
+ * Returns the value keys whose values have changed, along with their new values.
+ *
+ * <p> The values in here cannot have any dependencies. This is required in order to prevent
+ * conflation of injected values and derived values.
+ */
+ Map<SkyKey, ? extends SkyValue> changedKeysWithNewValues();
+ }
+
+ /**
+ * Returns the value keys that have changed between the two Versions.
+ */
+ Diff getDiff(Version fromVersion, Version toVersion) throws InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtiableGraph.java b/src/main/java/com/google/devtools/build/skyframe/DirtiableGraph.java
new file mode 100644
index 0000000..0781222
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/DirtiableGraph.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * Interface for classes that need to remove values from graph. Currently just used by {@link
+ * EagerInvalidator}.
+ *
+ * <p>This class is not intended for direct use, and is only exposed as public for use in
+ * evaluation implementations outside of this package.
+ */
+public interface DirtiableGraph extends QueryableGraph {
+ /**
+ * Remove the value with given name from the graph.
+ */
+ void remove(SkyKey key);
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTracker.java b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTracker.java
new file mode 100644
index 0000000..b0b5074
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTracker.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import java.util.Set;
+
+/**
+ * Interface for implementations that need to keep track of dirty SkyKeys.
+ */
+public interface DirtyKeyTracker {
+
+ /**
+ * Marks the {@code skyKey} as dirty.
+ */
+ @ThreadSafe
+ void dirty(SkyKey skyKey);
+
+ /**
+ * Marks the {@code skyKey} as not dirty.
+ */
+ @ThreadSafe
+ void notDirty(SkyKey skyKey);
+
+ /**
+ * Returns the set of keys k for which there was a call to dirty(k) but not a subsequent call
+ * to notDirty(k).
+ */
+ @ThreadSafe
+ Set<SkyKey> getDirtyKeys();
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTrackerImpl.java b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTrackerImpl.java
new file mode 100644
index 0000000..e3e070cb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTrackerImpl.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+/** Encapsulates a thread-safe set of SkyKeys. */
+public class DirtyKeyTrackerImpl implements DirtyKeyTracker {
+
+ private final Set<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
+
+ @Override
+ public void dirty(SkyKey skyKey) {
+ dirtyKeys.add(skyKey);
+ }
+
+ @Override
+ public void notDirty(SkyKey skyKey) {
+ dirtyKeys.remove(skyKey);
+ }
+
+ @Override
+ public Set<SkyKey> getDirtyKeys() {
+ return ImmutableSet.copyOf(dirtyKeys);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EagerInvalidator.java b/src/main/java/com/google/devtools/build/skyframe/EagerInvalidator.java
new file mode 100644
index 0000000..fc2a2c7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/EagerInvalidator.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DeletingNodeVisitor;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingNodeVisitor;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState;
+
+/**
+ * Utility class for performing eager invalidation on Skyframe graphs.
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+public final class EagerInvalidator {
+
+ private EagerInvalidator() {}
+
+ /**
+ * Deletes given values. The {@code traverseGraph} parameter controls whether this method deletes
+ * (transitive) dependents of these nodes and relevant graph edges, or just the nodes themselves.
+ * Deleting just the nodes is inconsistent unless the graph will not be used for incremental
+ * builds in the future, but unfortunately there is a case where we delete nodes intra-build. As
+ * long as the full upward transitive closure of the nodes is specified for deletion, the graph
+ * remains consistent.
+ */
+ public static void delete(DirtiableGraph graph, Iterable<SkyKey> diff,
+ EvaluationProgressReceiver invalidationReceiver, InvalidationState state,
+ boolean traverseGraph, DirtyKeyTracker dirtyKeyTracker) throws InterruptedException {
+ InvalidatingNodeVisitor visitor =
+ createVisitor(/*delete=*/true, graph, diff, invalidationReceiver, state, traverseGraph,
+ dirtyKeyTracker);
+ if (visitor != null) {
+ visitor.run();
+ }
+ }
+
+ /**
+ * Creates an invalidation visitor that is ready to run. Caller should call #run() on the visitor.
+ * Allows test classes to keep a reference to the visitor, and await exceptions/interrupts.
+ */
+ @VisibleForTesting
+ static InvalidatingNodeVisitor createVisitor(boolean delete, DirtiableGraph graph,
+ Iterable<SkyKey> diff, EvaluationProgressReceiver invalidationReceiver,
+ InvalidationState state, boolean traverseGraph, DirtyKeyTracker dirtyKeyTracker) {
+ state.update(diff);
+ if (state.isEmpty()) {
+ return null;
+ }
+ return delete
+ ? new DeletingNodeVisitor(graph, invalidationReceiver, state, traverseGraph,
+ dirtyKeyTracker)
+ : new DirtyingNodeVisitor(graph, invalidationReceiver, state, dirtyKeyTracker);
+ }
+
+ /**
+ * Invalidates given values and their upward transitive closure in the graph.
+ */
+ public static void invalidate(DirtiableGraph graph, Iterable<SkyKey> diff,
+ EvaluationProgressReceiver invalidationReceiver, InvalidationState state,
+ DirtyKeyTracker dirtyKeyTracker)
+ throws InterruptedException {
+ // If we are invalidating, we must be in an incremental build by definition, so we must
+ // maintain a consistent graph state by traversing the graph and invalidating transitive
+ // dependencies. If edges aren't present, it would be impossible to check the dependencies of
+ // a dirty node in any case.
+ InvalidatingNodeVisitor visitor =
+ createVisitor(/*delete=*/false, graph, diff, invalidationReceiver, state,
+ /*traverseGraph=*/true, dirtyKeyTracker);
+ if (visitor != null) {
+ visitor.run();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EdgelessNodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/EdgelessNodeEntry.java
new file mode 100644
index 0000000..98fb61e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/EdgelessNodeEntry.java
@@ -0,0 +1,32 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.skyframe;
+
+/**
+ * NodeEntry that does not store edges (directDeps and reverseDeps) when the node is done. Used to
+ * save memory when it is known that the graph will not be reused.
+ *
+ * <p>Graph edges must be stored for incremental builds, but if this program will terminate after a
+ * single run, edges can be thrown away in order to save memory. The edges will be stored in the
+ * {@link BuildingState} as usual while the node is being built, but will not be stored once the
+ * node is done and written to the graph. Any attempt to access the edges once the node is done will
+ * fail the build fast.
+ */
+class EdgelessNodeEntry extends NodeEntry {
+ @Override
+ protected boolean keepEdges() {
+ return false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java b/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java
new file mode 100644
index 0000000..6873d19
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java
@@ -0,0 +1,157 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.skyframe.SkyFunctionException.ReifiedSkyFunctionException;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+
+/**
+ * Information about why a {@link SkyValue} failed to evaluate successfully.
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+public class ErrorInfo implements Serializable {
+ /**
+ * The set of descendants of this value that failed to build
+ */
+ private final NestedSet<SkyKey> rootCauses;
+
+ /**
+ * An exception thrown upon a value's failure to build. The exception is used for reporting, and
+ * thus may ultimately be rethrown by the caller. As well, during a --nokeep_going evaluation, if
+ * an error value is encountered from an earlier --keep_going build, the exception to be thrown is
+ * taken from here.
+ */
+ @Nullable private final Exception exception;
+ private final SkyKey rootCauseOfException;
+
+ private final Iterable<CycleInfo> cycles;
+
+ private final boolean isTransient;
+ private final boolean isCatastrophic;
+
+ public ErrorInfo(ReifiedSkyFunctionException builderException) {
+ this.rootCauseOfException = builderException.getRootCauseSkyKey();
+ this.rootCauses = NestedSetBuilder.create(Order.STABLE_ORDER, rootCauseOfException);
+ this.exception = Preconditions.checkNotNull(builderException.getCause(), builderException);
+ this.cycles = ImmutableList.of();
+ this.isTransient = builderException.isTransient();
+ this.isCatastrophic = builderException.isCatastrophic();
+ }
+
+ ErrorInfo(CycleInfo cycleInfo) {
+ this.rootCauses = NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+ this.exception = null;
+ this.rootCauseOfException = null;
+ this.cycles = ImmutableList.of(cycleInfo);
+ this.isTransient = false;
+ this.isCatastrophic = false;
+ }
+
+ public ErrorInfo(SkyKey currentValue, Collection<ErrorInfo> childErrors) {
+ Preconditions.checkNotNull(currentValue);
+ Preconditions.checkState(!childErrors.isEmpty(),
+ "Error value %s with no exception must depend on another error value", currentValue);
+ NestedSetBuilder<SkyKey> builder = NestedSetBuilder.stableOrder();
+ ImmutableList.Builder<CycleInfo> cycleBuilder = ImmutableList.builder();
+ Exception firstException = null;
+ SkyKey firstChildKey = null;
+ boolean isTransient = false;
+ boolean isCatastrophic = false;
+ // Arbitrarily pick the first error.
+ for (ErrorInfo child : childErrors) {
+ if (firstException == null) {
+ firstException = child.getException();
+ firstChildKey = child.getRootCauseOfException();
+ }
+ builder.addTransitive(child.rootCauses);
+ cycleBuilder.addAll(CycleInfo.prepareCycles(currentValue, child.cycles));
+ isTransient |= child.isTransient();
+ isCatastrophic |= child.isCatastrophic();
+ }
+ this.rootCauses = builder.build();
+ this.exception = firstException;
+ this.rootCauseOfException = firstChildKey;
+ this.cycles = cycleBuilder.build();
+ this.isTransient = isTransient;
+ this.isCatastrophic = isCatastrophic;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("<ErrorInfo exception=%s rootCauses=%s cycles=%s>",
+ exception, rootCauses, cycles);
+ }
+
+ /**
+ * The root causes of a value that failed to build are its descendant values that failed to build.
+ * If a value's descendants all built successfully, but it failed to, its root cause will be
+ * itself. If a value depends on a cycle, but has no other errors, this method will return
+ * the empty set.
+ */
+ public Iterable<SkyKey> getRootCauses() {
+ return rootCauses;
+ }
+
+ /**
+ * The exception thrown when building a value. May be null if value's only error is depending
+ * on a cycle.
+ */
+ @Nullable public Exception getException() {
+ return exception;
+ }
+
+ public SkyKey getRootCauseOfException() {
+ return rootCauseOfException;
+ }
+
+ /**
+ * Any cycles found when building this value.
+ *
+ * <p>If there are a large number of cycles, only a limited number are returned here.
+ *
+ * <p>If this value has a child through which there are multiple paths to the same cycle, only one
+ * path is returned here. However, if there are multiple paths to the same cycle, each of which
+ * goes through a different child, each of them is returned here.
+ */
+ public Iterable<CycleInfo> getCycleInfo() {
+ return cycles;
+ }
+
+ /**
+ * Returns true iff the error is transient, i.e. if retrying the same computation could lead to a
+ * different result.
+ */
+ public boolean isTransient() {
+ return isTransient;
+ }
+
+
+ /**
+ * Returns true iff the error is catastrophic, i.e. it should halt even for a keepGoing update()
+ * call.
+ */
+ public boolean isCatastrophic() {
+ return isCatastrophic;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ErrorTransienceValue.java b/src/main/java/com/google/devtools/build/skyframe/ErrorTransienceValue.java
new file mode 100644
index 0000000..c0c445d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ErrorTransienceValue.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * A value that represents "error transience", i.e. anything which may have caused an unexpected
+ * failure.
+ */
+public final class ErrorTransienceValue implements SkyValue {
+ public static final SkyFunctionName FUNCTION_NAME =
+ new SkyFunctionName("ERROR_TRANSIENCE", false);
+
+ ErrorTransienceValue() {}
+
+ public static SkyKey key() {
+ return new SkyKey(FUNCTION_NAME, "ERROR_TRANSIENCE");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EvaluableGraph.java b/src/main/java/com/google/devtools/build/skyframe/EvaluableGraph.java
new file mode 100644
index 0000000..3d9a934
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/EvaluableGraph.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * Interface between a single version of the graph and the evaluator. Supports mutation of that
+ * single version of the graph.
+ */
+interface EvaluableGraph extends QueryableGraph {
+ /**
+ * Creates a new node with the specified key if it does not exist yet. Returns the node entry
+ * (either the existing one or the one just created), never {@code null}.
+ */
+ NodeEntry createIfAbsent(SkyKey key);
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EvaluationProgressReceiver.java b/src/main/java/com/google/devtools/build/skyframe/EvaluationProgressReceiver.java
new file mode 100644
index 0000000..7928878
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/EvaluationProgressReceiver.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+
+/**
+ * Receiver to inform callers which values have been invalidated. Values may be invalidated and then
+ * re-validated if they have been found not to be changed.
+ */
+public interface EvaluationProgressReceiver {
+ /**
+ * New state of the value entry after evaluation.
+ */
+ enum EvaluationState {
+ /** The value was successfully re-evaluated. */
+ BUILT,
+ /** The value is clean or re-validated. */
+ CLEAN,
+ }
+
+ /**
+ * New state of the value entry after invalidation.
+ */
+ enum InvalidationState {
+ /** The value is dirty, although it might get re-validated again. */
+ DIRTY,
+ /** The value is dirty and got deleted, cannot get re-validated again. */
+ DELETED,
+ }
+
+ /**
+ * Notifies that {@code value} has been invalidated.
+ *
+ * <p>{@code state} indicates the new state of the value.
+ *
+ * <p>This method is not called on invalidation of values which do not have a value (usually
+ * because they are in error).
+ *
+ * <p>May be called concurrently from multiple threads, possibly with the same {@code value}
+ * object.
+ */
+ @ThreadSafety.ThreadSafe
+ void invalidated(SkyValue value, InvalidationState state);
+
+ /**
+ * Notifies that {@code skyKey} is about to get queued for evaluation.
+ *
+ * <p>Note that we don't guarantee that it actually got enqueued or will, only that if
+ * everything "goes well" (e.g. no interrupts happen) it will.
+ *
+ * <p>This guarantee is intentionally vague to encourage writing robust implementations.
+ */
+ @ThreadSafety.ThreadSafe
+ void enqueueing(SkyKey skyKey);
+
+ /**
+ * Notifies that {@code value} has been evaluated.
+ *
+ * <p>{@code state} indicates the new state of the value.
+ *
+ * <p>This method is not called if the value builder threw an error when building this value.
+ */
+ @ThreadSafety.ThreadSafe
+ void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state);
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EvaluationResult.java b/src/main/java/com/google/devtools/build/skyframe/EvaluationResult.java
new file mode 100644
index 0000000..e518dca
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/EvaluationResult.java
@@ -0,0 +1,163 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The result of a Skyframe {@link Evaluator#eval} call. Will contain all the
+ * successfully evaluated values, retrievable through {@link #get}. As well, the {@link ErrorInfo}
+ * for the first value that failed to evaluate (in the non-keep-going case), or any remaining values
+ * that failed to evaluate (in the keep-going case) will be retrievable.
+ *
+ * @param <T> The type of the values that the caller has requested.
+ */
+public class EvaluationResult<T extends SkyValue> {
+
+ private final boolean hasError;
+
+ private final Map<SkyKey, T> resultMap;
+ private final Map<SkyKey, ErrorInfo> errorMap;
+
+ /**
+ * Constructor for the "completed" case. Used only by {@link Builder}.
+ */
+ private EvaluationResult(Map<SkyKey, T> result, Map<SkyKey, ErrorInfo> errorMap,
+ boolean hasError) {
+ Preconditions.checkState(errorMap.isEmpty() || hasError,
+ "result=%s, errorMap=%s", result, errorMap);
+ this.resultMap = Preconditions.checkNotNull(result);
+ this.errorMap = Preconditions.checkNotNull(errorMap);
+ this.hasError = hasError;
+ }
+
+ /**
+ * Get a successfully evaluated value.
+ */
+ public T get(SkyKey key) {
+ Preconditions.checkNotNull(resultMap, key);
+ return resultMap.get(key);
+ }
+
+ /**
+ * @return Whether or not the eval successfully evaluated all requested values. Note that this
+ * may return true even if all values returned are available in get(). This happens if a top-level
+ * value depends transitively on some value that recovered from a {@link SkyFunctionException}.
+ */
+ public boolean hasError() {
+ return hasError;
+ }
+
+ /**
+ * @return All successfully evaluated {@link SkyValue}s.
+ */
+ public Collection<T> values() {
+ return Collections.unmodifiableCollection(resultMap.values());
+ }
+
+ /**
+ * Returns {@link Map} of {@link SkyKey}s to {@link ErrorInfo}. Note that currently some
+ * of the returned SkyKeys may not be the ones requested by the user. Moreover, the SkyKey
+ * is not necessarily the cause of the error -- it is just the value that was being evaluated
+ * when the error was discovered. For the cause of the error, use
+ * {@link ErrorInfo#getRootCauses()} on each ErrorInfo.
+ */
+ public Map<SkyKey, ErrorInfo> errorMap() {
+ return ImmutableMap.copyOf(errorMap);
+ }
+
+ /**
+ * @param key {@link SkyKey} to get {@link ErrorInfo} for.
+ */
+ public ErrorInfo getError(SkyKey key) {
+ return Preconditions.checkNotNull(errorMap, key).get(key);
+ }
+
+ /**
+ * @return Names of all values that were successfully evaluated.
+ */
+ public <S> Collection<? extends S> keyNames() {
+ return this.<S>getNames(resultMap.keySet());
+ }
+
+ @SuppressWarnings("unchecked")
+ private <S> Collection<? extends S> getNames(Collection<SkyKey> keys) {
+ Collection<S> names = Lists.newArrayListWithCapacity(keys.size());
+ for (SkyKey key : keys) {
+ names.add((S) key.argument());
+ }
+ return names;
+ }
+
+ /**
+ * Returns some error info. Convenience method equivalent to
+ * Iterables.getFirst({@link #errorMap()}, null).getValue().
+ */
+ public ErrorInfo getError() {
+ return Iterables.getFirst(errorMap.entrySet(), null).getValue();
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public String toString() {
+ return Objects.toStringHelper(this) // MoreObjects is not in Guava
+ .add("hasError", hasError)
+ .add("errorMap", errorMap)
+ .add("resultMap", resultMap)
+ .toString();
+ }
+
+ public static <T extends SkyValue> Builder<T> builder() {
+ return new Builder<>();
+ }
+
+ /**
+ * Builder for {@link EvaluationResult}.
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+ public static class Builder<T extends SkyValue> {
+ private final Map<SkyKey, T> result = new HashMap<>();
+ private final Map<SkyKey, ErrorInfo> errors = new HashMap<>();
+ private boolean hasError = false;
+
+ @SuppressWarnings("unchecked")
+ public Builder<T> addResult(SkyKey key, SkyValue value) {
+ result.put(key, Preconditions.checkNotNull((T) value, key));
+ return this;
+ }
+
+ public Builder<T> addError(SkyKey key, ErrorInfo error) {
+ errors.put(key, Preconditions.checkNotNull(error, key));
+ return this;
+ }
+
+ public EvaluationResult<T> build() {
+ return new EvaluationResult<>(result, errors, hasError);
+ }
+
+ public void setHasError(boolean hasError) {
+ this.hasError = hasError;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Evaluator.java b/src/main/java/com/google/devtools/build/skyframe/Evaluator.java
new file mode 100644
index 0000000..342eff1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/Evaluator.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.devtools.build.lib.events.EventHandler;
+
+/**
+ * An interface for the evaluator for a particular graph version.
+ */
+public interface Evaluator {
+ /**
+ * Factory to create Evaluator instances.
+ */
+ interface Factory {
+ /**
+ * @param graph the graph to operate on
+ * @param graphVersion the version at which to write entries in the graph.
+ * @param reporter where to write warning/error/progress messages.
+ * @param keepGoing whether {@link #eval} should continue if building a {link Value} fails.
+ * Otherwise, we throw an exception on failure.
+ */
+ Evaluator create(ProcessableGraph graph, long graphVersion, EventHandler reporter,
+ boolean keepGoing);
+ }
+
+ /**
+ * Evaluates a set of values. Returns an {@link EvaluationResult}. All elements of skyKeys must
+ * be keys for Values of subtype T.
+ */
+ <T extends SkyValue> EvaluationResult<T> eval(Iterable<SkyKey> skyKeys)
+ throws InterruptedException;
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ImmutableDiff.java b/src/main/java/com/google/devtools/build/skyframe/ImmutableDiff.java
new file mode 100644
index 0000000..46ab29e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ImmutableDiff.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * Immutable implementation of {@link Differencer.Diff}.
+ */
+public class ImmutableDiff implements Differencer.Diff {
+
+ private final ImmutableList<SkyKey> valuesToInvalidate;
+ private final ImmutableMap<SkyKey, SkyValue> valuesToInject;
+
+ public ImmutableDiff(Iterable<SkyKey> valuesToInvalidate, Map<SkyKey, SkyValue> valuesToInject) {
+ this.valuesToInvalidate = ImmutableList.copyOf(valuesToInvalidate);
+ this.valuesToInject = ImmutableMap.copyOf(valuesToInject);
+ }
+
+ @Override
+ public Iterable<SkyKey> changedKeysWithoutNewValues() {
+ return valuesToInvalidate;
+ }
+
+ @Override
+ public Map<SkyKey, SkyValue> changedKeysWithNewValues() {
+ return valuesToInject;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java
new file mode 100644
index 0000000..44956da
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java
@@ -0,0 +1,126 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * An in-memory graph implementation. All operations are thread-safe with ConcurrentMap semantics.
+ * Also see {@link NodeEntry}.
+ *
+ * <p>This class is public only for use in alternative graph implementations.
+ */
+public class InMemoryGraph implements ProcessableGraph {
+
+ protected final ConcurrentMap<SkyKey, NodeEntry> nodeMap =
+ new MapMaker().initialCapacity(1024).concurrencyLevel(200).makeMap();
+ private final boolean keepEdges;
+
+ InMemoryGraph() {
+ this(/*keepEdges=*/true);
+ }
+
+ public InMemoryGraph(boolean keepEdges) {
+ this.keepEdges = keepEdges;
+ }
+
+ @Override
+ public void remove(SkyKey skyKey) {
+ nodeMap.remove(skyKey);
+ }
+
+ @Override
+ public NodeEntry get(SkyKey skyKey) {
+ return nodeMap.get(skyKey);
+ }
+
+ @Override
+ public NodeEntry createIfAbsent(SkyKey key) {
+ NodeEntry newval = keepEdges ? new NodeEntry() : new EdgelessNodeEntry();
+ NodeEntry oldval = nodeMap.putIfAbsent(key, newval);
+ return oldval == null ? newval : oldval;
+ }
+
+ /** Only done nodes exist to the outside world. */
+ private static final Predicate<NodeEntry> NODE_DONE_PREDICATE =
+ new Predicate<NodeEntry>() {
+ @Override
+ public boolean apply(NodeEntry entry) {
+ return entry != null && entry.isDone();
+ }
+ };
+
+ /**
+ * Returns a value, if it exists. If not, returns null.
+ */
+ @Nullable public SkyValue getValue(SkyKey key) {
+ NodeEntry entry = get(key);
+ return NODE_DONE_PREDICATE.apply(entry) ? entry.getValue() : null;
+ }
+
+ /**
+ * Returns a read-only live view of the nodes in the graph. All node are included. Dirty values
+ * include their Node value. Values in error have a null value.
+ */
+ Map<SkyKey, SkyValue> getValues() {
+ return Collections.unmodifiableMap(Maps.transformValues(
+ nodeMap,
+ new Function<NodeEntry, SkyValue>() {
+ @Override
+ public SkyValue apply(NodeEntry entry) {
+ return entry.toValue();
+ }
+ }));
+ }
+
+ /**
+ * Returns a read-only live view of the done values in the graph. Dirty, changed, and error values
+ * are not present in the returned map
+ */
+ Map<SkyKey, SkyValue> getDoneValues() {
+ return Collections.unmodifiableMap(Maps.filterValues(Maps.transformValues(
+ nodeMap,
+ new Function<NodeEntry, SkyValue>() {
+ @Override
+ public SkyValue apply(NodeEntry entry) {
+ return entry.isDone() ? entry.getValue() : null;
+ }
+ }), Predicates.notNull()));
+ }
+
+ // Only for use by MemoizingEvaluator#delete
+ Map<SkyKey, NodeEntry> getAllValues() {
+ return Collections.unmodifiableMap(nodeMap);
+ }
+
+ @VisibleForTesting
+ protected ConcurrentMap<SkyKey, NodeEntry> getNodeMap() {
+ return nodeMap;
+ }
+
+ boolean keepsEdges() {
+ return keepEdges;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java
new file mode 100644
index 0000000..827cc7b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java
@@ -0,0 +1,317 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.skyframe.Differencer.Diff;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DeletingInvalidationState;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingInvalidationState;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState;
+import com.google.devtools.build.skyframe.NodeEntry.DependencyState;
+
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.Nullable;
+
+/**
+ * An inmemory implementation that uses the eager invalidation strategy. This class is, by itself,
+ * not thread-safe. Neither is it thread-safe to use this class in parallel with any of the
+ * returned graphs. However, it is allowed to access the graph from multiple threads as long as
+ * that does not happen in parallel with an {@link #evaluate} call.
+ *
+ * <p>This memoizing evaluator requires a sequential versioning scheme. Evaluations
+ * must pass in a monotonically increasing {@link IntVersion}.
+ */
+public final class InMemoryMemoizingEvaluator implements MemoizingEvaluator {
+
+ private final ImmutableMap<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions;
+ @Nullable private final EvaluationProgressReceiver progressReceiver;
+ // Not final only for testing.
+ private InMemoryGraph graph;
+ private IntVersion lastGraphVersion = null;
+
+ // State related to invalidation and deletion.
+ private Set<SkyKey> valuesToDelete = new LinkedHashSet<>();
+ private Set<SkyKey> valuesToDirty = new LinkedHashSet<>();
+ private Map<SkyKey, SkyValue> valuesToInject = new HashMap<>();
+ private final DirtyKeyTracker dirtyKeyTracker = new DirtyKeyTrackerImpl();
+ private final InvalidationState deleterState = new DeletingInvalidationState();
+ private final Differencer differencer;
+
+ // Keep edges in graph. Can be false to save memory, in which case incremental builds are
+ // not possible.
+ private final boolean keepEdges;
+
+ // Values that the caller explicitly specified are assumed to be changed -- they will be
+ // re-evaluated even if none of their children are changed.
+ private final InvalidationState invalidatorState = new DirtyingInvalidationState();
+
+ private final EmittedEventState emittedEventState;
+
+ private final AtomicBoolean evaluating = new AtomicBoolean(false);
+
+ public InMemoryMemoizingEvaluator(
+ Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer) {
+ this(skyFunctions, differencer, null);
+ }
+
+ public InMemoryMemoizingEvaluator(
+ Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer,
+ @Nullable EvaluationProgressReceiver invalidationReceiver) {
+ this(skyFunctions, differencer, invalidationReceiver, new EmittedEventState(), true);
+ }
+
+ public InMemoryMemoizingEvaluator(
+ Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer,
+ @Nullable EvaluationProgressReceiver invalidationReceiver,
+ EmittedEventState emittedEventState, boolean keepEdges) {
+ this.skyFunctions = ImmutableMap.copyOf(skyFunctions);
+ this.differencer = Preconditions.checkNotNull(differencer);
+ this.progressReceiver = invalidationReceiver;
+ this.graph = new InMemoryGraph(keepEdges);
+ this.emittedEventState = emittedEventState;
+ this.keepEdges = keepEdges;
+ }
+
+ private void invalidate(Iterable<SkyKey> diff) {
+ Iterables.addAll(valuesToDirty, diff);
+ }
+
+ @Override
+ public void delete(final Predicate<SkyKey> deletePredicate) {
+ valuesToDelete.addAll(
+ Maps.filterEntries(graph.getAllValues(), new Predicate<Entry<SkyKey, NodeEntry>>() {
+ @Override
+ public boolean apply(Entry<SkyKey, NodeEntry> input) {
+ return input.getValue().isDirty() || deletePredicate.apply(input.getKey());
+ }
+ }).keySet());
+ }
+
+ @Override
+ public void deleteDirty(long versionAgeLimit) {
+ Preconditions.checkArgument(versionAgeLimit >= 0);
+ final Version threshold = new IntVersion(lastGraphVersion.getVal() - versionAgeLimit);
+ valuesToDelete.addAll(
+ Sets.filter(dirtyKeyTracker.getDirtyKeys(), new Predicate<SkyKey>() {
+ @Override
+ public boolean apply(SkyKey skyKey) {
+ NodeEntry entry = graph.get(skyKey);
+ Preconditions.checkNotNull(entry, skyKey);
+ Preconditions.checkState(entry.isDirty(), skyKey);
+ return entry.getVersion().atMost(threshold);
+ }
+ }));
+ }
+
+ @Override
+ public <T extends SkyValue> EvaluationResult<T> evaluate(Iterable<SkyKey> roots, Version version,
+ boolean keepGoing, int numThreads, EventHandler eventHandler)
+ throws InterruptedException {
+ // NOTE: Performance critical code. See bug "Null build performance parity".
+ IntVersion intVersion = (IntVersion) version;
+ Preconditions.checkState((lastGraphVersion == null && intVersion.getVal() == 0)
+ || version.equals(lastGraphVersion.next()),
+ "InMemoryGraph supports only monotonically increasing Integer versions: %s %s",
+ lastGraphVersion, version);
+ setAndCheckEvaluateState(true, roots);
+ try {
+ // The RecordingDifferencer implementation is not quite working as it should be at this point.
+ // It clears the internal data structures after getDiff is called and will not return
+ // diffs for historical versions. This makes the following code sensitive to interrupts.
+ // Ideally we would simply not update lastGraphVersion if an interrupt occurs.
+ Diff diff = differencer.getDiff(lastGraphVersion, version);
+ valuesToInject.putAll(diff.changedKeysWithNewValues());
+ invalidate(diff.changedKeysWithoutNewValues());
+ pruneInjectedValues(valuesToInject);
+ invalidate(valuesToInject.keySet());
+
+ performInvalidation();
+ injectValues(intVersion);
+
+ ParallelEvaluator evaluator = new ParallelEvaluator(graph, intVersion,
+ skyFunctions, eventHandler, emittedEventState, keepGoing, numThreads, progressReceiver,
+ dirtyKeyTracker);
+ return evaluator.eval(roots);
+ } finally {
+ lastGraphVersion = intVersion;
+ setAndCheckEvaluateState(false, roots);
+ }
+ }
+
+ /**
+ * Removes entries in {@code valuesToInject} whose values are equal to the present values in the
+ * graph.
+ */
+ private void pruneInjectedValues(Map<SkyKey, SkyValue> valuesToInject) {
+ for (Iterator<Entry<SkyKey, SkyValue>> it = valuesToInject.entrySet().iterator();
+ it.hasNext();) {
+ Entry<SkyKey, SkyValue> entry = it.next();
+ SkyKey key = entry.getKey();
+ SkyValue newValue = entry.getValue();
+ NodeEntry prevEntry = graph.get(key);
+ if (prevEntry != null && prevEntry.isDone()) {
+ Iterable<SkyKey> directDeps = prevEntry.getDirectDeps();
+ Preconditions.checkState(Iterables.isEmpty(directDeps),
+ "existing entry for %s has deps: %s", key, directDeps);
+ if (newValue.equals(prevEntry.getValue())
+ && !valuesToDirty.contains(key) && !valuesToDelete.contains(key)) {
+ it.remove();
+ }
+ }
+ }
+ }
+
+ /**
+ * Injects values in {@code valuesToInject} into the graph.
+ */
+ private void injectValues(IntVersion version) {
+ if (valuesToInject.isEmpty()) {
+ return;
+ }
+ for (Entry<SkyKey, SkyValue> entry : valuesToInject.entrySet()) {
+ SkyKey key = entry.getKey();
+ SkyValue value = entry.getValue();
+ Preconditions.checkState(value != null, key);
+ NodeEntry prevEntry = graph.createIfAbsent(key);
+ if (prevEntry.isDirty()) {
+ // There was an existing entry for this key in the graph.
+ // Get the node in the state where it is able to accept a value.
+ Preconditions.checkState(prevEntry.getTemporaryDirectDeps().isEmpty(), key);
+
+ DependencyState newState = prevEntry.addReverseDepAndCheckIfDone(null);
+ Preconditions.checkState(newState == DependencyState.NEEDS_SCHEDULING, key);
+
+ // Check that the previous node has no dependencies. Overwriting a value with deps with an
+ // injected value (which is by definition deps-free) needs a little additional bookkeeping
+ // (removing reverse deps from the dependencies), but more importantly it's something that
+ // we want to avoid, because it indicates confusion of input values and derived values.
+ Preconditions.checkState(prevEntry.noDepsLastBuild(),
+ "existing entry for %s has deps: %s", key, prevEntry);
+ }
+ prevEntry.setValue(value, version);
+ // The evaluate method previously invalidated all keys in valuesToInject that survived the
+ // pruneInjectedValues call. Now that this key's injected value is set, it is no longer dirty.
+ dirtyKeyTracker.notDirty(key);
+ }
+ // Start with a new map to avoid bloat since clear() does not downsize the map.
+ valuesToInject = new HashMap<>();
+ }
+
+ private void performInvalidation() throws InterruptedException {
+ EagerInvalidator.delete(graph, valuesToDelete, progressReceiver, deleterState, keepEdges,
+ dirtyKeyTracker);
+ // Note that clearing the valuesToDelete would not do an internal resizing. Therefore, if any
+ // build has a large set of dirty values, subsequent operations (even clearing) will be slower.
+ // Instead, just start afresh with a new LinkedHashSet.
+ valuesToDelete = new LinkedHashSet<>();
+
+ EagerInvalidator.invalidate(graph, valuesToDirty, progressReceiver, invalidatorState,
+ dirtyKeyTracker);
+ // Ditto.
+ valuesToDirty = new LinkedHashSet<>();
+ }
+
+ private void setAndCheckEvaluateState(boolean newValue, Object requestInfo) {
+ Preconditions.checkState(evaluating.getAndSet(newValue) != newValue,
+ "Re-entrant evaluation for request: %s", requestInfo);
+ }
+
+ @Override
+ public Map<SkyKey, SkyValue> getValues() {
+ return graph.getValues();
+ }
+
+ @Override
+ public Map<SkyKey, SkyValue> getDoneValues() {
+ return graph.getDoneValues();
+ }
+
+ @Override
+ @Nullable public SkyValue getExistingValueForTesting(SkyKey key) {
+ return graph.getValue(key);
+ }
+
+ @Override
+ @Nullable public ErrorInfo getExistingErrorForTesting(SkyKey key) {
+ NodeEntry entry = graph.get(key);
+ return (entry == null || !entry.isDone()) ? null : entry.getErrorInfo();
+ }
+
+ public void setGraphForTesting(InMemoryGraph graph) {
+ this.graph = graph;
+ }
+
+ @Override
+ public void dump(boolean summarize, PrintStream out) {
+ if (summarize) {
+ long nodes = 0;
+ long edges = 0;
+ for (NodeEntry entry : graph.getAllValues().values()) {
+ nodes++;
+ if (entry.isDone()) {
+ edges += Iterables.size(entry.getDirectDeps());
+ }
+ }
+ out.println("Node count: " + nodes);
+ out.println("Edge count: " + edges);
+ } else {
+ Function<SkyKey, String> keyFormatter =
+ new Function<SkyKey, String>() {
+ @Override
+ public String apply(SkyKey key) {
+ return String.format("%s:%s",
+ key.functionName(), key.argument().toString().replace('\n', '_'));
+ }
+ };
+
+ for (Entry<SkyKey, NodeEntry> mapPair : graph.getAllValues().entrySet()) {
+ SkyKey key = mapPair.getKey();
+ NodeEntry entry = mapPair.getValue();
+ if (entry.isDone()) {
+ out.print(keyFormatter.apply(key));
+ out.print("|");
+ out.println(Joiner.on('|').join(
+ Iterables.transform(entry.getDirectDeps(), keyFormatter)));
+ }
+ }
+ }
+ }
+
+ public static final EvaluatorSupplier SUPPLIER = new EvaluatorSupplier() {
+ @Override
+ public MemoizingEvaluator create(
+ Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer,
+ @Nullable EvaluationProgressReceiver invalidationReceiver,
+ EmittedEventState emittedEventState, boolean keepEdges) {
+ return new InMemoryMemoizingEvaluator(skyFunctions, differencer, invalidationReceiver,
+ emittedEventState, keepEdges);
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Injectable.java b/src/main/java/com/google/devtools/build/skyframe/Injectable.java
new file mode 100644
index 0000000..5325df3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/Injectable.java
@@ -0,0 +1,23 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import java.util.Map;
+
+/**
+ * An object that accepts Skyframe key / value mapping.
+ */
+public interface Injectable {
+ void inject(Map<SkyKey, ? extends SkyValue> values);
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/IntVersion.java b/src/main/java/com/google/devtools/build/skyframe/IntVersion.java
new file mode 100644
index 0000000..3d2a31d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/IntVersion.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * Versioning scheme based on integers.
+ */
+public final class IntVersion implements Version {
+
+ private final long val;
+
+ public IntVersion(long val) {
+ this.val = val;
+ }
+
+ public long getVal() {
+ return val;
+ }
+
+ public IntVersion next() {
+ return new IntVersion(val + 1);
+ }
+
+ @Override
+ public boolean atMost(Version other) {
+ if (!(other instanceof IntVersion)) {
+ return false;
+ }
+ return val <= ((IntVersion) other).val;
+ }
+
+ @Override
+ public int hashCode() {
+ return Long.valueOf(val).hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof IntVersion) {
+ IntVersion other = (IntVersion) obj;
+ return other.val == val;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "IntVersion: " + val;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InvalidatingNodeVisitor.java b/src/main/java/com/google/devtools/build/skyframe/InvalidatingNodeVisitor.java
new file mode 100644
index 0000000..7abf6c6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/InvalidatingNodeVisitor.java
@@ -0,0 +1,350 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.Nullable;
+
+/**
+ * A visitor that is useful for invalidating transitive dependencies of Skyframe nodes.
+ *
+ * <p>Interruptibility: It is safe to interrupt the invalidation process at any time. Consider a
+ * graph and a set of modified nodes. Then the reverse transitive closure of the modified nodes is
+ * the set of dirty nodes. We provide interruptibility by making sure that the following invariant
+ * holds at any time:
+ *
+ * <p>If a node is dirty, but not removed (or marked as dirty) yet, then either it or any of its
+ * transitive dependencies must be in the {@link #pendingVisitations} set. Furthermore, reverse dep
+ * pointers must always point to existing nodes.
+ *
+ * <p>Thread-safety: This class should only be instantiated and called on a single thread, but
+ * internally it spawns many worker threads to process the graph. The thread-safety of the workers
+ * on the graph can be delicate, and is documented below. Moreover, no other modifications to the
+ * graph can take place while invalidation occurs.
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+public abstract class InvalidatingNodeVisitor extends AbstractQueueVisitor {
+
+ // Default thread count is equal to the number of cores to exploit
+ // that level of hardware parallelism, since invalidation should be CPU-bound.
+ // We may consider increasing this in the future.
+ private static final int DEFAULT_THREAD_COUNT = Runtime.getRuntime().availableProcessors();
+
+ private static final boolean MUST_EXIST = true;
+
+ protected final DirtiableGraph graph;
+ @Nullable protected final EvaluationProgressReceiver invalidationReceiver;
+ protected final DirtyKeyTracker dirtyKeyTracker;
+ // Aliased to InvalidationState.pendingVisitations.
+ protected final Set<Pair<SkyKey, InvalidationType>> pendingVisitations;
+
+ protected InvalidatingNodeVisitor(
+ DirtiableGraph graph, @Nullable EvaluationProgressReceiver invalidationReceiver,
+ InvalidationState state, DirtyKeyTracker dirtyKeyTracker) {
+ super(/*concurrent*/true,
+ /*corePoolSize*/DEFAULT_THREAD_COUNT,
+ /*maxPoolSize*/DEFAULT_THREAD_COUNT,
+ 1, TimeUnit.SECONDS,
+ /*failFastOnException*/true,
+ /*failFastOnInterrupt*/true,
+ "skyframe-invalidator");
+ this.graph = Preconditions.checkNotNull(graph);
+ this.invalidationReceiver = invalidationReceiver;
+ this.dirtyKeyTracker = Preconditions.checkNotNull(dirtyKeyTracker);
+ this.pendingVisitations = state.pendingValues;
+ }
+
+ /**
+ * Initiates visitation and waits for completion.
+ */
+ void run() throws InterruptedException {
+ // Make a copy to avoid concurrent modification confusing us as to which nodes were passed by
+ // the caller, and which are added by other threads during the run. Since no tasks have been
+ // started yet (the queueDirtying calls start them), this is thread-safe.
+ for (Pair<SkyKey, InvalidationType> visitData : ImmutableList.copyOf(pendingVisitations)) {
+ // The caller may have specified non-existent SkyKeys, or there may be stale SkyKeys in
+ // pendingVisitations that have already been deleted. In both these cases, the nodes will not
+ // exist in the graph, so we must be tolerant of that case.
+ visit(visitData.first, visitData.second, !MUST_EXIST);
+ }
+ work(/*failFastOnInterrupt=*/true);
+ Preconditions.checkState(pendingVisitations.isEmpty(),
+ "All dirty nodes should have been processed: %s", pendingVisitations);
+ }
+
+ protected void informInvalidationReceiver(SkyValue value,
+ EvaluationProgressReceiver.InvalidationState state) {
+ if (invalidationReceiver != null && value != null) {
+ invalidationReceiver.invalidated(value, state);
+ }
+ }
+
+ /**
+ * Enqueues a node for invalidation.
+ */
+ @ThreadSafe
+ abstract void visit(SkyKey key, InvalidationType second, boolean mustExist);
+
+ @VisibleForTesting
+ enum InvalidationType {
+ /**
+ * The node is dirty and must be recomputed.
+ */
+ CHANGED,
+ /**
+ * The node is dirty, but may be marked clean later during change pruning.
+ */
+ DIRTIED,
+ /**
+ * The node is deleted.
+ */
+ DELETED;
+ }
+
+ /**
+ * Invalidation state object that keeps track of which nodes need to be invalidated, but have not
+ * been dirtied/deleted yet. This supports interrupts - by only deleting a node from this set
+ * when all its parents have been invalidated, we ensure that no information is lost when an
+ * interrupt comes in.
+ */
+ static class InvalidationState {
+ private final Set<Pair<SkyKey, InvalidationType>> pendingValues = Sets.newConcurrentHashSet();
+ private final InvalidationType defaultUpdateType;
+
+ private InvalidationState(InvalidationType defaultUpdateType) {
+ this.defaultUpdateType = Preconditions.checkNotNull(defaultUpdateType);
+ }
+
+ void update(Iterable<SkyKey> diff) {
+ Iterables.addAll(pendingValues, Iterables.transform(diff,
+ new Function<SkyKey, Pair<SkyKey, InvalidationType>>() {
+ @Override
+ public Pair<SkyKey, InvalidationType> apply(SkyKey skyKey) {
+ return Pair.of(skyKey, defaultUpdateType);
+ }
+ }));
+ }
+
+ @VisibleForTesting
+ boolean isEmpty() {
+ return pendingValues.isEmpty();
+ }
+
+ @VisibleForTesting
+ Set<Pair<SkyKey, InvalidationType>> getInvalidationsForTesting() {
+ return ImmutableSet.copyOf(pendingValues);
+ }
+ }
+
+ public static class DirtyingInvalidationState extends InvalidationState {
+ public DirtyingInvalidationState() {
+ super(InvalidationType.CHANGED);
+ }
+ }
+
+ static class DeletingInvalidationState extends InvalidationState {
+ public DeletingInvalidationState() {
+ super(InvalidationType.DELETED);
+ }
+ }
+
+ /**
+ * A node-deleting implementation.
+ */
+ static class DeletingNodeVisitor extends InvalidatingNodeVisitor {
+
+ private final Set<SkyKey> visitedValues = Sets.newConcurrentHashSet();
+ private final boolean traverseGraph;
+
+ protected DeletingNodeVisitor(DirtiableGraph graph,
+ EvaluationProgressReceiver invalidationReceiver, InvalidationState state,
+ boolean traverseGraph, DirtyKeyTracker dirtyKeyTracker) {
+ super(graph, invalidationReceiver, state, dirtyKeyTracker);
+ this.traverseGraph = traverseGraph;
+ }
+
+ @Override
+ public void visit(final SkyKey key, InvalidationType invalidationType, boolean mustExist) {
+ Preconditions.checkState(invalidationType == InvalidationType.DELETED, key);
+ if (!visitedValues.add(key)) {
+ return;
+ }
+ final Pair<SkyKey, InvalidationType> invalidationPair = Pair.of(key, invalidationType);
+ pendingVisitations.add(invalidationPair);
+ enqueue(new Runnable() {
+ @Override
+ public void run() {
+ NodeEntry entry = graph.get(key);
+ if (entry == null) {
+ pendingVisitations.remove(invalidationPair);
+ return;
+ }
+
+ if (traverseGraph) {
+ // Propagate deletion upwards.
+ for (SkyKey reverseDep : entry.getReverseDeps()) {
+ visit(reverseDep, InvalidationType.DELETED, !MUST_EXIST);
+ }
+ }
+
+ if (entry.isDone()) {
+ // Only process this node's value and children if it is done, since dirty nodes have
+ // no awareness of either.
+
+ // Unregister this node from direct deps, since reverse dep edges cannot point to
+ // non-existent nodes.
+ if (traverseGraph) {
+ for (SkyKey directDep : entry.getDirectDeps()) {
+ NodeEntry dep = graph.get(directDep);
+ if (dep != null) {
+ dep.removeReverseDep(key);
+ }
+ }
+ }
+ // Allow custom Value-specific logic to update dirtiness status.
+ informInvalidationReceiver(entry.getValue(),
+ EvaluationProgressReceiver.InvalidationState.DELETED);
+ }
+ if (traverseGraph) {
+ // Force reverseDeps consolidation (validates that attempts to remove reverse deps were
+ // really successful.
+ entry.getReverseDeps();
+ }
+ // Actually remove the node.
+ graph.remove(key);
+ dirtyKeyTracker.notDirty(key);
+
+ // Remove the node from the set as the last operation.
+ pendingVisitations.remove(invalidationPair);
+ }
+ });
+ }
+ }
+
+ /**
+ * A node-dirtying implementation.
+ */
+ static class DirtyingNodeVisitor extends InvalidatingNodeVisitor {
+
+ private final Set<Pair<SkyKey, InvalidationType>> visited = Sets.newConcurrentHashSet();
+
+ protected DirtyingNodeVisitor(DirtiableGraph graph,
+ EvaluationProgressReceiver invalidationReceiver, InvalidationState state,
+ DirtyKeyTracker dirtyKeyTracker) {
+ super(graph, invalidationReceiver, state, dirtyKeyTracker);
+ }
+
+ /**
+ * Queues a task to dirty the node named by {@code key}. May be called from multiple threads.
+ * It is possible that the same node is enqueued many times. However, we require that a node
+ * is only actually marked dirty/changed once, with two exceptions:
+ *
+ * (1) If a node is marked dirty, it can subsequently be marked changed. This can occur if, for
+ * instance, FileValue workspace/foo/foo.cc is marked dirty because FileValue workspace/foo is
+ * marked changed (and every FileValue depends on its parent). Then FileValue
+ * workspace/foo/foo.cc is itself changed (this can even happen on the same build).
+ *
+ * (2) If a node is going to be marked both dirty and changed, as, for example, in the previous
+ * case if both workspace/foo/foo.cc and workspace/foo have been changed in the same build, the
+ * thread marking workspace/foo/foo.cc dirty may race with the one marking it changed, and so
+ * try to mark it dirty after it has already been marked changed. In that case, the
+ * {@link NodeEntry} ignores the second marking.
+ *
+ * The invariant that we do not process a (SkyKey, InvalidationType) pair twice is enforced by
+ * the {@link #visited} set.
+ *
+ * The "invariant" is also enforced across builds by checking to see if the entry is already
+ * marked changed, or if it is already marked dirty and we are just going to mark it dirty
+ * again.
+ *
+ * If either of the above tests shows that we have already started a task to mark this entry
+ * dirty/changed, or that it is already marked dirty/changed, we do not continue this task.
+ */
+ @Override
+ @ThreadSafe
+ public void visit(final SkyKey key, final InvalidationType invalidationType,
+ final boolean mustExist) {
+ Preconditions.checkState(invalidationType != InvalidationType.DELETED, key);
+ final boolean isChanged = (invalidationType == InvalidationType.CHANGED);
+ final Pair<SkyKey, InvalidationType> invalidationPair = Pair.of(key, invalidationType);
+ if (!visited.add(invalidationPair)) {
+ return;
+ }
+ pendingVisitations.add(invalidationPair);
+ enqueue(new Runnable() {
+ @Override
+ public void run() {
+ NodeEntry entry = graph.get(key);
+
+ if (entry == null) {
+ Preconditions.checkState(!mustExist,
+ "%s does not exist in the graph but was enqueued for dirtying by another node",
+ key);
+ pendingVisitations.remove(invalidationPair);
+ return;
+ }
+
+ if (entry.isChanged() || (!isChanged && entry.isDirty())) {
+ // If this node is already marked changed, or we are only marking this node dirty, and
+ // it already is, move along.
+ pendingVisitations.remove(invalidationPair);
+ return;
+ }
+
+ // This entry remains in the graph in this dirty state until it is re-evaluated.
+ Pair<? extends Iterable<SkyKey>, ? extends SkyValue> depsAndValue =
+ entry.markDirty(isChanged);
+ // It is not safe to interrupt the logic from this point until the end of the method.
+ // Any exception thrown should be unrecoverable.
+ if (depsAndValue == null) {
+ // Another thread has already dirtied this node. Don't do anything in this thread.
+ pendingVisitations.remove(invalidationPair);
+ return;
+ }
+ // Propagate dirtiness upwards and mark this node dirty/changed. Reverse deps should only
+ // be marked dirty (because only a dependency of theirs has changed).
+ for (SkyKey reverseDep : entry.getReverseDeps()) {
+ visit(reverseDep, InvalidationType.DIRTIED, MUST_EXIST);
+ }
+
+ // Remove this node as a reverse dep from its children, since we have reset it and it no
+ // longer lists its children as direct deps.
+ for (SkyKey dep : depsAndValue.first) {
+ graph.get(dep).removeReverseDep(key);
+ }
+
+ SkyValue value = ValueWithMetadata.justValue(depsAndValue.second);
+ informInvalidationReceiver(value, EvaluationProgressReceiver.InvalidationState.DIRTY);
+ dirtyKeyTracker.dirty(key);
+ // Remove the node from the set as the last operation.
+ pendingVisitations.remove(invalidationPair);
+ }
+ });
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java
new file mode 100644
index 0000000..2c7f14e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicate;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile;
+import com.google.devtools.build.lib.events.EventHandler;
+
+import java.io.PrintStream;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A graph, defined by a set of functions that can construct values from value keys.
+ *
+ * <p>The value constructor functions ({@link SkyFunction}s) can declare dependencies on
+ * prerequisite {@link SkyValue}s. The {@link MemoizingEvaluator} implementation makes sure that
+ * those are created beforehand.
+ *
+ * <p>The graph caches previously computed value values. Arbitrary values can be invalidated between
+ * calls to {@link #evaluate}; they will be recreated the next time they are requested.
+ */
+public interface MemoizingEvaluator {
+
+ /**
+ * Computes the transitive closure of a given set of values at the given {@link Version}. See
+ * {@link EagerInvalidator#invalidate}.
+ *
+ * <p>The returned EvaluationResult is guaranteed to contain a result for at least one root if
+ * keepGoing is false. It will contain a result for every root if keepGoing is true, <i>unless</i>
+ * the evaluation failed with a "catastrophic" error. In that case, some or all results may be
+ * missing.
+ */
+ <T extends SkyValue> EvaluationResult<T> evaluate(
+ Iterable<SkyKey> roots,
+ Version version,
+ boolean keepGoing,
+ int numThreads,
+ EventHandler reporter)
+ throws InterruptedException;
+
+ /**
+ * Ensures that after the next completed {@link #evaluate} call the current values of any value
+ * matching this predicate (and all values that transitively depend on them) will be removed from
+ * the value cache. All values that were already marked dirty in the graph will also be deleted,
+ * regardless of whether or not they match the predicate.
+ *
+ * <p>If a later call to {@link #evaluate} requests some of the deleted values, those values will
+ * be recomputed and the new values stored in the cache again.
+ *
+ * <p>To delete all dirty values, you can specify a predicate that's always false.
+ */
+ void delete(Predicate<SkyKey> pred);
+
+ /**
+ * Marks dirty values for deletion if they have been dirty for at least as many graph versions
+ * as the specified limit.
+ *
+ * <p>This ensures that after the next completed {@link #evaluate} call, all such values, along
+ * with all values that transitively depend on them, will be removed from the value cache. Values
+ * that were marked dirty after the threshold version will not be affected by this call.
+ *
+ * <p>If a later call to {@link #evaluate} requests some of the deleted values, those values will
+ * be recomputed and the new values stored in the cache again.
+ *
+ * <p>To delete all dirty values, you can specify 0 for the limit.
+ */
+ void deleteDirty(long versionAgeLimit);
+
+ /**
+ * Returns the values in the graph.
+ *
+ * <p>The returned map may be a live view of the graph.
+ */
+ Map<SkyKey, SkyValue> getValues();
+
+
+ /**
+ * Returns the done (without error) values in the graph.
+ *
+ * <p>The returned map may be a live view of the graph.
+ */
+ Map<SkyKey, SkyValue> getDoneValues();
+
+ /**
+ * Returns a value if and only if an earlier call to {@link #evaluate} created it; null otherwise.
+ *
+ * <p>This method should only be used by tests that need to verify the presence of a value in the
+ * graph after an {@link #evaluate} call.
+ */
+ @VisibleForTesting
+ @Nullable
+ SkyValue getExistingValueForTesting(SkyKey key);
+
+ /**
+ * Returns an error if and only if an earlier call to {@link #evaluate} created it; null
+ * otherwise.
+ *
+ * <p>This method should only be used by tests that need to verify the presence of an error in the
+ * graph after an {@link #evaluate} call.
+ */
+ @VisibleForTesting
+ @Nullable
+ ErrorInfo getExistingErrorForTesting(SkyKey key);
+
+ /**
+ * Write the graph to the output stream. Not necessarily thread-safe. Use only for debugging
+ * purposes.
+ */
+ @ThreadHostile
+ void dump(boolean summarize, PrintStream out);
+
+ /**
+ * A supplier for creating instances of a particular evaluator implementation.
+ */
+ public static interface EvaluatorSupplier {
+ MemoizingEvaluator create(
+ Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer,
+ @Nullable EvaluationProgressReceiver invalidationReceiver,
+ EmittedEventState emittedEventState, boolean keepEdges);
+ }
+
+ /**
+ * Keeps track of already-emitted events. Users of the graph should instantiate an
+ * {@code EmittedEventState} first and pass it to the graph during creation. This allows them to
+ * determine whether or not to replay events.
+ */
+ public static class EmittedEventState extends NestedSetVisitor.VisitedState<TaggedEvents> {}
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/MinimalVersion.java b/src/main/java/com/google/devtools/build/skyframe/MinimalVersion.java
new file mode 100644
index 0000000..6f75c15
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/MinimalVersion.java
@@ -0,0 +1,31 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * A Version "less than" all other versions, other than itself.
+ *
+ * <p>Only use in custom evaluator implementations.
+ */
+public class MinimalVersion implements Version {
+ public static final MinimalVersion INSTANCE = new MinimalVersion();
+
+ private MinimalVersion() {
+ }
+
+ @Override
+ public boolean atMost(Version other) {
+ return true;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java
new file mode 100644
index 0000000..243189d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java
@@ -0,0 +1,581 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.util.GroupedList;
+import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A node in the graph. All operations on this class are thread-safe. Care was taken to provide
+ * certain compound operations to avoid certain check-then-act races. That means this class is
+ * somewhat closely tied to the exact Evaluator implementation.
+ *
+ * <p>Consider the example with two threads working on two nodes, where one depends on the other,
+ * say b depends on a. If a completes first, it's done. If it completes second, it needs to signal
+ * b, and potentially re-schedule it. If b completes first, it must exit, because it will be
+ * signaled (and re-scheduled) by a. If it completes second, it must signal (and re-schedule)
+ * itself. However, if the Evaluator supported re-entrancy for a node, then this wouldn't have to
+ * be so strict, because duplicate scheduling would be less problematic.
+ *
+ * <p>The transient state of a {@code NodeEntry} is kept in a {@link BuildingState} object. Many of
+ * the methods of {@code NodeEntry} are just wrappers around the corresponding
+ * {@link BuildingState} methods.
+ *
+ * <p>This class is non-final only for testing purposes.
+ * <p>This class is public only for the benefit of alternative graph implementations outside of the
+ * package.
+ */
+public class NodeEntry {
+ /**
+ * Return code for {@link #addReverseDepAndCheckIfDone(SkyKey)}.
+ */
+ enum DependencyState {
+ /** The node is done. */
+ DONE,
+
+ /**
+ * The node was just created and needs to be scheduled for its first evaluation pass. The
+ * evaluator is responsible for signaling the reverse dependency node.
+ */
+ NEEDS_SCHEDULING,
+
+ /**
+ * The node was already created, but isn't done yet. The evaluator is responsible for
+ * signaling the reverse dependency node.
+ */
+ ADDED_DEP;
+ }
+
+ /** Actual data stored in this entry when it is done. */
+ private SkyValue value = null;
+
+ /**
+ * The last version of the graph at which this node entry was changed. In {@link #setValue} it
+ * may be determined that the data being written to the graph at a given version is the same as
+ * the already-stored data. In that case, the version will remain the same. The version can be
+ * thought of as the latest timestamp at which this entry was changed.
+ */
+ private Version version = MinimalVersion.INSTANCE;
+
+ /**
+ * This object represents a {@link GroupedList}<SkyKey> in a memory-efficient way. It stores the
+ * direct dependencies of this node, in groups if the {@code SkyFunction} requested them that way.
+ */
+ private Object directDeps = null;
+
+ /**
+ * This list stores the reverse dependencies of this node that have been declared so far.
+ *
+ * <p>In case of a single object we store the object unwrapped, without the list, for
+ * memory-efficiency.
+ */
+ @VisibleForTesting
+ protected Object reverseDeps = ImmutableList.of();
+
+ /**
+ * We take advantage of memory alignment to avoid doing a nasty {@code instanceof} for knowing
+ * if {@code reverseDeps} is a single object or a list.
+ */
+ protected boolean reverseDepIsSingleObject = false;
+
+ /**
+ * During the invalidation we keep the reverse deps to be removed in this list instead of directly
+ * removing them from {@code reverseDeps}. That is because removals from reverseDeps are O(N).
+ * Originally reverseDeps was a HashSet, but because of memory consumption we switched to a list.
+ *
+ * <p>This requires that any usage of reverseDeps (contains, add, the list of reverse deps) call
+ * {@code consolidateReverseDepsRemovals} first. While this operation is not free, it can be done
+ * more effectively than trying to remove each dirty reverse dependency individually (O(N) each
+ * time).
+ */
+ private List<SkyKey> reverseDepsToRemove = null;
+
+ private static final ReverseDepsUtil<NodeEntry> REVERSE_DEPS_UTIL =
+ new ReverseDepsUtil<NodeEntry>() {
+ @Override
+ void setReverseDepsObject(NodeEntry container, Object object) {
+ container.reverseDeps = object;
+ }
+
+ @Override
+ void setSingleReverseDep(NodeEntry container, boolean singleObject) {
+ container.reverseDepIsSingleObject = singleObject;
+ }
+
+ @Override
+ void setReverseDepsToRemove(NodeEntry container, List<SkyKey> object) {
+ container.reverseDepsToRemove = object;
+ }
+
+ @Override
+ Object getReverseDepsObject(NodeEntry container) {
+ return container.reverseDeps;
+ }
+
+ @Override
+ boolean isSingleReverseDep(NodeEntry container) {
+ return container.reverseDepIsSingleObject;
+ }
+
+ @Override
+ List<SkyKey> getReverseDepsToRemove(NodeEntry container) {
+ return container.reverseDepsToRemove;
+ }
+ };
+
+ /**
+ * The transient state of this entry, after it has been created but before it is done. It allows
+ * us to keep the current state of the entry across invalidation and successive evaluations.
+ */
+ @VisibleForTesting
+ protected BuildingState buildingState = new BuildingState();
+
+ /**
+ * Construct a NodeEntry. Use ONLY in Skyframe evaluation and graph implementations.
+ */
+ public NodeEntry() {
+ }
+
+ protected boolean keepEdges() {
+ return true;
+ }
+
+ /** Returns whether the entry has been built and is finished evaluating. */
+ synchronized boolean isDone() {
+ return buildingState == null;
+ }
+
+ /**
+ * Returns the value stored in this entry. This method may only be called after the evaluation of
+ * this node is complete, i.e., after {@link #setValue} has been called.
+ */
+ synchronized SkyValue getValue() {
+ Preconditions.checkState(isDone(), "no value until done. ValueEntry: %s", this);
+ return ValueWithMetadata.justValue(value);
+ }
+
+ /**
+ * Returns the {@link SkyValue} for this entry and the metadata associated with it (Like events
+ * and errors). This method may only be called after the evaluation of this node is complete,
+ * i.e., after {@link #setValue} has been called.
+ */
+ synchronized ValueWithMetadata getValueWithMetadata() {
+ Preconditions.checkState(isDone(), "no value until done: %s", this);
+ return ValueWithMetadata.wrapWithMetadata(value);
+ }
+
+ /**
+ * Returns the value, even if dirty or changed. Returns null otherwise.
+ */
+ public synchronized SkyValue toValue() {
+ if (isDone()) {
+ return getErrorInfo() == null ? getValue() : null;
+ } else if (isChanged() || isDirty()) {
+ return (buildingState.getLastBuildValue() == null)
+ ? null
+ : ValueWithMetadata.justValue(buildingState.getLastBuildValue());
+ }
+ throw new AssertionError("Value in bad state: " + this);
+ }
+
+ /**
+ * Returns an immutable iterable of the direct deps of this node. This method may only be called
+ * after the evaluation of this node is complete, i.e., after {@link #setValue} has been called.
+ *
+ * <p>This method is not very efficient, but is only be called in limited circumstances --
+ * when the node is about to be deleted, or when the node is expected to have no direct deps (in
+ * which case the overhead is not so bad). It should not be called repeatedly for the same node,
+ * since each call takes time proportional to the number of direct deps of the node.
+ */
+ synchronized Iterable<SkyKey> getDirectDeps() {
+ assertKeepEdges();
+ Preconditions.checkState(isDone(), "no deps until done. ValueEntry: %s", this);
+ return GroupedList.<SkyKey>create(directDeps).toSet();
+ }
+
+ /**
+ * Returns the error, if any, associated to this node. This method may only be called after
+ * the evaluation of this node is complete, i.e., after {@link #setValue} has been called.
+ */
+ @Nullable
+ synchronized ErrorInfo getErrorInfo() {
+ Preconditions.checkState(isDone(), "no errors until done. ValueEntry: %s", this);
+ return ValueWithMetadata.getMaybeErrorInfo(value);
+ }
+
+ private synchronized Set<SkyKey> setStateFinishedAndReturnReverseDeps() {
+ // Get reverse deps that need to be signaled.
+ ImmutableSet<SkyKey> reverseDepsToSignal = buildingState.getReverseDepsToSignal();
+ REVERSE_DEPS_UTIL.consolidateReverseDepsRemovals(this);
+ REVERSE_DEPS_UTIL.addReverseDeps(this, reverseDepsToSignal);
+ this.directDeps = buildingState.getFinishedDirectDeps().compress();
+
+ // Set state of entry to done.
+ buildingState = null;
+
+ if (!keepEdges()) {
+ this.directDeps = null;
+ this.reverseDeps = null;
+ }
+ return reverseDepsToSignal;
+ }
+
+ /**
+ * Returns the set of reverse deps that have been declared so far this build. Only for use in
+ * debugging and when bubbling errors up in the --nokeep_going case, where we need to know what
+ * parents this entry has.
+ */
+ synchronized Set<SkyKey> getInProgressReverseDeps() {
+ Preconditions.checkState(!isDone(), this);
+ return buildingState.getReverseDepsToSignal();
+ }
+
+ /**
+ * Transitions the node from the EVALUATING to the DONE state and simultaneously sets it to the
+ * given value and error state. It then returns the set of reverse dependencies that need to be
+ * signaled.
+ *
+ * <p>This is an atomic operation to avoid a race where two threads work on two nodes, where one
+ * node depends on another (b depends on a). When a finishes, it signals <b>exactly</b> the set
+ * of reverse dependencies that are registered at the time of the {@code setValue} call. If b
+ * comes in before a, it is signaled (and re-scheduled) by a, otherwise it needs to do that
+ * itself.
+ *
+ * <p>{@code version} indicates the graph version at which this node is being written. If the
+ * entry determines that the new value is equal to the previous value, the entry will keep its
+ * current version. Callers can query that version to see if the node considers its value to have
+ * changed.
+ */
+ public synchronized Set<SkyKey> setValue(SkyValue value, Version version) {
+ Preconditions.checkState(isReady(), "%s %s", this, value);
+ // This check may need to be removed when we move to a non-linear versioning sequence.
+ Preconditions.checkState(this.version.atMost(version),
+ "%s %s %s", this, version, value);
+
+ if (isDirty() && buildingState.unchangedFromLastBuild(value)) {
+ // If the value is the same as before, just use the old value. Note that we don't use the new
+ // value, because preserving == equality is even better than .equals() equality.
+ this.value = buildingState.getLastBuildValue();
+ } else {
+ // If this is a new value, or it has changed since the last build, set the version to the
+ // current graph version.
+ this.version = version;
+ this.value = value;
+ }
+
+ return setStateFinishedAndReturnReverseDeps();
+ }
+
+ /**
+ * Queries if the node is done and adds the given key as a reverse dependency. The return code
+ * indicates whether a) the node is done, b) the reverse dependency is the first one, so the
+ * node needs to be scheduled, or c) the reverse dependency was added, and the node does not
+ * need to be scheduled.
+ *
+ * <p>This method <b>must</b> be called before any processing of the entry. This encourages
+ * callers to check that the entry is ready to be processed.
+ *
+ * <p>Adding the dependency and checking if the node needs to be scheduled is an atomic operation
+ * to avoid a race where two threads work on two nodes, where one depends on the other (b depends
+ * on a). In that case, we need to ensure that b is re-scheduled exactly once when a is done.
+ * However, a may complete first, in which case b has to re-schedule itself. Also see {@link
+ * #setValue}.
+ *
+ * <p>If the parameter is {@code null}, then no reverse dependency is added, but we still check
+ * if the node needs to be scheduled.
+ */
+ synchronized DependencyState addReverseDepAndCheckIfDone(SkyKey reverseDep) {
+ if (reverseDep != null) {
+ if (keepEdges()) {
+ REVERSE_DEPS_UTIL.consolidateReverseDepsRemovals(this);
+ REVERSE_DEPS_UTIL.maybeCheckReverseDepNotPresent(this, reverseDep);
+ }
+ if (isDone()) {
+ if (keepEdges()) {
+ REVERSE_DEPS_UTIL.addReverseDeps(this, ImmutableList.of(reverseDep));
+ }
+ } else {
+ // Parent should never register itself twice in the same build.
+ buildingState.addReverseDepToSignal(reverseDep);
+ }
+ }
+ if (isDone()) {
+ return DependencyState.DONE;
+ }
+ return buildingState.startEvaluating() ? DependencyState.NEEDS_SCHEDULING
+ : DependencyState.ADDED_DEP;
+ }
+
+ /**
+ * Removes a reverse dependency.
+ */
+ synchronized void removeReverseDep(SkyKey reverseDep) {
+ if (!keepEdges()) {
+ return;
+ }
+ REVERSE_DEPS_UTIL.removeReverseDep(this, reverseDep);
+ if (!isDone()) {
+ // This is currently unnecessary -- the only time we remove a reverse dep that was added this
+ // build is during the clean following a build failure. In that case, this node that is not
+ // done will be deleted soon, so clearing the reverse dep is not required.
+ buildingState.removeReverseDepToSignal(reverseDep);
+ }
+ }
+
+ /**
+ * Returns a copy of the set of reverse dependencies. Note that this introduces a potential
+ * check-then-act race; {@link #removeReverseDep} may fail for a key that is returned here.
+ */
+ synchronized Iterable<SkyKey> getReverseDeps() {
+ assertKeepEdges();
+ Preconditions.checkState(isDone() || buildingState.getReverseDepsToSignal().isEmpty(),
+ "Reverse deps should only be queried before the build has begun "
+ + "or after the node is done %s", this);
+ return REVERSE_DEPS_UTIL.getReverseDeps(this);
+ }
+
+ /**
+ * Tell this node that one of its dependencies is now done. Callers must check the return value,
+ * and if true, they must re-schedule this node for evaluation. Equivalent to
+ * {@code #signalDep(Long.MAX_VALUE)}. Since this entry's version is less than
+ * {@link Long#MAX_VALUE}, informing this entry that a child of it has version
+ * {@link Long#MAX_VALUE} will force it to re-evaluate.
+ */
+ synchronized boolean signalDep() {
+ return signalDep(/*childVersion=*/new IntVersion(Long.MAX_VALUE));
+ }
+
+ /**
+ * Tell this entry that one of its dependencies is now done. Callers must check the return value,
+ * and if true, they must re-schedule this node for evaluation.
+ *
+ * @param childVersion If this entry {@link #isDirty()} and {@code childVersion} is not at most
+ * {@link #getVersion()}, then this entry records that one of its children has changed since it
+ * was last evaluated (namely, it was last evaluated at version {@link #getVersion()} and the
+ * child was last evaluated at {@code childVersion}. Thus, the next call to
+ * {@link #getDirtyState()} will return {@link BuildingState.DirtyState#REBUILDING}.
+ */
+ synchronized boolean signalDep(Version childVersion) {
+ Preconditions.checkState(!isDone(), "Value must not be done in signalDep %s", this);
+ return buildingState.signalDep(/*childChanged=*/!childVersion.atMost(getVersion()));
+ }
+
+ /**
+ * Returns true if the entry is marked dirty, meaning that at least one of its transitive
+ * dependencies is marked changed.
+ */
+ public synchronized boolean isDirty() {
+ return !isDone() && buildingState.isDirty();
+ }
+
+ /**
+ * Returns true if the entry is marked changed, meaning that it must be re-evaluated even if its
+ * dependencies' values have not changed.
+ */
+ synchronized boolean isChanged() {
+ return !isDone() && buildingState.isChanged();
+ }
+
+ /** Checks that a caller is not trying to access not-stored graph edges. */
+ private void assertKeepEdges() {
+ Preconditions.checkState(keepEdges(), "Graph edges not stored. %s", this);
+ }
+
+ /**
+ * Marks this node dirty, or changed if {@code isChanged} is true. The node is put in the
+ * just-created state. It will be re-evaluated if necessary during the evaluation phase,
+ * but if it has not changed, it will not force a re-evaluation of its parents.
+ *
+ * @return The direct deps and value of this entry, or null if the entry has already been marked
+ * dirty. In the latter case, the caller should abort its handling of this node, since another
+ * thread is already dealing with it.
+ */
+ @Nullable
+ synchronized Pair<? extends Iterable<SkyKey>, ? extends SkyValue> markDirty(boolean isChanged) {
+ assertKeepEdges();
+ if (isDone()) {
+ GroupedList<SkyKey> lastDirectDeps = GroupedList.create(directDeps);
+ buildingState = BuildingState.newDirtyState(isChanged, lastDirectDeps, value);
+ Pair<? extends Iterable<SkyKey>, ? extends SkyValue> result =
+ Pair.of(lastDirectDeps.toSet(), value);
+ value = null;
+ directDeps = null;
+ return result;
+ }
+ // The caller may be simultaneously trying to mark this node dirty and changed, and the dirty
+ // thread may have lost the race, but it is the caller's responsibility not to try to mark
+ // this node changed twice. The end result of racing markers must be a changed node, since one
+ // of the markers is trying to mark the node changed.
+ Preconditions.checkState(isChanged != isChanged(),
+ "Cannot mark node dirty twice or changed twice: %s", this);
+ Preconditions.checkState(value == null, "Value should have been reset already %s", this);
+ Preconditions.checkState(directDeps == null, "direct deps not already reset %s", this);
+ if (isChanged) {
+ // If the changed marker lost the race, we just need to mark changed in this method -- all
+ // other work was done by the dirty marker.
+ buildingState.markChanged();
+ }
+ return null;
+ }
+
+ /**
+ * Marks this entry as up-to-date at this version.
+ *
+ * @return {@link Set} of reverse dependencies to signal that this node is done.
+ */
+ synchronized Set<SkyKey> markClean() {
+ this.value = buildingState.getLastBuildValue();
+ // This checks both the value and the direct deps, but since we're passing in the same value,
+ // the value check should be trivial.
+ Preconditions.checkState(buildingState.unchangedFromLastBuild(this.value),
+ "Direct deps must be the same as those found last build for node to be marked clean: %s",
+ this);
+ Preconditions.checkState(isDirty(), this);
+ Preconditions.checkState(!buildingState.isChanged(), "shouldn't be changed: %s", this);
+ return setStateFinishedAndReturnReverseDeps();
+ }
+
+ /**
+ * Forces this node to be reevaluated, even if none of its dependencies are known to have
+ * changed.
+ *
+ * <p>Used when an external caller has reason to believe that re-evaluating may yield a new
+ * result. This method should not be used if one of the normal deps of this node has changed,
+ * the usual change-pruning process should work in that case.
+ */
+ synchronized void forceRebuild() {
+ buildingState.forceChanged();
+ }
+
+ /**
+ * Gets the current version of this entry.
+ */
+ synchronized Version getVersion() {
+ return version;
+ }
+
+ /**
+ * Gets the current state of checking this dirty entry to see if it must be re-evaluated. Must be
+ * called each time evaluation of a dirty entry starts to find the proper action to perform next,
+ * as enumerated by {@link BuildingState.DirtyState}.
+ *
+ * @see BuildingState#getDirtyState()
+ */
+ synchronized BuildingState.DirtyState getDirtyState() {
+ return buildingState.getDirtyState();
+ }
+
+ /**
+ * Should only be called if the entry is dirty. During the examination to see if the entry must be
+ * re-evaluated, this method returns the next group of children to be checked. Callers should
+ * have already called {@link #getDirtyState} and received a return value of
+ * {@link BuildingState.DirtyState#CHECK_DEPENDENCIES} before calling this method -- any other
+ * return value from {@link #getDirtyState} means that this method must not be called, since
+ * whether or not the node needs to be rebuilt is already known.
+ *
+ * <p>Deps are returned in groups. The deps in each group were requested in parallel by the
+ * {@code SkyFunction} last build, meaning independently of the values of any other deps in this
+ * group (although possibly depending on deps in earlier groups). Thus the caller may check all
+ * the deps in this group in parallel, since the deps in all previous groups are verified
+ * unchanged. See {@link SkyFunction.Environment#getValues} for more on dependency groups.
+ *
+ * <p>The caller should register these as deps of this entry using {@link #addTemporaryDirectDeps}
+ * before checking them.
+ *
+ * @see BuildingState#getNextDirtyDirectDeps()
+ */
+ synchronized Collection<SkyKey> getNextDirtyDirectDeps() {
+ return buildingState.getNextDirtyDirectDeps();
+ }
+
+ /**
+ * Returns the set of direct dependencies. This may only be called while the node is being
+ * evaluated, that is, before {@link #setValue} and after {@link #markDirty}.
+ */
+ synchronized Set<SkyKey> getTemporaryDirectDeps() {
+ Preconditions.checkState(!isDone(), "temporary shouldn't be done: %s", this);
+ return buildingState.getDirectDepsForBuild();
+ }
+
+ synchronized boolean noDepsLastBuild() {
+ return buildingState.noDepsLastBuild();
+ }
+
+ /**
+ * Remove dep from direct deps. This should only be called if this entry is about to be
+ * committed as a cycle node, but some of its children were not checked for cycles, either
+ * because the cycle was discovered before some children were checked; some children didn't have a
+ * chance to finish before the evaluator aborted; or too many cycles were found when it came time
+ * to check the children.
+ */
+ synchronized void removeUnfinishedDeps(Set<SkyKey> unfinishedDeps) {
+ buildingState.removeDirectDeps(unfinishedDeps);
+ }
+
+ synchronized void addTemporaryDirectDeps(GroupedListHelper<SkyKey> helper) {
+ Preconditions.checkState(!isDone(), "add temp shouldn't be done: %s %s", helper, this);
+ buildingState.addDirectDeps(helper);
+ }
+
+ /**
+ * Returns true if the node is ready to be evaluated, i.e., it has been signaled exactly as many
+ * times as it has temporary dependencies. This may only be called while the node is being
+ * evaluated, that is, before {@link #setValue} and after {@link #markDirty}.
+ */
+ synchronized boolean isReady() {
+ Preconditions.checkState(!isDone(), "can't be ready if done: %s", this);
+ return buildingState.isReady();
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public String toString() {
+ return Objects.toStringHelper(this) // MoreObjects is not in Guava
+ .add("value", value)
+ .add("version", version)
+ .add("directDeps", directDeps == null ? null : GroupedList.create(directDeps))
+ .add("reverseDeps", REVERSE_DEPS_UTIL.toString(this))
+ .add("buildingState", buildingState).toString();
+ }
+
+ /**
+ * Do not use except in custom evaluator implementations! Added only temporarily.
+ *
+ * <p>Clones a NodeEntry iff it is a done node. Otherwise it fails.
+ */
+ @Deprecated
+ public synchronized NodeEntry cloneNodeEntry() {
+ // As this is temporary, for now lets limit to done nodes
+ Preconditions.checkState(isDone(), "Only done nodes can be copied");
+ NodeEntry nodeEntry = new NodeEntry();
+ nodeEntry.value = value;
+ nodeEntry.version = this.version;
+ REVERSE_DEPS_UTIL.addReverseDeps(nodeEntry, REVERSE_DEPS_UTIL.getReverseDeps(this));
+ nodeEntry.directDeps = directDeps;
+ nodeEntry.buildingState = null;
+ return nodeEntry;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/NullDirtyKeyTrackerImpl.java b/src/main/java/com/google/devtools/build/skyframe/NullDirtyKeyTrackerImpl.java
new file mode 100644
index 0000000..937f1cb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/NullDirtyKeyTrackerImpl.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Set;
+
+/**
+ * Tracks nothing. Should be used by evaluators that don't do dirty node garbage collection.
+ */
+public class NullDirtyKeyTrackerImpl implements DirtyKeyTracker {
+
+ @Override
+ public void dirty(SkyKey skyKey) {
+ }
+
+ @Override
+ public void notDirty(SkyKey skyKey) {
+ }
+
+ @Override
+ public Set<SkyKey> getDirtyKeys() {
+ return ImmutableSet.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java
new file mode 100644
index 0000000..39f11d7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java
@@ -0,0 +1,1786 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor;
+import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
+import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper;
+import com.google.devtools.build.skyframe.BuildingState.DirtyState;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver.EvaluationState;
+import com.google.devtools.build.skyframe.NodeEntry.DependencyState;
+import com.google.devtools.build.skyframe.Scheduler.SchedulerException;
+import com.google.devtools.build.skyframe.SkyFunctionException.ReifiedSkyFunctionException;
+import com.google.devtools.build.skyframe.ValueOrExceptionUtils.BottomException;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.Nullable;
+
+/**
+ * Evaluates a set of given functions ({@code SkyFunction}s) with arguments ({@code SkyKey}s).
+ * Cycles are not allowed and are detected during the traversal.
+ *
+ * <p>This class implements multi-threaded evaluation. This is a fairly complex process that has
+ * strong consistency requirements between the {@link ProcessableGraph}, the nodes in the graph of
+ * type {@link NodeEntry}, the work queue, and the set of in-flight nodes.
+ *
+ * <p>The basic invariants are:
+ *
+ * <p>A node can be in one of three states: ready, waiting, and done. A node is ready if and only
+ * if all of its dependencies have been signaled. A node is done if it has a value. It is waiting
+ * if not all of its dependencies have been signaled.
+ *
+ * <p>A node must be in the work queue if and only if it is ready. It is an error for a node to be
+ * in the work queue twice at the same time.
+ *
+ * <p>A node is considered in-flight if it has been created, and is not done yet. In case of an
+ * interrupt, the work queue is discarded, and the in-flight set is used to remove partially
+ * computed values.
+ *
+ * <p>Each evaluation of the graph takes place at a "version," which is currently given by a
+ * non-negative {@code long}. The version can also be thought of as an "mtime." Each node in the
+ * graph has a version, which is the last version at which its value changed. This version data is
+ * used to avoid unnecessary re-evaluation of values. If a node is re-evaluated and found to have
+ * the same data as before, its version (mtime) remains the same. If all of a node's children's
+ * have the same version as before, its re-evaluation can be skipped.
+ *
+ * <p>This class is not intended for direct use, and is only exposed as public for use in
+ * evaluation implementations outside of this package.
+ */
+public final class ParallelEvaluator implements Evaluator {
+ private final ProcessableGraph graph;
+ private final Version graphVersion;
+
+ private final Predicate<SkyKey> nodeEntryIsDone = new Predicate<SkyKey>() {
+ @Override
+ public boolean apply(SkyKey skyKey) {
+ return isDoneForBuild(graph.get(skyKey));
+ }
+ };
+
+ private final ImmutableMap<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions;
+
+ private final EventHandler reporter;
+ private final NestedSetVisitor<TaggedEvents> replayingNestedSetEventVisitor;
+ private final boolean keepGoing;
+ private final int threadCount;
+ @Nullable private final EvaluationProgressReceiver progressReceiver;
+ private final DirtyKeyTracker dirtyKeyTracker;
+ private final AtomicBoolean errorEncountered = new AtomicBoolean(false);
+
+ private static final Interner<SkyKey> KEY_CANONICALIZER = Interners.newWeakInterner();
+
+ public ParallelEvaluator(ProcessableGraph graph, Version graphVersion,
+ ImmutableMap<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions,
+ final EventHandler reporter,
+ MemoizingEvaluator.EmittedEventState emittedEventState,
+ boolean keepGoing, int threadCount,
+ @Nullable EvaluationProgressReceiver progressReceiver,
+ DirtyKeyTracker dirtyKeyTracker) {
+ this.graph = graph;
+ this.skyFunctions = skyFunctions;
+ this.graphVersion = graphVersion;
+ this.reporter = Preconditions.checkNotNull(reporter);
+ this.keepGoing = keepGoing;
+ this.threadCount = threadCount;
+ this.progressReceiver = progressReceiver;
+ this.dirtyKeyTracker = Preconditions.checkNotNull(dirtyKeyTracker);
+ this.replayingNestedSetEventVisitor =
+ new NestedSetVisitor<>(new NestedSetEventReceiver(reporter), emittedEventState);
+ }
+
+ /**
+ * Receives the events from the NestedSet and delegates to the reporter.
+ */
+ private static class NestedSetEventReceiver implements NestedSetVisitor.Receiver<TaggedEvents> {
+
+ private final EventHandler reporter;
+
+ public NestedSetEventReceiver(EventHandler reporter) {
+ this.reporter = reporter;
+ }
+ @Override
+ public void accept(TaggedEvents events) {
+ String tag = events.getTag();
+ for (Event e : events.getEvents()) {
+ reporter.handle(e.withTag(tag));
+ }
+ }
+ }
+
+ /**
+ * A suitable {@link SkyFunction.Environment} implementation.
+ */
+ class SkyFunctionEnvironment implements SkyFunction.Environment {
+ private boolean building = true;
+ private boolean valuesMissing = false;
+ private SkyKey depErrorKey = null;
+ private final SkyKey skyKey;
+ private SkyValue value = null;
+ private ErrorInfo errorInfo = null;
+ private final Map<SkyKey, ValueWithMetadata> bubbleErrorInfo;
+ /** The set of values previously declared as dependencies. */
+ private final Set<SkyKey> directDeps;
+
+ /**
+ * The grouped list of values requested during this build as dependencies. On a subsequent
+ * build, if this value is dirty, all deps in the same dependency group can be checked in
+ * parallel for changes. In other words, if dep1 and dep2 are in the same group, then dep1 will
+ * be checked in parallel with dep2. See {@link #getValues} for more.
+ */
+ private final GroupedListHelper<SkyKey> newlyRequestedDeps = new GroupedListHelper<>();
+
+ /**
+ * The value visitor managing the thread pool. Used to enqueue parents when this value is
+ * finished, and, during testing, to block until an exception is thrown if a value builder
+ * requests that.
+ */
+ private final ValueVisitor visitor;
+
+ /** The set of errors encountered while fetching children. */
+ private final Collection<ErrorInfo> childErrorInfos = new LinkedHashSet<>();
+ private final StoredEventHandler eventHandler = new StoredEventHandler() {
+ @Override
+ public void handle(Event e) {
+ checkActive();
+ switch (e.getKind()) {
+ case INFO:
+ throw new UnsupportedOperationException("Values should not display INFO messages: " +
+ skyKey + " printed " + e.getLocation() + ": " + e.getMessage());
+ case PROGRESS:
+ reporter.handle(e);
+ break;
+ default:
+ super.handle(e);
+ }
+ }
+ };
+
+ private SkyFunctionEnvironment(SkyKey skyKey, Set<SkyKey> directDeps, ValueVisitor visitor) {
+ this(skyKey, directDeps, null, visitor);
+ }
+
+ private SkyFunctionEnvironment(SkyKey skyKey, Set<SkyKey> directDeps,
+ @Nullable Map<SkyKey, ValueWithMetadata> bubbleErrorInfo, ValueVisitor visitor) {
+ this.skyKey = skyKey;
+ this.directDeps = Collections.unmodifiableSet(directDeps);
+ this.bubbleErrorInfo = bubbleErrorInfo;
+ this.childErrorInfos.addAll(childErrorInfos);
+ this.visitor = visitor;
+ }
+
+ private void checkActive() {
+ Preconditions.checkState(building, skyKey);
+ }
+
+ private NestedSet<TaggedEvents> buildEvents(boolean missingChildren) {
+ // Aggregate the nested set of events from the direct deps, also adding the events from
+ // building this value.
+ NestedSetBuilder<TaggedEvents> eventBuilder = NestedSetBuilder.stableOrder();
+ ImmutableList<Event> events = eventHandler.getEvents();
+ if (!events.isEmpty()) {
+ eventBuilder.add(new TaggedEvents(getTagFromKey(), events));
+ }
+ for (SkyKey dep : graph.get(skyKey).getTemporaryDirectDeps()) {
+ ValueWithMetadata value = getValueMaybeFromError(dep, bubbleErrorInfo);
+ if (value != null) {
+ eventBuilder.addTransitive(value.getTransitiveEvents());
+ } else {
+ Preconditions.checkState(missingChildren, "", dep, skyKey);
+ }
+ }
+ return eventBuilder.build();
+ }
+
+ /**
+ * If this node has an error, that is, if errorInfo is non-null, do nothing. Otherwise, set
+ * errorInfo to the union of the child errors that were recorded earlier by getValueOrException,
+ * if there are any.
+ */
+ private void finalizeErrorInfo() {
+ if (errorInfo == null && !childErrorInfos.isEmpty()) {
+ errorInfo = new ErrorInfo(skyKey, childErrorInfos);
+ }
+ }
+
+ private void setValue(SkyValue newValue) {
+ Preconditions.checkState(errorInfo == null && bubbleErrorInfo == null,
+ "%s %s %s %s", skyKey, newValue, errorInfo, bubbleErrorInfo);
+ Preconditions.checkState(value == null, "%s %s %s", skyKey, value, newValue);
+ value = newValue;
+ }
+
+ /**
+ * Set this node to be in error. The node's value must not have already been set. However, all
+ * dependencies of this node <i>must</i> already have been registered, since this method may
+ * register a dependence on the error transience node, which should always be the last dep.
+ */
+ private void setError(ErrorInfo errorInfo) {
+ Preconditions.checkState(value == null, "%s %s %s", skyKey, value, errorInfo);
+ Preconditions.checkState(this.errorInfo == null,
+ "%s %s %s", skyKey, this.errorInfo, errorInfo);
+
+ if (errorInfo.isTransient()) {
+ DependencyState triState =
+ graph.get(ErrorTransienceValue.key()).addReverseDepAndCheckIfDone(skyKey);
+ Preconditions.checkState(triState == DependencyState.DONE,
+ "%s %s %s", skyKey, triState, errorInfo);
+
+ final NodeEntry state = graph.get(skyKey);
+ state.addTemporaryDirectDeps(
+ GroupedListHelper.create(ImmutableList.of(ErrorTransienceValue.key())));
+ state.signalDep();
+ }
+
+ this.errorInfo = Preconditions.checkNotNull(errorInfo, skyKey);
+ }
+
+ /** Get a child of the value being evaluated, for use by the value builder. */
+ private ValueOrUntypedException getValueOrUntypedException(SkyKey depKey) {
+ checkActive();
+ depKey = KEY_CANONICALIZER.intern(depKey); // Canonicalize SkyKeys to save memory.
+ ValueWithMetadata value = getValueMaybeFromError(depKey, bubbleErrorInfo);
+ if (value == null) {
+ // If this entry is not yet done then (optionally) record the missing dependency and return
+ // null.
+ valuesMissing = true;
+ if (bubbleErrorInfo != null) {
+ // Values being built just for their errors don't get to request new children.
+ return ValueOrExceptionUtils.ofNull();
+ }
+ Preconditions.checkState(!directDeps.contains(depKey), "%s %s %s", skyKey, depKey, value);
+ addDep(depKey);
+ valuesMissing = true;
+ return ValueOrExceptionUtils.ofNull();
+ }
+
+ if (!directDeps.contains(depKey)) {
+ // If this child is done, we will return it, but also record that it was newly requested so
+ // that the dependency can be properly registered in the graph.
+ addDep(depKey);
+ }
+
+ replayingNestedSetEventVisitor.visit(value.getTransitiveEvents());
+ ErrorInfo errorInfo = value.getErrorInfo();
+
+ if (errorInfo != null) {
+ childErrorInfos.add(errorInfo);
+ }
+
+ if (value.getValue() != null && (keepGoing || errorInfo == null)) {
+ // The caller is given the value of the value if there was no error computing the value, or
+ // if this is a keepGoing build (in which case each value should get child values even if
+ // there are also errors).
+ return ValueOrExceptionUtils.ofValueUntyped(value.getValue());
+ }
+
+ // There was an error building the value, which we will either report by throwing an exception
+ // or insulate the caller from by returning null.
+ Preconditions.checkNotNull(errorInfo, "%s %s %s", skyKey, depKey, value);
+
+ if (!keepGoing && errorInfo.getException() != null && bubbleErrorInfo == null) {
+ // Child errors should not be propagated in noKeepGoing mode (except during error bubbling).
+ // Instead we should fail fast.
+
+ // We arbitrarily record the first child error.
+ if (depErrorKey == null) {
+ depErrorKey = depKey;
+ }
+ valuesMissing = true;
+ return ValueOrExceptionUtils.ofNull();
+ }
+
+ if (bubbleErrorInfo != null) {
+ // Set interrupted status, so that builder doesn't try anything fancy after this.
+ Thread.currentThread().interrupt();
+ }
+ if (errorInfo.getException() != null) {
+ // Give builder a chance to handle this exception.
+ Exception e = errorInfo.getException();
+ return ValueOrExceptionUtils.ofExn(e);
+ }
+ // In a cycle.
+ Preconditions.checkState(!Iterables.isEmpty(errorInfo.getCycleInfo()), "%s %s %s %s", skyKey,
+ depKey, errorInfo, value);
+ valuesMissing = true;
+ return ValueOrExceptionUtils.ofNull();
+ }
+
+ private <E extends Exception> ValueOrException<E> getValueOrException(SkyKey depKey,
+ Class<E> exceptionClass) {
+ return ValueOrExceptionUtils.downcovert(getValueOrException(depKey, exceptionClass,
+ BottomException.class), exceptionClass);
+ }
+
+ private <E1 extends Exception, E2 extends Exception> ValueOrException2<E1, E2>
+ getValueOrException(SkyKey depKey, Class<E1> exceptionClass1, Class<E2> exceptionClass2) {
+ return ValueOrExceptionUtils.downconvert(getValueOrException(depKey, exceptionClass1,
+ exceptionClass2, BottomException.class), exceptionClass1, exceptionClass2);
+ }
+
+ private <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ ValueOrException3<E1, E2, E3> getValueOrException(SkyKey depKey, Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2, Class<E3> exceptionClass3) {
+ return ValueOrExceptionUtils.downconvert(getValueOrException(depKey, exceptionClass1,
+ exceptionClass2, exceptionClass3, BottomException.class), exceptionClass1,
+ exceptionClass2, exceptionClass3);
+ }
+
+ private <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> getValueOrException(SkyKey depKey,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4) {
+ SkyFunctionException.validateExceptionType(exceptionClass1);
+ SkyFunctionException.validateExceptionType(exceptionClass2);
+ SkyFunctionException.validateExceptionType(exceptionClass3);
+ SkyFunctionException.validateExceptionType(exceptionClass4);
+ ValueOrUntypedException voe = getValueOrUntypedException(depKey);
+ SkyValue value = voe.getValue();
+ if (value != null) {
+ return ValueOrExceptionUtils.ofValue(value);
+ }
+ Exception e = voe.getException();
+ if (e != null) {
+ if (exceptionClass1.isInstance(e)) {
+ return ValueOrExceptionUtils.ofExn1(exceptionClass1.cast(e));
+ }
+ if (exceptionClass2.isInstance(e)) {
+ return ValueOrExceptionUtils.ofExn2(exceptionClass2.cast(e));
+ }
+ if (exceptionClass3.isInstance(e)) {
+ return ValueOrExceptionUtils.ofExn3(exceptionClass3.cast(e));
+ }
+ if (exceptionClass4.isInstance(e)) {
+ return ValueOrExceptionUtils.ofExn4(exceptionClass4.cast(e));
+ }
+ }
+ valuesMissing = true;
+ return ValueOrExceptionUtils.ofNullValue();
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue(SkyKey depKey) {
+ try {
+ return getValueOrThrow(depKey, BottomException.class);
+ } catch (BottomException e) {
+ throw new IllegalStateException("shouldn't reach here");
+ }
+ }
+
+ @Override
+ @Nullable
+ public <E extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E> exceptionClass)
+ throws E {
+ return getValueOrException(depKey, exceptionClass).get();
+ }
+
+ @Override
+ @Nullable
+ public <E1 extends Exception, E2 extends Exception> SkyValue getValueOrThrow(SkyKey depKey,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2) throws E1, E2 {
+ return getValueOrException(depKey, exceptionClass1, exceptionClass2).get();
+ }
+
+ @Override
+ @Nullable
+ public <E1 extends Exception, E2 extends Exception,
+ E3 extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2, Class<E3> exceptionClass3) throws E1, E2, E3 {
+ return getValueOrException(depKey, exceptionClass1, exceptionClass2, exceptionClass3).get();
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2, Class<E3> exceptionClass3, Class<E4> exceptionClass4) throws E1,
+ E2, E3, E4 {
+ return getValueOrException(depKey, exceptionClass1, exceptionClass2, exceptionClass3,
+ exceptionClass4).get();
+ }
+
+ @Override
+ public Map<SkyKey, SkyValue> getValues(Iterable<SkyKey> depKeys) {
+ return Maps.transformValues(getValuesOrThrow(depKeys, BottomException.class),
+ GET_VALUE_FROM_VOE);
+ }
+
+ @Override
+ public <E extends Exception> Map<SkyKey, ValueOrException<E>> getValuesOrThrow(
+ Iterable<SkyKey> depKeys, Class<E> exceptionClass) {
+ return Maps.transformValues(getValuesOrThrow(depKeys, exceptionClass, BottomException.class),
+ makeSafeDowncastToVOEFunction(exceptionClass));
+ }
+
+ @Override
+ public <E1 extends Exception,
+ E2 extends Exception> Map<SkyKey, ValueOrException2<E1, E2>> getValuesOrThrow(
+ Iterable<SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2) {
+ return Maps.transformValues(getValuesOrThrow(depKeys, exceptionClass1, exceptionClass2,
+ BottomException.class), makeSafeDowncastToVOE2Function(exceptionClass1,
+ exceptionClass2));
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception> Map<SkyKey,
+ ValueOrException3<E1, E2, E3>> getValuesOrThrow(Iterable<SkyKey> depKeys,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3) {
+ return Maps.transformValues(getValuesOrThrow(depKeys, exceptionClass1, exceptionClass2,
+ exceptionClass3, BottomException.class), makeSafeDowncastToVOE3Function(exceptionClass1,
+ exceptionClass2, exceptionClass3));
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> Map<SkyKey, ValueOrException4<E1, E2, E3, E4>> getValuesOrThrow(
+ Iterable<SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3, Class<E4> exceptionClass4) {
+ Map<SkyKey, ValueOrException4<E1, E2, E3, E4>> result = new HashMap<>();
+ newlyRequestedDeps.startGroup();
+ for (SkyKey depKey : depKeys) {
+ if (result.containsKey(depKey)) {
+ continue;
+ }
+ result.put(depKey, getValueOrException(depKey, exceptionClass1, exceptionClass2,
+ exceptionClass3, exceptionClass4));
+ }
+ newlyRequestedDeps.endGroup();
+ return Collections.unmodifiableMap(result);
+ }
+
+ private void addDep(SkyKey key) {
+ if (!newlyRequestedDeps.contains(key)) {
+ // dep may have been requested already this evaluation. If not, add it.
+ newlyRequestedDeps.add(key);
+ }
+ }
+
+ @Override
+ public boolean valuesMissing() {
+ return valuesMissing;
+ }
+
+ /**
+ * If {@code !keepGoing} and there is at least one dep in error, returns a dep in error.
+ * Otherwise returns {@code null}.
+ */
+ @Nullable
+ private SkyKey getDepErrorKey() {
+ return depErrorKey;
+ }
+
+ @Override
+ public EventHandler getListener() {
+ checkActive();
+ return eventHandler;
+ }
+
+ private void doneBuilding() {
+ building = false;
+ }
+
+ /**
+ * Apply the change to the graph (mostly) atomically and signal all nodes that are waiting for
+ * this node to complete. Adding nodes and signaling is not atomic, but may need to be changed
+ * for interruptibility.
+ *
+ * <p>Parents are only enqueued if {@code enqueueParents} holds. Parents should be enqueued
+ * unless (1) this node is being built after the main evaluation has aborted, or (2) this node
+ * is being built with --nokeep_going, and so we are about to shut down the main evaluation
+ * anyway.
+ *
+ * <p>The node entry is informed if the node's value and error are definitive via the flag
+ * {@code completeValue}.
+ */
+ void commit(boolean enqueueParents) {
+ NodeEntry primaryEntry = Preconditions.checkNotNull(graph.get(skyKey), skyKey);
+ // Construct the definitive error info, if there is one.
+ finalizeErrorInfo();
+
+ // We have the following implications:
+ // errorInfo == null => value != null => enqueueParents.
+ // All these implications are strict:
+ // (1) errorInfo != null && value != null happens for values with recoverable errors.
+ // (2) value == null && enqueueParents happens for values that are found to have errors
+ // during a --keep_going build.
+
+ NestedSet<TaggedEvents> events = buildEvents(/*missingChildren=*/false);
+ if (value == null) {
+ Preconditions.checkNotNull(errorInfo, "%s %s", skyKey, primaryEntry);
+ // We could consider using max(childVersions) here instead of graphVersion. When full
+ // versioning is implemented, this would allow evaluation at a version between
+ // max(childVersions) and graphVersion to re-use this result.
+ Set<SkyKey> reverseDeps = primaryEntry.setValue(
+ ValueWithMetadata.error(errorInfo, events), graphVersion);
+ signalValuesAndEnqueueIfReady(enqueueParents ? visitor : null, reverseDeps, graphVersion);
+ } else {
+ // We must be enqueueing parents if we have a value.
+ Preconditions.checkState(enqueueParents, "%s %s", skyKey, primaryEntry);
+ Set<SkyKey> reverseDeps;
+ Version valueVersion;
+ // If this entry is dirty, setValue may not actually change it, if it determines that
+ // the data being written now is the same as the data already present in the entry.
+ // We could consider using max(childVersions) here instead of graphVersion. When full
+ // versioning is implemented, this would allow evaluation at a version between
+ // max(childVersions) and graphVersion to re-use this result.
+ reverseDeps = primaryEntry.setValue(
+ ValueWithMetadata.normal(value, errorInfo, events), graphVersion);
+ // Note that if this update didn't actually change the value entry, this version may not
+ // be the graph version.
+ valueVersion = primaryEntry.getVersion();
+ Preconditions.checkState(valueVersion.atMost(graphVersion),
+ "%s should be at most %s in the version partial ordering",
+ valueVersion, graphVersion);
+ if (progressReceiver != null) {
+ // Tell the receiver that this value was built. If valueVersion.equals(graphVersion), it
+ // was evaluated this run, and so was changed. Otherwise, it is less than graphVersion,
+ // by the Preconditions check above, and was not actually changed this run -- when it was
+ // written above, its version stayed below this update's version, so its value remains the
+ // same as before.
+ progressReceiver.evaluated(skyKey, value,
+ valueVersion.equals(graphVersion) ? EvaluationState.BUILT : EvaluationState.CLEAN);
+ }
+ signalValuesAndEnqueueIfReady(visitor, reverseDeps, valueVersion);
+ }
+
+ visitor.notifyDone(skyKey);
+ replayingNestedSetEventVisitor.visit(events);
+ }
+
+ @Nullable
+ private String getTagFromKey() {
+ return skyFunctions.get(skyKey.functionName()).extractTag(skyKey);
+ }
+
+ /**
+ * Gets the latch that is counted down when an exception is thrown in {@code
+ * AbstractQueueVisitor}. For use in tests to check if an exception actually was thrown. Calling
+ * {@code AbstractQueueVisitor#awaitExceptionForTestingOnly} can throw a spurious {@link
+ * InterruptedException} because {@link CountDownLatch#await} checks the interrupted bit before
+ * returning, even if the latch is already at 0. See bug "testTwoErrors is flaky".
+ */
+ CountDownLatch getExceptionLatchForTesting() {
+ return visitor.getExceptionLatchForTestingOnly();
+ }
+
+ @Override
+ public boolean inErrorBubblingForTesting() {
+ return bubbleErrorInfo != null;
+ }
+ }
+
+ private static final Function<ValueOrException<BottomException>, SkyValue> GET_VALUE_FROM_VOE =
+ new Function<ValueOrException<BottomException>, SkyValue>() {
+ @Override
+ public SkyValue apply(ValueOrException<BottomException> voe) {
+ return ValueOrExceptionUtils.downcovert(voe);
+ }
+ };
+
+ private static <E extends Exception>
+ Function<ValueOrException2<E, BottomException>, ValueOrException<E>>
+ makeSafeDowncastToVOEFunction(final Class<E> exceptionClass) {
+ return new Function<ValueOrException2<E, BottomException>, ValueOrException<E>>() {
+ @Override
+ public ValueOrException<E> apply(ValueOrException2<E, BottomException> voe) {
+ return ValueOrExceptionUtils.downcovert(voe, exceptionClass);
+ }
+ };
+ }
+
+ private static <E1 extends Exception, E2 extends Exception>
+ Function<ValueOrException3<E1, E2, BottomException>, ValueOrException2<E1, E2>>
+ makeSafeDowncastToVOE2Function(final Class<E1> exceptionClass1,
+ final Class<E2> exceptionClass2) {
+ return new Function<ValueOrException3<E1, E2, BottomException>,
+ ValueOrException2<E1, E2>>() {
+ @Override
+ public ValueOrException2<E1, E2> apply(ValueOrException3<E1, E2, BottomException> voe) {
+ return ValueOrExceptionUtils.downconvert(voe, exceptionClass1, exceptionClass2);
+ }
+ };
+ }
+
+ private static <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ Function<ValueOrException4<E1, E2, E3, BottomException>, ValueOrException3<E1, E2, E3>>
+ makeSafeDowncastToVOE3Function(final Class<E1> exceptionClass1,
+ final Class<E2> exceptionClass2, final Class<E3> exceptionClass3) {
+ return new Function<ValueOrException4<E1, E2, E3, BottomException>,
+ ValueOrException3<E1, E2, E3>>() {
+ @Override
+ public ValueOrException3<E1, E2, E3> apply(ValueOrException4<E1, E2, E3,
+ BottomException> voe) {
+ return ValueOrExceptionUtils.downconvert(voe, exceptionClass1, exceptionClass2,
+ exceptionClass3);
+ }
+ };
+ }
+
+ private class ValueVisitor extends AbstractQueueVisitor {
+ private AtomicBoolean preventNewEvaluations = new AtomicBoolean(false);
+ private final Set<SkyKey> inflightNodes = Sets.newConcurrentHashSet();
+
+ private ValueVisitor(int threadCount) {
+ super(/*concurrent*/true,
+ threadCount,
+ threadCount,
+ 1, TimeUnit.SECONDS,
+ /*failFastOnException*/true,
+ /*failFastOnInterrupt*/true,
+ "skyframe-evaluator");
+ }
+
+ @Override
+ protected boolean isCriticalError(Throwable e) {
+ return e instanceof RuntimeException;
+ }
+
+ protected void waitForCompletion() throws InterruptedException {
+ work(/*failFastOnInterrupt=*/true);
+ }
+
+ public void enqueueEvaluation(final SkyKey key) {
+ // We unconditionally add the key to the set of in-flight nodes because even if evaluation is
+ // never scheduled we still want to remove the previously created NodeEntry from the graph.
+ // Otherwise we would leave the graph in a weird state (wasteful garbage in the best case and
+ // inconsistent in the worst case).
+ boolean newlyEnqueued = inflightNodes.add(key);
+ // All nodes enqueued for evaluation will be either verified clean, re-evaluated, or cleaned
+ // up after being in-flight when an error happens in nokeep_going mode or in the event of an
+ // interrupt. In any of these cases, they won't be dirty anymore.
+ if (newlyEnqueued) {
+ dirtyKeyTracker.notDirty(key);
+ }
+ if (preventNewEvaluations.get()) {
+ return;
+ }
+ if (newlyEnqueued && progressReceiver != null) {
+ progressReceiver.enqueueing(key);
+ }
+ enqueue(new Evaluate(this, key));
+ }
+
+ public void preventNewEvaluations() {
+ preventNewEvaluations.set(true);
+ }
+
+ public void notifyDone(SkyKey key) {
+ inflightNodes.remove(key);
+ }
+
+ private boolean isInflight(SkyKey key) {
+ return inflightNodes.contains(key);
+ }
+ }
+
+ /**
+ * An action that evaluates a value.
+ */
+ private class Evaluate implements Runnable {
+ private final ValueVisitor visitor;
+ /** The name of the value to be evaluated. */
+ private final SkyKey skyKey;
+
+ private Evaluate(ValueVisitor visitor, SkyKey skyKey) {
+ this.visitor = visitor;
+ this.skyKey = skyKey;
+ }
+
+ private void enqueueChild(SkyKey skyKey, NodeEntry entry, SkyKey child) {
+ Preconditions.checkState(!entry.isDone(), "%s %s", skyKey, entry);
+ Preconditions.checkState(!ErrorTransienceValue.key().equals(child),
+ "%s cannot request ErrorTransienceValue as a dep: %s", skyKey, entry);
+
+ NodeEntry depEntry = graph.createIfAbsent(child);
+ switch (depEntry.addReverseDepAndCheckIfDone(skyKey)) {
+ case DONE :
+ if (entry.signalDep(depEntry.getVersion())) {
+ // This can only happen if there are no more children to be added.
+ visitor.enqueueEvaluation(skyKey);
+ }
+ break;
+ case ADDED_DEP :
+ break;
+ case NEEDS_SCHEDULING :
+ visitor.enqueueEvaluation(child);
+ break;
+ }
+ }
+
+ /**
+ * Returns true if this depGroup consists of the error transience value and the error transience
+ * value is newer than the entry, meaning that the entry must be re-evaluated.
+ */
+ private boolean invalidatedByErrorTransience(Collection<SkyKey> depGroup, NodeEntry entry) {
+ return depGroup.size() == 1
+ && depGroup.contains(ErrorTransienceValue.key())
+ && !graph.get(ErrorTransienceValue.key()).getVersion().atMost(entry.getVersion());
+ }
+
+ @Override
+ public void run() {
+ NodeEntry state = graph.get(skyKey);
+ Preconditions.checkNotNull(state, "%s %s", skyKey, state);
+ Preconditions.checkState(state.isReady(), "%s %s", skyKey, state);
+
+ if (state.isDirty()) {
+ switch (state.getDirtyState()) {
+ case CHECK_DEPENDENCIES:
+ // Evaluating a dirty node for the first time, and checking its children to see if any
+ // of them have changed. Note that there must be dirty children for this to happen.
+
+ // Check the children group by group -- we don't want to evaluate a value that is no
+ // longer needed because an earlier dependency changed. For example, //foo:foo depends
+ // on target //bar:bar and is built. Then foo/BUILD is modified to remove the dependence
+ // on bar, and bar/BUILD is deleted. Reloading //bar:bar would incorrectly throw an
+ // exception. To avoid this, we must reload foo/BUILD first, at which point we will
+ // discover that it has changed, and re-evaluate target //foo:foo from scratch.
+ // On the other hand, when an action requests all of its inputs, we can safely check all
+ // of them in parallel on a subsequent build. So we allow checking an entire group in
+ // parallel here, if the node builder requested a group last build.
+ Collection<SkyKey> directDepsToCheck = state.getNextDirtyDirectDeps();
+
+ if (invalidatedByErrorTransience(directDepsToCheck, state)) {
+ // If this dep is the ErrorTransienceValue and the ErrorTransienceValue has been
+ // updated then we need to force a rebuild. We would like to just signal the entry as
+ // usual, but we can't, because then the ErrorTransienceValue would remain as a dep,
+ // which would be incorrect if, for instance, the value re-evaluated to a non-error.
+ state.forceRebuild();
+ break; // Fall through to re-evaluation.
+ } else {
+ // If this isn't the error transience value, it is safe to add these deps back to the
+ // node -- even if one of them has changed, the contract of pruning is that the node
+ // will request these deps again when it rebuilds. We must add these deps before
+ // enqueuing them, so that the node knows that it depends on them.
+ state.addTemporaryDirectDeps(GroupedListHelper.create(directDepsToCheck));
+ }
+
+ for (SkyKey directDep : directDepsToCheck) {
+ enqueueChild(skyKey, state, directDep);
+ }
+ return;
+ case VERIFIED_CLEAN:
+ // No child has a changed value. This node can be marked done and its parents signaled
+ // without any re-evaluation.
+ visitor.notifyDone(skyKey);
+ Set<SkyKey> reverseDeps = state.markClean();
+ SkyValue value = state.getValue();
+ if (progressReceiver != null && value != null) {
+ // Tell the receiver that the value was not actually changed this run.
+ progressReceiver.evaluated(skyKey, value, EvaluationState.CLEAN);
+ }
+ signalValuesAndEnqueueIfReady(visitor, reverseDeps, state.getVersion());
+ return;
+ case REBUILDING:
+ // Nothing to be done if we are already rebuilding.
+ }
+ }
+
+ // TODO(bazel-team): Once deps are requested in a deterministic order within a group, or the
+ // framework is resilient to rearranging group order, change this so that
+ // SkyFunctionEnvironment "follows along" as the node builder runs, iterating through the
+ // direct deps that were requested on a previous run. This would allow us to avoid the
+ // conversion of the direct deps into a set.
+ Set<SkyKey> directDeps = state.getTemporaryDirectDeps();
+ Preconditions.checkState(!directDeps.contains(ErrorTransienceValue.key()),
+ "%s cannot have a dep on ErrorTransienceValue during building: %s", skyKey, state);
+ // Get the corresponding SkyFunction and call it on this value.
+ SkyFunctionEnvironment env = new SkyFunctionEnvironment(skyKey, directDeps, visitor);
+ SkyFunctionName functionName = skyKey.functionName();
+ SkyFunction factory = skyFunctions.get(functionName);
+ Preconditions.checkState(factory != null, "%s %s", functionName, state);
+
+ SkyValue value = null;
+ Profiler.instance().startTask(ProfilerTask.SKYFUNCTION, skyKey);
+ try {
+ // TODO(bazel-team): count how many of these calls returns null vs. non-null
+ value = factory.compute(skyKey, env);
+ } catch (final SkyFunctionException builderException) {
+ ReifiedSkyFunctionException reifiedBuilderException =
+ new ReifiedSkyFunctionException(builderException, skyKey);
+ // Propagated transitive errors are treated the same as missing deps.
+ if (reifiedBuilderException.getRootCauseSkyKey().equals(skyKey)) {
+ boolean shouldFailFast = !keepGoing || builderException.isCatastrophic();
+ if (shouldFailFast) {
+ // After we commit this error to the graph but before the eval call completes with the
+ // error there is a race-like opportunity for the error to be used, either by an
+ // in-flight computation or by a future computation.
+ if (errorEncountered.compareAndSet(false, true)) {
+ // This is the first error encountered.
+ visitor.preventNewEvaluations();
+ } else {
+ // This is not the first error encountered, so we ignore it so that we can terminate
+ // with the first error.
+ return;
+ }
+ }
+
+ registerNewlyDiscoveredDepsForDoneEntry(skyKey, state, env);
+ ErrorInfo errorInfo = new ErrorInfo(reifiedBuilderException);
+ env.setError(errorInfo);
+ env.commit(/*enqueueParents=*/keepGoing);
+ if (!shouldFailFast) {
+ return;
+ }
+ throw SchedulerException.ofError(errorInfo, skyKey);
+ }
+ } catch (InterruptedException ie) {
+ // InterruptedException cannot be thrown by Runnable.run, so we must wrap it.
+ // Interrupts can be caught by both the Evaluator and the AbstractQueueVisitor.
+ // The former will unwrap the IE and propagate it as is; the latter will throw a new IE.
+ throw SchedulerException.ofInterruption(ie, skyKey);
+ } catch (RuntimeException re) {
+ // Programmer error (most likely NPE or a failed precondition in a SkyFunction). Output
+ // some context together with the exception.
+ String msg = prepareCrashMessage(skyKey, state.getInProgressReverseDeps());
+ throw new RuntimeException(msg, re);
+ } finally {
+ env.doneBuilding();
+ Profiler.instance().completeTask(ProfilerTask.SKYFUNCTION);
+ }
+
+ GroupedListHelper<SkyKey> newDirectDeps = env.newlyRequestedDeps;
+
+ if (value != null) {
+ Preconditions.checkState(!env.valuesMissing,
+ "%s -> %s, ValueEntry: %s", skyKey, newDirectDeps, state);
+ env.setValue(value);
+ registerNewlyDiscoveredDepsForDoneEntry(skyKey, state, env);
+ env.commit(/*enqueueParents=*/true);
+ return;
+ }
+
+ if (!newDirectDeps.isEmpty() && env.getDepErrorKey() != null) {
+ Preconditions.checkState(!keepGoing);
+ // We encountered a child error in noKeepGoing mode, so we want to fail fast. But we first
+ // need to add the edge between the current node and the child error it requested so that
+ // error bubbling can occur. Note that this edge will subsequently be removed during graph
+ // cleaning (since the current node will never be committed to the graph).
+ SkyKey childErrorKey = env.getDepErrorKey();
+ NodeEntry childErrorEntry = Preconditions.checkNotNull(graph.get(childErrorKey),
+ "skyKey: %s, state: %s childErrorKey: %s", skyKey, state, childErrorKey);
+ if (!state.getTemporaryDirectDeps().contains(childErrorKey)) {
+ // This means the cached error was freshly requested (e.g. the parent has never been
+ // built before).
+ Preconditions.checkState(newDirectDeps.contains(childErrorKey), "%s %s %s", state,
+ childErrorKey, newDirectDeps);
+ state.addTemporaryDirectDeps(GroupedListHelper.create(ImmutableList.of(childErrorKey)));
+ DependencyState childErrorState = childErrorEntry.addReverseDepAndCheckIfDone(skyKey);
+ Preconditions.checkState(childErrorState == DependencyState.DONE,
+ "skyKey: %s, state: %s childErrorKey: %s", skyKey, state, childErrorKey,
+ childErrorEntry);
+ } else {
+ // This means the cached error was previously requested, and was then subsequently (after
+ // a restart) requested along with another sibling dep. This can happen on an incremental
+ // eval call when the parent is dirty and the child error is in a separate dependency
+ // group from the sibling dep.
+ Preconditions.checkState(!newDirectDeps.contains(childErrorKey), "%s %s %s", state,
+ childErrorKey, newDirectDeps);
+ Preconditions.checkState(childErrorEntry.isDone(),
+ "skyKey: %s, state: %s childErrorKey: %s", skyKey, state, childErrorKey,
+ childErrorEntry);
+ }
+ ErrorInfo childErrorInfo = Preconditions.checkNotNull(childErrorEntry.getErrorInfo());
+ throw SchedulerException.ofError(childErrorInfo, childErrorKey);
+ }
+
+ // TODO(bazel-team): This code is not safe to interrupt, because we would lose the state in
+ // newDirectDeps.
+
+ // TODO(bazel-team): An ill-behaved SkyFunction can throw us into an infinite loop where we
+ // add more dependencies on every run. [skyframe-core]
+
+ // Add all new keys to the set of known deps.
+ state.addTemporaryDirectDeps(newDirectDeps);
+
+ // If there were no newly requested dependencies, at least one of them was in error or there
+ // is a bug in the SkyFunction implementation. The environment has collected its errors, so we
+ // just order it to be built.
+ if (newDirectDeps.isEmpty()) {
+ // TODO(bazel-team): This means a bug in the SkyFunction. What to do?
+ Preconditions.checkState(!env.childErrorInfos.isEmpty(), "%s %s", skyKey, state);
+ env.commit(/*enqueueParents=*/keepGoing);
+ if (!keepGoing) {
+ throw SchedulerException.ofError(state.getErrorInfo(), skyKey);
+ }
+ return;
+ }
+
+ for (SkyKey newDirectDep : newDirectDeps) {
+ enqueueChild(skyKey, state, newDirectDep);
+ }
+ // It is critical that there is no code below this point.
+ }
+
+ private String prepareCrashMessage(SkyKey skyKey, Iterable<SkyKey> reverseDeps) {
+ StringBuilder reverseDepDump = new StringBuilder();
+ for (SkyKey key : reverseDeps) {
+ if (reverseDepDump.length() > MAX_REVERSEDEP_DUMP_LENGTH) {
+ reverseDepDump.append(", ...");
+ break;
+ }
+ if (reverseDepDump.length() > 0) {
+ reverseDepDump.append(", ");
+ }
+ reverseDepDump.append("'");
+ reverseDepDump.append(key.toString());
+ reverseDepDump.append("'");
+ }
+
+ return String.format(
+ "Unrecoverable error while evaluating node '%s' (requested by nodes %s)",
+ skyKey, reverseDepDump);
+ }
+
+ private static final int MAX_REVERSEDEP_DUMP_LENGTH = 1000;
+ }
+
+ /**
+ * Signals all parents that this node is finished. If visitor is not null, also enqueues any
+ * parents that are ready. If visitor is null, indicating that we are building this node after
+ * the main build aborted, then skip any parents that are already done (that can happen with
+ * cycles).
+ */
+ private void signalValuesAndEnqueueIfReady(@Nullable ValueVisitor visitor, Iterable<SkyKey> keys,
+ Version version) {
+ if (visitor != null) {
+ for (SkyKey key : keys) {
+ if (graph.get(key).signalDep(version)) {
+ visitor.enqueueEvaluation(key);
+ }
+ }
+ } else {
+ for (SkyKey key : keys) {
+ NodeEntry entry = Preconditions.checkNotNull(graph.get(key), key);
+ if (!entry.isDone()) {
+ // In cycles, we can have parents that are already done.
+ entry.signalDep(version);
+ }
+ }
+ }
+ }
+
+ /**
+ * If child is not done, removes key from child's reverse deps. Returns whether child should be
+ * removed from key's entry's direct deps.
+ */
+ private boolean removeIncompleteChild(SkyKey key, SkyKey child) {
+ NodeEntry childEntry = graph.get(child);
+ if (!isDoneForBuild(childEntry)) {
+ childEntry.removeReverseDep(key);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Add any additional deps that were registered during the run of a builder that finished by
+ * creating a node or throwing an error. Builders may throw errors even if all their deps were
+ * not provided -- we trust that a SkyFunction may be know it should throw an error even if not
+ * all of its requested deps are done. However, that means we're assuming the SkyFunction would
+ * throw that same error if all of its requested deps were done. Unfortunately, there is no way to
+ * enforce that condition.
+ */
+ private void registerNewlyDiscoveredDepsForDoneEntry(SkyKey skyKey, NodeEntry entry,
+ SkyFunctionEnvironment env) {
+ Set<SkyKey> unfinishedDeps = new HashSet<>();
+ Iterables.addAll(unfinishedDeps,
+ Iterables.filter(env.newlyRequestedDeps, Predicates.not(nodeEntryIsDone)));
+ env.newlyRequestedDeps.remove(unfinishedDeps);
+ entry.addTemporaryDirectDeps(env.newlyRequestedDeps);
+ for (SkyKey newDep : env.newlyRequestedDeps) {
+ NodeEntry depEntry = graph.get(newDep);
+ DependencyState triState = depEntry.addReverseDepAndCheckIfDone(skyKey);
+ Preconditions.checkState(DependencyState.DONE == triState,
+ "new dep %s was not already done for %s. ValueEntry: %s. DepValueEntry: %s",
+ newDep, skyKey, entry, depEntry);
+ entry.signalDep();
+ }
+ Preconditions.checkState(entry.isReady(), "%s %s %s", skyKey, entry, env.newlyRequestedDeps);
+ }
+
+ private void informProgressReceiverThatValueIsDone(SkyKey key) {
+ if (progressReceiver != null) {
+ NodeEntry entry = graph.get(key);
+ Preconditions.checkState(entry.isDone(), entry);
+ SkyValue value = entry.getValue();
+ if (value != null) {
+ Version valueVersion = entry.getVersion();
+ Preconditions.checkState(valueVersion.atMost(graphVersion),
+ "%s should be at most %s in the version partial ordering", valueVersion, graphVersion);
+ // Nodes with errors will have no value. Don't inform the receiver in that case.
+ // For most nodes we do not inform the progress receiver if they were already done
+ // when we retrieve them, but top-level nodes are presumably of more interest.
+ // If valueVersion is not equal to graphVersion, it must be less than it (by the
+ // Preconditions check above), and so the node is clean.
+ progressReceiver.evaluated(key, value, valueVersion.equals(graphVersion)
+ ? EvaluationState.BUILT
+ : EvaluationState.CLEAN);
+ }
+ }
+ }
+
+ @Override
+ @ThreadCompatible
+ public <T extends SkyValue> EvaluationResult<T> eval(Iterable<SkyKey> skyKeys)
+ throws InterruptedException {
+ ImmutableSet<SkyKey> skyKeySet = ImmutableSet.copyOf(skyKeys);
+
+ // Optimization: if all required node values are already present in the cache, return them
+ // directly without launching the heavy machinery, spawning threads, etc.
+ // Inform progressReceiver that these nodes are done to be consistent with the main code path.
+ if (Iterables.all(skyKeySet, nodeEntryIsDone)) {
+ for (SkyKey skyKey : skyKeySet) {
+ informProgressReceiverThatValueIsDone(skyKey);
+ }
+ // Note that the 'catastrophe' parameter doesn't really matter here (it's only used for
+ // sanity checking).
+ return constructResult(null, skyKeySet, null, /*catastrophe=*/false);
+ }
+
+ if (!keepGoing) {
+ Set<SkyKey> cachedErrorKeys = new HashSet<>();
+ for (SkyKey skyKey : skyKeySet) {
+ NodeEntry entry = graph.get(skyKey);
+ if (entry == null) {
+ continue;
+ }
+ if (entry.isDone() && entry.getErrorInfo() != null) {
+ informProgressReceiverThatValueIsDone(skyKey);
+ cachedErrorKeys.add(skyKey);
+ }
+ }
+
+ // Errors, even cached ones, should halt evaluations not in keepGoing mode.
+ if (!cachedErrorKeys.isEmpty()) {
+ // Note that the 'catastrophe' parameter doesn't really matter here (it's only used for
+ // sanity checking).
+ return constructResult(null, cachedErrorKeys, null, /*catastrophe=*/false);
+ }
+ }
+
+ // We delay this check until we know that some kind of evaluation is necessary, since !keepGoing
+ // and !keepsEdges are incompatible only in the case of a failed evaluation -- there is no
+ // need to be overly harsh to callers who are just trying to retrieve a cached result.
+ Preconditions.checkState(keepGoing || !(graph instanceof InMemoryGraph)
+ || ((InMemoryGraph) graph).keepsEdges(),
+ "nokeep_going evaluations are not allowed if graph edges are not kept: %s", skyKeys);
+
+ Profiler.instance().startTask(ProfilerTask.SKYFRAME_EVAL, skyKeySet);
+ try {
+ return eval(skyKeySet, new ValueVisitor(threadCount));
+ } finally {
+ Profiler.instance().completeTask(ProfilerTask.SKYFRAME_EVAL);
+ }
+ }
+
+ @ThreadCompatible
+ private <T extends SkyValue> EvaluationResult<T> eval(ImmutableSet<SkyKey> skyKeys,
+ ValueVisitor visitor) throws InterruptedException {
+ // We unconditionally add the ErrorTransienceValue here, to ensure that it will be created, and
+ // in the graph, by the time that it is needed. Creating it on demand in a parallel context sets
+ // up a race condition, because there is no way to atomically create a node and set its value.
+ NodeEntry errorTransienceEntry = graph.createIfAbsent(ErrorTransienceValue.key());
+ DependencyState triState = errorTransienceEntry.addReverseDepAndCheckIfDone(null);
+ Preconditions.checkState(triState != DependencyState.ADDED_DEP,
+ "%s %s", errorTransienceEntry, triState);
+ if (triState != DependencyState.DONE) {
+ errorTransienceEntry.setValue(new ErrorTransienceValue(), graphVersion);
+ // The error transience entry is always invalidated by the RecordingDifferencer.
+ // Now that the entry's value is set, it is no longer dirty.
+ dirtyKeyTracker.notDirty(ErrorTransienceValue.key());
+
+ Preconditions.checkState(
+ errorTransienceEntry.addReverseDepAndCheckIfDone(null) != DependencyState.ADDED_DEP,
+ errorTransienceEntry);
+ }
+ for (SkyKey skyKey : skyKeys) {
+ NodeEntry entry = graph.createIfAbsent(skyKey);
+ // This must be equivalent to the code in enqueueChild above, in order to be thread-safe.
+ switch (entry.addReverseDepAndCheckIfDone(null)) {
+ case NEEDS_SCHEDULING:
+ visitor.enqueueEvaluation(skyKey);
+ break;
+ case DONE:
+ informProgressReceiverThatValueIsDone(skyKey);
+ break;
+ case ADDED_DEP:
+ break;
+ default:
+ throw new IllegalStateException(entry + " for " + skyKey + " in unknown state");
+ }
+ }
+ try {
+ return waitForCompletionAndConstructResult(visitor, skyKeys);
+ } finally {
+ // TODO(bazel-team): In nokeep_going mode or in case of an interrupt, we need to remove
+ // partial values from the graph. Find a better way to handle those cases.
+ clean(visitor.inflightNodes);
+ }
+ }
+
+ private void clean(Set<SkyKey> inflightNodes) throws InterruptedException {
+ boolean alreadyInterrupted = Thread.interrupted();
+ // This parallel computation is fully cpu-bound, so we use a thread for each processor.
+ ExecutorService executor = Executors.newFixedThreadPool(
+ Runtime.getRuntime().availableProcessors(),
+ new ThreadFactoryBuilder().setNameFormat("ParallelEvaluator#clean %d").build());
+ ThrowableRecordingRunnableWrapper wrapper =
+ new ThrowableRecordingRunnableWrapper("ParallelEvaluator#clean");
+ for (final SkyKey key : inflightNodes) {
+ final NodeEntry entry = graph.get(key);
+ if (entry.isDone()) {
+ // Entry may be done in case of a RuntimeException or other programming bug. Do nothing,
+ // since (a) we're about to crash anyway, and (b) getTemporaryDirectDeps cannot be called
+ // on a done node, so the call below would crash, which would mask the actual exception
+ // that caused this state.
+ continue;
+ }
+ executor.execute(wrapper.wrap(new Runnable() {
+ @Override
+ public void run() {
+ cleanInflightNode(key, entry);
+ }
+ }));
+ }
+ // We uninterruptibly wait for all nodes to be cleaned because we want to make sure the graph
+ // is left in a good state.
+ //
+ // TODO(bazel-team): Come up with a better design for graph cleaning such that we can respond
+ // to interrupts in constant time.
+ boolean newlyInterrupted = ExecutorShutdownUtil.uninterruptibleShutdown(executor);
+ Throwables.propagateIfPossible(wrapper.getFirstThrownError());
+ if (newlyInterrupted || alreadyInterrupted) {
+ throw new InterruptedException();
+ }
+ }
+
+ private void cleanInflightNode(SkyKey key, NodeEntry entry) {
+ Set<SkyKey> temporaryDeps = entry.getTemporaryDirectDeps();
+ graph.remove(key);
+ for (SkyKey dep : temporaryDeps) {
+ NodeEntry nodeEntry = graph.get(dep);
+ // The direct dep might have already been cleaned from the graph.
+ if (nodeEntry != null) {
+ // Only bother removing the reverse dep on done nodes since other in-flight nodes will be
+ // cleaned too.
+ if (nodeEntry.isDone()) {
+ nodeEntry.removeReverseDep(key);
+ }
+ }
+ }
+ }
+
+ private <T extends SkyValue> EvaluationResult<T> waitForCompletionAndConstructResult(
+ ValueVisitor visitor, Iterable<SkyKey> skyKeys) throws InterruptedException {
+ Map<SkyKey, ValueWithMetadata> bubbleErrorInfo = null;
+ boolean catastrophe = false;
+ try {
+ visitor.waitForCompletion();
+ } catch (final SchedulerException e) {
+ Throwables.propagateIfPossible(e.getCause(), InterruptedException.class);
+ if (Thread.interrupted()) {
+ // As per the contract of AbstractQueueVisitor#work, if an unchecked exception is thrown and
+ // the build is interrupted, the thrown exception is what will be rethrown. Since the user
+ // presumably wanted to interrupt the build, we ignore the thrown SchedulerException (which
+ // doesn't indicate a programming bug) and throw an InterruptedException.
+ throw new InterruptedException();
+ }
+
+ SkyKey errorKey = Preconditions.checkNotNull(e.getFailedValue(), e);
+ // ErrorInfo could only be null if SchedulerException wrapped an InterruptedException, but
+ // that should have been propagated.
+ ErrorInfo errorInfo = Preconditions.checkNotNull(e.getErrorInfo(), errorKey);
+ catastrophe = errorInfo.isCatastrophic();
+ if (!catastrophe || !keepGoing) {
+ bubbleErrorInfo = bubbleErrorUp(errorInfo, errorKey, skyKeys, visitor);
+ } else {
+ // Bubbling the error up requires that graph edges are present for done nodes. This is not
+ // always the case in a keepGoing evaluation, since it is assumed that done nodes do not
+ // need to be traversed. In this case, we hope the caller is tolerant of a possibly empty
+ // result, and return prematurely.
+ bubbleErrorInfo = ImmutableMap.of(errorKey, graph.get(errorKey).getValueWithMetadata());
+ }
+ }
+
+ // Successful evaluation, either because keepGoing or because we actually did succeed.
+ // TODO(bazel-team): Maybe report root causes during the build for lower latency.
+ return constructResult(visitor, skyKeys, bubbleErrorInfo, catastrophe);
+ }
+
+ /**
+ * Walk up graph to find a top-level node (without parents) that wanted this failure. Store
+ * the failed nodes along the way in a map, with ErrorInfos that are appropriate for that layer.
+ * Example:
+ * foo bar
+ * \ /
+ * unrequested baz
+ * \ |
+ * failed-node
+ * User requests foo, bar. When failed-node fails, we look at its parents. unrequested is not
+ * in-flight, so we replace failed-node by baz and repeat. We look at baz's parents. foo is
+ * in-flight, so we replace baz by foo. Since foo is a top-level node and doesn't have parents,
+ * we then break, since we know a top-level node, foo, that depended on the failed node.
+ *
+ * There's the potential for a weird "track jump" here in the case:
+ * foo
+ * / \
+ * fail1 fail2
+ * If fail1 and fail2 fail simultaneously, fail2 may start propagating up in the loop below.
+ * However, foo requests fail1 first, and then throws an exception based on that. This is not
+ * incorrect, but may be unexpected.
+ *
+ * <p>Returns a map of errors that have been constructed during the bubbling up, so that the
+ * appropriate error can be returned to the caller, even though that error was not written to the
+ * graph. If a cycle is detected during the bubbling, this method aborts and returns null so that
+ * the normal cycle detection can handle the cycle.
+ *
+ * <p>Note that we are not propagating error to the first top-level node but to the highest one,
+ * because during this process we can add useful information about error from other nodes.
+ */
+ private Map<SkyKey, ValueWithMetadata> bubbleErrorUp(final ErrorInfo leafFailure,
+ SkyKey errorKey, Iterable<SkyKey> skyKeys, ValueVisitor visitor) {
+ Set<SkyKey> rootValues = ImmutableSet.copyOf(skyKeys);
+ ErrorInfo error = leafFailure;
+ Map<SkyKey, ValueWithMetadata> bubbleErrorInfo = new HashMap<>();
+ boolean externalInterrupt = false;
+ while (true) {
+ NodeEntry errorEntry = graph.get(errorKey);
+ Iterable<SkyKey> reverseDeps = errorEntry.isDone()
+ ? errorEntry.getReverseDeps()
+ : errorEntry.getInProgressReverseDeps();
+ // We should break from loop only when node doesn't have any parents.
+ if (Iterables.isEmpty(reverseDeps)) {
+ Preconditions.checkState(rootValues.contains(errorKey),
+ "Current key %s has to be a top-level key: %s", errorKey, rootValues);
+ break;
+ }
+ SkyKey parent = null;
+ NodeEntry parentEntry = null;
+ for (SkyKey bubbleParent : reverseDeps) {
+ if (bubbleErrorInfo.containsKey(bubbleParent)) {
+ // We are in a cycle. Don't try to bubble anything up -- cycle detection will kick in.
+ return null;
+ }
+ NodeEntry bubbleParentEntry = Preconditions.checkNotNull(graph.get(bubbleParent),
+ "parent %s of %s not in graph", bubbleParent, errorKey);
+ // Might be the parent that requested the error.
+ if (bubbleParentEntry.isDone()) {
+ // This parent is cached from a previous evaluate call. We shouldn't bubble up to it
+ // since any error message produced won't be meaningful to this evaluate call.
+ // The child error must also be cached from a previous build.
+ Preconditions.checkState(errorEntry.isDone(), "%s %s", errorEntry, bubbleParentEntry);
+ Version parentVersion = bubbleParentEntry.getVersion();
+ Version childVersion = errorEntry.getVersion();
+ Preconditions.checkState(childVersion.atMost(graphVersion)
+ && !childVersion.equals(graphVersion),
+ "child entry is not older than the current graph version, but had a done parent. "
+ + "child: %s childEntry: %s, childVersion: %s"
+ + "bubbleParent: %s bubbleParentEntry: %s, parentVersion: %s, graphVersion: %s",
+ errorKey, errorEntry, childVersion,
+ bubbleParent, bubbleParentEntry, parentVersion, graphVersion);
+ Preconditions.checkState(parentVersion.atMost(graphVersion)
+ && !parentVersion.equals(graphVersion),
+ "parent entry is not older than the current graph version. "
+ + "child: %s childEntry: %s, childVersion: %s"
+ + "bubbleParent: %s bubbleParentEntry: %s, parentVersion: %s, graphVersion: %s",
+ errorKey, errorEntry, childVersion,
+ bubbleParent, bubbleParentEntry, parentVersion, graphVersion);
+ continue;
+ }
+ // Arbitrarily pick the first in-flight parent.
+ Preconditions.checkState(visitor.isInflight(bubbleParent),
+ "errorKey: %s, errorEntry: %s, bubbleParent: %s, bubbleParentEntry: %s", errorKey,
+ errorEntry, bubbleParent, bubbleParentEntry);
+ parent = bubbleParent;
+ parentEntry = bubbleParentEntry;
+ break;
+ }
+ Preconditions.checkNotNull(parent, "", errorKey, bubbleErrorInfo);
+ errorKey = parent;
+ SkyFunction factory = skyFunctions.get(parent.functionName());
+ if (parentEntry.isDirty()) {
+ switch (parentEntry.getDirtyState()) {
+ case CHECK_DEPENDENCIES:
+ // If this value's child was bubbled up to, it did not signal this value, and so we must
+ // manually make it ready to build.
+ parentEntry.signalDep();
+ // Fall through to REBUILDING, since state is now REBUILDING.
+ case REBUILDING:
+ // Nothing to be done.
+ break;
+ default:
+ throw new AssertionError(parent + " not in valid dirty state: " + parentEntry);
+ }
+ }
+ SkyFunctionEnvironment env =
+ new SkyFunctionEnvironment(parent, parentEntry.getTemporaryDirectDeps(),
+ bubbleErrorInfo, visitor);
+ externalInterrupt = externalInterrupt || Thread.currentThread().isInterrupted();
+ try {
+ // This build is only to check if the parent node can give us a better error. We don't
+ // care about a return value.
+ factory.compute(parent, env);
+ } catch (SkyFunctionException builderException) {
+ ReifiedSkyFunctionException reifiedBuilderException =
+ new ReifiedSkyFunctionException(builderException, parent);
+ if (reifiedBuilderException.getRootCauseSkyKey().equals(parent)) {
+ error = new ErrorInfo(reifiedBuilderException);
+ bubbleErrorInfo.put(errorKey,
+ ValueWithMetadata.error(new ErrorInfo(errorKey, ImmutableSet.of(error)),
+ env.buildEvents(/*missingChildren=*/true)));
+ continue;
+ }
+ } catch (InterruptedException interruptedException) {
+ // Do nothing.
+ // This throw happens if the builder requested the failed node, and then checked the
+ // interrupted state later -- getValueOrThrow sets the interrupted bit after the failed
+ // value is requested, to prevent the builder from doing too much work.
+ } finally {
+ // Clear interrupted status. We're not listening to interrupts here.
+ Thread.interrupted();
+ }
+ // Builder didn't throw an exception, so just propagate this one up.
+ bubbleErrorInfo.put(errorKey,
+ ValueWithMetadata.error(new ErrorInfo(errorKey, ImmutableSet.of(error)),
+ env.buildEvents(/*missingChildren=*/true)));
+ }
+
+ // Reset the interrupt bit if there was an interrupt from outside this evaluator interrupt.
+ // Note that there are internal interrupts set in the node builder environment if an error
+ // bubbling node calls getValueOrThrow() on a node in error.
+ if (externalInterrupt) {
+ Thread.currentThread().interrupt();
+ }
+ return bubbleErrorInfo;
+ }
+
+ /**
+ * Constructs an {@link EvaluationResult} from the {@link #graph}. Looks for cycles if there
+ * are unfinished nodes but no error was already found through bubbling up
+ * (as indicated by {@code bubbleErrorInfo} being null).
+ *
+ * <p>{@code visitor} may be null, but only in the case where all graph entries corresponding to
+ * {@code skyKeys} are known to be in the DONE state ({@code entry.isDone()} returns true).
+ */
+ private <T extends SkyValue> EvaluationResult<T> constructResult(
+ @Nullable ValueVisitor visitor, Iterable<SkyKey> skyKeys,
+ Map<SkyKey, ValueWithMetadata> bubbleErrorInfo, boolean catastrophe) {
+ Preconditions.checkState(!keepGoing || catastrophe || bubbleErrorInfo == null,
+ "", skyKeys, bubbleErrorInfo);
+ EvaluationResult.Builder<T> result = EvaluationResult.builder();
+ List<SkyKey> cycleRoots = new ArrayList<>();
+ boolean hasError = false;
+ for (SkyKey skyKey : skyKeys) {
+ ValueWithMetadata valueWithMetadata = getValueMaybeFromError(skyKey, bubbleErrorInfo);
+ // Cycle checking: if there is a cycle, evaluation cannot progress, therefore,
+ // the final values will not be in DONE state when the work runs out.
+ if (valueWithMetadata == null) {
+ // Don't look for cycles if the build failed for a known reason.
+ if (bubbleErrorInfo == null) {
+ cycleRoots.add(skyKey);
+ }
+ hasError = true;
+ continue;
+ }
+ SkyValue value = valueWithMetadata.getValue();
+ // TODO(bazel-team): Verify that message replay is fast and works in failure
+ // modes [skyframe-core]
+ // Note that replaying events here is only necessary on null builds, because otherwise we
+ // would have already printed the transitive messages after building these values.
+ replayingNestedSetEventVisitor.visit(valueWithMetadata.getTransitiveEvents());
+ ErrorInfo errorInfo = valueWithMetadata.getErrorInfo();
+ Preconditions.checkState(value != null || errorInfo != null, skyKey);
+ hasError = hasError || (errorInfo != null);
+ if (!keepGoing && errorInfo != null) {
+ // value will be null here unless the value was already built on a prior keepGoing build.
+ result.addError(skyKey, errorInfo);
+ continue;
+ }
+ if (value == null) {
+ // Note that we must be in the keepGoing case. Only make this value an error if it doesn't
+ // have a value. The error shouldn't matter to the caller since the value succeeded after a
+ // fashion.
+ result.addError(skyKey, errorInfo);
+ } else {
+ result.addResult(skyKey, value);
+ }
+ }
+ if (!cycleRoots.isEmpty()) {
+ Preconditions.checkState(visitor != null, skyKeys);
+ checkForCycles(cycleRoots, result, visitor, keepGoing);
+ }
+ Preconditions.checkState(bubbleErrorInfo == null || hasError,
+ "If an error bubbled up, some top-level node must be in error", bubbleErrorInfo, skyKeys);
+ result.setHasError(hasError);
+ return result.build();
+ }
+
+ private <T extends SkyValue> void checkForCycles(
+ Iterable<SkyKey> badRoots, EvaluationResult.Builder<T> result, final ValueVisitor visitor,
+ boolean keepGoing) {
+ for (SkyKey root : badRoots) {
+ ErrorInfo errorInfo = checkForCycles(root, visitor, keepGoing);
+ if (errorInfo == null) {
+ // This node just wasn't finished when evaluation aborted -- there were no cycles below it.
+ Preconditions.checkState(!keepGoing, "", root, badRoots);
+ continue;
+ }
+ Preconditions.checkState(!Iterables.isEmpty(errorInfo.getCycleInfo()),
+ "%s was not evaluated, but was not part of a cycle", root);
+ result.addError(root, errorInfo);
+ if (!keepGoing) {
+ return;
+ }
+ }
+ }
+
+ /**
+ * Marker value that we push onto a stack before we push a node's children on. When the marker
+ * value is popped, we know that all the children are finished. We would use null instead, but
+ * ArrayDeque does not permit null elements.
+ */
+ private static final SkyKey CHILDREN_FINISHED =
+ new SkyKey(new SkyFunctionName("MARKER", false), "MARKER");
+
+ /** The max number of cycles we will report to the user for a given root, to avoid OOMing. */
+ private static final int MAX_CYCLES = 20;
+
+ /**
+ * The algorithm for this cycle detector is as follows. We visit the graph depth-first, keeping
+ * track of the path we are currently on. We skip any DONE nodes (they are transitively
+ * error-free). If we come to a node already on the path, we immediately construct a cycle. If
+ * we are in the noKeepGoing case, we return ErrorInfo with that cycle to the caller. Otherwise,
+ * we continue. Once all of a node's children are done, we construct an error value for it, based
+ * on those children. Finally, when the original root's node is constructed, we return its
+ * ErrorInfo.
+ */
+ private ErrorInfo checkForCycles(SkyKey root, ValueVisitor visitor, boolean keepGoing) {
+ // The number of cycles found. Do not keep on searching for more cycles after this many were
+ // found.
+ int cyclesFound = 0;
+ // The path through the graph currently being visited.
+ List<SkyKey> graphPath = new ArrayList<>();
+ // Set of nodes on the path, to avoid expensive searches through the path for cycles.
+ Set<SkyKey> pathSet = new HashSet<>();
+
+ // Maintain a stack explicitly instead of recursion to avoid stack overflows
+ // on extreme graphs (with long dependency chains).
+ Deque<SkyKey> toVisit = new ArrayDeque<>();
+
+ toVisit.push(root);
+
+ // The procedure for this check is as follows: we visit a node, push it onto the graph stack,
+ // push a marker value onto the toVisit stack, and then push all of its children onto the
+ // toVisit stack. Thus, when the marker node comes to the top of the toVisit stack, we have
+ // visited the downward transitive closure of the value. At that point, all of its children must
+ // be finished, and so we can build the definitive error info for the node, popping it off the
+ // graph stack.
+ while (!toVisit.isEmpty()) {
+ SkyKey key = toVisit.pop();
+ NodeEntry entry = graph.get(key);
+
+ if (key == CHILDREN_FINISHED) {
+ // A marker node means we are done with all children of a node. Since all nodes have
+ // errors, we must have found errors in the children when that happens.
+ key = graphPath.remove(graphPath.size() - 1);
+ entry = graph.get(key);
+ pathSet.remove(key);
+ // Skip this node if it was first/last node of a cycle, and so has already been processed.
+ if (entry.isDone()) {
+ continue;
+ }
+ if (!keepGoing) {
+ // in the --nokeep_going mode, we would have already returned if we'd found a cycle below
+ // this node. The fact that we haven't means that there were no cycles below this node
+ // -- it just hadn't finished evaluating. So skip it.
+ continue;
+ }
+ if (cyclesFound < MAX_CYCLES) {
+ // Value must be ready, because all of its children have finished, so we can build its
+ // error.
+ Preconditions.checkState(entry.isReady(), "%s not ready. ValueEntry: %s", key, entry);
+ } else if (!entry.isReady()) {
+ removeIncompleteChildrenForCycle(key, entry, entry.getTemporaryDirectDeps());
+ }
+ Set<SkyKey> directDeps = entry.getTemporaryDirectDeps();
+ // Find out which children have errors. Similar logic to that in Evaluate#run().
+ List<ErrorInfo> errorDeps = getChildrenErrorsForCycle(directDeps);
+ Preconditions.checkState(!errorDeps.isEmpty(),
+ "Value %s was not successfully evaluated, but had no child errors. ValueEntry: %s", key,
+ entry);
+ SkyFunctionEnvironment env = new SkyFunctionEnvironment(key, directDeps, visitor);
+ env.setError(new ErrorInfo(key, errorDeps));
+ env.commit(/*enqueueParents=*/false);
+ }
+
+ // Nothing to be done for this node if it already has an entry.
+ if (entry.isDone()) {
+ continue;
+ }
+ if (cyclesFound == MAX_CYCLES) {
+ // Do not keep on searching for cycles indefinitely, to avoid excessive runtime/OOMs.
+ continue;
+ }
+
+ if (pathSet.contains(key)) {
+ int cycleStart = graphPath.indexOf(key);
+ // Found a cycle!
+ cyclesFound++;
+ Iterable<SkyKey> cycle = graphPath.subList(cycleStart, graphPath.size());
+ // Put this node into a consistent state for building if it is dirty.
+ if (entry.isDirty() && entry.getDirtyState() == DirtyState.CHECK_DEPENDENCIES) {
+ // In the check deps state, entry has exactly one child not done yet. Note that this node
+ // must be part of the path to the cycle we have found (since done nodes cannot be in
+ // cycles, and this is the only missing one). Thus, it will not be removed below in
+ // removeDescendantsOfCycleValue, so it is safe here to signal that it is done.
+ entry.signalDep();
+ }
+ if (keepGoing) {
+ // Any children of this node that we haven't already visited are not worth visiting,
+ // since this node is about to be done. Thus, the only child worth visiting is the one in
+ // this cycle, the cycleChild (which may == key if this cycle is a self-edge).
+ SkyKey cycleChild = selectCycleChild(key, graphPath, cycleStart);
+ removeDescendantsOfCycleValue(key, entry, cycleChild, toVisit,
+ graphPath.size() - cycleStart);
+ ValueWithMetadata dummyValue = ValueWithMetadata.wrapWithMetadata(new SkyValue() {});
+
+
+ SkyFunctionEnvironment env =
+ new SkyFunctionEnvironment(key, entry.getTemporaryDirectDeps(),
+ ImmutableMap.of(cycleChild, dummyValue), visitor);
+
+ // Construct error info for this node. Get errors from children, which are all done
+ // except possibly for the cycleChild.
+ List<ErrorInfo> allErrors =
+ getChildrenErrors(entry.getTemporaryDirectDeps(), /*unfinishedChild=*/cycleChild);
+ CycleInfo cycleInfo = new CycleInfo(cycle);
+ // Add in this cycle.
+ allErrors.add(new ErrorInfo(cycleInfo));
+ env.setError(new ErrorInfo(key, allErrors));
+ env.commit(/*enqueueParents=*/false);
+ continue;
+ } else {
+ // We need to return right away in the noKeepGoing case, so construct the cycle (with the
+ // path) and return.
+ Preconditions.checkState(graphPath.get(0).equals(root),
+ "%s not reached from %s. ValueEntry: %s", key, root, entry);
+ return new ErrorInfo(new CycleInfo(graphPath.subList(0, cycleStart), cycle));
+ }
+ }
+
+ // This node is not yet known to be in a cycle. So process its children.
+ Iterable<? extends SkyKey> children = graph.get(key).getTemporaryDirectDeps();
+ if (Iterables.isEmpty(children)) {
+ continue;
+ }
+
+ // This marker flag will tell us when all this node's children have been processed.
+ toVisit.push(CHILDREN_FINISHED);
+ // This node is now part of the path through the graph.
+ graphPath.add(key);
+ pathSet.add(key);
+ for (SkyKey nextValue : children) {
+ toVisit.push(nextValue);
+ }
+ }
+ return keepGoing ? getAndCheckDone(root).getErrorInfo() : null;
+ }
+
+ /**
+ * Returns the child of this node that is in the cycle that was just found. If the cycle is a
+ * self-edge, returns the node itself.
+ */
+ private static SkyKey selectCycleChild(SkyKey key, List<SkyKey> graphPath, int cycleStart) {
+ return cycleStart + 1 == graphPath.size() ? key : graphPath.get(cycleStart + 1);
+ }
+
+ /**
+ * Get all the errors of child nodes. There must be at least one cycle amongst them.
+ *
+ * @param children child nodes to query for errors.
+ * @return List of ErrorInfos from all children that had errors.
+ */
+ private List<ErrorInfo> getChildrenErrorsForCycle(Iterable<SkyKey> children) {
+ List<ErrorInfo> allErrors = new ArrayList<>();
+ boolean foundCycle = false;
+ for (SkyKey child : children) {
+ ErrorInfo errorInfo = getAndCheckDone(child).getErrorInfo();
+ if (errorInfo != null) {
+ foundCycle |= !Iterables.isEmpty(errorInfo.getCycleInfo());
+ allErrors.add(errorInfo);
+ }
+ }
+ Preconditions.checkState(foundCycle, "", children, allErrors);
+ return allErrors;
+ }
+
+ /**
+ * Get all the errors of child nodes.
+ *
+ * @param children child nodes to query for errors.
+ * @param unfinishedChild child which is allowed to not be done.
+ * @return List of ErrorInfos from all children that had errors.
+ */
+ private List<ErrorInfo> getChildrenErrors(Iterable<SkyKey> children, SkyKey unfinishedChild) {
+ List<ErrorInfo> allErrors = new ArrayList<>();
+ for (SkyKey child : children) {
+ ErrorInfo errorInfo = getErrorMaybe(child, /*allowUnfinished=*/child.equals(unfinishedChild));
+ if (errorInfo != null) {
+ allErrors.add(errorInfo);
+ }
+ }
+ return allErrors;
+ }
+
+ @Nullable
+ private ErrorInfo getErrorMaybe(SkyKey key, boolean allowUnfinished) {
+ if (!allowUnfinished) {
+ return getAndCheckDone(key).getErrorInfo();
+ }
+ NodeEntry entry = Preconditions.checkNotNull(graph.get(key), key);
+ return entry.isDone() ? entry.getErrorInfo() : null;
+ }
+
+ /**
+ * Removes direct children of key from toVisit and from the entry itself, and makes the entry
+ * ready if necessary. We must do this because it would not make sense to try to build the
+ * children after building the entry. It would violate the invariant that a parent can only be
+ * built after its children are built; See bug "Precondition error while evaluating a Skyframe
+ * graph with a cycle".
+ *
+ * @param key SkyKey of node in a cycle.
+ * @param entry NodeEntry of node in a cycle.
+ * @param cycleChild direct child of key in the cycle, or key itself if the cycle is a self-edge.
+ * @param toVisit list of remaining nodes to visit by the cycle-checker.
+ * @param cycleLength the length of the cycle found.
+ */
+ private void removeDescendantsOfCycleValue(SkyKey key, NodeEntry entry,
+ @Nullable SkyKey cycleChild, Iterable<SkyKey> toVisit, int cycleLength) {
+ Set<SkyKey> unvisitedDeps = new HashSet<>(entry.getTemporaryDirectDeps());
+ unvisitedDeps.remove(cycleChild);
+ // Remove any children from this node that are not part of the cycle we just found. They are
+ // irrelevant to the node as it stands, and if they are deleted from the graph because they are
+ // not built by the end of cycle-checking, we would have dangling references.
+ removeIncompleteChildrenForCycle(key, entry, unvisitedDeps);
+ if (!entry.isReady()) {
+ // The entry has at most one undone dep now, its cycleChild. Signal to make entry ready. Note
+ // that the entry can conceivably be ready if its cycleChild already found a different cycle
+ // and was built.
+ entry.signalDep();
+ }
+ Preconditions.checkState(entry.isReady(), "%s %s %s", key, cycleChild, entry);
+ Iterator<SkyKey> it = toVisit.iterator();
+ while (it.hasNext()) {
+ SkyKey descendant = it.next();
+ if (descendant == CHILDREN_FINISHED) {
+ // Marker value, delineating the end of a group of children that were enqueued.
+ cycleLength--;
+ if (cycleLength == 0) {
+ // We have seen #cycleLength-1 marker values, and have arrived at the one for this value,
+ // so we are done.
+ return;
+ }
+ continue; // Don't remove marker values.
+ }
+ if (cycleLength == 1) {
+ // Remove the direct children remaining to visit of the cycle node.
+ Preconditions.checkState(unvisitedDeps.contains(descendant),
+ "%s %s %s %s %s", key, descendant, cycleChild, unvisitedDeps, entry);
+ it.remove();
+ }
+ }
+ throw new IllegalStateException("There were not " + cycleLength + " groups of children in "
+ + toVisit + " when trying to remove children of " + key + " other than " + cycleChild);
+ }
+
+ private void removeIncompleteChildrenForCycle(SkyKey key, NodeEntry entry,
+ Iterable<SkyKey> children) {
+ Set<SkyKey> unfinishedDeps = new HashSet<>();
+ for (SkyKey child : children) {
+ if (removeIncompleteChild(key, child)) {
+ unfinishedDeps.add(child);
+ }
+ }
+ entry.removeUnfinishedDeps(unfinishedDeps);
+ }
+
+ private NodeEntry getAndCheckDone(SkyKey key) {
+ NodeEntry entry = graph.get(key);
+ Preconditions.checkNotNull(entry, key);
+ Preconditions.checkState(entry.isDone(), "%s %s", key, entry);
+ return entry;
+ }
+
+ private ValueWithMetadata getValueMaybeFromError(SkyKey key,
+ @Nullable Map<SkyKey, ValueWithMetadata> bubbleErrorInfo) {
+ SkyValue value = bubbleErrorInfo == null ? null : bubbleErrorInfo.get(key);
+ NodeEntry entry = graph.get(key);
+ if (value != null) {
+ Preconditions.checkNotNull(entry,
+ "Value cannot have error before evaluation started", key, value);
+ return ValueWithMetadata.wrapWithMetadata(value);
+ }
+ return isDoneForBuild(entry) ? entry.getValueWithMetadata() : null;
+ }
+
+ /**
+ * Return true if the entry does not need to be re-evaluated this build. The entry will need to
+ * be re-evaluated if it is not done, but also if it was not completely evaluated last build and
+ * this build is keepGoing.
+ */
+ private boolean isDoneForBuild(@Nullable NodeEntry entry) {
+ return entry != null && entry.isDone();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ProcessableGraph.java b/src/main/java/com/google/devtools/build/skyframe/ProcessableGraph.java
new file mode 100644
index 0000000..8bf8a38
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ProcessableGraph.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * A graph that is both Dirtiable (values can be deleted) and Evaluable (values can be added). All
+ * methods in this interface (as inherited from super-interfaces) should be thread-safe.
+ *
+ * <p>This class is not intended for direct use, and is only exposed as public for use in
+ * evaluation implementations outside of this package.
+ */
+public interface ProcessableGraph extends DirtiableGraph, EvaluableGraph {
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java b/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java
new file mode 100644
index 0000000..e1cfc0a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * A graph that exposes its entries and structure, for use by classes that must traverse it.
+ */
+public interface QueryableGraph {
+ /**
+ * Returns the node with the given name, or {@code null} if the node does not exist.
+ */
+ NodeEntry get(SkyKey key);
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java
new file mode 100644
index 0000000..3ebbf33
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A simple Differencer which just records the invalidated values it's been given.
+ */
+@ThreadSafety.ThreadCompatible
+public class RecordingDifferencer implements Differencer, Injectable {
+
+ private List<SkyKey> valuesToInvalidate;
+ private Map<SkyKey, SkyValue> valuesToInject;
+
+ public RecordingDifferencer() {
+ clear();
+ }
+
+ private void clear() {
+ valuesToInvalidate = new ArrayList<>();
+ valuesToInject = new HashMap<>();
+ }
+
+ @Override
+ public Diff getDiff(Version fromVersion, Version toVersion) {
+ Diff diff = new ImmutableDiff(valuesToInvalidate, valuesToInject);
+ clear();
+ return diff;
+ }
+
+ /**
+ * Store the given values for invalidation.
+ */
+ public void invalidate(Iterable<SkyKey> values) {
+ Iterables.addAll(valuesToInvalidate, values);
+ }
+
+ /**
+ * Invalidates the cached values of any values in error transiently.
+ *
+ * <p>If a future call to {@link MemoizingEvaluator#evaluate} requests a value that transitively
+ * depends on any value that was in an error state (or is one of these), they will be re-computed.
+ */
+ public void invalidateTransientErrors() {
+ // All transient error values have a dependency on the single global ERROR_TRANSIENCE value,
+ // so we only have to invalidate that one value to catch everything.
+ invalidate(ImmutableList.of(ErrorTransienceValue.key()));
+ }
+
+ /**
+ * Store the given values for injection.
+ */
+ @Override
+ public void inject(Map<SkyKey, ? extends SkyValue> values) {
+ valuesToInject.putAll(values);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ReverseDepsUtil.java b/src/main/java/com/google/devtools/build/skyframe/ReverseDepsUtil.java
new file mode 100644
index 0000000..13d8c4b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ReverseDepsUtil.java
@@ -0,0 +1,211 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A utility class that allows us to keep the reverse dependencies as an array list instead of a
+ * set. This is more memory-efficient. At the same time it allows us to group the removals and
+ * uniqueness checks so that it also performs well.
+ *
+ * <p>The reason of this class it to share non-trivial code between BuildingState and NodeEntry. We
+ * could simply make those two classes extend this class instead, but we would be less
+ * memory-efficient since object memory alignment does not cross classes ( you would have two memory
+ * alignments, one for the base class and one for the extended one).
+ */
+abstract class ReverseDepsUtil<T> {
+
+ static final int MAYBE_CHECK_THRESHOLD = 10;
+
+ abstract void setReverseDepsObject(T container, Object object);
+
+ abstract void setSingleReverseDep(T container, boolean singleObject);
+
+ abstract void setReverseDepsToRemove(T container, List<SkyKey> object);
+
+ abstract Object getReverseDepsObject(T container);
+
+ abstract boolean isSingleReverseDep(T container);
+
+ abstract List<SkyKey> getReverseDepsToRemove(T container);
+
+ /**
+ * We check that the reverse dependency is not already present. We only do that if reverseDeps is
+ * small, so that it does not impact performance.
+ */
+ void maybeCheckReverseDepNotPresent(T container, SkyKey reverseDep) {
+ if (isSingleReverseDep(container)) {
+ Preconditions.checkState(!getReverseDepsObject(container).equals(reverseDep),
+ "Reverse dep %s already present", reverseDep);
+ return;
+ }
+ @SuppressWarnings("unchecked")
+ List<SkyKey> asList = (List<SkyKey>) getReverseDepsObject(container);
+ if (asList.size() < MAYBE_CHECK_THRESHOLD) {
+ Preconditions.checkState(!asList.contains(reverseDep), "Reverse dep %s already present"
+ + " in %s", reverseDep, asList);
+ }
+ }
+
+ /**
+ * We use a memory-efficient trick to keep reverseDeps memory usage low. Edges in Bazel are
+ * dominant over the number of nodes.
+ *
+ * <p>Most of the nodes have zero or one reverse dep. That is why we use immutable versions of the
+ * lists for those cases. In case of the size being > 1 we switch to an ArrayList. That is because
+ * we also have a decent number of nodes for which the reverseDeps are huge (for example almost
+ * everything depends on BuildInfo node).
+ *
+ * <p>We also optimize for the case where we have only one dependency. In that case we keep the
+ * object directly instead of a wrapper list.
+ */
+ @SuppressWarnings("unchecked")
+ void addReverseDeps(T container, Collection<SkyKey> newReverseDeps) {
+ if (newReverseDeps.isEmpty()) {
+ return;
+ }
+ Object reverseDeps = getReverseDepsObject(container);
+ int reverseDepsSize = isSingleReverseDep(container) ? 1 : ((List<SkyKey>) reverseDeps).size();
+ int newSize = reverseDepsSize + newReverseDeps.size();
+ if (newSize == 1) {
+ overwriteReverseDepsWithObject(container, Iterables.getOnlyElement(newReverseDeps));
+ } else if (reverseDepsSize == 0) {
+ overwriteReverseDepsList(container, Lists.newArrayList(newReverseDeps));
+ } else if (reverseDepsSize == 1) {
+ List<SkyKey> newList = Lists.newArrayListWithExpectedSize(newSize);
+ newList.add((SkyKey) reverseDeps);
+ newList.addAll(newReverseDeps);
+ overwriteReverseDepsList(container, newList);
+ } else {
+ ((List<SkyKey>) reverseDeps).addAll(newReverseDeps);
+ }
+ }
+
+ /**
+ * See {@code addReverseDeps} method.
+ */
+ void removeReverseDep(T container, SkyKey reverseDep) {
+ if (isSingleReverseDep(container)) {
+ // This removal is cheap so let's do it and not keep it in reverseDepsToRemove.
+ // !equals should only happen in case of catastrophe.
+ if (getReverseDepsObject(container).equals(reverseDep)) {
+ overwriteReverseDepsList(container, ImmutableList.<SkyKey>of());
+ }
+ return;
+ }
+ @SuppressWarnings("unchecked")
+ List<SkyKey> reverseDepsAsList = (List<SkyKey>) getReverseDepsObject(container);
+ if (reverseDepsAsList.isEmpty()) {
+ return;
+ }
+ List<SkyKey> reverseDepsToRemove = getReverseDepsToRemove(container);
+ if (reverseDepsToRemove == null) {
+ reverseDepsToRemove = Lists.newArrayListWithExpectedSize(1);
+ setReverseDepsToRemove(container, reverseDepsToRemove);
+ }
+ reverseDepsToRemove.add(reverseDep);
+ }
+
+ ImmutableSet<SkyKey> getReverseDeps(T container) {
+ consolidateReverseDepsRemovals(container);
+
+ // TODO(bazel-team): Unfortunately, we need to make a copy here right now to be on the safe side
+ // wrt. thread-safety. The parents of a node get modified when any of the parents is deleted,
+ // and we can't handle that right now.
+ if (isSingleReverseDep(container)) {
+ return ImmutableSet.of((SkyKey) getReverseDepsObject(container));
+ } else {
+ @SuppressWarnings("unchecked")
+ List<SkyKey> reverseDeps = (List<SkyKey>) getReverseDepsObject(container);
+ ImmutableSet<SkyKey> set = ImmutableSet.copyOf(reverseDeps);
+ Preconditions.checkState(set.size() == reverseDeps.size(),
+ "Duplicate reverse deps present in %s: %s. %s", this, reverseDeps, container);
+ return set;
+ }
+ }
+
+ void consolidateReverseDepsRemovals(T container) {
+ List<SkyKey> reverseDepsToRemove = getReverseDepsToRemove(container);
+ Object reverseDeps = getReverseDepsObject(container);
+ if (reverseDepsToRemove == null) {
+ return;
+ }
+ Preconditions.checkState(!isSingleReverseDep(container),
+ "We do not use reverseDepsToRemove for single lists: %s", container);
+ // Should not happen, as we only create reverseDepsToRemove in case we have at least one
+ // reverse dep to remove.
+ Preconditions.checkState((!((List<?>) reverseDeps).isEmpty()),
+ "Could not remove %s elements from %s.\nReverse deps to remove: %s. %s",
+ reverseDepsToRemove.size(), reverseDeps, reverseDepsToRemove, container);
+
+ Set<SkyKey> toRemove = Sets.newHashSet(reverseDepsToRemove);
+ int expectedRemovals = toRemove.size();
+ Preconditions.checkState(expectedRemovals == reverseDepsToRemove.size(),
+ "A reverse dependency tried to remove itself twice: %s. %s", reverseDepsToRemove,
+ container);
+
+ @SuppressWarnings("unchecked")
+ List<SkyKey> reverseDepsAsList = (List<SkyKey>) reverseDeps;
+ List<SkyKey> newReverseDeps = Lists
+ .newArrayListWithExpectedSize(Math.max(0, reverseDepsAsList.size() - expectedRemovals));
+
+ for (SkyKey reverseDep : reverseDepsAsList) {
+ if (!toRemove.contains(reverseDep)) {
+ newReverseDeps.add(reverseDep);
+ }
+ }
+ Preconditions.checkState(newReverseDeps.size() == reverseDepsAsList.size() - expectedRemovals,
+ "Could not remove some elements from %s.\nReverse deps to remove: %s. %s", reverseDeps,
+ toRemove, container);
+
+ if (newReverseDeps.isEmpty()) {
+ overwriteReverseDepsList(container, ImmutableList.<SkyKey>of());
+ } else if (newReverseDeps.size() == 1) {
+ overwriteReverseDepsWithObject(container, newReverseDeps.get(0));
+ } else {
+ overwriteReverseDepsList(container, newReverseDeps);
+ }
+ setReverseDepsToRemove(container, null);
+ }
+
+ @SuppressWarnings("deprecation")
+ String toString(T container) {
+ return Objects.toStringHelper("ReverseDeps") // MoreObjects is not in Guava
+ .add("reverseDeps", getReverseDepsObject(container))
+ .add("singleReverseDep", isSingleReverseDep(container))
+ .add("reverseDepsToRemove", getReverseDepsToRemove(container))
+ .toString();
+ }
+
+ private void overwriteReverseDepsWithObject(T container, SkyKey newObject) {
+ setReverseDepsObject(container, newObject);
+ setSingleReverseDep(container, true);
+ }
+
+ private void overwriteReverseDepsList(T container, List<SkyKey> list) {
+ setReverseDepsObject(container, list);
+ setSingleReverseDep(container, false);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Scheduler.java b/src/main/java/com/google/devtools/build/skyframe/Scheduler.java
new file mode 100644
index 0000000..f05860f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/Scheduler.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+
+import javax.annotation.Nullable;
+
+/**
+ * A work queue -- takes {@link Runnable}s and runs them when requested.
+ */
+interface Scheduler {
+ /**
+ * Schedules a new action to be eventually done.
+ */
+ void schedule(Runnable action);
+
+ /**
+ * Runs the actions that have been scheduled. These actions can in turn schedule new actions,
+ * which will be run as well.
+ *
+ * @throw SchedulerException wrapping a scheduled action's exception.
+ */
+ void run() throws SchedulerException;
+
+ /**
+ * Wrapper exception that {@link Runnable}s can throw, to be caught and handled
+ * by callers of {@link #run}.
+ */
+ static class SchedulerException extends RuntimeException {
+ private final SkyKey failedValue;
+ private final ErrorInfo errorInfo;
+
+ private SchedulerException(@Nullable Throwable cause, @Nullable ErrorInfo errorInfo,
+ SkyKey failedValue) {
+ super(errorInfo != null ? errorInfo.getException() : cause);
+ this.errorInfo = errorInfo;
+ this.failedValue = Preconditions.checkNotNull(failedValue, errorInfo);
+ }
+
+ /**
+ * Returns a SchedulerException wrapping an expected error, e.g. an error describing an expected
+ * build failure when trying to evaluate the given value, that should cause Skyframe to produce
+ * useful error information to the user.
+ */
+ static SchedulerException ofError(ErrorInfo errorInfo, SkyKey failedValue) {
+ Preconditions.checkNotNull(errorInfo);
+ return new SchedulerException(errorInfo.getException(), errorInfo, failedValue);
+ }
+
+ /**
+ * Returns a SchedulerException wrapping an InterruptedException, e.g. if the user interrupts
+ * the build, that should cause Skyframe to exit as soon as possible.
+ */
+ static SchedulerException ofInterruption(InterruptedException cause, SkyKey failedValue) {
+ return new SchedulerException(cause, null, failedValue);
+ }
+
+ SkyKey getFailedValue() {
+ return failedValue;
+ }
+
+ @Nullable ErrorInfo getErrorInfo() {
+ return errorInfo;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java b/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java
new file mode 100644
index 0000000..9b7f036
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.events.EventHandler;
+
+/**
+ * A driver for auto-updating graphs which operate over monotonically increasing integer versions.
+ */
+public class SequentialBuildDriver implements BuildDriver {
+ private final MemoizingEvaluator memoizingEvaluator;
+ private IntVersion curVersion;
+
+ public SequentialBuildDriver(MemoizingEvaluator evaluator) {
+ this.memoizingEvaluator = Preconditions.checkNotNull(evaluator);
+ this.curVersion = new IntVersion(0);
+ }
+
+ @Override
+ public <T extends SkyValue> EvaluationResult<T> evaluate(
+ Iterable<SkyKey> roots, boolean keepGoing, int numThreads, EventHandler reporter)
+ throws InterruptedException {
+ try {
+ return memoizingEvaluator.evaluate(roots, curVersion, keepGoing, numThreads, reporter);
+ } finally {
+ curVersion = curVersion.next();
+ }
+ }
+
+ @Override
+ public MemoizingEvaluator getGraphForTesting() {
+ return memoizingEvaluator;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java
new file mode 100644
index 0000000..324c03d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java
@@ -0,0 +1,187 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.events.EventHandler;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Machinery to evaluate a single value.
+ *
+ * <p>The builder is supposed to access only direct dependencies of the value. However, the direct
+ * dependencies need not be known in advance. The builder can request arbitrary values using
+ * {@link Environment#getValue}. If the values are not ready, the call will return null; in that
+ * case the builder can either try to proceed (and potentially indicate more dependencies by
+ * additional {@code getValue} calls), or just return null, in which case the missing dependencies
+ * will be computed and the builder will be started again.
+ */
+public interface SkyFunction {
+
+ /**
+ * When a value is requested, this method is called with the name of the value and a value
+ * building environment.
+ *
+ * <p>This method should return a constructed value, or null if any dependencies were missing
+ * ({@link Environment#valuesMissing} was true before returning). In that case the missing
+ * dependencies will be computed and the value builder restarted.
+ *
+ * <p>Implementations must be threadsafe and reentrant.
+ *
+ * @throws SkyFunctionException on failure
+ * @throws InterruptedException when the user interrupts the build
+ */
+ @Nullable SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException;
+
+ /**
+ * Extracts a tag (target label) from a SkyKey if it has one. Otherwise return null.
+ *
+ * <p>The tag is used for filtering out non-error event messages that do not match --output_filter
+ * flag. If a SkyFunction returns null in this method it means that all the info/warning messages
+ * associated with this value will be shown, no matter what --output_filter says.
+ */
+ @Nullable
+ String extractTag(SkyKey skyKey);
+
+ /**
+ * The services provided to the value builder by the graph implementation.
+ */
+ interface Environment {
+ /**
+ * Returns a direct dependency. If the specified value is not in the set of already evaluated
+ * direct dependencies, returns null. Also returns null if the specified value has already been
+ * evaluated and found to be in error.
+ *
+ * <p>On a subsequent build, if any of this value's dependencies have changed they will be
+ * re-evaluated in the same order as originally requested by the {@code SkyFunction} using
+ * this {@code getValue} call (see {@link #getValues} for when preserving the order is not
+ * important).
+ */
+ @Nullable
+ SkyValue getValue(SkyKey valueName);
+
+ /**
+ * Returns a direct dependency. If the specified value is not in the set of already evaluated
+ * direct dependencies, returns null. If the specified value has already been evaluated and
+ * found to be in error, throws the exception coming from the error. Value builders may
+ * use this method to continue evaluation even if one of their children is in error by catching
+ * the thrown exception and proceeding. The caller must specify the exception that might be
+ * thrown using the {@code exceptionClass} argument. If the child's exception is not an instance
+ * of {@code exceptionClass}, returns null without throwing.
+ *
+ * <p>The exception class given cannot be a supertype or a subtype of {@link RuntimeException},
+ * or a subtype of {@link InterruptedException}. See
+ * {@link SkyFunctionException#validateExceptionType} for details.
+ */
+ @Nullable
+ <E extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E> exceptionClass) throws E;
+ @Nullable
+ <E1 extends Exception, E2 extends Exception> SkyValue getValueOrThrow(SkyKey depKey,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2) throws E1, E2;
+ @Nullable
+ <E1 extends Exception, E2 extends Exception, E3 extends Exception> SkyValue getValueOrThrow(
+ SkyKey depKey, Class<E1> exceptionClass1, Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3) throws E1, E2, E3;
+ @Nullable
+ <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ SkyValue getValueOrThrow(SkyKey depKey, Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2, Class<E3> exceptionClass3, Class<E4> exceptionClass4)
+ throws E1, E2, E3, E4;
+
+ /**
+ * Returns true iff any of the past {@link #getValue}(s) or {@link #getValueOrThrow} method
+ * calls for this instance returned null (because the value was not yet present and done in the
+ * graph).
+ *
+ * <p>If this returns true, the {@link SkyFunction} must return {@code null}.
+ */
+ boolean valuesMissing();
+
+ /**
+ * Requests {@code depKeys} "in parallel", independent of each others' values. These keys may be
+ * thought of as a "dependency group" -- they are requested together by this value.
+ *
+ * <p>In general, if the result of one getValue call can affect the argument of a later getValue
+ * call, the two calls cannot be merged into a single getValues call, since the result of the
+ * first call might change on a later build. Inversely, if the result of one getValue call
+ * cannot affect the parameters of the next getValue call, the two keys can form a dependency
+ * group and the two getValue calls merged into one getValues call.
+ *
+ * <p>This means that on subsequent builds, when checking to see if a value requires rebuilding,
+ * all the values in this group may be simultaneously checked. A SkyFunction should request a
+ * dependency group if checking the deps serially on a subsequent build would take too long, and
+ * if the builder would request all deps anyway as long as no earlier deps had changed.
+ * SkyFunction.Environment implementations may also choose to request these deps in
+ * parallel on the first build, potentially speeding up the build.
+ *
+ * <p>While re-evaluating every value in the group may take longer than re-evaluating just the
+ * first one and finding that it has changed, no extra work is done: the contract of the
+ * dependency group means that the builder, when called to rebuild this value, will request all
+ * values in the group again anyway, so they would have to have been built in any case.
+ *
+ * <p>Example of when to use getValues: A ListProcessor value is built with key inputListRef.
+ * The builder first calls getValue(InputList.key(inputListRef)), and retrieves inputList. It
+ * then iterates through inputList, calling getValue on each input. Finally, it processes the
+ * whole list and returns. Say inputList is (a, b, c). Since the builder will unconditionally
+ * call getValue(a), getValue(b), and getValue(c), the builder can instead just call
+ * getValues({a, b, c}). If the value is later dirtied the evaluator will build a, b, and c in
+ * parallel (assuming the inputList value was unchanged), and re-evaluate the ListProcessor
+ * value only if at least one of them was changed. On the other hand, if the InputList changes
+ * to be (a, b, d), then the evaluator will see that the first dep has changed, and call the
+ * builder to rebuild from scratch, without considering the dep group of {a, b, c}.
+ *
+ * <p>Example of when not to use getValues: A BestMatch value is built with key
+ * <potentialMatchesRef, matchCriterion>. The builder first calls
+ * getValue(PotentialMatches.key(potentialMatchesRef) and retrieves potentialMatches. It then
+ * iterates through potentialMatches, calling getValue on each potential match until it finds
+ * one that satisfies matchCriterion. In this case, if potentialMatches is (a, b, c), it would
+ * be <i>incorrect</i> to call getValues({a, b, c}), because it is not known yet whether
+ * requesting b or c will be necessary -- if a matches, then we will never call b or c.
+ */
+ Map<SkyKey, SkyValue> getValues(Iterable<SkyKey> depKeys);
+
+ /**
+ * The same as {@link #getValues} but the returned objects may throw when attempting to retrieve
+ * their value. Note that even if the requested values can throw different kinds of exceptions,
+ * only exceptions of type {@code E} will be preserved in the returned objects. All others will
+ * be null.
+ */
+ <E extends Exception> Map<SkyKey, ValueOrException<E>> getValuesOrThrow(
+ Iterable<SkyKey> depKeys, Class<E> exceptionClass);
+ <E1 extends Exception, E2 extends Exception> Map<SkyKey, ValueOrException2<E1, E2>>
+ getValuesOrThrow(Iterable<SkyKey> depKeys, Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2);
+ <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ Map<SkyKey, ValueOrException3<E1, E2, E3>> getValuesOrThrow(Iterable<SkyKey> depKeys,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3);
+ <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ Map<SkyKey, ValueOrException4<E1, E2, E3, E4>> getValuesOrThrow(Iterable<SkyKey> depKeys,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4);
+
+ /**
+ * Returns the {@link EventHandler} that a SkyFunction should use to print any errors,
+ * warnings, or progress messages while building.
+ */
+ EventHandler getListener();
+
+ /** Returns whether we are currently in error bubbling. */
+ @VisibleForTesting
+ boolean inErrorBubblingForTesting();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunctionException.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionException.java
new file mode 100644
index 0000000..71b4710
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionException.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+
+import com.google.common.base.Preconditions;
+
+import javax.annotation.Nullable;
+
+/**
+ * Base class of exceptions thrown by {@link SkyFunction#compute} on failure.
+ *
+ * SkyFunctions should declare a subclass {@code C} of {@link SkyFunctionException} whose
+ * constructors forward fine-grained exception types (e.g. {@link IOException}) to
+ * {@link SkyFunctionException}'s constructor, and they should also declare
+ * {@link SkyFunction#compute} to throw {@code C}. This way the type system checks that no
+ * unexpected exceptions are thrown by the {@link SkyFunction}.
+ *
+ * <p>We took this approach over using a generic exception class since Java disallows it because of
+ * type erasure
+ * (see http://docs.oracle.com/javase/tutorial/java/generics/restrictions.html#cannotCatch).
+ *
+ * <p> Note that there are restrictions on what Exception types are allowed to be wrapped in this
+ * manner. See {@link SkyFunctionException#validateExceptionType}.
+ *
+ * <p>Failures are explicitly either transient or persistent. The transience of the failure from
+ * {@link SkyFunction#compute} should be influenced only by the computations done, and not by the
+ * transience of the failures from computations requested via
+ * {@link SkyFunction.Environment#getValueOrThrow}.
+ */
+public abstract class SkyFunctionException extends Exception {
+
+ /** The transience of the error. */
+ public enum Transience {
+ // An error that may or may not occur again if the computation were re-run. If a computation
+ // results in a transient error and is needed on a subsequent MemoizingEvaluator#evaluate call,
+ // it will be re-executed.
+ TRANSIENT,
+
+ // An error that is completely deterministic and persistent in terms of the computation's
+ // inputs. Persistent errors may be cached.
+ PERSISTENT;
+ }
+
+ private final Transience transience;
+ @Nullable
+ private final SkyKey rootCause;
+
+ public SkyFunctionException(Exception cause, Transience transience) {
+ this(cause, transience, null);
+ }
+
+ /** Used to rethrow a child error that the parent cannot handle. */
+ public SkyFunctionException(Exception cause, SkyKey childKey) {
+ this(cause, Transience.PERSISTENT, childKey);
+ }
+
+ private SkyFunctionException(Exception cause, Transience transience, SkyKey rootCause) {
+ super(Preconditions.checkNotNull(cause));
+ SkyFunctionException.validateExceptionType(cause.getClass());
+ this.transience = transience;
+ this.rootCause = rootCause;
+ }
+
+ @Nullable
+ final SkyKey getRootCauseSkyKey() {
+ return rootCause;
+ }
+
+ final boolean isTransient() {
+ return transience == Transience.TRANSIENT;
+ }
+
+ /**
+ * Catastrophic failures halt the build even when in keepGoing mode.
+ */
+ public boolean isCatastrophic() {
+ return false;
+ }
+
+ @Override
+ public Exception getCause() {
+ return (Exception) super.getCause();
+ }
+
+ static <E extends Throwable> void validateExceptionType(Class<E> exceptionClass) {
+ if (exceptionClass.equals(ValueOrExceptionUtils.BottomException.class)) {
+ return;
+ }
+
+ if (exceptionClass.isAssignableFrom(RuntimeException.class)) {
+ throw new IllegalStateException(exceptionClass.getSimpleName() + " is a supertype of "
+ + "RuntimeException. Don't do this since then you would potentially swallow all "
+ + "RuntimeExceptions, even those from Skyframe");
+ }
+ if (RuntimeException.class.isAssignableFrom(exceptionClass)) {
+ throw new IllegalStateException(exceptionClass.getSimpleName() + " is a subtype of "
+ + "RuntimeException. You should rewrite your code to use checked exceptions.");
+ }
+ if (InterruptedException.class.isAssignableFrom(exceptionClass)) {
+ throw new IllegalStateException(exceptionClass.getSimpleName() + " is a subtype of "
+ + "InterruptedException. Don't do this; Skyframe handles interrupts separately from the "
+ + "general SkyFunctionException mechanism.");
+ }
+ }
+
+ /** A {@link SkyFunctionException} with a definite root cause. */
+ static class ReifiedSkyFunctionException extends SkyFunctionException {
+ private final boolean isCatastrophic;
+
+ ReifiedSkyFunctionException(SkyFunctionException e, SkyKey key) {
+ super(e.getCause(), e.transience, Preconditions.checkNotNull(e.getRootCauseSkyKey() == null
+ ? key : e.getRootCauseSkyKey()));
+ this.isCatastrophic = e.isCatastrophic();
+ }
+
+ @Override
+ public boolean isCatastrophic() {
+ return isCatastrophic;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunctionName.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionName.java
new file mode 100644
index 0000000..389d4d8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionName.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Predicate;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * An identifier for a {@code SkyFunction}.
+ */
+public final class SkyFunctionName implements Serializable {
+ public static SkyFunctionName computed(String name) {
+ return new SkyFunctionName(name, true);
+ }
+
+ private final String name;
+ private final boolean isComputed;
+
+ public SkyFunctionName(String name, boolean isComputed) {
+ this.name = name;
+ this.isComputed = isComputed;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SkyFunctionName)) {
+ return false;
+ }
+ SkyFunctionName other = (SkyFunctionName) obj;
+ return name.equals(other.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ /**
+ * Returns whether the values of this type are computed. The computation of a computed value must
+ * be deterministic and may only access requested dependencies.
+ */
+ public boolean isComputed() {
+ return isComputed;
+ }
+
+ /**
+ * A predicate that returns true for {@link SkyKey}s that have the given {@link SkyFunctionName}.
+ */
+ public static Predicate<SkyKey> functionIs(final SkyFunctionName functionName) {
+ return new Predicate<SkyKey>() {
+ @Override
+ public boolean apply(SkyKey skyKey) {
+ return functionName.equals(skyKey.functionName());
+ }
+ };
+ }
+
+ /**
+ * A predicate that returns true for {@link SkyKey}s that have the given {@link SkyFunctionName}.
+ */
+ public static Predicate<SkyKey> functionIsIn(final Set<SkyFunctionName> functionNames) {
+ return new Predicate<SkyKey>() {
+ @Override
+ public boolean apply(SkyKey skyKey) {
+ return functionNames.contains(skyKey.functionName());
+ }
+ };
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyKey.java b/src/main/java/com/google/devtools/build/skyframe/SkyKey.java
new file mode 100644
index 0000000..cc1dd1f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyKey.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+
+import java.io.Serializable;
+
+/**
+ * A {@link SkyKey} is effectively a pair (type, name) that identifies a Skyframe value.
+ */
+public final class SkyKey implements Serializable {
+ private final SkyFunctionName functionName;
+
+ /**
+ * The name of the value.
+ *
+ * <p>This is deliberately an untyped Object so that we can use arbitrary value types (e.g.,
+ * Labels, PathFragments, BuildConfigurations, etc.) as value names without incurring
+ * serialization costs in the in-memory implementation of the graph.
+ */
+ private final Object argument;
+
+ /**
+ * Cache the hash code for this object. It might be expensive to compute.
+ */
+ private final int hashCode;
+
+ public SkyKey(SkyFunctionName functionName, Object valueName) {
+ this.functionName = Preconditions.checkNotNull(functionName);
+ this.argument = Preconditions.checkNotNull(valueName);
+ this.hashCode = 31 * functionName.hashCode() + argument.hashCode();
+ }
+
+ public SkyFunctionName functionName() {
+ return functionName;
+ }
+
+ public Object argument() {
+ return argument;
+ }
+
+ @Override
+ public String toString() {
+ return functionName + ":" + argument;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ SkyKey other = (SkyKey) obj;
+ return argument.equals(other.argument) && functionName.equals(other.functionName);
+ }
+
+ public static final Function<SkyKey, Object> NODE_NAME = new Function<SkyKey, Object>() {
+ @Override
+ public Object apply(SkyKey input) {
+ return input.argument();
+ }
+ };
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyValue.java b/src/main/java/com/google/devtools/build/skyframe/SkyValue.java
new file mode 100644
index 0000000..7cfaa78
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyValue.java
@@ -0,0 +1,22 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import java.io.Serializable;
+
+/**
+ * A return value of a {@code SkyFunction}.
+ */
+public interface SkyValue extends Serializable {
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/TaggedEvents.java b/src/main/java/com/google/devtools/build/skyframe/TaggedEvents.java
new file mode 100644
index 0000000..056175e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/TaggedEvents.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * A wrapper of {@link Event} that contains a tag of the label where the event was generated. This
+ * class allows us to tell where the events are coming from when we group all the tags in a
+ * NestedSet.
+ *
+ * <p>The only usage of this code for now is to be able to use --output_filter in Skyframe
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+@Immutable
+public final class TaggedEvents {
+
+ @Nullable
+ private final String tag;
+ private final ImmutableCollection<Event> events;
+
+ TaggedEvents(@Nullable String tag, ImmutableCollection<Event> events) {
+
+ this.tag = tag;
+ this.events = events;
+ }
+
+ @Nullable
+ String getTag() {
+ return tag;
+ }
+
+ ImmutableCollection<Event> getEvents() {
+ return events;
+ }
+
+ /**
+ * Returns <i>some</i> moderately sane representation of the events. Should never be used in
+ * user-visible places, only for debugging and testing.
+ */
+ @Override
+ public String toString() {
+ return tag == null ? "<unknown>" : tag + ": " + Iterables.toString(events);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException.java
new file mode 100644
index 0000000..d682095
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import javax.annotation.Nullable;
+
+/** Wrapper for a value or the typed exception thrown when trying to compute it. */
+public abstract class ValueOrException<E extends Exception> extends ValueOrUntypedException {
+
+ /** Gets the stored value. Throws an exception if one was thrown when computing this value. */
+ @Nullable
+ public abstract SkyValue get() throws E;
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException2.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException2.java
new file mode 100644
index 0000000..deedbb1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException2.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import javax.annotation.Nullable;
+
+/** Wrapper for a value or the typed exception thrown when trying to compute it. */
+public abstract class ValueOrException2<E1 extends Exception, E2 extends Exception>
+ extends ValueOrUntypedException {
+
+ /** Gets the stored value. Throws an exception if one was thrown when computing this value. */
+ @Nullable
+ public abstract SkyValue get() throws E1, E2;
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException3.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException3.java
new file mode 100644
index 0000000..e737c55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException3.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import javax.annotation.Nullable;
+
+/** Wrapper for a value or the typed exception thrown when trying to compute it. */
+public abstract class ValueOrException3<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception> extends ValueOrUntypedException {
+
+ /** Gets the stored value. Throws an exception if one was thrown when computing this value. */
+ @Nullable
+ public abstract SkyValue get() throws E1, E2, E3;
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException4.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException4.java
new file mode 100644
index 0000000..176f405
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException4.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import javax.annotation.Nullable;
+
+/** Wrapper for a value or the typed exception thrown when trying to compute it. */
+public abstract class ValueOrException4<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception, E4 extends Exception> extends ValueOrUntypedException {
+
+ /** Gets the stored value. Throws an exception if one was thrown when computing this value. */
+ @Nullable
+ public abstract SkyValue get() throws E1, E2, E3, E4;
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrExceptionUtils.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrExceptionUtils.java
new file mode 100644
index 0000000..e66f4fa
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrExceptionUtils.java
@@ -0,0 +1,520 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import javax.annotation.Nullable;
+
+/** Utilities for producing and consuming ValueOrException(2|3|4)? instances. */
+class ValueOrExceptionUtils {
+
+ /** The bottom exception type. */
+ class BottomException extends Exception {
+ }
+
+ @Nullable
+ public static SkyValue downcovert(ValueOrException<BottomException> voe) {
+ return voe.getValue();
+ }
+
+ public static <E1 extends Exception> ValueOrException<E1> downcovert(
+ ValueOrException2<E1, BottomException> voe, Class<E1> exceptionClass1) {
+ Exception e = voe.getException();
+ if (e == null) {
+ return new ValueOrExceptionValueImpl<>(voe.getValue());
+ }
+ // Here and below, we use type-safe casts for performance reasons. Another approach would be
+ // cascading try-catch-rethrow blocks, but that has a higher performance penalty.
+ if (exceptionClass1.isInstance(e)) {
+ return new ValueOrExceptionExnImpl<>(exceptionClass1.cast(e));
+ }
+ throw new IllegalStateException("shouldn't reach here " + e.getClass() + " " + exceptionClass1,
+ e);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception> ValueOrException2<E1, E2> downconvert(
+ ValueOrException3<E1, E2, BottomException> voe, Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2) {
+ Exception e = voe.getException();
+ if (e == null) {
+ return new ValueOrException2ValueImpl<>(voe.getValue());
+ }
+ if (exceptionClass1.isInstance(e)) {
+ return new ValueOrException2Exn1Impl<>(exceptionClass1.cast(e));
+ }
+ if (exceptionClass2.isInstance(e)) {
+ return new ValueOrException2Exn2Impl<>(exceptionClass2.cast(e));
+ }
+ throw new IllegalStateException("shouldn't reach here " + e.getClass() + " " + exceptionClass1
+ + " " + exceptionClass2, e);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ ValueOrException3<E1, E2, E3> downconvert(ValueOrException4<E1, E2, E3, BottomException> voe,
+ Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3) {
+ Exception e = voe.getException();
+ if (e == null) {
+ return new ValueOrException3ValueImpl<>(voe.getValue());
+ }
+ if (exceptionClass1.isInstance(e)) {
+ return new ValueOrException3Exn1Impl<>(exceptionClass1.cast(e));
+ }
+ if (exceptionClass2.isInstance(e)) {
+ return new ValueOrException3Exn2Impl<>(exceptionClass2.cast(e));
+ }
+ if (exceptionClass3.isInstance(e)) {
+ return new ValueOrException3Exn3Impl<>(exceptionClass3.cast(e));
+ }
+ throw new IllegalStateException("shouldn't reach here " + e.getClass() + " " + exceptionClass1
+ + " " + exceptionClass2 + " " + exceptionClass3, e);
+ }
+
+ public static <E extends Exception> ValueOrException<E> ofNull() {
+ return ValueOrExceptionValueImpl.ofNull();
+ }
+
+ public static ValueOrUntypedException ofValueUntyped(SkyValue value) {
+ return new ValueOrUntypedExceptionImpl(value);
+ }
+
+ public static <E extends Exception> ValueOrException<E> ofExn(E e) {
+ return new ValueOrExceptionExnImpl<>(e);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofNullValue() {
+ return ValueOrException4ValueImpl.ofNullValue();
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofValue(SkyValue value) {
+ return new ValueOrException4ValueImpl<>(value);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn1(E1 e) {
+ return new ValueOrException4Exn1Impl<>(e);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn2(E2 e) {
+ return new ValueOrException4Exn2Impl<>(e);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn3(E3 e) {
+ return new ValueOrException4Exn3Impl<>(e);
+ }
+
+ public static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn4(E4 e) {
+ return new ValueOrException4Exn4Impl<>(e);
+ }
+
+ private static class ValueOrUntypedExceptionImpl extends ValueOrUntypedException {
+ @Nullable
+ private final SkyValue value;
+
+ ValueOrUntypedExceptionImpl(@Nullable SkyValue value) {
+ this.value = value;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return value;
+ }
+
+ @Override
+ public Exception getException() {
+ return null;
+ }
+ }
+
+ private static class ValueOrExceptionValueImpl<E extends Exception> extends ValueOrException<E> {
+ private static final ValueOrExceptionValueImpl<Exception> NULL =
+ new ValueOrExceptionValueImpl<Exception>((SkyValue) null);
+
+ @Nullable
+ private final SkyValue value;
+
+ private ValueOrExceptionValueImpl(@Nullable SkyValue value) {
+ this.value = value;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue get() {
+ return value;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return value;
+ }
+
+ @Override
+ @Nullable
+ public Exception getException() {
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <E extends Exception> ValueOrExceptionValueImpl<E> ofNull() {
+ return (ValueOrExceptionValueImpl<E>) NULL;
+ }
+ }
+
+ private static class ValueOrExceptionExnImpl<E extends Exception> extends ValueOrException<E> {
+ private final E e;
+
+ private ValueOrExceptionExnImpl(E e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E {
+ throw e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+ }
+
+ private static class ValueOrException2ValueImpl<E1 extends Exception, E2 extends Exception>
+ extends ValueOrException2<E1, E2> {
+ @Nullable
+ private final SkyValue value;
+
+ ValueOrException2ValueImpl(@Nullable SkyValue value) {
+ this.value = value;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue get() throws E1, E2 {
+ return value;
+ }
+
+ @Override
+ @Nullable
+ public Exception getException() {
+ return null;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return value;
+ }
+ }
+
+ private static class ValueOrException2Exn1Impl<E1 extends Exception, E2 extends Exception>
+ extends ValueOrException2<E1, E2> {
+ private final E1 e;
+
+ private ValueOrException2Exn1Impl(E1 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E1 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException2Exn2Impl<E1 extends Exception, E2 extends Exception>
+ extends ValueOrException2<E1, E2> {
+ private final E2 e;
+
+ private ValueOrException2Exn2Impl(E2 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E2 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException3ValueImpl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception> extends ValueOrException3<E1, E2, E3> {
+ @Nullable
+ private final SkyValue value;
+
+ ValueOrException3ValueImpl(@Nullable SkyValue value) {
+ this.value = value;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue get() throws E1, E2 {
+ return value;
+ }
+
+ @Override
+ @Nullable
+ public Exception getException() {
+ return null;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return value;
+ }
+ }
+
+ private static class ValueOrException3Exn1Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception> extends ValueOrException3<E1, E2, E3> {
+ private final E1 e;
+
+ private ValueOrException3Exn1Impl(E1 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E1 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException3Exn2Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception> extends ValueOrException3<E1, E2, E3> {
+ private final E2 e;
+
+ private ValueOrException3Exn2Impl(E2 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E2 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException3Exn3Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception> extends ValueOrException3<E1, E2, E3> {
+ private final E3 e;
+
+ private ValueOrException3Exn3Impl(E3 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E3 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException4ValueImpl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> {
+ private static final ValueOrException4ValueImpl<Exception, Exception, Exception,
+ Exception> NULL = new ValueOrException4ValueImpl<>((SkyValue) null);
+
+ @Nullable
+ private final SkyValue value;
+
+ ValueOrException4ValueImpl(@Nullable SkyValue value) {
+ this.value = value;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue get() throws E1, E2 {
+ return value;
+ }
+
+ @Override
+ @Nullable
+ public Exception getException() {
+ return null;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return value;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <E1 extends Exception, E2 extends Exception, E3 extends Exception,
+ E4 extends Exception>ValueOrException4ValueImpl<E1, E2, E3, E4> ofNullValue() {
+ return (ValueOrException4ValueImpl<E1, E2, E3, E4>) NULL;
+ }
+ }
+
+ private static class ValueOrException4Exn1Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> {
+ private final E1 e;
+
+ private ValueOrException4Exn1Impl(E1 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E1 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException4Exn2Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> {
+ private final E2 e;
+
+ private ValueOrException4Exn2Impl(E2 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E2 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException4Exn3Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> {
+ private final E3 e;
+
+ private ValueOrException4Exn3Impl(E3 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E3 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+
+ private static class ValueOrException4Exn4Impl<E1 extends Exception, E2 extends Exception,
+ E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> {
+ private final E4 e;
+
+ private ValueOrException4Exn4Impl(E4 e) {
+ this.e = e;
+ }
+
+ @Override
+ public SkyValue get() throws E4 {
+ throw e;
+ }
+
+ @Override
+ public Exception getException() {
+ return e;
+ }
+
+ @Override
+ @Nullable
+ public SkyValue getValue() {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrUntypedException.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrUntypedException.java
new file mode 100644
index 0000000..c7ea7d4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrUntypedException.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import javax.annotation.Nullable;
+
+/**
+ * Wrapper for a value or the untyped exception thrown when trying to compute it.
+ *
+ * <p>This is an implementation detail of {@link ParallelEvaluator} and
+ * {@link ValueOrExceptionUtils}. It's an abstract class (as opposed to an interface) to avoid
+ * exposing the methods outside the package.
+ */
+abstract class ValueOrUntypedException {
+
+ /** Returns the stored value, if there was one. */
+ @Nullable
+ abstract SkyValue getValue();
+
+ /** Returns the stored exception, if there was one. */
+ @Nullable
+ abstract Exception getException();
+}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueWithMetadata.java b/src/main/java/com/google/devtools/build/skyframe/ValueWithMetadata.java
new file mode 100644
index 0000000..956e404
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/ValueWithMetadata.java
@@ -0,0 +1,209 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Encapsulation of data stored by {@link NodeEntry} when the value has finished building.
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+public abstract class ValueWithMetadata implements SkyValue {
+ protected final SkyValue value;
+
+ private static final NestedSet<TaggedEvents> NO_EVENTS =
+ NestedSetBuilder.<TaggedEvents>emptySet(Order.STABLE_ORDER);
+
+ public ValueWithMetadata(SkyValue value) {
+ this.value = value;
+ }
+
+ /** Builds a value entry value that has an error (and no value value).
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+ public static ValueWithMetadata error(ErrorInfo errorInfo,
+ NestedSet<TaggedEvents> transitiveEvents) {
+ return new ErrorInfoValue(errorInfo, null, transitiveEvents);
+ }
+
+ /**
+ * Builds a value entry value that has a value value, and possibly an error (constructed from its
+ * children's errors).
+ *
+ * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations.
+ */
+ static SkyValue normal(@Nullable SkyValue value, @Nullable ErrorInfo errorInfo,
+ NestedSet<TaggedEvents> transitiveEvents) {
+ Preconditions.checkState(value != null || errorInfo != null,
+ "Value and error cannot both be null");
+ if (errorInfo == null) {
+ return transitiveEvents.isEmpty()
+ ? value
+ : new ValueWithEvents(value, transitiveEvents);
+ }
+ return new ErrorInfoValue(errorInfo, value, transitiveEvents);
+ }
+
+
+ @Nullable SkyValue getValue() {
+ return value;
+ }
+
+ @Nullable
+ abstract ErrorInfo getErrorInfo();
+
+ abstract NestedSet<TaggedEvents> getTransitiveEvents();
+
+ static final class ValueWithEvents extends ValueWithMetadata {
+
+ private final NestedSet<TaggedEvents> transitiveEvents;
+
+ ValueWithEvents(SkyValue value, NestedSet<TaggedEvents> transitiveEvents) {
+ super(Preconditions.checkNotNull(value));
+ this.transitiveEvents = Preconditions.checkNotNull(transitiveEvents);
+ }
+
+ @Nullable
+ @Override
+ ErrorInfo getErrorInfo() { return null; }
+
+ @Override
+ NestedSet<TaggedEvents> getTransitiveEvents() { return transitiveEvents; }
+
+ /**
+ * We override equals so that if the same value is written to a {@link NodeEntry} twice, it can
+ * verify that the two values are equal, and avoid incrementing its version.
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ ValueWithEvents that = (ValueWithEvents) o;
+
+ // Shallow equals is a middle ground between using default equals, which might miss
+ // nested sets with the same elements, and deep equality checking, which would be expensive.
+ // All three choices are sound, since shallow equals and default equals are more
+ // conservative than deep equals. Using shallow equals means that we may unnecessarily
+ // consider some values unequal that are actually equal, but this is still a net win over
+ // deep equals.
+ return value.equals(that.value) && transitiveEvents.shallowEquals(that.transitiveEvents);
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * value.hashCode() + transitiveEvents.hashCode();
+ }
+
+ @Override
+ public String toString() { return value.toString(); }
+ }
+
+ static final class ErrorInfoValue extends ValueWithMetadata {
+
+ private final ErrorInfo errorInfo;
+ private final NestedSet<TaggedEvents> transitiveEvents;
+
+ ErrorInfoValue(ErrorInfo errorInfo, @Nullable SkyValue value,
+ NestedSet<TaggedEvents> transitiveEvents) {
+ super(value);
+ this.errorInfo = Preconditions.checkNotNull(errorInfo);
+ this.transitiveEvents = Preconditions.checkNotNull(transitiveEvents);
+ }
+
+ @Nullable
+ @Override
+ ErrorInfo getErrorInfo() { return errorInfo; }
+
+ @Override
+ NestedSet<TaggedEvents> getTransitiveEvents() { return transitiveEvents; }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ ErrorInfoValue that = (ErrorInfoValue) o;
+
+ // Shallow equals is a middle ground between using default equals, which might miss
+ // nested sets with the same elements, and deep equality checking, which would be expensive.
+ // All three choices are sound, since shallow equals and default equals are more
+ // conservative than deep equals. Using shallow equals means that we may unnecessarily
+ // consider some values unequal that are actually equal, but this is still a net win over
+ // deep equals.
+ return Objects.equals(this.value, that.value)
+ && Objects.equals(this.errorInfo, that.errorInfo)
+ && transitiveEvents.shallowEquals(that.transitiveEvents);
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * Objects.hash(value, errorInfo) + transitiveEvents.shallowHashCode();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ if (value != null) {
+ result.append("Value: ").append(value);
+ }
+ if (errorInfo != null) {
+ if (result.length() > 0) {
+ result.append("; ");
+ }
+ result.append("Error: ").append(errorInfo);
+ }
+ return result.toString();
+ }
+ }
+
+ static SkyValue justValue(SkyValue value) {
+ if (value instanceof ValueWithMetadata) {
+ return ((ValueWithMetadata) value).getValue();
+ }
+ return value;
+ }
+
+ static ValueWithMetadata wrapWithMetadata(SkyValue value) {
+ if (value instanceof ValueWithMetadata) {
+ return (ValueWithMetadata) value;
+ }
+ return new ValueWithEvents(value, NO_EVENTS);
+ }
+
+ @Nullable
+ public static ErrorInfo getMaybeErrorInfo(SkyValue value) {
+ if (value.getClass() == ErrorInfoValue.class) {
+ return ((ValueWithMetadata) value).getErrorInfo();
+ }
+ return null;
+
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/skyframe/Version.java b/src/main/java/com/google/devtools/build/skyframe/Version.java
new file mode 100644
index 0000000..90a6020
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/skyframe/Version.java
@@ -0,0 +1,32 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * A Version defines a value in a version tree used in persistent data structures.
+ * See http://en.wikipedia.org/wiki/Persistent_data_structure.
+ */
+public interface Version {
+ /**
+ * Defines a partial order relation on versions. Returns true if this object is at most
+ * {@code other} in that partial order. If x.equals(y), then x.atMost(y).
+ *
+ * <p>If x.atMost(y) returns false, then there are two possibilities: y < x in the partial order,
+ * so y.atMost(x) returns true and !x.equals(y), or x and y are incomparable in this partial
+ * order. This may be because x and y are instances of different Version implementations (although
+ * it is legal for different Version implementations to be comparable as well).
+ * See http://en.wikipedia.org/wiki/Partially_ordered_set.
+ */
+ boolean atMost(Version other);
+}
diff --git a/src/main/java/com/google/devtools/common/options/Converter.java b/src/main/java/com/google/devtools/common/options/Converter.java
new file mode 100644
index 0000000..867ef82
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/Converter.java
@@ -0,0 +1,33 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+/**
+ * A converter is a little helper object that can take a String and
+ * turn it into an instance of type T (the type parameter to the converter).
+ */
+public interface Converter<T> {
+
+ /**
+ * Convert a string into type T.
+ */
+ T convert(String input) throws OptionsParsingException;
+
+ /**
+ * The type description appears in usage messages. E.g.: "a string",
+ * "a path", etc.
+ */
+ String getTypeDescription();
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/Converters.java b/src/main/java/com/google/devtools/common/options/Converters.java
new file mode 100644
index 0000000..e8c69ec
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/Converters.java
@@ -0,0 +1,326 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Some convenient converters used by blaze. Note: These are specific to
+ * blaze.
+ */
+public final class Converters {
+
+ /**
+ * Join a list of words as in English. Examples:
+ * "nothing"
+ * "one"
+ * "one or two"
+ * "one and two"
+ * "one, two or three".
+ * "one, two and three".
+ * The toString method of each element is used.
+ */
+ static String joinEnglishList(Iterable<?> choices) {
+ StringBuilder buf = new StringBuilder();
+ for (Iterator<?> ii = choices.iterator(); ii.hasNext(); ) {
+ Object choice = ii.next();
+ if (buf.length() > 0) {
+ buf.append(ii.hasNext() ? ", " : " or ");
+ }
+ buf.append(choice);
+ }
+ return buf.length() == 0 ? "nothing" : buf.toString();
+ }
+
+ public static class SeparatedOptionListConverter
+ implements Converter<List<String>> {
+
+ private final String separatorDescription;
+ private final Splitter splitter;
+
+ protected SeparatedOptionListConverter(char separator,
+ String separatorDescription) {
+ this.separatorDescription = separatorDescription;
+ this.splitter = Splitter.on(separator);
+ }
+
+ @Override
+ public List<String> convert(String input) {
+ return input.equals("")
+ ? ImmutableList.<String>of()
+ : ImmutableList.copyOf(splitter.split(input));
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return separatorDescription + "-separated list of options";
+ }
+ }
+
+ public static class CommaSeparatedOptionListConverter
+ extends SeparatedOptionListConverter {
+ public CommaSeparatedOptionListConverter() {
+ super(',', "comma");
+ }
+ }
+
+ public static class ColonSeparatedOptionListConverter extends SeparatedOptionListConverter {
+ public ColonSeparatedOptionListConverter() {
+ super(':', "colon");
+ }
+ }
+
+ public static class LogLevelConverter implements Converter<Level> {
+
+ public static Level[] LEVELS = new Level[] {
+ Level.OFF, Level.SEVERE, Level.WARNING, Level.INFO, Level.FINE,
+ Level.FINER, Level.FINEST
+ };
+
+ @Override
+ public Level convert(String input) throws OptionsParsingException {
+ try {
+ int level = Integer.parseInt(input);
+ return LEVELS[level];
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("Not a log level: " + input);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new OptionsParsingException("Not a log level: " + input);
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "0 <= an integer <= " + (LEVELS.length - 1);
+ }
+
+ }
+
+ /**
+ * Checks whether a string is part of a set of strings.
+ */
+ public static class StringSetConverter implements Converter<String> {
+
+ // TODO(bazel-team): if this class never actually contains duplicates, we could s/List/Set/
+ // here.
+ private final List<String> values;
+
+ public StringSetConverter(String... values) {
+ this.values = ImmutableList.copyOf(values);
+ }
+
+ @Override
+ public String convert(String input) throws OptionsParsingException {
+ if (values.contains(input)) {
+ return input;
+ }
+
+ throw new OptionsParsingException("Not one of " + values);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return joinEnglishList(values);
+ }
+ }
+
+ /**
+ * Checks whether a string is a valid regex pattern and compiles it.
+ */
+ public static class RegexPatternConverter implements Converter<Pattern> {
+
+ @Override
+ public Pattern convert(String input) throws OptionsParsingException {
+ try {
+ return Pattern.compile(input);
+ } catch (PatternSyntaxException e) {
+ throw new OptionsParsingException("Not a valid regular expression: " + e.getMessage());
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a valid Java regular expression";
+ }
+ }
+
+ /**
+ * Limits the length of a string argument.
+ */
+ public static class LengthLimitingConverter implements Converter<String> {
+ private final int maxSize;
+
+ public LengthLimitingConverter(int maxSize) {
+ this.maxSize = maxSize;
+ }
+
+ @Override
+ public String convert(String input) throws OptionsParsingException {
+ if (input.length() > maxSize) {
+ throw new OptionsParsingException("Input must be " + getTypeDescription());
+ }
+ return input;
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a string <= " + maxSize + " characters";
+ }
+ }
+
+ /**
+ * Checks whether an integer is in the given range.
+ */
+ public static class RangeConverter implements Converter<Integer> {
+ final int minValue;
+ final int maxValue;
+
+ public RangeConverter(int minValue, int maxValue) {
+ this.minValue = minValue;
+ this.maxValue = maxValue;
+ }
+
+ @Override
+ public Integer convert(String input) throws OptionsParsingException {
+ try {
+ Integer value = Integer.parseInt(input);
+ if (value < minValue) {
+ throw new OptionsParsingException("'" + input + "' should be >= " + minValue);
+ } else if (value < minValue || value > maxValue) {
+ throw new OptionsParsingException("'" + input + "' should be <= " + maxValue);
+ }
+ return value;
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("'" + input + "' is not an int");
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ if (minValue == Integer.MIN_VALUE) {
+ if (maxValue == Integer.MAX_VALUE) {
+ return "an integer";
+ } else {
+ return "an integer, <= " + maxValue;
+ }
+ } else if (maxValue == Integer.MAX_VALUE) {
+ return "an integer, >= " + minValue;
+ } else {
+ return "an integer in "
+ + (minValue < 0 ? "(" + minValue + ")" : minValue) + "-" + maxValue + " range";
+ }
+ }
+ }
+
+ /**
+ * A converter for variable assignments from the parameter list of a blaze
+ * command invocation. Assignments are expected to have the form "name=value",
+ * where names and values are defined to be as permissive as possible.
+ */
+ public static class AssignmentConverter implements Converter<Map.Entry<String, String>> {
+
+ @Override
+ public Map.Entry<String, String> convert(String input)
+ throws OptionsParsingException {
+ int pos = input.indexOf("=");
+ if (pos <= 0) {
+ throw new OptionsParsingException("Variable definitions must be in the form of a "
+ + "'name=value' assignment");
+ }
+ String name = input.substring(0, pos);
+ String value = input.substring(pos + 1);
+ return Maps.immutableEntry(name, value);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a 'name=value' assignment";
+ }
+
+ }
+
+ /**
+ * A converter for variable assignments from the parameter list of a blaze
+ * command invocation. Assignments are expected to have the form "name[=value]",
+ * where names and values are defined to be as permissive as possible and value
+ * part can be optional (in which case it is considered to be null).
+ */
+ public static class OptionalAssignmentConverter implements Converter<Map.Entry<String, String>> {
+
+ @Override
+ public Map.Entry<String, String> convert(String input)
+ throws OptionsParsingException {
+ int pos = input.indexOf("=");
+ if (pos == 0 || input.length() == 0) {
+ throw new OptionsParsingException("Variable definitions must be in the form of a "
+ + "'name=value' or 'name' assignment");
+ } else if (pos < 0) {
+ return Maps.immutableEntry(input, null);
+ }
+ String name = input.substring(0, pos);
+ String value = input.substring(pos + 1);
+ return Maps.immutableEntry(name, value);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a 'name=value' assignment with an optional value part";
+ }
+
+ }
+
+ public static class HelpVerbosityConverter extends EnumConverter<OptionsParser.HelpVerbosity> {
+ public HelpVerbosityConverter() {
+ super(OptionsParser.HelpVerbosity.class, "--help_verbosity setting");
+ }
+ }
+
+ /**
+ * A converter for boolean values. This is already one of the defaults, so clients
+ * should not typically need to add this.
+ */
+ public static class BooleanConverter implements Converter<Boolean> {
+ @Override
+ public Boolean convert(String input) throws OptionsParsingException {
+ if (input == null) {
+ return false;
+ }
+ input = input.toLowerCase();
+ if (input.equals("true") || input.equals("1") || input.equals("yes") ||
+ input.equals("t") || input.equals("y")) {
+ return true;
+ }
+ if (input.equals("false") || input.equals("0") || input.equals("no") ||
+ input.equals("f") || input.equals("n")) {
+ return false;
+ }
+ throw new OptionsParsingException("'" + input + "' is not a boolean");
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a boolean";
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/DuplicateOptionDeclarationException.java b/src/main/java/com/google/devtools/common/options/DuplicateOptionDeclarationException.java
new file mode 100644
index 0000000..b4e572e
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/DuplicateOptionDeclarationException.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+/**
+ * Indicates that an option is declared in more than one class.
+ */
+public class DuplicateOptionDeclarationException extends RuntimeException {
+
+ DuplicateOptionDeclarationException(String message) {
+ super(message);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/EnumConverter.java b/src/main/java/com/google/devtools/common/options/EnumConverter.java
new file mode 100644
index 0000000..f65241a
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/EnumConverter.java
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import java.util.Arrays;
+
+/**
+ * A converter superclass for converters that parse enums.
+ *
+ * Just subclass this class, creating a zero aro argument constructor that
+ * calls {@link #EnumConverter(Class, String)}.
+ *
+ * This class compares the input string to the string returned by the toString()
+ * method of each enum member in a case-insensitive way. Usually, this is the
+ * name of the symbol, but beware if you override toString()!
+ */
+public abstract class EnumConverter<T extends Enum<T>>
+ implements Converter<T> {
+
+ private final Class<T> enumType;
+ private final String typeName;
+
+ /**
+ * Creates a new enum converter. You *must* implement a zero-argument
+ * constructor that delegates to this constructor, passing in the appropriate
+ * parameters.
+ *
+ * @param enumType The type of your enumeration; usually a class literal
+ * like MyEnum.class
+ * @param typeName The intuitive name of your enumeration, for example, the
+ * type name for CompilationMode might be "compilation mode".
+ */
+ protected EnumConverter(Class<T> enumType, String typeName) {
+ this.enumType = enumType;
+ this.typeName = typeName;
+ }
+
+ /**
+ * Implements {@link #convert(String)}.
+ */
+ @Override
+ public final T convert(String input) throws OptionsParsingException {
+ for (T value : enumType.getEnumConstants()) {
+ if (value.toString().equalsIgnoreCase(input)) {
+ return value;
+ }
+ }
+ throw new OptionsParsingException("Not a valid " + typeName + ": '"
+ + input + "' (should be "
+ + getTypeDescription() + ")");
+ }
+
+ /**
+ * Implements {@link #getTypeDescription()}.
+ */
+ @Override
+ public final String getTypeDescription() {
+ return Converters.joinEnglishList(
+ Arrays.asList(enumType.getEnumConstants())).toLowerCase();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/GenericTypeHelper.java b/src/main/java/com/google/devtools/common/options/GenericTypeHelper.java
new file mode 100644
index 0000000..2240860
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/GenericTypeHelper.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Primitives;
+import com.google.common.reflect.TypeToken;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.lang.reflect.TypeVariable;
+
+/**
+ * A helper class for {@link OptionsParserImpl} to help checking the return type
+ * of a {@link Converter} against the type of a field or the element type of a
+ * list.
+ *
+ * <p>This class has to go through considerable contortion to get the correct result
+ * from the Java reflection system, unfortunately. If the generic reflection part
+ * had been better designed, some of this would not be necessary.
+ */
+class GenericTypeHelper {
+
+ /**
+ * Returns the raw type of t, if t is either a raw or parameterized type.
+ * Otherwise, this method throws an {@link AssertionError}.
+ */
+ @VisibleForTesting
+ static Class<?> getRawType(Type t) {
+ if (t instanceof Class<?>) {
+ return (Class<?>) t;
+ } else if (t instanceof ParameterizedType) {
+ return (Class<?>) ((ParameterizedType) t).getRawType();
+ } else {
+ throw new AssertionError("A known concrete type is not concrete");
+ }
+ }
+
+ /**
+ * If type is a parameterized type, searches the given type variable in the list
+ * of declared type variables, and then returns the corresponding actual type.
+ * Returns null if the type variable is not defined by type.
+ */
+ private static Type matchTypeVariable(Type type, TypeVariable<?> variable) {
+ if (type instanceof ParameterizedType) {
+ Class<?> rawInterfaceType = getRawType(type);
+ TypeVariable<?>[] typeParameters = rawInterfaceType.getTypeParameters();
+ for (int i = 0; i < typeParameters.length; i++) {
+ if (variable.equals(typeParameters[i])) {
+ return ((ParameterizedType) type).getActualTypeArguments()[i];
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Resolves the return type of a method, in particular if the generic return
+ * type ({@link Method#getGenericReturnType()}) is a type variable
+ * ({@link TypeVariable}), by checking all super-classes and directly
+ * implemented interfaces.
+ *
+ * <p>The method m must be defined by the given type or by its raw class type.
+ *
+ * @throws AssertionError if the generic return type could not be resolved
+ */
+ // TODO(bazel-team): also check enclosing classes and indirectly implemented
+ // interfaces, which can also contribute type variables. This doesn't happen
+ // in the existing use cases.
+ public static Type getActualReturnType(Type type, Method method) {
+ Type returnType = method.getGenericReturnType();
+ if (returnType instanceof Class<?>) {
+ return returnType;
+ } else if (returnType instanceof ParameterizedType) {
+ return returnType;
+ } else if (returnType instanceof TypeVariable<?>) {
+ TypeVariable<?> variable = (TypeVariable<?>) returnType;
+ while (type != null) {
+ Type candidate = matchTypeVariable(type, variable);
+ if (candidate != null) {
+ return candidate;
+ }
+
+ Class<?> rawType = getRawType(type);
+ for (Type interfaceType : rawType.getGenericInterfaces()) {
+ candidate = matchTypeVariable(interfaceType, variable);
+ if (candidate != null) {
+ return candidate;
+ }
+ }
+
+ type = rawType.getGenericSuperclass();
+ }
+ }
+ throw new AssertionError("The type " + returnType
+ + " is not a Class, ParameterizedType, or TypeVariable");
+ }
+
+ /**
+ * Determines if a value of a particular type (from) is assignable to a field of
+ * a particular type (to). Also allows assigning wrapper types to primitive
+ * types.
+ *
+ * <p>The checks done here should be identical to the checks done by
+ * {@link java.lang.reflect.Field#set}. I.e., if this method returns true, a
+ * subsequent call to {@link java.lang.reflect.Field#set} should succeed.
+ */
+ public static boolean isAssignableFrom(Type to, Type from) {
+ if (to instanceof Class<?>) {
+ Class<?> toClass = (Class<?>) to;
+ if (toClass.isPrimitive()) {
+ return Primitives.wrap(toClass).equals(from);
+ }
+ }
+ return TypeToken.of(to).isAssignableFrom(from);
+ }
+
+ private GenericTypeHelper() {
+ // Prevents Java from creating a public constructor.
+ }
+}
diff --git a/src/main/java/com/google/devtools/common/options/Option.java b/src/main/java/com/google/devtools/common/options/Option.java
new file mode 100644
index 0000000..e244736
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/Option.java
@@ -0,0 +1,127 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An interface for annotating fields in classes (derived from OptionsBase)
+ * that are options.
+ */
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Option {
+
+ /**
+ * The name of the option ("--name").
+ */
+ String name();
+
+ /**
+ * The single-character abbreviation of the option ("-abbrev").
+ */
+ char abbrev() default '\0';
+
+ /**
+ * A help string for the usage information.
+ */
+ String help() default "";
+
+ /**
+ * The default value for the option. This method should only be invoked
+ * directly by the parser implementation. Any access to default values
+ * should go via the parser to allow for application specific defaults.
+ *
+ * <p>There are two reasons this is a string. Firstly, it ensures that
+ * explicitly specifying this option at its default value (as printed in the
+ * usage message) has the same behavior as not specifying the option at all;
+ * this would be very hard to achieve if the default value was an instance of
+ * type T, since we'd need to ensure that {@link #toString()} and {@link
+ * #converter} were dual to each other. The second reason is more mundane
+ * but also more restrictive: annotation values must be compile-time
+ * constants.
+ *
+ * <p>If an option's defaultValue() is the string "null", the option's
+ * converter will not be invoked to interpret it; a null reference will be
+ * used instead. (It would be nice if defaultValue could simply return null,
+ * but bizarrely, the Java Language Specification does not consider null to
+ * be a compile-time constant.) This special interpretation of the string
+ * "null" is only applicable when computing the default value; if specified
+ * on the command-line, this string will have its usual literal meaning.
+ */
+ String defaultValue();
+
+ /**
+ * A string describing the category of options that this belongs to. {@link
+ * OptionsParser#describeOptions} prints options of the same category grouped
+ * together.
+ */
+ String category() default "misc";
+
+ /**
+ * The converter that we'll use to convert this option into an object or
+ * a simple type. The default is to use the builtin converters.
+ * Custom converters must implement the {@link Converter} interface.
+ */
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ // Can't figure out how to coerce Converter.class into Class<? extends Converter<?>>
+ Class<? extends Converter> converter() default Converter.class;
+
+ /**
+ * A flag indicating whether the option type should be allowed to occur
+ * multiple times in a single option list.
+ *
+ * <p>If the command can occur multiple times, then the attribute value
+ * <em>must</em> be a list type {@code List<T>}, and the result type of the
+ * converter for this option must either match the parameter {@code T} or
+ * {@code List<T>}. In the latter case the individual lists are concatenated
+ * to form the full options value.
+ */
+ boolean allowMultiple() default false;
+
+ /**
+ * If the option is actually an abbreviation for other options, this field will
+ * contain the strings to expand this option into. The original option is dropped
+ * and the replacement used in its stead. It is recommended that such an option be
+ * of type {@link Void}.
+ *
+ * An expanded option overrides previously specified options of the same name,
+ * even if it is explicitly specified. This is the original behavior and can
+ * be surprising if the user is not aware of it, which has led to several
+ * requests to change this behavior. This was discussed in the blaze team and
+ * it was decided that it is not a strong enough case to change the behavior.
+ */
+ String[] expansion() default {};
+
+ /**
+ * If the option requires that additional options be implicitly appended, this field
+ * will contain the additional options. Implicit dependencies are parsed at the end
+ * of each {@link OptionsParser#parse} invocation, and override options specified in
+ * the same call. However, they can be overridden by options specified in a later
+ * call or by options with a higher priority.
+ *
+ * @see OptionPriority
+ */
+ String[] implicitRequirements() default {};
+
+ /**
+ * If this field is a non-empty string, the option is deprecated, and a
+ * deprecation warning is added to the list of warnings when such an option
+ * is used.
+ */
+ String deprecationWarning() default "";
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionPriority.java b/src/main/java/com/google/devtools/common/options/OptionPriority.java
new file mode 100644
index 0000000..6e90008
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionPriority.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+/**
+ * The priority of option values, in order of increasing priority.
+ *
+ * <p>In general, new values for options can only override values with a lower or
+ * equal priority. Option values provided in annotations in an options class are
+ * implicitly at the priority {@code DEFAULT}.
+ *
+ * <p>The ordering of the priorities is the source-code order. This is consistent
+ * with the automatically generated {@code compareTo} method as specified by the
+ * Java Language Specification. DO NOT change the source-code order of these
+ * values, or you will break code that relies on the ordering.
+ */
+public enum OptionPriority {
+
+ /**
+ * The priority of values specified in the {@link Option} annotation. This
+ * should never be specified in calls to {@link OptionsParser#parse}.
+ */
+ DEFAULT,
+
+ /**
+ * Overrides default options at runtime, while still allowing the values to be
+ * overridden manually.
+ */
+ COMPUTED_DEFAULT,
+
+ /**
+ * For options coming from a configuration file or rc file.
+ */
+ RC_FILE,
+
+ /**
+ * For options coming from the command line.
+ */
+ COMMAND_LINE,
+
+ /**
+ * This priority can be used to unconditionally override any user-provided options.
+ * This should be used rarely and with caution!
+ */
+ SOFTWARE_REQUIREMENT;
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/Options.java b/src/main/java/com/google/devtools/common/options/Options.java
new file mode 100644
index 0000000..171be2e
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/Options.java
@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Interface for parsing options from a single options specification class.
+ *
+ * The {@link Options#parse(Class, String...)} method in this class has no clear
+ * use case. Instead, use the {@link OptionsParser} class directly, as in this
+ * code snippet:
+ *
+ * <pre>
+ * OptionsParser parser = OptionsParser.newOptionsParser(FooOptions.class);
+ * try {
+ * parser.parse(FooOptions.class, args);
+ * } catch (OptionsParsingException e) {
+ * System.err.print("Error parsing options: " + e.getMessage());
+ * System.err.print(options.getUsage());
+ * System.exit(1);
+ * }
+ * FooOptions foo = parser.getOptions(FooOptions.class);
+ * List<String> otherArguments = parser.getResidue();
+ * </pre>
+ *
+ * Using this class in this case actually results in more code.
+ *
+ * @see OptionsParser for parsing options from multiple options specification classes.
+ */
+public class Options<O extends OptionsBase> {
+
+ /**
+ * Parse the options provided in args, given the specification in
+ * optionsClass.
+ */
+ public static <O extends OptionsBase> Options<O> parse(Class<O> optionsClass, String... args)
+ throws OptionsParsingException {
+ OptionsParser parser = OptionsParser.newOptionsParser(optionsClass);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList(args));
+ List<String> remainingArgs = parser.getResidue();
+ return new Options<O>(parser.getOptions(optionsClass),
+ remainingArgs.toArray(new String[0]));
+ }
+
+ /**
+ * Returns an options object at its default values. The returned object may
+ * be freely modified by the caller, by assigning its fields.
+ */
+ public static <O extends OptionsBase> O getDefaults(Class<O> optionsClass) {
+ try {
+ return parse(optionsClass, new String[0]).getOptions();
+ } catch (OptionsParsingException e) {
+ String message = "Error while parsing defaults: " + e.getMessage();
+ throw new AssertionError(message);
+ }
+ }
+
+ /**
+ * Returns a usage string (renders the help information, the defaults, and
+ * of course the option names).
+ */
+ public static String getUsage(Class<? extends OptionsBase> optionsClass) {
+ StringBuilder usage = new StringBuilder();
+ OptionsUsage.getUsage(optionsClass, usage);
+ return usage.toString();
+ }
+
+ private O options;
+ private String[] remainingArgs;
+
+ private Options(O options, String[] remainingArgs) {
+ this.options = options;
+ this.remainingArgs = remainingArgs;
+ }
+
+ /**
+ * Returns an instance of options class O.
+ */
+ public O getOptions() {
+ return options;
+ }
+
+ /**
+ * Returns the arguments that we didn't parse.
+ */
+ public String[] getRemainingArgs() {
+ return remainingArgs;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsBase.java b/src/main/java/com/google/devtools/common/options/OptionsBase.java
new file mode 100644
index 0000000..ed9f215
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsBase.java
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import com.google.common.escape.CharEscaperBuilder;
+import com.google.common.escape.Escaper;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Base class for all options classes. Extend this class, adding public
+ * instance fields annotated with @Option. Then you can create instances
+ * either programmatically:
+ *
+ * <pre>
+ * X x = Options.getDefaults(X.class);
+ * x.host = "localhost";
+ * x.port = 80;
+ * </pre>
+ *
+ * or from an array of command-line arguments:
+ *
+ * <pre>
+ * OptionsParser parser = OptionsParser.newOptionsParser(X.class);
+ * parser.parse("--host", "localhost", "--port", "80");
+ * X x = parser.getOptions(X.class);
+ * </pre>
+ *
+ * <p>Subclasses of OptionsBase <b>must</b> be constructed reflectively,
+ * i.e. using not {@code new MyOptions}, but one of the two methods above
+ * instead. (Direct construction creates an empty instance, not containing
+ * default values. This leads to surprising behavior and often
+ * NullPointerExceptions, etc.)
+ */
+public abstract class OptionsBase {
+
+ private static final Escaper ESCAPER = new CharEscaperBuilder()
+ .addEscape('\\', "\\\\").addEscape('"', "\\\"").toEscaper();
+
+ /**
+ * Subclasses must provide a default (no argument) constructor.
+ */
+ protected OptionsBase() {
+ // There used to be a sanity check here that checks the stack trace of this constructor
+ // invocation; unfortunately, that makes the options construction about 10x slower. So be
+ // careful with how you construct options classes.
+ }
+
+ /**
+ * Returns this options object in the form of a (new) mapping from option
+ * names, including inherited ones, to option values. If the public fields
+ * are mutated, this will be reflected in subsequent calls to {@code asMap}.
+ * Mutation of this map by the caller does not affect this options object.
+ */
+ public final Map<String, Object> asMap() {
+ return OptionsParserImpl.optionsAsMap(this);
+ }
+
+ @Override
+ public final String toString() {
+ return getClass().getName() + asMap();
+ }
+
+ /**
+ * Returns a string that uniquely identifies the options. This value is
+ * intended for analysis caching.
+ */
+ public final String cacheKey() {
+ StringBuilder result = new StringBuilder(getClass().getName()).append("{");
+
+ for (Entry<String, Object> entry : asMap().entrySet()) {
+ result.append(entry.getKey()).append("=");
+
+ Object value = entry.getValue();
+ // This special case is needed because List.toString() prints the same
+ // ("[]") for an empty list and for a list with a single empty string.
+ if (value instanceof List<?> && ((List<?>) value).isEmpty()) {
+ result.append("EMPTY");
+ } else if (value == null) {
+ result.append("NULL");
+ } else {
+ result
+ .append('"')
+ .append(ESCAPER.escape(value.toString()))
+ .append('"');
+ }
+ result.append(", ");
+ }
+
+ return result.append("}").toString();
+ }
+
+ @Override
+ public final boolean equals(Object that) {
+ return that != null &&
+ this.getClass() == that.getClass() &&
+ this.asMap().equals(((OptionsBase) that).asMap());
+ }
+
+ @Override
+ public final int hashCode() {
+ return this.getClass().hashCode() + asMap().hashCode();
+ }
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsClassProvider.java b/src/main/java/com/google/devtools/common/options/OptionsClassProvider.java
new file mode 100644
index 0000000..1868e23
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsClassProvider.java
@@ -0,0 +1,29 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+/**
+ * A read-only interface for options parser results, which only allows to query the options of
+ * a specific class, but not e.g. the residue any other information pertaining to the command line.
+ */
+public interface OptionsClassProvider {
+ /**
+ * Returns the options instance for the given {@code optionsClass}, that is,
+ * the parsed options, or null if it is not among those available.
+ *
+ * <p>The returned options should be treated by library code as immutable and
+ * a provider is permitted to return the same options instance multiple times.
+ */
+ <O extends OptionsBase> O getOptions(Class<O> optionsClass);
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsData.java b/src/main/java/com/google/devtools/common/options/OptionsData.java
new file mode 100644
index 0000000..e9b6574
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsData.java
@@ -0,0 +1,264 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.concurrent.Immutable;
+
+/**
+ * An immutable selection of options data corresponding to a set of options
+ * classes. The data is collected using reflection, which can be expensive.
+ * Therefore this class can be used internally to cache the results.
+ */
+@Immutable
+final class OptionsData {
+
+ /**
+ * These are the options-declaring classes which are annotated with
+ * {@link Option} annotations.
+ */
+ private final Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses;
+
+ /** Maps option name to Option-annotated Field. */
+ private final Map<String, Field> nameToField;
+
+ /** Maps option abbreviation to Option-annotated Field. */
+ private final Map<Character, Field> abbrevToField;
+
+ /**
+ * For each options class, contains a list of all Option-annotated fields in
+ * that class.
+ */
+ private final Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields;
+
+ /**
+ * Mapping from each Option-annotated field to the default value for that
+ * field.
+ */
+ private final Map<Field, Object> optionDefaults;
+
+ /**
+ * Mapping from each Option-annotated field to the proper converter.
+ *
+ * @see OptionsParserImpl#findConverter
+ */
+ private final Map<Field, Converter<?>> converters;
+
+ private OptionsData(Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses,
+ Map<String, Field> nameToField,
+ Map<Character, Field> abbrevToField,
+ Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields,
+ Map<Field, Object> optionDefaults,
+ Map<Field, Converter<?>> converters) {
+ this.optionsClasses = ImmutableMap.copyOf(optionsClasses);
+ this.allOptionsFields = ImmutableMap.copyOf(allOptionsFields);
+ this.nameToField = ImmutableMap.copyOf(nameToField);
+ this.abbrevToField = ImmutableMap.copyOf(abbrevToField);
+ // Can't use an ImmutableMap here because of null values.
+ this.optionDefaults = Collections.unmodifiableMap(optionDefaults);
+ this.converters = ImmutableMap.copyOf(converters);
+ }
+
+ public Collection<Class<? extends OptionsBase>> getOptionsClasses() {
+ return optionsClasses.keySet();
+ }
+
+ @SuppressWarnings("unchecked") // The construction ensures that the case is always valid.
+ public <T extends OptionsBase> Constructor<T> getConstructor(Class<T> clazz) {
+ return (Constructor<T>) optionsClasses.get(clazz);
+ }
+
+ public Field getFieldFromName(String name) {
+ return nameToField.get(name);
+ }
+
+ public Iterable<Map.Entry<String, Field>> getAllNamedFields() {
+ return nameToField.entrySet();
+ }
+
+ public Field getFieldForAbbrev(char abbrev) {
+ return abbrevToField.get(abbrev);
+ }
+
+ public List<Field> getFieldsForClass(Class<? extends OptionsBase> optionsClass) {
+ return allOptionsFields.get(optionsClass);
+ }
+
+ public Object getDefaultValue(Field field) {
+ return optionDefaults.get(field);
+ }
+
+ public Converter<?> getConverter(Field field) {
+ return converters.get(field);
+ }
+
+ private static List<Field> getAllAnnotatedFields(Class<? extends OptionsBase> optionsClass) {
+ List<Field> allFields = Lists.newArrayList();
+ for (Field field : optionsClass.getFields()) {
+ if (field.isAnnotationPresent(Option.class)) {
+ allFields.add(field);
+ }
+ }
+ if (allFields.isEmpty()) {
+ throw new IllegalStateException(optionsClass + " has no public @Option-annotated fields");
+ }
+ return ImmutableList.copyOf(allFields);
+ }
+
+ private static Object retrieveDefaultFromAnnotation(Field optionField) {
+ Option annotation = optionField.getAnnotation(Option.class);
+ // If an option can be specified multiple times, its default value is a new empty list.
+ if (annotation.allowMultiple()) {
+ return Collections.emptyList();
+ }
+ String defaultValueString = OptionsParserImpl.getDefaultOptionString(optionField);
+ try {
+ return OptionsParserImpl.isSpecialNullDefault(defaultValueString, optionField)
+ ? null
+ : OptionsParserImpl.findConverter(optionField).convert(defaultValueString);
+ } catch (OptionsParsingException e) {
+ throw new IllegalStateException("OptionsParsingException while "
+ + "retrieving default for " + optionField.getName() + ": "
+ + e.getMessage());
+ }
+ }
+
+ static OptionsData of(Collection<Class<? extends OptionsBase>> classes) {
+ Map<Class<? extends OptionsBase>, Constructor<?>> constructorBuilder = Maps.newHashMap();
+ Map<Class<? extends OptionsBase>, List<Field>> allOptionsFieldsBuilder = Maps.newHashMap();
+ Map<String, Field> nameToFieldBuilder = Maps.newHashMap();
+ Map<Character, Field> abbrevToFieldBuilder = Maps.newHashMap();
+ Map<Field, Object> optionDefaultsBuilder = Maps.newHashMap();
+ Map<Field, Converter<?>> convertersBuilder = Maps.newHashMap();
+
+ // Read all Option annotations:
+ for (Class<? extends OptionsBase> parsedOptionsClass : classes) {
+ try {
+ Constructor<? extends OptionsBase> constructor =
+ parsedOptionsClass.getConstructor(new Class[0]);
+ constructorBuilder.put(parsedOptionsClass, constructor);
+ } catch (NoSuchMethodException e) {
+ throw new IllegalArgumentException(parsedOptionsClass
+ + " lacks an accessible default constructor");
+ }
+ List<Field> fields = getAllAnnotatedFields(parsedOptionsClass);
+ allOptionsFieldsBuilder.put(parsedOptionsClass, fields);
+
+ for (Field field : fields) {
+ Option annotation = field.getAnnotation(Option.class);
+
+ // Check that the field type is a List, and that the converter
+ // type matches the element type of the list.
+ Type fieldType = field.getGenericType();
+ if (annotation.allowMultiple()) {
+ if (!(fieldType instanceof ParameterizedType)) {
+ throw new AssertionError("Type of multiple occurrence option must be a List<...>");
+ }
+ ParameterizedType pfieldType = (ParameterizedType) fieldType;
+ if (pfieldType.getRawType() != List.class) {
+ // Throw an assertion, because this indicates an undetected type
+ // error in the code.
+ throw new AssertionError("Type of multiple occurrence option must be a List<...>");
+ }
+ fieldType = pfieldType.getActualTypeArguments()[0];
+ }
+
+ // Get the converter return type.
+ @SuppressWarnings("rawtypes")
+ Class<? extends Converter> converter = annotation.converter();
+ if (converter == Converter.class) {
+ Converter<?> actualConverter = OptionsParserImpl.DEFAULT_CONVERTERS.get(fieldType);
+ if (actualConverter == null) {
+ throw new AssertionError("Cannot find converter for field of type "
+ + field.getType() + " named " + field.getName()
+ + " in class " + field.getDeclaringClass().getName());
+ }
+ converter = actualConverter.getClass();
+ }
+ if (Modifier.isAbstract(converter.getModifiers())) {
+ throw new AssertionError("The converter type (" + converter
+ + ") must be a concrete type");
+ }
+ Type converterResultType;
+ try {
+ Method convertMethod = converter.getMethod("convert", String.class);
+ converterResultType = GenericTypeHelper.getActualReturnType(converter, convertMethod);
+ } catch (NoSuchMethodException e) {
+ throw new AssertionError("A known converter object doesn't implement the convert"
+ + " method");
+ }
+
+ if (annotation.allowMultiple()) {
+ if (GenericTypeHelper.getRawType(converterResultType) == List.class) {
+ Type elementType =
+ ((ParameterizedType) converterResultType).getActualTypeArguments()[0];
+ if (!GenericTypeHelper.isAssignableFrom(fieldType, elementType)) {
+ throw new AssertionError("If the converter return type of a multiple occurance " +
+ "option is a list, then the type of list elements (" + fieldType + ") must be " +
+ "assignable from the converter list element type (" + elementType + ")");
+ }
+ } else {
+ if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) {
+ throw new AssertionError("Type of list elements (" + fieldType +
+ ") for multiple occurrence option must be assignable from the converter " +
+ "return type (" + converterResultType + ")");
+ }
+ }
+ } else {
+ if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) {
+ throw new AssertionError("Type of field (" + fieldType +
+ ") must be assignable from the converter " +
+ "return type (" + converterResultType + ")");
+ }
+ }
+
+ if (annotation.name() == null) {
+ throw new AssertionError(
+ "Option cannot have a null name");
+ }
+ if (nameToFieldBuilder.put(annotation.name(), field) != null) {
+ throw new DuplicateOptionDeclarationException(
+ "Duplicate option name: --" + annotation.name());
+ }
+ if (annotation.abbrev() != '\0') {
+ if (abbrevToFieldBuilder.put(annotation.abbrev(), field) != null) {
+ throw new DuplicateOptionDeclarationException(
+ "Duplicate option abbrev: -" + annotation.abbrev());
+ }
+ }
+ optionDefaultsBuilder.put(field, retrieveDefaultFromAnnotation(field));
+
+ convertersBuilder.put(field, OptionsParserImpl.findConverter(field));
+ }
+ }
+ return new OptionsData(constructorBuilder, nameToFieldBuilder, abbrevToFieldBuilder,
+ allOptionsFieldsBuilder, optionDefaultsBuilder, convertersBuilder);
+ }
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParser.java b/src/main/java/com/google/devtools/common/options/OptionsParser.java
new file mode 100644
index 0000000..9564daa
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsParser.java
@@ -0,0 +1,526 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A parser for options. Typical use case in a main method:
+ *
+ * <pre>
+ * OptionsParser parser = OptionsParser.newOptionsParser(FooOptions.class, BarOptions.class);
+ * parser.parseAndExitUponError(args);
+ * FooOptions foo = parser.getOptions(FooOptions.class);
+ * BarOptions bar = parser.getOptions(BarOptions.class);
+ * List<String> otherArguments = parser.getResidue();
+ * </pre>
+ *
+ * <p>FooOptions and BarOptions would be options specification classes, derived
+ * from OptionsBase, that contain fields annotated with @Option(...).
+ *
+ * <p>Alternatively, rather than calling
+ * {@link #parseAndExitUponError(OptionPriority, String, String[])},
+ * client code may call {@link #parse(OptionPriority,String,List)}, and handle
+ * parser exceptions usage messages themselves.
+ *
+ * <p>This options parsing implementation has (at least) one design flaw. It
+ * allows both '--foo=baz' and '--foo baz' for all options except void, boolean
+ * and tristate options. For these, the 'baz' in '--foo baz' is not treated as
+ * a parameter to the option, making it is impossible to switch options between
+ * void/boolean/tristate and everything else without breaking backwards
+ * compatibility.
+ *
+ * @see Options a simpler class which you can use if you only have one options
+ * specification class
+ */
+public class OptionsParser implements OptionsProvider {
+
+ /**
+ * A cache for the parsed options data. Both keys and values are immutable, so
+ * this is always safe. Only access this field through the {@link
+ * #getOptionsData} method for thread-safety! The cache is very unlikely to
+ * grow to a significant amount of memory, because there's only a fixed set of
+ * options classes on the classpath.
+ */
+ private static final Map<ImmutableList<Class<? extends OptionsBase>>, OptionsData> optionsData =
+ Maps.newHashMap();
+
+ private static synchronized OptionsData getOptionsData(
+ ImmutableList<Class<? extends OptionsBase>> optionsClasses) {
+ OptionsData result = optionsData.get(optionsClasses);
+ if (result == null) {
+ result = OptionsData.of(optionsClasses);
+ optionsData.put(optionsClasses, result);
+ }
+ return result;
+ }
+
+ /**
+ * Returns all the annotated fields for the given class, including inherited
+ * ones.
+ */
+ static Collection<Field> getAllAnnotatedFields(Class<? extends OptionsBase> optionsClass) {
+ OptionsData data = getOptionsData(ImmutableList.<Class<? extends OptionsBase>>of(optionsClass));
+ return data.getFieldsForClass(optionsClass);
+ }
+
+ /**
+ * @see #newOptionsParser(Iterable)
+ */
+ public static OptionsParser newOptionsParser(Class<? extends OptionsBase> class1) {
+ return newOptionsParser(ImmutableList.<Class<? extends OptionsBase>>of(class1));
+ }
+
+ /**
+ * @see #newOptionsParser(Iterable)
+ */
+ public static OptionsParser newOptionsParser(Class<? extends OptionsBase> class1,
+ Class<? extends OptionsBase> class2) {
+ return newOptionsParser(ImmutableList.of(class1, class2));
+ }
+
+ /**
+ * Create a new {@link OptionsParser}.
+ */
+ public static OptionsParser newOptionsParser(
+ Iterable<Class<? extends OptionsBase>> optionsClasses) {
+ return new OptionsParser(getOptionsData(ImmutableList.copyOf(optionsClasses)));
+ }
+
+ /**
+ * Canonicalizes a list of options using the given option classes. The
+ * contract is that if the returned set of options is passed to an options
+ * parser with the same options classes, then that will have the same effect
+ * as using the original args (which are passed in here), except for cosmetic
+ * differences.
+ */
+ public static List<String> canonicalize(
+ Collection<Class<? extends OptionsBase>> optionsClasses, List<String> args)
+ throws OptionsParsingException {
+ OptionsParser parser = new OptionsParser(optionsClasses);
+ parser.setAllowResidue(false);
+ parser.parse(args);
+ return parser.impl.asCanonicalizedList();
+ }
+
+ private final OptionsParserImpl impl;
+ private final List<String> residue = new ArrayList<String>();
+ private boolean allowResidue = true;
+
+ OptionsParser(Collection<Class<? extends OptionsBase>> optionsClasses) {
+ this(OptionsData.of(optionsClasses));
+ }
+
+ OptionsParser(OptionsData optionsData) {
+ impl = new OptionsParserImpl(optionsData);
+ }
+
+ /**
+ * Indicates whether or not the parser will allow a non-empty residue; that
+ * is, iff this value is true then a call to one of the {@code parse}
+ * methods will throw {@link OptionsParsingException} unless
+ * {@link #getResidue()} is empty after parsing.
+ */
+ public void setAllowResidue(boolean allowResidue) {
+ this.allowResidue = allowResidue;
+ }
+
+ /**
+ * Indicates whether or not the parser will allow long options with a
+ * single-dash, instead of the usual double-dash, too, eg. -example instead of just --example.
+ */
+ public void setAllowSingleDashLongOptions(boolean allowSingleDashLongOptions) {
+ this.impl.setAllowSingleDashLongOptions(allowSingleDashLongOptions);
+ }
+
+ public void parseAndExitUponError(String[] args) {
+ parseAndExitUponError(OptionPriority.COMMAND_LINE, "unknown", args);
+ }
+
+ /**
+ * A convenience function for use in main methods. Parses the command line
+ * parameters, and exits upon error. Also, prints out the usage message
+ * if "--help" appears anywhere within {@code args}.
+ */
+ public void parseAndExitUponError(OptionPriority priority, String source, String[] args) {
+ try {
+ parse(priority, source, Arrays.asList(args));
+ } catch (OptionsParsingException e) {
+ System.err.println("Error parsing command line: " + e.getMessage());
+ System.err.println("Try --help.");
+ System.exit(2);
+ }
+ for (String arg : args) {
+ if (arg.equals("--help")) {
+ System.out.println(describeOptions(Collections.<String, String>emptyMap(),
+ HelpVerbosity.LONG));
+ System.exit(0);
+ }
+ }
+ }
+
+ /**
+ * The name and value of an option with additional metadata describing its
+ * priority, source, whether it was set via an implicit dependency, and if so,
+ * by which other option.
+ */
+ public static class OptionValueDescription {
+ private final String name;
+ private final Object value;
+ private final OptionPriority priority;
+ private final String source;
+ private final String implicitDependant;
+ private final String expandedFrom;
+
+ public OptionValueDescription(String name, Object value,
+ OptionPriority priority, String source, String implicitDependant, String expandedFrom) {
+ this.name = name;
+ this.value = value;
+ this.priority = priority;
+ this.source = source;
+ this.implicitDependant = implicitDependant;
+ this.expandedFrom = expandedFrom;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ public OptionPriority getPriority() {
+ return priority;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public String getImplicitDependant() {
+ return implicitDependant;
+ }
+
+ public boolean isImplicitDependency() {
+ return implicitDependant != null;
+ }
+
+ public String getExpansionParent() {
+ return expandedFrom;
+ }
+
+ public boolean isExpansion() {
+ return expandedFrom != null;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ result.append("option '").append(name).append("' ");
+ result.append("set to '").append(value).append("' ");
+ result.append("with priority ").append(priority);
+ if (source != null) {
+ result.append(" and source '").append(source).append("'");
+ }
+ if (implicitDependant != null) {
+ result.append(" implicitly by ");
+ }
+ return result.toString();
+ }
+ }
+
+ /**
+ * The name and unparsed value of an option with additional metadata describing its
+ * priority, source, whether it was set via an implicit dependency, and if so,
+ * by which other option.
+ *
+ * <p>Note that the unparsed value and the source parameters can both be null.
+ */
+ public static class UnparsedOptionValueDescription {
+ private final String name;
+ private final Field field;
+ private final String unparsedValue;
+ private final OptionPriority priority;
+ private final String source;
+ private final boolean explicit;
+
+ public UnparsedOptionValueDescription(String name, Field field, String unparsedValue,
+ OptionPriority priority, String source, boolean explicit) {
+ this.name = name;
+ this.field = field;
+ this.unparsedValue = unparsedValue;
+ this.priority = priority;
+ this.source = source;
+ this.explicit = explicit;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ Field getField() {
+ return field;
+ }
+
+ public boolean isBooleanOption() {
+ return field.getType().equals(boolean.class);
+ }
+
+ private DocumentationLevel documentationLevel() {
+ Option option = field.getAnnotation(Option.class);
+ return OptionsParser.documentationLevel(option.category());
+ }
+
+ public boolean isDocumented() {
+ return documentationLevel() == DocumentationLevel.DOCUMENTED;
+ }
+
+ public boolean isHidden() {
+ return documentationLevel() == DocumentationLevel.HIDDEN;
+ }
+
+ boolean isExpansion() {
+ Option option = field.getAnnotation(Option.class);
+ return option.expansion().length > 0;
+ }
+
+ boolean isImplicitRequirement() {
+ Option option = field.getAnnotation(Option.class);
+ return option.implicitRequirements().length > 0;
+ }
+
+ boolean allowMultiple() {
+ Option option = field.getAnnotation(Option.class);
+ return option.allowMultiple();
+ }
+
+ public String getUnparsedValue() {
+ return unparsedValue;
+ }
+
+ OptionPriority getPriority() {
+ return priority;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public boolean isExplicit() {
+ return explicit;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ result.append("option '").append(name).append("' ");
+ result.append("set to '").append(unparsedValue).append("' ");
+ result.append("with priority ").append(priority);
+ if (source != null) {
+ result.append(" and source '").append(source).append("'");
+ }
+ return result.toString();
+ }
+ }
+
+ /**
+ * The verbosity with which option help messages are displayed: short (just
+ * the name), medium (name, type, default, abbreviation), and long (full
+ * description).
+ */
+ public enum HelpVerbosity { LONG, MEDIUM, SHORT }
+
+ /**
+ * The level of documentation. Only documented options are output as part of
+ * the help.
+ *
+ * <p>We use 'hidden' so that options that form the protocol between the
+ * client and the server are not logged.
+ */
+ enum DocumentationLevel {
+ DOCUMENTED, UNDOCUMENTED, HIDDEN
+ }
+
+ /**
+ * Returns a description of all the options this parser can digest.
+ * In addition to {@link Option} annotations, this method also
+ * interprets {@link OptionsUsage} annotations which give an intuitive short
+ * description for the options.
+ *
+ * @param categoryDescriptions a mapping from category names to category
+ * descriptions. Options of the same category (see {@link
+ * Option#category}) will be grouped together, preceded by the description
+ * of the category.
+ * @param helpVerbosity if {@code long}, the options will be described
+ * verbosely, including their types, defaults and descriptions. If {@code
+ * medium}, the descriptions are omitted, and if {@code short}, the options
+ * are just enumerated.
+ */
+ public String describeOptions(Map<String, String> categoryDescriptions,
+ HelpVerbosity helpVerbosity) {
+ StringBuilder desc = new StringBuilder();
+ if (!impl.getOptionsClasses().isEmpty()) {
+
+ List<Field> allFields = Lists.newArrayList();
+ for (Class<? extends OptionsBase> optionsClass : impl.getOptionsClasses()) {
+ allFields.addAll(impl.getAnnotatedFieldsFor(optionsClass));
+ }
+ Collections.sort(allFields, OptionsUsage.BY_CATEGORY);
+ String prevCategory = null;
+
+ for (Field optionField : allFields) {
+ String category = optionField.getAnnotation(Option.class).category();
+ if (!category.equals(prevCategory)) {
+ prevCategory = category;
+ String description = categoryDescriptions.get(category);
+ if (description == null) {
+ description = "Options category '" + category + "'";
+ }
+ if (documentationLevel(category) == DocumentationLevel.DOCUMENTED) {
+ desc.append("\n").append(description).append(":\n");
+ }
+ }
+
+ if (documentationLevel(prevCategory) == DocumentationLevel.DOCUMENTED) {
+ OptionsUsage.getUsage(optionField, desc, helpVerbosity);
+ }
+ }
+ }
+ return desc.toString().trim();
+ }
+
+ /**
+ * Returns a description of the option value set by the last previous call to
+ * {@link #parse(OptionPriority, String, List)} that successfully set the given
+ * option. If the option is of type {@link List}, the description will
+ * correspond to any one of the calls, but not necessarily the last.
+ */
+ public OptionValueDescription getOptionValueDescription(String name) {
+ return impl.getOptionValueDescription(name);
+ }
+
+ static DocumentationLevel documentationLevel(String category) {
+ if ("undocumented".equals(category)) {
+ return DocumentationLevel.UNDOCUMENTED;
+ } else if ("hidden".equals(category)) {
+ return DocumentationLevel.HIDDEN;
+ } else {
+ return DocumentationLevel.DOCUMENTED;
+ }
+ }
+
+ /**
+ * A convenience method, equivalent to
+ * {@code parse(OptionPriority.COMMAND_LINE, null, Arrays.asList(args))}.
+ */
+ public void parse(String... args) throws OptionsParsingException {
+ parse(OptionPriority.COMMAND_LINE, (String) null, Arrays.asList(args));
+ }
+
+ /**
+ * A convenience method, equivalent to
+ * {@code parse(OptionPriority.COMMAND_LINE, null, args)}.
+ */
+ public void parse(List<String> args) throws OptionsParsingException {
+ parse(OptionPriority.COMMAND_LINE, (String) null, args);
+ }
+
+ /**
+ * Parses {@code args}, using the classes registered with this parser.
+ * {@link #getOptions(Class)} and {@link #getResidue()} return the results.
+ * May be called multiple times; later options override existing ones if they
+ * have equal or higher priority. The source of options is a free-form string
+ * that can be used for debugging. Strings that cannot be parsed as options
+ * accumulates as residue, if this parser allows it.
+ *
+ * @see OptionPriority
+ */
+ public void parse(OptionPriority priority, String source,
+ List<String> args) throws OptionsParsingException {
+ parseWithSourceFunction(priority, Functions.constant(source), args);
+ }
+
+ /**
+ * Parses {@code args}, using the classes registered with this parser.
+ * {@link #getOptions(Class)} and {@link #getResidue()} return the results. May be called
+ * multiple times; later options override existing ones if they have equal or higher priority.
+ * The source of options is given as a function that maps option names to the source of the
+ * option. Strings that cannot be parsed as options accumulates as* residue, if this parser
+ * allows it.
+ */
+ public void parseWithSourceFunction(OptionPriority priority,
+ Function<? super String, String> sourceFunction, List<String> args)
+ throws OptionsParsingException {
+ Preconditions.checkNotNull(priority);
+ Preconditions.checkArgument(priority != OptionPriority.DEFAULT);
+ residue.addAll(impl.parse(priority, sourceFunction, args));
+ if (!allowResidue && !residue.isEmpty()) {
+ String errorMsg = "Unrecognized arguments: " + Joiner.on(' ').join(residue);
+ throw new OptionsParsingException(errorMsg);
+ }
+ }
+
+ @Override
+ public List<String> getResidue() {
+ return ImmutableList.copyOf(residue);
+ }
+
+ /**
+ * Returns a list of warnings about problems encountered by previous parse calls.
+ */
+ public List<String> getWarnings() {
+ return impl.getWarnings();
+ }
+
+ @Override
+ public <O extends OptionsBase> O getOptions(Class<O> optionsClass) {
+ return impl.getParsedOptions(optionsClass);
+ }
+
+ @Override
+ public boolean containsExplicitOption(String name) {
+ return impl.containsExplicitOption(name);
+ }
+
+ @Override
+ public List<UnparsedOptionValueDescription> asListOfUnparsedOptions() {
+ return impl.asListOfUnparsedOptions();
+ }
+
+ @Override
+ public List<UnparsedOptionValueDescription> asListOfExplicitOptions() {
+ return impl.asListOfExplicitOptions();
+ }
+
+ @Override
+ public List<OptionValueDescription> asListOfEffectiveOptions() {
+ return impl.asListOfEffectiveOptions();
+ }
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParserImpl.java b/src/main/java/com/google/devtools/common/options/OptionsParserImpl.java
new file mode 100644
index 0000000..e339dcd
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsParserImpl.java
@@ -0,0 +1,722 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.devtools.common.options.OptionsParser.OptionValueDescription;
+import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The implementation of the options parser. This is intentionally package
+ * private for full flexibility. Use {@link OptionsParser} or {@link Options}
+ * if you're a consumer.
+ */
+class OptionsParserImpl {
+
+ /**
+ * A bunch of default converters in case the user doesn't specify a
+ * different one in the field annotation.
+ */
+ static final Map<Class<?>, Converter<?>> DEFAULT_CONVERTERS = Maps.newHashMap();
+
+ static {
+ DEFAULT_CONVERTERS.put(String.class, new Converter<String>() {
+ @Override
+ public String convert(String input) {
+ return input;
+ }
+ @Override
+ public String getTypeDescription() {
+ return "a string";
+ }});
+ DEFAULT_CONVERTERS.put(int.class, new Converter<Integer>() {
+ @Override
+ public Integer convert(String input) throws OptionsParsingException {
+ try {
+ return Integer.decode(input);
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("'" + input + "' is not an int");
+ }
+ }
+ @Override
+ public String getTypeDescription() {
+ return "an integer";
+ }});
+ DEFAULT_CONVERTERS.put(double.class, new Converter<Double>() {
+ @Override
+ public Double convert(String input) throws OptionsParsingException {
+ try {
+ return Double.parseDouble(input);
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("'" + input + "' is not a double");
+ }
+ }
+ @Override
+ public String getTypeDescription() {
+ return "a double";
+ }});
+ DEFAULT_CONVERTERS.put(boolean.class, new Converters.BooleanConverter());
+ DEFAULT_CONVERTERS.put(TriState.class, new Converter<TriState>() {
+ @Override
+ public TriState convert(String input) throws OptionsParsingException {
+ if (input == null) {
+ return TriState.AUTO;
+ }
+ input = input.toLowerCase();
+ if (input.equals("auto")) {
+ return TriState.AUTO;
+ }
+ if (input.equals("true") || input.equals("1") || input.equals("yes") ||
+ input.equals("t") || input.equals("y")) {
+ return TriState.YES;
+ }
+ if (input.equals("false") || input.equals("0") || input.equals("no") ||
+ input.equals("f") || input.equals("n")) {
+ return TriState.NO;
+ }
+ throw new OptionsParsingException("'" + input + "' is not a boolean");
+ }
+ @Override
+ public String getTypeDescription() {
+ return "a tri-state (auto, yes, no)";
+ }});
+ DEFAULT_CONVERTERS.put(Void.class, new Converter<Void>() {
+ @Override
+ public Void convert(String input) throws OptionsParsingException {
+ if (input == null) {
+ return null; // expected input, return is unused so null is fine.
+ }
+ throw new OptionsParsingException("'" + input + "' unexpected");
+ }
+ @Override
+ public String getTypeDescription() {
+ return "";
+ }});
+ DEFAULT_CONVERTERS.put(long.class, new Converter<Long>() {
+ @Override
+ public Long convert(String input) throws OptionsParsingException {
+ try {
+ return Long.decode(input);
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException("'" + input + "' is not a long");
+ }
+ }
+ @Override
+ public String getTypeDescription() {
+ return "a long integer";
+ }});
+ }
+
+ /**
+ * For every value, this class keeps track of its priority, its free-form source
+ * description, whether it was set as an implicit dependency, and the value.
+ */
+ private static final class ParsedOptionEntry {
+ private final Object value;
+ private final OptionPriority priority;
+ private final String source;
+ private final String implicitDependant;
+ private final String expandedFrom;
+ private final boolean allowMultiple;
+
+ ParsedOptionEntry(Object value,
+ OptionPriority priority, String source, String implicitDependant, String expandedFrom,
+ boolean allowMultiple) {
+ this.value = value;
+ this.priority = priority;
+ this.source = source;
+ this.implicitDependant = implicitDependant;
+ this.expandedFrom = expandedFrom;
+ this.allowMultiple = allowMultiple;
+ }
+
+ // Need to suppress unchecked warnings, because the "multiple occurrence"
+ // options use unchecked ListMultimaps due to limitations of Java generics.
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ Object getValue() {
+ if (allowMultiple) {
+ // Sort the results by option priority and return them in a new list.
+ // The generic type of the list is not known at runtime, so we can't
+ // use it here. It was already checked in the constructor, so this is
+ // type-safe.
+ List result = Lists.newArrayList();
+ ListMultimap realValue = (ListMultimap) value;
+ for (OptionPriority priority : OptionPriority.values()) {
+ // If there is no mapping for this key, this check avoids object creation (because
+ // ListMultimap has to return a new object on get) and also an unnecessary addAll call.
+ if (realValue.containsKey(priority)) {
+ result.addAll(realValue.get(priority));
+ }
+ }
+ return result;
+ }
+ return value;
+ }
+
+ // Need to suppress unchecked warnings, because the "multiple occurrence"
+ // options use unchecked ListMultimaps due to limitations of Java generics.
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ void addValue(OptionPriority addedPriority, Object addedValue) {
+ Preconditions.checkState(allowMultiple);
+ ListMultimap optionValueList = (ListMultimap) value;
+ if (addedValue instanceof List<?>) {
+ for (Object element : (List<?>) addedValue) {
+ optionValueList.put(addedPriority, element);
+ }
+ } else {
+ optionValueList.put(addedPriority, addedValue);
+ }
+ }
+
+ OptionValueDescription asOptionValueDescription(String fieldName) {
+ return new OptionValueDescription(fieldName, getValue(), priority,
+ source, implicitDependant, expandedFrom);
+ }
+ }
+
+ private final OptionsData optionsData;
+
+ /**
+ * We store the results of parsing the arguments in here. It'll look like
+ * <pre>
+ * Field("--host") -> "www.google.com"
+ * Field("--port") -> 80
+ * </pre>
+ * This map is modified by repeated calls to
+ * {@link #parse(OptionPriority,Function,List)}.
+ */
+ private final Map<Field, ParsedOptionEntry> parsedValues = Maps.newHashMap();
+
+ /**
+ * We store the pre-parsed, explicit options for each priority in here.
+ * We use partially preparsed options, which can be different from the original
+ * representation, e.g. "--nofoo" becomes "--foo=0".
+ */
+ private final List<UnparsedOptionValueDescription> unparsedValues =
+ Lists.newArrayList();
+
+ private final List<String> warnings = Lists.newArrayList();
+
+ private boolean allowSingleDashLongOptions = false;
+
+ /**
+ * Create a new parser object
+ */
+ OptionsParserImpl(OptionsData optionsData) {
+ this.optionsData = optionsData;
+ }
+
+ /**
+ * Indicates whether or not the parser will allow long options with a
+ * single-dash, instead of the usual double-dash, too, eg. -example instead of just --example.
+ */
+ void setAllowSingleDashLongOptions(boolean allowSingleDashLongOptions) {
+ this.allowSingleDashLongOptions = allowSingleDashLongOptions;
+ }
+
+ /**
+ * The implementation of {@link OptionsBase#asMap}.
+ */
+ static Map<String, Object> optionsAsMap(OptionsBase optionsInstance) {
+ Map<String, Object> map = Maps.newHashMap();
+ for (Field field : OptionsParser.getAllAnnotatedFields(optionsInstance.getClass())) {
+ try {
+ String name = field.getAnnotation(Option.class).name();
+ Object value = field.get(optionsInstance);
+ map.put(name, value);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(e); // unreachable
+ }
+ }
+ return map;
+ }
+
+ List<Field> getAnnotatedFieldsFor(Class<? extends OptionsBase> clazz) {
+ return optionsData.getFieldsForClass(clazz);
+ }
+
+ /**
+ * Implements {@link OptionsParser#asListOfUnparsedOptions()}.
+ */
+ List<UnparsedOptionValueDescription> asListOfUnparsedOptions() {
+ List<UnparsedOptionValueDescription> result = Lists.newArrayList(unparsedValues);
+ // It is vital that this sort is stable so that options on the same priority are not reordered.
+ Collections.sort(result, new Comparator<UnparsedOptionValueDescription>() {
+ @Override
+ public int compare(UnparsedOptionValueDescription o1,
+ UnparsedOptionValueDescription o2) {
+ return o1.getPriority().compareTo(o2.getPriority());
+ }
+ });
+ return result;
+ }
+
+ /**
+ * Implements {@link OptionsParser#asListOfExplicitOptions()}.
+ */
+ List<UnparsedOptionValueDescription> asListOfExplicitOptions() {
+ List<UnparsedOptionValueDescription> result = Lists.newArrayList(Iterables.filter(
+ unparsedValues,
+ new Predicate<UnparsedOptionValueDescription>() {
+ @Override
+ public boolean apply(UnparsedOptionValueDescription input) {
+ return input.isExplicit();
+ }
+ }));
+ // It is vital that this sort is stable so that options on the same priority are not reordered.
+ Collections.sort(result, new Comparator<UnparsedOptionValueDescription>() {
+ @Override
+ public int compare(UnparsedOptionValueDescription o1,
+ UnparsedOptionValueDescription o2) {
+ return o1.getPriority().compareTo(o2.getPriority());
+ }
+ });
+ return result;
+ }
+
+ /**
+ * Implements {@link OptionsParser#canonicalize}.
+ */
+ List<String> asCanonicalizedList() {
+ List<UnparsedOptionValueDescription> processed = Lists.newArrayList(unparsedValues);
+ Collections.sort(processed, new Comparator<UnparsedOptionValueDescription>() {
+ // This Comparator sorts implicit requirement options to the end, keeping their existing
+ // order, and sorts the other options alphabetically.
+ @Override
+ public int compare(UnparsedOptionValueDescription o1,
+ UnparsedOptionValueDescription o2) {
+ if (o1.isImplicitRequirement()) {
+ return o2.isImplicitRequirement() ? 0 : 1;
+ }
+ if (o2.isImplicitRequirement()) {
+ return -1;
+ }
+ return o1.getName().compareTo(o2.getName());
+ }
+ });
+
+ List<String> result = Lists.newArrayList();
+ for (int i = 0; i < processed.size(); i++) {
+ UnparsedOptionValueDescription value = processed.get(i);
+ // Skip an option if the next option is the same, but only if the option does not allow
+ // multiple values.
+ if (!value.allowMultiple()) {
+ if ((i < processed.size() - 1) && value.getName().equals(processed.get(i + 1).getName())) {
+ continue;
+ }
+ }
+
+ // Ignore expansion options.
+ if (value.isExpansion()) {
+ continue;
+ }
+
+ result.add("--" + value.getName() + "=" + value.getUnparsedValue());
+ }
+ return result;
+ }
+
+ /**
+ * Implements {@link OptionsParser#asListOfEffectiveOptions()}.
+ */
+ List<OptionValueDescription> asListOfEffectiveOptions() {
+ List<OptionValueDescription> result = Lists.newArrayList();
+ for (Map.Entry<String,Field> mapEntry : optionsData.getAllNamedFields()) {
+ String fieldName = mapEntry.getKey();
+ Field field = mapEntry.getValue();
+ ParsedOptionEntry entry = parsedValues.get(field);
+ if (entry == null) {
+ Object value = optionsData.getDefaultValue(field);
+ result.add(new OptionValueDescription(fieldName, value, OptionPriority.DEFAULT,
+ null, null, null));
+ } else {
+ result.add(entry.asOptionValueDescription(fieldName));
+ }
+ }
+ return result;
+ }
+
+ Collection<Class<? extends OptionsBase>> getOptionsClasses() {
+ return optionsData.getOptionsClasses();
+ }
+
+ private void maybeAddDeprecationWarning(Field field) {
+ Option option = field.getAnnotation(Option.class);
+ // Continue to support the old behavior for @Deprecated options.
+ String warning = option.deprecationWarning();
+ if (!warning.equals("") || (field.getAnnotation(Deprecated.class) != null)) {
+ warnings.add("Option '" + option.name() + "' is deprecated"
+ + (warning.equals("") ? "" : ": " + warning));
+ }
+ }
+
+ // Warnings should not end with a '.' because the internal reporter adds one automatically.
+ private void setValue(Field field, String name, Object value,
+ OptionPriority priority, String source, String implicitDependant, String expandedFrom) {
+ ParsedOptionEntry entry = parsedValues.get(field);
+ if (entry != null) {
+ // Override existing option if the new value has higher or equal priority.
+ if (priority.compareTo(entry.priority) >= 0) {
+ // Output warnings:
+ if ((implicitDependant != null) && (entry.implicitDependant != null)) {
+ if (!implicitDependant.equals(entry.implicitDependant)) {
+ warnings.add("Option '" + name + "' is implicitly defined by both option '" +
+ entry.implicitDependant + "' and option '" + implicitDependant + "'");
+ }
+ } else if ((implicitDependant != null) && priority.equals(entry.priority)) {
+ warnings.add("Option '" + name + "' is implicitly defined by option '" +
+ implicitDependant + "'; the implicitly set value overrides the previous one");
+ } else if (entry.implicitDependant != null) {
+ warnings.add("A new value for option '" + name + "' overrides a previous " +
+ "implicit setting of that option by option '" + entry.implicitDependant + "'");
+ } else if ((priority == entry.priority) &&
+ ((entry.expandedFrom == null) && (expandedFrom != null))) {
+ // Create a warning if an expansion option overrides an explicit option:
+ warnings.add("The option '" + expandedFrom + "' was expanded and now overrides a "
+ + "previous explicitly specified option '" + name + "'");
+ }
+
+ // Record the new value:
+ parsedValues.put(field,
+ new ParsedOptionEntry(value, priority, source, implicitDependant, expandedFrom, false));
+ }
+ } else {
+ parsedValues.put(field,
+ new ParsedOptionEntry(value, priority, source, implicitDependant, expandedFrom, false));
+ maybeAddDeprecationWarning(field);
+ }
+ }
+
+ private void addListValue(Field field, String name, Object value,
+ OptionPriority priority, String source, String implicitDependant, String expandedFrom) {
+ ParsedOptionEntry entry = parsedValues.get(field);
+ if (entry == null) {
+ entry = new ParsedOptionEntry(ArrayListMultimap.create(), priority, source,
+ implicitDependant, expandedFrom, true);
+ parsedValues.put(field, entry);
+ maybeAddDeprecationWarning(field);
+ }
+ entry.addValue(priority, value);
+ }
+
+ private Object getValue(Field field) {
+ ParsedOptionEntry entry = parsedValues.get(field);
+ return entry == null ? null : entry.getValue();
+ }
+
+ OptionValueDescription getOptionValueDescription(String name) {
+ Field field = optionsData.getFieldFromName(name);
+ if (field == null) {
+ throw new IllegalArgumentException("No such option '" + name + "'");
+ }
+ ParsedOptionEntry entry = parsedValues.get(field);
+ if (entry == null) {
+ return null;
+ }
+ return entry.asOptionValueDescription(name);
+ }
+
+ boolean containsExplicitOption(String name) {
+ Field field = optionsData.getFieldFromName(name);
+ if (field == null) {
+ throw new IllegalArgumentException("No such option '" + name + "'");
+ }
+ return parsedValues.get(field) != null;
+ }
+
+ /**
+ * Parses the args, and returns what it doesn't parse. May be called multiple
+ * times, and may be called recursively. In each call, there may be no
+ * duplicates, but separate calls may contain intersecting sets of options; in
+ * that case, the arg seen last takes precedence.
+ */
+ List<String> parse(OptionPriority priority, Function<? super String, String> sourceFunction,
+ List<String> args) throws OptionsParsingException {
+ return parse(priority, sourceFunction, null, null, args);
+ }
+
+ /**
+ * Parses the args, and returns what it doesn't parse. May be called multiple
+ * times, and may be called recursively. Calls may contain intersecting sets
+ * of options; in that case, the arg seen last takes precedence.
+ *
+ * <p>The method uses the invariant that if an option has neither an implicit
+ * dependant nor an expanded from value, then it must have been explicitly
+ * set.
+ */
+ private List<String> parse(OptionPriority priority,
+ final Function<? super String, String> sourceFunction, String implicitDependant,
+ String expandedFrom, List<String> args) throws OptionsParsingException {
+ List<String> unparsedArgs = Lists.newArrayList();
+ LinkedHashMap<String,List<String>> implicitRequirements = Maps.newLinkedHashMap();
+ for (int pos = 0; pos < args.size(); pos++) {
+ String arg = args.get(pos);
+ if (!arg.startsWith("-")) {
+ unparsedArgs.add(arg);
+ continue; // not an option arg
+ }
+ if (arg.equals("--")) { // "--" means all remaining args aren't options
+ while (++pos < args.size()) {
+ unparsedArgs.add(args.get(pos));
+ }
+ break;
+ }
+
+ String value = null;
+ Field field;
+ boolean booleanValue = true;
+
+ if (arg.length() == 2) { // -l (may be nullary or unary)
+ field = optionsData.getFieldForAbbrev(arg.charAt(1));
+ booleanValue = true;
+
+ } else if (arg.length() == 3 && arg.charAt(2) == '-') { // -l- (boolean)
+ field = optionsData.getFieldForAbbrev(arg.charAt(1));
+ booleanValue = false;
+
+ } else if (allowSingleDashLongOptions // -long_option
+ || arg.startsWith("--")) { // or --long_option
+ int equalsAt = arg.indexOf('=');
+ int nameStartsAt = arg.startsWith("--") ? 2 : 1;
+ String name =
+ equalsAt == -1 ? arg.substring(nameStartsAt) : arg.substring(nameStartsAt, equalsAt);
+ if (name.trim().equals("")) {
+ throw new OptionsParsingException("Invalid options syntax: " + arg, arg);
+ }
+ value = equalsAt == -1 ? null : arg.substring(equalsAt + 1);
+ field = optionsData.getFieldFromName(name);
+
+ // look for a "no"-prefixed option name: "no<optionname>";
+ // (Undocumented: we also allow --no_foo. We're generous like that.)
+ if (field == null && name.startsWith("no")) {
+ String realname = name.substring(name.startsWith("no_") ? 3 : 2);
+ field = optionsData.getFieldFromName(realname);
+ booleanValue = false;
+ if (field != null) {
+ // TODO(bazel-team): Add tests for these cases.
+ if (!OptionsParserImpl.isBooleanField(field)) {
+ throw new OptionsParsingException(
+ "Illegal use of 'no' prefix on non-boolean option: " + arg, arg);
+ }
+ if (value != null) {
+ throw new OptionsParsingException(
+ "Unexpected value after boolean option: " + arg, arg);
+ }
+ // "no<optionname>" signifies a boolean option w/ false value
+ value = "0";
+ }
+ }
+
+ } else {
+ throw new OptionsParsingException("Invalid options syntax: " + arg, arg);
+ }
+
+ if (field == null) {
+ throw new OptionsParsingException("Unrecognized option: " + arg, arg);
+ }
+
+ if (value == null) {
+ // special case boolean to supply value based on presence of "no" prefix
+ if (OptionsParserImpl.isBooleanField(field)) {
+ value = booleanValue ? "1" : "0";
+ } else if (field.getType().equals(Void.class)) {
+ // this is expected, Void type options have no args
+ } else if (pos != args.size() - 1) {
+ value = args.get(++pos); // "--flag value" form
+ } else {
+ throw new OptionsParsingException("Expected value after " + arg);
+ }
+ }
+
+ Option option = field.getAnnotation(Option.class);
+ final String originalName = option.name();
+ if (implicitDependant == null) {
+ // Log explicit options and expanded options in the order they are parsed (can be sorted
+ // later). Also remember whether they were expanded or not. This information is needed to
+ // correctly canonicalize flags.
+ unparsedValues.add(new UnparsedOptionValueDescription(originalName, field, value,
+ priority, sourceFunction.apply(originalName), expandedFrom == null));
+ }
+
+ // Handle expansion options.
+ if (option.expansion().length > 0) {
+ Function<Object, String> expansionSourceFunction = Functions.<String>constant(
+ "expanded from option --" + originalName + " from " +
+ sourceFunction.apply(originalName));
+ maybeAddDeprecationWarning(field);
+ List<String> unparsed = parse(priority, expansionSourceFunction, null, originalName,
+ ImmutableList.copyOf(option.expansion()));
+ if (!unparsed.isEmpty()) {
+ // Throw an assertion, because this indicates an error in the code that specified the
+ // expansion for the current option.
+ throw new AssertionError("Unparsed options remain after parsing expansion of " +
+ arg + ":" + Joiner.on(' ').join(unparsed));
+ }
+ } else {
+ Converter<?> converter = optionsData.getConverter(field);
+ Object convertedValue;
+ try {
+ convertedValue = converter.convert(value);
+ } catch (OptionsParsingException e) {
+ // The converter doesn't know the option name, so we supply it here by
+ // re-throwing:
+ throw new OptionsParsingException("While parsing option " + arg
+ + ": " + e.getMessage(), e);
+ }
+
+ // ...but allow duplicates of single-use options across separate calls to
+ // parse(); latest wins:
+ if (!option.allowMultiple()) {
+ setValue(field, originalName, convertedValue,
+ priority, sourceFunction.apply(originalName), implicitDependant, expandedFrom);
+ } else {
+ // But if it's a multiple-use option, then just accumulate the
+ // values, in the order in which they were seen.
+ // Note: The type of the list member is not known; Java introspection
+ // only makes it available in String form via the signature string
+ // for the field declaration.
+ addListValue(field, originalName, convertedValue,
+ priority, sourceFunction.apply(originalName), implicitDependant, expandedFrom);
+ }
+ }
+
+ // Collect any implicit requirements.
+ if (option.implicitRequirements().length > 0) {
+ implicitRequirements.put(option.name(), Arrays.asList(option.implicitRequirements()));
+ }
+ }
+
+ // Now parse any implicit requirements that were collected.
+ // TODO(bazel-team): this should happen when the option is encountered.
+ if (!implicitRequirements.isEmpty()) {
+ for (Map.Entry<String,List<String>> entry : implicitRequirements.entrySet()) {
+ Function<Object, String> requirementSourceFunction = Functions.<String>constant(
+ "implicit requirement of option --" + entry.getKey() + " from " +
+ sourceFunction.apply(entry.getKey()));
+
+ List<String> unparsed = parse(priority, requirementSourceFunction, entry.getKey(), null,
+ entry.getValue());
+ if (!unparsed.isEmpty()) {
+ // Throw an assertion, because this indicates an error in the code that specified in the
+ // implicit requirements for the option(s).
+ throw new AssertionError("Unparsed options remain after parsing implicit options:"
+ + Joiner.on(' ').join(unparsed));
+ }
+ }
+ }
+
+ return unparsedArgs;
+ }
+
+ /**
+ * Gets the result of parsing the options.
+ */
+ <O extends OptionsBase> O getParsedOptions(Class<O> optionsClass) {
+ // Create the instance:
+ O optionsInstance;
+ try {
+ Constructor<O> constructor = optionsData.getConstructor(optionsClass);
+ if (constructor == null) {
+ return null;
+ }
+ optionsInstance = constructor.newInstance(new Object[0]);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+
+ // Set the fields
+ for (Field field : optionsData.getFieldsForClass(optionsClass)) {
+ Object value = getValue(field);
+ if (value == null) {
+ value = optionsData.getDefaultValue(field);
+ }
+ try {
+ field.set(optionsInstance, value);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ return optionsInstance;
+ }
+
+ List<String> getWarnings() {
+ return ImmutableList.copyOf(warnings);
+ }
+
+ static String getDefaultOptionString(Field optionField) {
+ Option annotation = optionField.getAnnotation(Option.class);
+ return annotation.defaultValue();
+ }
+
+ static boolean isBooleanField(Field field) {
+ return field.getType().equals(boolean.class) || field.getType().equals(TriState.class);
+ }
+
+ static boolean isSpecialNullDefault(String defaultValueString, Field optionField) {
+ return defaultValueString.equals("null") && !optionField.getType().isPrimitive();
+ }
+
+ static Converter<?> findConverter(Field optionField) {
+ Option annotation = optionField.getAnnotation(Option.class);
+ if (annotation.converter() == Converter.class) {
+ Type type;
+ if (annotation.allowMultiple()) {
+ // The OptionParserImpl already checked that the type is List<T> for some T;
+ // here we extract the type T.
+ type = ((ParameterizedType) optionField.getGenericType()).getActualTypeArguments()[0];
+ } else {
+ type = optionField.getType();
+ }
+ Converter<?> converter = DEFAULT_CONVERTERS.get(type);
+ if (converter == null) {
+ throw new AssertionError("No converter found for "
+ + type + "; possible fix: add "
+ + "converter=... to @Option annotation for "
+ + optionField.getName());
+ }
+ return converter;
+ }
+ try {
+ Class<?> converter = annotation.converter();
+ Constructor<?> constructor = converter.getConstructor(new Class<?>[0]);
+ return (Converter<?>) constructor.newInstance(new Object[0]);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParsingException.java b/src/main/java/com/google/devtools/common/options/OptionsParsingException.java
new file mode 100644
index 0000000..9d2916a
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsParsingException.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+/**
+ * An exception that's thrown when the {@link OptionsParser} fails.
+ *
+ * @see OptionsParser#parse(OptionPriority,String,java.util.List)
+ */
+public class OptionsParsingException extends Exception {
+ private final String invalidArgument;
+
+ public OptionsParsingException(String message) {
+ this(message, (String) null);
+ }
+
+ public OptionsParsingException(String message, String argument) {
+ super(message);
+ this.invalidArgument = argument;
+ }
+
+ public OptionsParsingException(String message, Throwable throwable) {
+ this(message, null, throwable);
+ }
+
+ public OptionsParsingException(String message, String argument, Throwable throwable) {
+ super(message, throwable);
+ this.invalidArgument = argument;
+ }
+
+ /**
+ * Gets the name of the invalid argument or {@code null} if the exception
+ * can not determine the exact invalid arguments
+ */
+ public String getInvalidArgument() {
+ return invalidArgument;
+ }
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsProvider.java b/src/main/java/com/google/devtools/common/options/OptionsProvider.java
new file mode 100644
index 0000000..be399a7
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsProvider.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import com.google.devtools.common.options.OptionsParser.OptionValueDescription;
+import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription;
+
+import java.util.List;
+
+/**
+ * A read-only interface for options parser results, which does not allow any
+ * further parsing of options.
+ */
+public interface OptionsProvider extends OptionsClassProvider {
+
+ /**
+ * Returns an immutable copy of the residue, that is, the arguments that
+ * have not been parsed.
+ */
+ List<String> getResidue();
+
+ /**
+ * Returns if the named option was specified explicitly in a call to parse.
+ */
+ boolean containsExplicitOption(String string);
+
+ /**
+ * Returns a mutable copy of the list of all options that were specified
+ * either explicitly or implicitly. These options are sorted by priority, and
+ * by the order in which they were specified. If an option was specified
+ * multiple times, it is included in the result multiple times. Does not
+ * include the residue.
+ *
+ * <p>The returned list can be filtered if undocumented, hidden or implicit
+ * options should not be displayed.
+ */
+ List<UnparsedOptionValueDescription> asListOfUnparsedOptions();
+
+ /**
+ * Returns a list of all explicitly specified options, suitable for logging
+ * or for displaying back to the user. These options are sorted by priority,
+ * and by the order in which they were specified. If an option was
+ * explicitly specified multiple times, it is included in the result
+ * multiple times. Does not include the residue.
+ *
+ * <p>The list includes undocumented options.
+ */
+ public List<UnparsedOptionValueDescription> asListOfExplicitOptions();
+
+ /**
+ * Returns a list of all options, including undocumented ones, and their
+ * effective values. There is no guaranteed ordering for the result.
+ */
+ public List<OptionValueDescription> asListOfEffectiveOptions();
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsUsage.java b/src/main/java/com/google/devtools/common/options/OptionsUsage.java
new file mode 100644
index 0000000..c48a532
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/OptionsUsage.java
@@ -0,0 +1,156 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import static com.google.devtools.common.options.OptionsParserImpl.findConverter;
+
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+import java.lang.reflect.Field;
+import java.text.BreakIterator;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * A renderer for usage messages. For now this is very simple.
+ */
+class OptionsUsage {
+
+ private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n');
+
+ /**
+ * Given an options class, render the usage string into the usage,
+ * which is passed in as an argument.
+ */
+ static void getUsage(Class<? extends OptionsBase> optionsClass, StringBuilder usage) {
+ List<Field> optionFields =
+ Lists.newArrayList(OptionsParser.getAllAnnotatedFields(optionsClass));
+ Collections.sort(optionFields, BY_NAME);
+ for (Field optionField : optionFields) {
+ getUsage(optionField, usage, OptionsParser.HelpVerbosity.LONG);
+ }
+ }
+
+ /**
+ * Paragraph-fill the specified input text, indenting lines to 'indent' and
+ * wrapping lines at 'width'. Returns the formatted result.
+ */
+ static String paragraphFill(String in, int indent, int width) {
+ String indentString = Strings.repeat(" ", indent);
+ StringBuilder out = new StringBuilder();
+ String sep = "";
+ for (String paragraph : NEWLINE_SPLITTER.split(in)) {
+ BreakIterator boundary = BreakIterator.getLineInstance(); // (factory)
+ boundary.setText(paragraph);
+ out.append(sep).append(indentString);
+ int cursor = indent;
+ for (int start = boundary.first(), end = boundary.next();
+ end != BreakIterator.DONE;
+ start = end, end = boundary.next()) {
+ String word =
+ paragraph.substring(start, end); // (may include trailing space)
+ if (word.length() + cursor > width) {
+ out.append('\n').append(indentString);
+ cursor = indent;
+ }
+ out.append(word);
+ cursor += word.length();
+ }
+ sep = "\n";
+ }
+ return out.toString();
+ }
+
+ /**
+ * Append the usage message for a single option-field message to 'usage'.
+ */
+ static void getUsage(Field optionField, StringBuilder usage,
+ OptionsParser.HelpVerbosity helpVerbosity) {
+ String flagName = getFlagName(optionField);
+ String typeDescription = getTypeDescription(optionField);
+ Option annotation = optionField.getAnnotation(Option.class);
+ usage.append(" --" + flagName);
+ if (helpVerbosity == OptionsParser.HelpVerbosity.SHORT) { // just the name
+ usage.append('\n');
+ return;
+ }
+ if (annotation.abbrev() != '\0') {
+ usage.append(" [-").append(annotation.abbrev()).append(']');
+ }
+ if (!typeDescription.equals("")) {
+ usage.append(" (" + typeDescription + "; ");
+ if (annotation.allowMultiple()) {
+ usage.append("may be used multiple times");
+ } else {
+ // Don't call the annotation directly (we must allow overrides to certain defaults)
+ String defaultValueString = OptionsParserImpl.getDefaultOptionString(optionField);
+ if (OptionsParserImpl.isSpecialNullDefault(defaultValueString, optionField)) {
+ usage.append("default: see description");
+ } else {
+ usage.append("default: \"" + defaultValueString + "\"");
+ }
+ }
+ usage.append(")");
+ }
+ usage.append("\n");
+ if (helpVerbosity == OptionsParser.HelpVerbosity.MEDIUM) { // just the name and type.
+ return;
+ }
+ if (!annotation.help().equals("")) {
+ usage.append(paragraphFill(annotation.help(), 4, 80)); // (indent, width)
+ usage.append('\n');
+ }
+ if (annotation.expansion().length > 0) {
+ StringBuilder expandsMsg = new StringBuilder("Expands to: ");
+ for (String exp : annotation.expansion()) {
+ expandsMsg.append(exp).append(" ");
+ }
+ usage.append(paragraphFill(expandsMsg.toString(), 4, 80)); // (indent, width)
+ usage.append('\n');
+ }
+ }
+
+ private static final Comparator<Field> BY_NAME = new Comparator<Field>() {
+ @Override
+ public int compare(Field left, Field right) {
+ return left.getName().compareTo(right.getName());
+ }
+ };
+
+ /**
+ * An ordering relation for option-field fields that first groups together
+ * options of the same category, then sorts by name within the category.
+ */
+ static final Comparator<Field> BY_CATEGORY = new Comparator<Field>() {
+ @Override
+ public int compare(Field left, Field right) {
+ int r = left.getAnnotation(Option.class).category().compareTo(
+ right.getAnnotation(Option.class).category());
+ return r == 0 ? BY_NAME.compare(left, right) : r;
+ }
+ };
+
+ private static String getTypeDescription(Field optionsField) {
+ return findConverter(optionsField).getTypeDescription();
+ }
+
+ static String getFlagName(Field field) {
+ String name = field.getAnnotation(Option.class).name();
+ return OptionsParserImpl.isBooleanField(field) ? "[no]" + name : name;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/common/options/TriState.java b/src/main/java/com/google/devtools/common/options/TriState.java
new file mode 100644
index 0000000..9e873ea
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/TriState.java
@@ -0,0 +1,21 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+/**
+ * Enum used to represent tri-state options (yes/no/auto).
+ */
+public enum TriState {
+ YES, NO, AUTO
+}
diff --git a/src/main/native/BUILD b/src/main/native/BUILD
new file mode 100644
index 0000000..b1679b7
--- /dev/null
+++ b/src/main/native/BUILD
@@ -0,0 +1,57 @@
+genrule(
+ name = "copy_link_jni_md_header",
+ srcs = select({
+ "//src:darwin": ["//tools/jdk:jni_md_header-darwin"],
+ "//conditions:default": ["//tools/jdk:jni_md_header-linux"],
+ }),
+ outs = ["jni_md.h"],
+ cmd = "cp -f $< $@",
+)
+
+genrule(
+ name = "copy_link_jni_header",
+ srcs = ["//tools/jdk:jni_header"],
+ outs = ["jni.h"],
+ cmd = "cp -f $< $@",
+)
+
+filegroup(
+ name = "jni_os",
+ srcs = select({
+ "//src:darwin": ["unix_jni_darwin.cc"],
+ "//conditions:default": ["unix_jni_linux.cc"],
+ }),
+)
+
+cc_binary(
+ name = "libunix.so",
+ srcs = [
+ "localsocket.cc",
+ "process.cc",
+ "unix_jni.cc",
+ ":jni.h",
+ ":jni_md.h",
+ ":jni_os",
+ ],
+ copts = [
+ "-fPIC",
+ "-DBLAZE_JAVA_CPU=\"k8\"",
+ "-DBLAZE_OPENSOURCE=1",
+ ],
+ includes = ["."], # For jni headers.
+ linkshared = 1,
+ visibility = ["//src:__subpackages__"],
+ deps = [
+ "//src/main/cpp:md5",
+ ],
+)
+
+# HACK for Mac: copy libunix.so to libunix.dylib. We'll need to come up with a
+# way to support platform-specific dynamic library extensions.
+genrule(
+ name = "mac-compat",
+ srcs = ["libunix.so"],
+ outs = ["libunix.dylib"],
+ cmd = "cp $< $@",
+ visibility = ["//src:__subpackages__"],
+)
diff --git a/src/main/native/localsocket.cc b/src/main/native/localsocket.cc
new file mode 100644
index 0000000..56c4817
--- /dev/null
+++ b/src/main/native/localsocket.cc
@@ -0,0 +1,312 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <jni.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <poll.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include <string>
+
+#include "unix_jni.h"
+
+
+// Returns the field ID for FileDescriptor.fd.
+static jfieldID GetFileDescriptorField(JNIEnv *env) {
+ // See http://java.sun.com/docs/books/jni/html/fldmeth.html#26855
+ static jclass fd_class = NULL;
+ if (fd_class == NULL) { /* note: harmless race condition */
+ jclass local = env->FindClass("java/io/FileDescriptor");
+ CHECK(local != NULL);
+ fd_class = static_cast<jclass>(env->NewGlobalRef(local));
+ }
+ static jfieldID fieldId = NULL;
+ if (fieldId == NULL) { /* note: harmless race condition */
+ fieldId = env->GetFieldID(fd_class, "fd", "I");
+ CHECK(fieldId != NULL);
+ }
+ return fieldId;
+}
+
+// Returns the UNIX filedescriptor from a java.io.FileDescriptor instance.
+static jint GetUnixFileDescriptor(JNIEnv *env, jobject fd_obj) {
+ return env->GetIntField(fd_obj, GetFileDescriptorField(env));
+}
+
+// Sets the UNIX filedescriptor of a java.io.FileDescriptor instance.
+static void SetUnixFileDescriptor(JNIEnv *env, jobject fd_obj, jint fd) {
+ env->SetIntField(fd_obj, GetFileDescriptorField(env), fd);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: socket
+ * Signature: (Ljava/io/FileDescriptor;)V
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_socket(JNIEnv *env,
+ jclass clazz,
+ jobject fd_sock) {
+ int sock;
+ if ((sock = ::socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ return;
+ }
+ SetUnixFileDescriptor(env, fd_sock, sock);
+}
+
+// Initialize "addr" from "name_chars", reporting error and returning
+// false on failure.
+static bool InitializeSockaddr(JNIEnv *env,
+ struct sockaddr_un *addr,
+ const char* name_chars) {
+ memset(addr, 0, sizeof *addr);
+ addr->sun_family = AF_UNIX;
+ // Note: UNIX_PATH_MAX is quite small!
+ if (strlen(name_chars) >= sizeof(addr->sun_path)) {
+ ::PostFileException(env, ENAMETOOLONG, name_chars);
+ return false;
+ }
+ strcpy((char*) &addr->sun_path, name_chars);
+ return true;
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: bind
+ * Signature: (Ljava/io/FileDescriptor;Ljava/lang/String;)V
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_bind(JNIEnv *env,
+ jclass clazz,
+ jobject fd_svr,
+ jstring name) {
+ int svr_sock = GetUnixFileDescriptor(env, fd_svr);
+ const char* name_chars = env->GetStringUTFChars(name, NULL);
+ struct sockaddr_un addr;
+ if (InitializeSockaddr(env, &addr, name_chars) &&
+ ::bind(svr_sock, (struct sockaddr *) &addr, sizeof addr) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ }
+ env->ReleaseStringUTFChars(name, name_chars);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: listen
+ * Signature: (Ljava/io/FileDescriptor;I)V
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_listen(JNIEnv *env,
+ jclass clazz,
+ jobject fd_svr,
+ jint backlog) {
+ int svr_sock = GetUnixFileDescriptor(env, fd_svr);
+ if (::listen(svr_sock, backlog) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ }
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: select
+ * Signature: (L[java/io/FileDescriptor;[java/io/FileDescriptor;[java/io/FileDescriptor;J)Ljava/io/FileDescriptor
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_poll(JNIEnv *env,
+ jclass clazz,
+ jobject rfds_svr,
+ jlong timeoutMillis) {
+ // TODO(bazel-team): Handle Unix signals, namely SIGTERM.
+
+ // Copy Java FD into pollfd
+ pollfd pollfd;
+ pollfd.fd = GetUnixFileDescriptor(env, rfds_svr);
+ pollfd.events = POLLIN;
+ pollfd.revents = 0;
+
+ int count = poll(&pollfd, 1, timeoutMillis);
+ if (count == 0) {
+ // throws a timeout exception.
+ ::PostException(env, ETIMEDOUT, ::ErrorMessage(ETIMEDOUT));
+ } else if (count < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ }
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: accept
+ * Signature: (Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;)V
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_accept(JNIEnv *env,
+ jclass clazz,
+ jobject fd_svr,
+ jobject fd_cli) {
+ int svr_sock = GetUnixFileDescriptor(env, fd_svr);
+ int cli_sock;
+ if ((cli_sock = ::accept(svr_sock, NULL, NULL)) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ return;
+ }
+ SetUnixFileDescriptor(env, fd_cli, cli_sock);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: close
+ * Signature: (Ljava/io/FileDescriptor;)V
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_close(JNIEnv *env,
+ jclass clazz,
+ jobject fd_svr) {
+ int svr_sock = GetUnixFileDescriptor(env, fd_svr);
+ if (::close(svr_sock) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ }
+ SetUnixFileDescriptor(env, fd_svr, -1);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: connect
+ * Signature: (Ljava/io/FileDescriptor;Ljava/lang/String;)V
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_connect(JNIEnv *env,
+ jclass clazz,
+ jobject fd_cli,
+ jstring name) {
+ const char* name_chars = env->GetStringUTFChars(name, NULL);
+ jint cli_sock = GetUnixFileDescriptor(env, fd_cli);
+ if (cli_sock == -1) {
+ ::PostFileException(env, ENOTSOCK, name_chars);
+ } else {
+ struct sockaddr_un addr;
+ if (InitializeSockaddr(env, &addr, name_chars)) {
+ if (::connect(cli_sock, (struct sockaddr *) &addr, sizeof addr) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ }
+ }
+ }
+ env->ReleaseStringUTFChars(name, name_chars);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.LocalSocket
+ * Method: shutdown()
+ * Signature: (Ljava/io/FileDescriptor;I)V
+ * Parameters: code: 0 to shutdown input and 1 to shutdown output.
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocket_shutdown(JNIEnv *env,
+ jclass clazz,
+ jobject fd,
+ jint code) {
+ int action;
+ if (code == 0) {
+ action = SHUT_RD;
+ } else {
+ CHECK(code == 1);
+ action = SHUT_WR;
+ }
+
+ int sock = GetUnixFileDescriptor(env, fd);
+ if (shutdown(sock, action) < 0) {
+ ::PostException(env, errno, ::ErrorMessage(errno));
+ }
+}
+
+// TODO(bazel-team): These methods were removed in JDK8, so they
+// can be removed when we are no longer using JDK7. See note in
+// LocalSocketImpl.
+static jmethodID increment_use_count_;
+static jmethodID decrement_use_count_;
+
+// >=JDK8
+static jmethodID fd_attach_;
+static jmethodID fd_close_all_;
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocketImpl_init(JNIEnv *env, jclass ignored) {
+ jclass cls = env->FindClass("java/io/FileDescriptor");
+ if (cls == NULL) {
+ cls = env->FindClass("java/lang/NoClassDefFoundError");
+ env->ThrowNew(cls, "FileDescriptor class not found");
+ return;
+ }
+
+ // JDK7
+ increment_use_count_ =
+ env->GetMethodID(cls, "incrementAndGetUseCount", "()I");
+ if (increment_use_count_ != NULL) {
+ decrement_use_count_ =
+ env->GetMethodID(cls, "decrementAndGetUseCount", "()I");
+ } else {
+ // JDK8
+ env->ExceptionClear(); // The pending exception from increment_use_count_
+
+ fd_attach_ = env->GetMethodID(cls, "attach", "(Ljava/io/Closeable;)V");
+ fd_close_all_ = env->GetMethodID(cls, "closeAll", "(Ljava/io/Closeable;)V");
+
+ if (fd_attach_ == NULL || fd_close_all_ == NULL) {
+ cls = env->FindClass("java/lang/NoSuchMethodError");
+ env->ThrowNew(cls, "FileDescriptor methods not found");
+ return;
+ }
+ }
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocketImpl_ref(JNIEnv *env, jclass clazz,
+ jobject fd, jobject closer) {
+ if (increment_use_count_ != NULL) {
+ env->CallIntMethod(fd, increment_use_count_);
+ }
+
+ if (fd_attach_ != NULL) {
+ env->CallVoidMethod(fd, fd_attach_, closer);
+ }
+}
+
+extern "C" JNIEXPORT jboolean JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocketImpl_unref(JNIEnv *env, jclass clazz,
+ jobject fd) {
+ if (decrement_use_count_ != NULL) {
+ env->CallIntMethod(fd, decrement_use_count_);
+ return true;
+ }
+ return false;
+}
+
+extern "C" JNIEXPORT jboolean JNICALL
+Java_com_google_devtools_build_lib_unix_LocalSocketImpl_close0(JNIEnv *env, jclass clazz,
+ jobject fd,
+ jobject closeable) {
+ if (fd_close_all_ != NULL) {
+ env->CallVoidMethod(fd, fd_close_all_, closeable);
+ return true;
+ }
+ // This will happen if fd_close_all_ is NULL, which means we are running in
+ // <=JDK7, which means that the caller needs to invoke close() explicitly.
+ return false;
+}
diff --git a/src/main/native/macros.h b/src/main/native/macros.h
new file mode 100644
index 0000000..567ccd9
--- /dev/null
+++ b/src/main/native/macros.h
@@ -0,0 +1,21 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MACROS_H__
+#define MACROS_H__
+
+// TODO(bazel-team): Use the proper annotation for clang.
+#define FALLTHROUGH_INTENDED do { } while (0)
+
+#endif // MACROS_H__
diff --git a/src/main/native/process.cc b/src/main/native/process.cc
new file mode 100644
index 0000000..e42e34d
--- /dev/null
+++ b/src/main/native/process.cc
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <jni.h>
+
+#include <unistd.h>
+
+/*
+ * Class: com.google.devtools.build.lib.unix.ProcessUtils
+ * Method: getgid
+ * Signature: ()I
+ */
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_devtools_build_lib_unix_ProcessUtils_getgid(JNIEnv *env, jclass clazz) {
+ return getgid();
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.ProcessUtils
+ * Method: getpid
+ * Signature: ()I
+ */
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_devtools_build_lib_unix_ProcessUtils_getpid(JNIEnv *env, jclass clazz) {
+ return getpid();
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.ProcessUtils
+ * Method: getuid
+ * Signature: ()I
+ */
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_devtools_build_lib_unix_ProcessUtils_getuid(JNIEnv *env, jclass clazz) {
+ return getuid();
+}
diff --git a/src/main/native/unix_jni.cc b/src/main/native/unix_jni.cc
new file mode 100644
index 0000000..2a195e9
--- /dev/null
+++ b/src/main/native/unix_jni.cc
@@ -0,0 +1,827 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <jni.h>
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <utime.h>
+
+#include <string>
+#include <vector>
+
+#include "macros.h"
+#include "util/md5.h"
+#include "unix_jni.h"
+
+namespace {
+ const int kMd5DigestLength = 16;
+}
+
+////////////////////////////////////////////////////////////////////////
+// Latin1 <--> java.lang.String conversion functions.
+// Derived from similar routines in Sun JDK. See:
+// j2se/src/solaris/native/java/io/UnixFileSystem_md.c
+// j2se/src/share/native/common/jni_util.c
+//
+// Like the Sun JDK in its usual configuration, we assume all UNIX
+// filenames are Latin1 encoded.
+
+/**
+ * Returns a new Java String for the specified Latin1 characters.
+ */
+static jstring NewStringLatin1(JNIEnv *env, const char *str) {
+ int len = strlen(str);
+ jchar buf[512];
+ jchar *str1;
+
+ if (len > 512) {
+ str1 = reinterpret_cast<jchar *>(malloc(len * sizeof(jchar)));
+ if (str1 == 0) {
+ ::PostException(env, ENOMEM, "Out of memory in NewStringLatin1");
+ return NULL;
+ }
+ } else {
+ str1 = buf;
+ }
+
+ for (int i = 0; i < len ; i++) {
+ str1[i] = (unsigned char) str[i];
+ }
+ jstring result = env->NewString(str1, len);
+ if (str1 != buf) {
+ free(str1);
+ }
+ return result;
+}
+
+/**
+ * Returns a nul-terminated Latin1-encoded byte array for the
+ * specified Java string, or null on failure. Unencodable characters
+ * are replaced by '?'. Must be followed by a call to
+ * ReleaseStringLatin1Chars.
+ */
+static const char *GetStringLatin1Chars(JNIEnv *env, jstring jstr) {
+ jint len = env->GetStringLength(jstr);
+ const jchar *str = env->GetStringCritical(jstr, NULL);
+ if (str == NULL) {
+ return NULL;
+ }
+
+ char *result = reinterpret_cast<char *>(malloc(len + 1));
+ if (result == NULL) {
+ env->ReleaseStringCritical(jstr, str);
+ ::PostException(env, ENOMEM, "Out of memory in GetStringLatin1Chars");
+ return NULL;
+ }
+
+ for (int i = 0; i < len; i++) {
+ jchar unicode = str[i]; // (unsigned)
+ result[i] = unicode <= 0x00ff ? unicode : '?';
+ }
+
+ result[len] = 0;
+ env->ReleaseStringCritical(jstr, str);
+ return result;
+}
+
+/**
+ * Release the Latin1 chars returned by a prior call to
+ * GetStringLatin1Chars.
+ */
+static void ReleaseStringLatin1Chars(const char *s) {
+ if (s != NULL) {
+ free(const_cast<char *>(s));
+ }
+}
+
+////////////////////////////////////////////////////////////////////////
+
+// See unix_jni.h.
+void PostException(JNIEnv *env, int error_number, const std::string& message) {
+ // Keep consistent with package-info.html!
+ //
+ // See /usr/include/asm/errno.h for UNIX error messages.
+ // Select the most appropriate Java exception for a given UNIX error number.
+ // (Consistent with errors generated by java.io package.)
+ const char *exception_classname;
+ switch (error_number) {
+ case EFAULT: // Illegal pointer--not likely
+ case EBADF: // Bad file number
+ exception_classname = "java/lang/IllegalArgumentException";
+ break;
+ case ETIMEDOUT: // Local socket timed out
+ exception_classname = "java/net/SocketTimeoutException";
+ break;
+ case ENOENT: // No such file or directory
+ exception_classname = "java/io/FileNotFoundException";
+ break;
+ case EACCES: // Permission denied
+ exception_classname = "com/google/devtools/build/lib/unix/FileAccessException";
+ break;
+ case EPERM: // Operation not permitted
+ exception_classname = "com/google/devtools/build/lib/unix/FilePermissionException";
+ break;
+ case EINTR: // Interrupted system call
+ exception_classname = "java/io/InterruptedIOException";
+ break;
+ case ENOMEM: // Out of memory
+ exception_classname = "java/lang/OutOfMemoryError";
+ break;
+ case ENOSYS: // Function not implemented
+ case ENOTSUP: // Operation not supported on transport endpoint
+ // (aka EOPNOTSUPP)
+ exception_classname = "java/lang/UnsupportedOperationException";
+ break;
+ case ENAMETOOLONG: // File name too long
+ case ENODATA: // No data available
+ case EINVAL: // Invalid argument
+ case EMULTIHOP: // Multihop attempted
+ case ENOLINK: // Link has been severed
+ case EIO: // I/O error
+ case EAGAIN: // Try again
+ case EFBIG: // File too large
+ case EPIPE: // Broken pipe
+ case ENOSPC: // No space left on device
+ case EXDEV: // Cross-device link
+ case EROFS: // Read-only file system
+ case EEXIST: // File exists
+ case EMLINK: // Too many links
+ case ELOOP: // Too many symbolic links encountered
+ case EISDIR: // Is a directory
+ case ENOTDIR: // Not a directory
+ case ENOTEMPTY: // Directory not empty
+ case EBUSY: // Device or resource busy
+ case ENFILE: // File table overflow
+ case EMFILE: // Too many open files
+ default:
+ exception_classname = "java/io/IOException";
+ }
+ jclass exception_class = env->FindClass(exception_classname);
+ if (exception_class != NULL) {
+ env->ThrowNew(exception_class, message.c_str());
+ } else {
+ abort(); // panic!
+ }
+}
+
+// Throws RuntimeExceptions for IO operations which fail unexpectedly.
+// See package-info.html.
+// Returns true iff an exception was thrown.
+static bool PostRuntimeException(JNIEnv *env, int error_number,
+ const char *file_path) {
+ const char *exception_classname;
+ switch (error_number) {
+ case EFAULT: // Illegal pointer--not likely
+ case EBADF: // Bad file number
+ exception_classname = "java/lang/IllegalArgumentException";
+ break;
+ case ENOMEM: // Out of memory
+ exception_classname = "java/lang/OutOfMemoryError";
+ break;
+ case ENOTSUP: // Operation not supported on transport endpoint
+ // (aka EOPNOTSUPP)
+ exception_classname = "java/lang/UnsupportedOperationException";
+ break;
+ default:
+ exception_classname = NULL;
+ }
+
+ if (exception_classname == NULL) {
+ return false;
+ }
+
+ jclass exception_class = env->FindClass(exception_classname);
+ if (exception_class != NULL) {
+ std::string message(file_path);
+ message += " (";
+ message += ErrorMessage(error_number);
+ message += ")";
+ env->ThrowNew(exception_class, message.c_str());
+ return true;
+ } else {
+ abort(); // panic!
+ return false; // Not reachable.
+ }
+}
+
+// See unix_jni.h.
+void PostFileException(JNIEnv *env, int error_number, const char *filename) {
+ ::PostException(env, error_number,
+ std::string(filename) + " (" + ErrorMessage(error_number)
+ + ")");
+}
+
+// TODO(bazel-team): split out all the FileSystem class's native methods
+// into a separate source file, fsutils.cc.
+
+extern "C" JNIEXPORT jstring JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_readlink(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ char target[PATH_MAX] = "";
+ jstring r = NULL;
+ if (readlink(path_chars, target, arraysize(target)) == -1) {
+ ::PostFileException(env, errno, path_chars);
+ } else {
+ r = NewStringLatin1(env, target);
+ }
+ ReleaseStringLatin1Chars(path_chars);
+ return r;
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_chmod(JNIEnv *env,
+ jclass clazz,
+ jstring path,
+ jint mode) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ if (chmod(path_chars, static_cast<int>(mode)) == -1) {
+ ::PostFileException(env, errno, path_chars);
+ }
+ ReleaseStringLatin1Chars(path_chars);
+}
+
+static void link_common(JNIEnv *env,
+ jstring oldpath,
+ jstring newpath,
+ int (*link_function)(const char *, const char *)) {
+ const char *oldpath_chars = GetStringLatin1Chars(env, oldpath);
+ const char *newpath_chars = GetStringLatin1Chars(env, newpath);
+ if (link_function(oldpath_chars, newpath_chars) == -1) {
+ ::PostFileException(env, errno, newpath_chars);
+ }
+ ReleaseStringLatin1Chars(oldpath_chars);
+ ReleaseStringLatin1Chars(newpath_chars);
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_link(JNIEnv *env,
+ jclass clazz,
+ jstring oldpath,
+ jstring newpath) {
+ link_common(env, oldpath, newpath, ::link);
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_symlink(JNIEnv *env,
+ jclass clazz,
+ jstring oldpath,
+ jstring newpath) {
+ link_common(env, oldpath, newpath, ::symlink);
+}
+
+static jobject NewFileStatus(JNIEnv *env,
+ const struct stat &stat_ref) {
+ static jclass file_status_class = NULL;
+ if (file_status_class == NULL) { // note: harmless race condition
+ jclass local = env->FindClass("com/google/devtools/build/lib/unix/FileStatus");
+ CHECK(local != NULL);
+ file_status_class = static_cast<jclass>(env->NewGlobalRef(local));
+ }
+
+ static jmethodID method = NULL;
+ if (method == NULL) { // note: harmless race condition
+ method = env->GetMethodID(file_status_class, "<init>", "(IIIIIIIJIJ)V");
+ CHECK(method != NULL);
+ }
+
+ return env->NewObject(
+ file_status_class, method, stat_ref.st_mode,
+ StatSeconds(stat_ref, STAT_ATIME), StatNanoSeconds(stat_ref, STAT_ATIME),
+ StatSeconds(stat_ref, STAT_MTIME), StatNanoSeconds(stat_ref, STAT_MTIME),
+ StatSeconds(stat_ref, STAT_CTIME), StatNanoSeconds(stat_ref, STAT_CTIME),
+ stat_ref.st_size,
+ static_cast<int>(stat_ref.st_dev), static_cast<jlong>(stat_ref.st_ino));
+}
+
+static jobject NewErrnoFileStatus(JNIEnv *env,
+ int saved_errno,
+ const struct stat &stat_ref) {
+ static jclass errno_file_status_class = NULL;
+ if (errno_file_status_class == NULL) { // note: harmless race condition
+ jclass local = env->FindClass("com/google/devtools/build/lib/unix/ErrnoFileStatus");
+ CHECK(local != NULL);
+ errno_file_status_class = static_cast<jclass>(env->NewGlobalRef(local));
+ }
+
+ static jmethodID no_error_ctor = NULL;
+ if (no_error_ctor == NULL) { // note: harmless race condition
+ no_error_ctor = env->GetMethodID(errno_file_status_class,
+ "<init>", "(IIIIIIIJIJ)V");
+ CHECK(no_error_ctor != NULL);
+ }
+
+ static jmethodID errorno_ctor = NULL;
+ if (errorno_ctor == NULL) { // note: harmless race condition
+ errorno_ctor = env->GetMethodID(errno_file_status_class, "<init>", "(I)V");
+ CHECK(errorno_ctor != NULL);
+ }
+
+ if (saved_errno != 0) {
+ return env->NewObject(errno_file_status_class, errorno_ctor, errno);
+ }
+ return env->NewObject(
+ errno_file_status_class, no_error_ctor, stat_ref.st_mode,
+ StatSeconds(stat_ref, STAT_ATIME), StatNanoSeconds(stat_ref, STAT_ATIME),
+ StatSeconds(stat_ref, STAT_MTIME), StatNanoSeconds(stat_ref, STAT_MTIME),
+ StatSeconds(stat_ref, STAT_CTIME), StatNanoSeconds(stat_ref, STAT_CTIME),
+ stat_ref.st_size, static_cast<int>(stat_ref.st_dev),
+ static_cast<jlong>(stat_ref.st_ino));
+}
+
+static void SetIntField(JNIEnv *env,
+ const jclass &clazz,
+ const jobject &object,
+ const char *name,
+ int val) {
+ jfieldID fid = env->GetFieldID(clazz, name, "I");
+ CHECK(fid != NULL);
+ env->SetIntField(object, fid, val);
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_ErrnoFileStatus_00024ErrnoConstants_initErrnoConstants( // NOLINT
+ JNIEnv *env, jobject errno_constants) {
+ jclass clazz = env->GetObjectClass(errno_constants);
+ SetIntField(env, clazz, errno_constants, "ENOENT", ENOENT);
+ SetIntField(env, clazz, errno_constants, "EACCES", EACCES);
+ SetIntField(env, clazz, errno_constants, "ELOOP", ELOOP);
+ SetIntField(env, clazz, errno_constants, "ENOTDIR", ENOTDIR);
+ SetIntField(env, clazz, errno_constants, "ENAMETOOLONG", ENAMETOOLONG);
+}
+
+static jobject StatCommon(JNIEnv *env,
+ jstring path,
+ int (*stat_function)(const char *, struct stat *),
+ bool should_throw) {
+ struct stat statbuf;
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ int r;
+ int saved_errno = 0;
+ while ((r = stat_function(path_chars, &statbuf)) == -1 && errno == EINTR) { }
+ if (r == -1) {
+ // EACCES ENOENT ENOTDIR ELOOP -> IOException
+ // ENAMETOOLONGEFAULT -> RuntimeException
+ // ENOMEM -> OutOfMemoryError
+
+ if (PostRuntimeException(env, errno, path_chars)) {
+ ::ReleaseStringLatin1Chars(path_chars);
+ return NULL;
+ } else if (should_throw) {
+ ::PostFileException(env, errno, path_chars);
+ ::ReleaseStringLatin1Chars(path_chars);
+ return NULL;
+ } else {
+ saved_errno = errno;
+ }
+ }
+ ::ReleaseStringLatin1Chars(path_chars);
+
+ return should_throw
+ ? NewFileStatus(env, statbuf)
+ : NewErrnoFileStatus(env, saved_errno, statbuf);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: stat
+ * Signature: (Ljava/lang/String;)Lcom/google/devtools/build/lib/unix/FileStatus;
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT jobject JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_stat(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ return ::StatCommon(env, path, ::stat, true);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: lstat
+ * Signature: (Ljava/lang/String;)Lcom/google/devtools/build/lib/unix/FileStatus;
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT jobject JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_lstat(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ return ::StatCommon(env, path, ::lstat, true);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: statNullable
+ * Signature: (Ljava/lang/String;)Lcom/google/devtools/build/lib/unix/FileStatus;
+ */
+extern "C" JNIEXPORT jobject JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_errnoStat(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ return ::StatCommon(env, path, ::stat, false);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: lstatNullable
+ * Signature: (Ljava/lang/String;)Lcom/google/devtools/build/lib/unix/FileStatus;
+ */
+extern "C" JNIEXPORT jobject JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_errnoLstat(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ return ::StatCommon(env, path, ::lstat, false);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: utime
+ * Signature: (Ljava/lang/String;ZII)V
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_utime(JNIEnv *env,
+ jclass clazz,
+ jstring path,
+ jboolean now,
+ jint actime,
+ jint modtime) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ struct utimbuf buf = { actime, modtime };
+ struct utimbuf *bufptr = now ? NULL : &buf;
+ if (::utime(path_chars, bufptr) == -1) {
+ // EACCES ENOENT EMULTIHOP ELOOP EINTR
+ // ENOTDIR ENOLINK EPERM EROFS -> IOException
+ // EFAULT ENAMETOOLONG -> RuntimeException
+ ::PostFileException(env, errno, path_chars);
+ }
+ ReleaseStringLatin1Chars(path_chars);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: umask
+ * Signature: (I)I
+ */
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_umask(JNIEnv *env,
+ jclass clazz,
+ jint new_umask) {
+ return ::umask(new_umask);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: mkdir
+ * Signature: (Ljava/lang/String;I)Z
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT jboolean JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_mkdir(JNIEnv *env,
+ jclass clazz,
+ jstring path,
+ jint mode) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ jboolean result = true;
+ if (::mkdir(path_chars, mode) == -1) {
+ // EACCES ENOENT ELOOP
+ // ENOSPC ENOTDIR EPERM EROFS -> IOException
+ // EFAULT ENAMETOOLONG -> RuntimeException
+ // ENOMEM -> OutOfMemoryError
+ // EEXIST -> return false
+ if (errno == EEXIST) {
+ result = false;
+ } else {
+ ::PostFileException(env, errno, path_chars);
+ }
+ }
+ ReleaseStringLatin1Chars(path_chars);
+ return result;
+}
+
+static jobject NewDirents(JNIEnv *env,
+ jobjectArray names,
+ jbyteArray types) {
+ // See http://java.sun.com/docs/books/jni/html/fldmeth.html#26855
+ static jclass dirents_class = NULL;
+ if (dirents_class == NULL) { // note: harmless race condition
+ jclass local = env->FindClass("com/google/devtools/build/lib/unix/FilesystemUtils$Dirents");
+ CHECK(local != NULL);
+ dirents_class = static_cast<jclass>(env->NewGlobalRef(local));
+ }
+
+ static jmethodID ctor = NULL;
+ if (ctor == NULL) { // note: harmless race condition
+ ctor = env->GetMethodID(dirents_class, "<init>", "([Ljava/lang/String;[B)V");
+ CHECK(ctor != NULL);
+ }
+
+ return env->NewObject(dirents_class, ctor, names, types);
+}
+
+static char GetDirentType(struct dirent *entry,
+ int dirfd,
+ bool follow_symlinks) {
+ switch (entry->d_type) {
+ case DT_REG:
+ return 'f';
+ case DT_DIR:
+ return 'd';
+ case DT_LNK:
+ if (!follow_symlinks) {
+ return 's';
+ }
+ FALLTHROUGH_INTENDED;
+ case DT_UNKNOWN:
+ struct stat statbuf;
+ if (portable_fstatat(dirfd, entry->d_name, &statbuf, 0) == 0) {
+ if (S_ISREG(statbuf.st_mode)) return 'f';
+ if (S_ISDIR(statbuf.st_mode)) return 'd';
+ }
+ // stat failed or returned something weird; fall through
+ FALLTHROUGH_INTENDED;
+ default:
+ return '?';
+ }
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: readdir
+ * Signature: (Ljava/lang/String;Z)Lcom/google/devtools/build/lib/unix/Dirents;
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT jobject JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_readdir(JNIEnv *env,
+ jclass clazz,
+ jstring path,
+ jchar read_types) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ DIR *dirh;
+ while ((dirh = ::opendir(path_chars)) == NULL && errno == EINTR) { }
+ if (dirh == NULL) {
+ // EACCES EMFILE ENFILE ENOENT ENOTDIR -> IOException
+ // ENOMEM -> OutOfMemoryError
+ ::PostFileException(env, errno, path_chars);
+ }
+ ReleaseStringLatin1Chars(path_chars);
+ if (dirh == NULL) {
+ return NULL;
+ }
+ int fd = dirfd(dirh);
+
+ std::vector<std::string> entries;
+ std::vector<jbyte> types;
+ for (;;) {
+ // Clear errno beforehand. Because readdir() is not required to clear it at
+ // EOF, this is the only way to reliably distinguish EOF from error.
+ errno = 0;
+ struct dirent *entry = ::readdir(dirh);
+ if (entry == NULL) {
+ if (errno == 0) break; // EOF
+ // It is unclear whether an error can also skip some records.
+ // That does not appear to happen with glibc, at least.
+ if (errno == EINTR) continue; // interrupted by a signal
+ if (errno == EIO) continue; // glibc returns this on transient errors
+ // Otherwise, this is a real error we should report.
+ ::PostFileException(env, errno, path_chars);
+ ::closedir(dirh);
+ return NULL;
+ }
+ // Omit . and .. from results.
+ if (entry->d_name[0] == '.') {
+ if (entry->d_name[1] == '\0') continue;
+ if (entry->d_name[1] == '.' && entry->d_name[2] == '\0') continue;
+ }
+ entries.push_back(entry->d_name);
+ if (read_types != 'n') {
+ types.push_back(GetDirentType(entry, fd, read_types == 'f'));
+ }
+ }
+
+ if (::closedir(dirh) < 0 && errno != EINTR) {
+ ::PostFileException(env, errno, path_chars);
+ return NULL;
+ }
+
+ size_t len = entries.size();
+ jclass jlStringClass = env->GetObjectClass(path);
+ jobjectArray names_obj = env->NewObjectArray(len, jlStringClass, NULL);
+ if (names_obj == NULL && env->ExceptionOccurred()) {
+ return NULL; // async exception!
+ }
+
+ for (int ii = 0; ii < len; ++ii) {
+ jstring s = NewStringLatin1(env, entries[ii].c_str());
+ if (s == NULL && env->ExceptionOccurred()) {
+ return NULL; // async exception!
+ }
+ env->SetObjectArrayElement(names_obj, ii, s);
+ }
+
+ jbyteArray types_obj = NULL;
+ if (read_types != 'n') {
+ CHECK(len == types.size());
+ types_obj = env->NewByteArray(len);
+ CHECK(types_obj);
+ if (len > 0) {
+ env->SetByteArrayRegion(types_obj, 0, len, &types[0]);
+ }
+ }
+
+ return NewDirents(env, names_obj, types_obj);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: rename
+ * Signature: (Ljava/lang/String;Ljava/lang/String;)V
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_rename(JNIEnv *env,
+ jclass clazz,
+ jstring oldpath,
+ jstring newpath) {
+ const char *oldpath_chars = GetStringLatin1Chars(env, oldpath);
+ const char *newpath_chars = GetStringLatin1Chars(env, newpath);
+ if (::rename(oldpath_chars, newpath_chars) == -1) {
+ // EISDIR EXDEV ENOTEMPTY EEXIST EBUSY
+ // EINVAL EMLINK ENOTDIR EACCES EPERM
+ // ENOENT EROFS ELOOP ENOSPC -> IOException
+ // EFAULT ENAMETOOLONG -> RuntimeException
+ // ENOMEM -> OutOfMemoryError
+ std::string filename(std::string(oldpath_chars) + " -> " + newpath_chars);
+ ::PostFileException(env, errno, filename.c_str());
+ }
+ ReleaseStringLatin1Chars(oldpath_chars);
+ ReleaseStringLatin1Chars(newpath_chars);
+}
+
+static bool delete_common(JNIEnv *env,
+ jstring path,
+ int (*delete_function)(const char *),
+ bool (*error_function)(int)) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ if (path_chars == NULL) {
+ return false;
+ }
+ bool ok = delete_function(path_chars) != -1;
+ if (!ok) {
+ if (!error_function(errno)) {
+ ::PostFileException(env, errno, path_chars);
+ }
+ }
+ ReleaseStringLatin1Chars(path_chars);
+ return ok;
+}
+
+static bool unlink_err(int err) { return err == ENOENT; }
+static bool remove_err(int err) { return err == ENOENT || err == ENOTDIR; }
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: unlink
+ * Signature: (Ljava/lang/String;)V
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT bool JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_unlink(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ return ::delete_common(env, path, ::unlink, ::unlink_err);
+}
+
+/*
+ * Class: com.google.devtools.build.lib.unix.FilesystemUtils
+ * Method: remove
+ * Signature: (Ljava/lang/String;)V
+ * Throws: java.io.IOException
+ */
+extern "C" JNIEXPORT bool JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_remove(JNIEnv *env,
+ jclass clazz,
+ jstring path) {
+ return ::delete_common(env, path, ::remove, ::remove_err);
+}
+
+
+////////////////////////////////////////////////////////////////////////
+// Linux extended file attributes
+
+typedef ssize_t getxattr_func(const char *path, const char *name,
+ void *value, size_t size);
+
+static jbyteArray getxattr_common(JNIEnv *env,
+ jstring path,
+ jstring name,
+ getxattr_func getxattr) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ const char *name_chars = GetStringLatin1Chars(env, name);
+
+ // TODO(bazel-team): on ERANGE, try again with larger buffer.
+ jbyte value[4096];
+ jbyteArray result = NULL;
+ ssize_t size = getxattr(path_chars, name_chars, value, arraysize(value));
+ if (size == -1) {
+ if (errno != ENODATA) {
+ ::PostFileException(env, errno, path_chars);
+ }
+ } else {
+ result = env->NewByteArray(size);
+ env->SetByteArrayRegion(result, 0, size, value);
+ }
+ ReleaseStringLatin1Chars(path_chars);
+ ReleaseStringLatin1Chars(name_chars);
+ return result;
+}
+
+extern "C" JNIEXPORT jbyteArray JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_getxattr(JNIEnv *env,
+ jclass clazz,
+ jstring path,
+ jstring name) {
+ return ::getxattr_common(env, path, name, ::portable_getxattr);
+}
+
+extern "C" JNIEXPORT jbyteArray JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_lgetxattr(JNIEnv *env,
+ jclass clazz,
+ jstring path,
+ jstring name) {
+ return ::getxattr_common(env, path, name, ::portable_lgetxattr);
+}
+
+
+// Computes MD5 digest of "file", writes result in "result", which
+// must be of length kMd5DigestLength. Returns zero on success, or
+// -1 (and sets errno) otherwise.
+static int md5sumAsBytes(const char *file, jbyte result[kMd5DigestLength]) {
+ blaze_util::Md5Digest digest;
+ // OPT: Using a 32k buffer would give marginally better performance,
+ // but what is the stack size here?
+ jbyte buf[4096];
+ int fd;
+ while ((fd = open(file, O_RDONLY)) == -1 && errno == EINTR) { }
+ if (fd == -1) {
+ return -1;
+ }
+ for (ssize_t len = read(fd, buf, arraysize(buf));
+ len != 0;
+ len = read(fd, buf, arraysize(buf))) {
+ if (len == -1) {
+ if (errno == EINTR) {
+ continue;
+ } else {
+ int read_errno = errno;
+ close(fd); // prefer read() errors over close().
+ errno = read_errno;
+ return -1;
+ }
+ }
+ digest.Update(buf, len);
+ }
+ if (close(fd) < 0 && errno != EINTR) {
+ return -1;
+ }
+ digest.Finish(reinterpret_cast<unsigned char*>(result));
+ return 0;
+}
+
+
+extern "C" JNIEXPORT jbyteArray JNICALL
+Java_com_google_devtools_build_lib_unix_FilesystemUtils_md5sumAsBytes(
+ JNIEnv *env, jclass clazz, jstring path) {
+ const char *path_chars = GetStringLatin1Chars(env, path);
+ jbyte value[kMd5DigestLength];
+ jbyteArray result = NULL;
+ if (md5sumAsBytes(path_chars, value) == 0) {
+ result = env->NewByteArray(kMd5DigestLength);
+ env->SetByteArrayRegion(result, 0, kMd5DigestLength, value);
+ } else {
+ ::PostFileException(env, errno, path_chars);
+ }
+ ReleaseStringLatin1Chars(path_chars);
+ return result;
+}
diff --git a/src/main/native/unix_jni.h b/src/main/native/unix_jni.h
new file mode 100644
index 0000000..4078119
--- /dev/null
+++ b/src/main/native/unix_jni.h
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// INTERNAL header file for use by C++ code in this package.
+
+#ifndef JAVA_COM_GOOGLE_DEVTOOLS_BUILD_LIB_UNIX_UNIX_JNI_H__
+#define JAVA_COM_GOOGLE_DEVTOOLS_BUILD_LIB_UNIX_UNIX_JNI_H__
+
+#include <jni.h>
+
+#include <string>
+
+#define CHECK(condition) \
+ do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: check failed: %s\n", \
+ __FILE__, __LINE__, #condition); \
+ abort(); \
+ } \
+ } while (0)
+
+// Posts a JNI exception to the current thread with the specified
+// message; the exception's class is determined by the specified UNIX
+// error number. See package-info.html for details.
+extern void PostException(JNIEnv *env, int error_number,
+ const std::string &message);
+
+// Like PostException, but the exception message includes both the
+// specified filename and the standard UNIX error message for the
+// error number.
+// (Consistent with errors generated by java.io package.)
+extern void PostFileException(JNIEnv *env, int error_number,
+ const char *filename);
+
+// Returns the standard error message for a given UNIX error number.
+extern std::string ErrorMessage(int error_number);
+
+// Runs fstatat(2), if available, or sets errno to ENOSYS if not.
+int portable_fstatat(int dirfd, char *name, struct stat *statbuf, int flags);
+
+// Encoding for different timestamps in a struct stat{}.
+enum StatTimes {
+ STAT_ATIME, // access
+ STAT_MTIME, // modification
+ STAT_CTIME, // status change
+};
+
+// Returns seconds from a stat buffer.
+int StatSeconds(const struct stat &statbuf, StatTimes t);
+
+// Returns nanoseconds from a stat buffer.
+int StatNanoSeconds(const struct stat &statbuf, StatTimes t);
+
+// Runs getxattr(2), if available. If not, sets errno to ENOSYS.
+ssize_t portable_getxattr(const char *path, const char *name, void *value,
+ size_t size);
+
+// Run lgetxattr(2), if available. If not, sets errno to ENOSYS.
+ssize_t portable_lgetxattr(const char *path, const char *name, void *value,
+ size_t size);
+
+#endif // JAVA_COM_GOOGLE_DEVTOOLS_BUILD_LIB_UNIX_UNIX_JNI_H__
diff --git a/src/main/native/unix_jni_darwin.cc b/src/main/native/unix_jni_darwin.cc
new file mode 100644
index 0000000..3d7063d
--- /dev/null
+++ b/src/main/native/unix_jni_darwin.cc
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "unix_jni.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include <string>
+
+using std::string;
+
+// See unix_jni.h.
+string ErrorMessage(int error_number) {
+ char buf[1024] = "";
+ if (strerror_r(error_number, buf, sizeof buf) < 0) {
+ snprintf(buf, sizeof buf, "strerror_r(%d): errno %d", error_number, errno);
+ }
+
+ return string(buf);
+}
+
+int portable_fstatat(int dirfd, char *name, struct stat *statbuf, int flags) {
+ errno = ENOSYS;
+ return -1;
+}
+
+int StatSeconds(const struct stat &statbuf, StatTimes t) {
+ switch (t) {
+ case STAT_ATIME:
+ return statbuf.st_atime;
+ case STAT_CTIME:
+ return statbuf.st_ctime;
+ case STAT_MTIME:
+ return statbuf.st_mtime;
+ default:
+ CHECK(false);
+ }
+}
+
+int StatNanoSeconds(const struct stat &statbuf, StatTimes t) {
+ switch (t) {
+ case STAT_ATIME:
+ return statbuf.st_atimespec.tv_nsec;
+ case STAT_CTIME:
+ return statbuf.st_ctimespec.tv_nsec;
+ case STAT_MTIME:
+ return statbuf.st_mtimespec.tv_nsec;
+ default:
+ CHECK(false);
+ }
+}
+
+ssize_t portable_getxattr(const char *path, const char *name, void *value,
+ size_t size) {
+ errno = ENOSYS;
+ return -1;
+}
+
+ssize_t portable_lgetxattr(const char *path, const char *name, void *value,
+ size_t size) {
+ errno = ENOSYS;
+ return -1;
+}
diff --git a/src/main/native/unix_jni_linux.cc b/src/main/native/unix_jni_linux.cc
new file mode 100644
index 0000000..3562adb
--- /dev/null
+++ b/src/main/native/unix_jni_linux.cc
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "unix_jni.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <sys/xattr.h>
+
+#include <string>
+
+std::string ErrorMessage(int error_number) {
+ char buf[1024] = "";
+
+ // In its infinite wisdom, GNU libc defines strerror_r with extended
+ // functionality which is not compatible with not the
+ // SUSv3-conformant one which returns an error code; see DESCRIPTION
+ // at strerror(1).
+ return std::string(strerror_r(error_number, buf, sizeof buf));
+}
+
+int portable_fstatat(int dirfd, char *name, struct stat *statbuf, int flags) {
+ return fstatat(dirfd, name, statbuf, flags);
+}
+
+int StatSeconds(const struct stat &statbuf, StatTimes t) {
+ switch (t) {
+ case STAT_ATIME:
+ return statbuf.st_atim.tv_sec;
+ case STAT_CTIME:
+ return statbuf.st_ctim.tv_sec;
+ case STAT_MTIME:
+ return statbuf.st_mtim.tv_sec;
+ default:
+ CHECK(false);
+ }
+ return 0;
+}
+
+int StatNanoSeconds(const struct stat &statbuf, StatTimes t) {
+ switch (t) {
+ case STAT_ATIME:
+ return statbuf.st_atim.tv_nsec;
+ case STAT_CTIME:
+ return statbuf.st_ctim.tv_nsec;
+ case STAT_MTIME:
+ return statbuf.st_mtim.tv_nsec;
+ default:
+ CHECK(false);
+ }
+ return 0;
+}
+
+ssize_t portable_getxattr(const char *path, const char *name, void *value,
+ size_t size) {
+ return ::getxattr(path, name, value, size);
+}
+
+ssize_t portable_lgetxattr(const char *path, const char *name, void *value,
+ size_t size) {
+ return ::lgetxattr(path, name, value, size);
+}
diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD
new file mode 100644
index 0000000..3a82b24
--- /dev/null
+++ b/src/main/protobuf/BUILD
@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+load("tools/build_rules/genproto", "genproto")
+
+[genproto(
+ name = "proto_" + proto_file,
+ src = proto_file + ".proto",
+) for proto_file in [
+ "build",
+ "deps",
+ "crosstool_config",
+ "extra_actions_base",
+ "test_status",
+ "bundlemerge",
+ "xcodegen",
+]]
diff --git a/src/main/protobuf/build.proto b/src/main/protobuf/build.proto
new file mode 100644
index 0000000..b5d6346
--- /dev/null
+++ b/src/main/protobuf/build.proto
@@ -0,0 +1,467 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This file contains the protocol buffer representation of a build
+// file or 'blaze query --output=proto' call.
+
+syntax = "proto2";
+
+package blaze_query;
+
+
+option java_package = "com.google.devtools.build.lib.query2.proto.proto2api";
+
+message License {
+ repeated string license_type = 1;
+ repeated string exception = 2;
+}
+
+message StringDictEntry {
+ required string key = 1;
+ required string value = 2;
+}
+
+message StringDictUnaryEntry {
+ required string key = 1;
+ required string value = 2;
+}
+
+message LabelListDictEntry {
+ required string key = 1;
+ repeated string value = 2;
+}
+
+message StringListDictEntry {
+ required string key = 1;
+ repeated string value = 2;
+}
+
+// Represents an entry attribute of a Fileset rule in a build file.
+message FilesetEntry {
+ // Indicates what to do when a source file is actually a symlink.
+ enum SymlinkBehavior {
+ COPY = 1;
+ DEREFERENCE = 2;
+ }
+
+ // The label pointing to the source target where files are copied from.
+ required string source = 1;
+
+ // The relative path within the fileset rule where files will be mapped.
+ required string destination_directory = 2;
+
+ // Whether the files= attribute was specified. This is necessary because
+ // no files= attribute and files=[] mean different things.
+ optional bool files_present = 7;
+
+ // A list of file labels to include from the source directory.
+ repeated string file = 3;
+
+ // If this is a fileset entry representing files within the rule
+ // package, this lists relative paths to files that should be excluded from
+ // the set. This cannot contain values if 'file' also has values.
+ repeated string exclude = 4;
+
+ // This field is optional because there will be some time when the new
+ // PB is used by tools depending on blaze query, but the new blaze version
+ // is not yet released.
+ // TODO(bazel-team): Make this field required once a version of Blaze is
+ // released that outputs this field.
+ optional SymlinkBehavior symlink_behavior = 5 [ default=COPY ];
+
+ // The prefix to strip from the path of the files in this FilesetEntry. Note
+ // that no value and the empty string as the value mean different things here.
+ optional string strip_prefix = 6;
+}
+
+// A rule attribute. Each attribute must have a type and can only have one of
+// the various value fields populated. By checking the type, the appropriate
+// value can be extracted - see the comments on each type for the associated
+// value. The order of lists comes from the blaze parsing and if an attribute
+// is of a list type, the associated list should never be empty.
+message Attribute {
+ // Indicates the type of attribute.
+ enum Discriminator {
+ INTEGER = 1; // int_value
+ STRING = 2; // string_value
+ LABEL = 3; // string_value
+ OUTPUT = 4; // string_value
+ STRING_LIST = 5; // string_list_value
+ LABEL_LIST = 6; // string_list_value
+ OUTPUT_LIST = 7; // string_list_value
+ DISTRIBUTION_SET = 8; // string_list_value - order is unimportant
+ LICENSE = 9; // license
+ STRING_DICT = 10; // string_dict_value
+ FILESET_ENTRY_LIST = 11; // fileset_list_value
+ LABEL_LIST_DICT = 12; // label_list_dict_value
+ STRING_LIST_DICT = 13; // string_list_dict_value
+ BOOLEAN = 14; // int, bool and string value
+ TRISTATE = 15; // tristate, int and string value
+ INTEGER_LIST = 16; // int_list_value
+ STRING_DICT_UNARY = 17; // string_dict_unary_value
+ UNKNOWN = 18; // unknown type, use only for build extensions
+ }
+
+ // Values for the TriState field type.
+ enum Tristate {
+ NO = 0;
+ YES = 1;
+ AUTO = 2;
+ }
+
+ // The name of the attribute
+ required string name = 1;
+
+ // The location of the target in the BUILD file in a machine-parseable form.
+ optional Location parseable_location = 12;
+
+ // Whether the attribute was explicitly specified
+ optional bool explicitly_specified = 13;
+
+ // The type of attribute. This message is used for all of the different
+ // attribute types so the discriminator helps for figuring out what is
+ // stored in the message.
+ required Discriminator type = 2;
+
+ // If this attribute has an integer value this will be populated.
+ // Boolean and TriState also use this field as [0,1] and [-1,0,1]
+ // for [false, true] and [auto, no, yes] respectively.
+ optional int32 int_value = 3;
+
+ // If the attribute has a string value this will be populated. Label and
+ // path attributes use this field as the value even though the type may
+ // be LABEL or something else other than STRING.
+ optional string string_value = 5;
+
+ // If the attribute has a boolean value this will be populated
+ optional bool boolean_value = 14;
+
+ // If the attribute is a Tristate value, this will be populated.
+ optional Tristate tristate_value = 15;
+
+ // The value of the attribute has a list of string values (label and path
+ // note from STRING applies here as well).
+ repeated string string_list_value = 6;
+
+ // If this is a license attribute, the license information is stored here.
+ optional License license = 7;
+
+ // If this is a string dict, each entry will be stored here.
+ repeated StringDictEntry string_dict_value = 8;
+
+ // If the attribute is part of a Fileset, the fileset entries are stored in
+ // this field.
+ repeated FilesetEntry fileset_list_value = 9;
+
+ // If this is a label list dict, each entry will be stored here.
+ repeated LabelListDictEntry label_list_dict_value = 10;
+
+ // If this is a string list dict, each entry will be stored here.
+ repeated StringListDictEntry string_list_dict_value = 11;
+
+ // The glob criteria. This is non-empty if
+ // 1. This attribute is a list of strings or labels
+ // 2. It contained a glob() expression
+ repeated GlobCriteria glob_criteria = 16;
+
+ // The value of the attribute has a list of int32 values
+ repeated int32 int_list_value = 17;
+
+ // If this is a string dict unary, each entry will be stored here.
+ repeated StringDictUnaryEntry string_dict_unary_value = 18;
+}
+
+// A rule from a BUILD file (e.g., cc_library, java_binary). The rule class
+// is the actual name of the rule (e.g., cc_library) and the name is the full
+// label of the rule (e.g., //foo/bar:baz).
+message Rule {
+ // The name of the rule
+ required string name = 1;
+
+ // The rule class (e.g., java_library)
+ required string rule_class = 2;
+
+ // The BUILD file and line number of the rule.
+ optional string location = 3;
+
+ // All of the attributes that describe the rule.
+ repeated Attribute attribute = 4;
+
+ // All of the inputs to the rule. These are predecessors in the dependency
+ // graph. A rule_input for a rule should always be described as a
+ // source_file in some package (either the rule's package or some other one).
+ repeated string rule_input = 5;
+
+ // All of the outputs of the rule. These are the successors in the
+ // dependency graph.
+ repeated string rule_output = 6;
+
+ // The set of all default settings affecting this rule. The name of a default
+ // setting is "<setting type>_<setting name>". There currently defined setting
+ // types are:
+ //
+ // - 'blaze': settings implemented in Blaze itself
+ repeated string default_setting = 7;
+
+ // The location of the target in the BUILD file in a machine-parseable form.
+ optional Location parseable_location = 8;
+}
+
+// Summary of all transitive dependencies of 'rule,' where each dependent
+// rule is included only once in the 'dependency' field. Gives complete
+// information to analyze the single build target labeled rule.name,
+// including optional location of target in BUILD file.
+message RuleSummary {
+ required Rule rule = 1;
+ repeated Rule dependency = 2;
+ optional string location = 3;
+}
+
+// A package group. Aside from the name, it contains the list of packages
+// present in the group (as specified in the BUILD file).
+message PackageGroup {
+ // The name of the package group
+ required string name = 1;
+
+ // The list of packages as specified in the BUILD file. Currently this is
+ // only a list of packages, but some time in the future, there might be
+ // some type of wildcard mechanism.
+ repeated string contained_package = 2;
+
+ // The list of sub package groups included in this one.
+ repeated string included_package_group = 3;
+
+ // The location of the target in the BUILD file in a machine-parseable form.
+ optional Location parseable_location = 4;
+}
+
+// A file that is an input into the build system.
+// Next-Id: 8
+message SourceFile {
+ // The name of the source file (a label).
+ required string name = 1;
+
+ // The location of the source file. This is a path with line numbers, not
+ // a label in the build system.
+ optional string location = 2;
+
+ // The location of the corresponding label in the BUILD file in a
+ // machine-parseable form.
+ optional Location parseable_location = 7;
+
+ // Labels of files that are transitively subincluded in this BUILD file. This
+ // is present only when the SourceFile represents a BUILD file that
+ // subincludes other files. The subincluded file can be either a Python
+ // preprocessed build extension or a Skylark file.
+ repeated string subinclude = 3;
+
+ // Labels of package groups that are mentioned in the visibility declaration
+ // for this source file.
+ repeated string package_group = 4;
+
+ // Labels mentioned in the visibility declaration (including :__pkg__ and
+ // //visibility: ones)
+ repeated string visibility_label = 5;
+
+ // The package-level features enabled for this package. Only present if the
+ // SourceFile represents a BUILD file.
+ repeated string feature = 6;
+
+ // License attribute for the file.
+ optional License license = 8;
+}
+
+// A file that is the output of a build rule.
+message GeneratedFile {
+ // The name of the generated file (a label).
+ required string name = 1;
+
+ // The label of the target that generates the file.
+ required string generating_rule = 2;
+
+ // The path of the output file (not a label).
+ optional string location = 3;
+}
+
+// A target from a blaze query execution. Similar to the Attribute message,
+// the Discriminator is used to determine which field contains information.
+// For any given type, only one of rule, input_file or output_file can be
+// populated in a single Target.
+message Target {
+ enum Discriminator {
+ RULE = 1;
+ SOURCE_FILE = 2;
+ GENERATED_FILE = 3;
+ PACKAGE_GROUP = 4;
+ }
+
+ // The type of target contained in the message.
+ required Discriminator type = 1;
+
+ // If this target represents a rule, the rule is stored here.
+ optional Rule rule = 2;
+
+ // A file that is not generated by the build system (version controlled
+ // or created by the test harness)
+ optional SourceFile source_file = 3;
+
+ // A generated file that is the output of a rule
+ optional GeneratedFile generated_file = 4;
+
+ // A package group
+ optional PackageGroup package_group = 5;
+}
+
+// Container for all of the blaze query results.
+message QueryResult {
+ // All of the targets returned by the blaze query.
+ repeated Target target = 1;
+}
+
+////////////////////////////////////////////////////////////////////////////
+// Messages dealing with querying the BUILD language itself. For now, this is
+// quite simplistic: Blaze can only tell the names of the rule classes, their
+// attributes with their type.
+
+// Information about allowed rule classes for a specific attribute of a rule.
+message AllowedRuleClassInfo {
+ enum AllowedRuleClasses {
+ ANY = 1; // Any rule is allowed to be in this attribute
+ SPECIFIED = 2; // Only the explicitly listed rules are allowed
+ }
+
+ required AllowedRuleClasses policy = 1;
+
+ // Rule class names of rules allowed in this attribute, e.g "cc_library",
+ // "py_binary". Only present if the allowed_rule_classes field is set to
+ // SPECIFIED.
+ repeated string allowed_rule_class = 2;
+}
+
+// This message represents a single attribute of a single rule.
+message AttributeDefinition {
+
+ // Rule name, e.g. "cc_library"
+ required string name = 1;
+ required Attribute.Discriminator type = 2;
+ required bool mandatory = 3;
+
+ // Only present for attributes of type LABEL and LABEL_LIST.
+ optional AllowedRuleClassInfo allowed_rule_classes = 4;
+
+ optional string documentation = 5;
+}
+
+message RuleDefinition {
+ required string name = 1;
+ // Only contains documented attributes
+ repeated AttributeDefinition attribute = 2;
+ optional string documentation = 3;
+ // Only for build extensions: label to file that defines the extension
+ optional string label = 4;
+}
+
+message BuildLanguage {
+ // Only contains documented rule definitions
+ repeated RuleDefinition rule = 1;
+}
+
+message Location {
+ optional int32 start_offset = 1;
+ optional int32 start_line = 2;
+ optional int32 start_column = 3;
+ optional int32 end_offset = 4;
+ optional int32 end_line = 5;
+ optional int32 end_column = 6;
+}
+
+message MakeVarBinding {
+ required string value = 1;
+ required string platform_set_regexp = 2;
+}
+
+message MakeVar {
+ required string name = 1;
+ repeated MakeVarBinding binding = 2;
+}
+
+message GlobCriteria {
+ // List of includes (or items if this criteria did not come from a glob)
+ repeated string include = 1;
+
+ // List of exclude expressions
+ repeated string exclude = 2;
+
+ // Whether this message came from a glob
+ optional bool glob = 3;
+}
+
+message Event {
+ enum EventKind {
+ ERROR = 1;
+ WARNING = 2;
+ INFO = 3;
+ PROGRESS = 4;
+ }
+
+ required EventKind kind = 1;
+ optional Location location = 2;
+ optional string message = 3;
+}
+
+message Package {
+ enum State {
+ PRESENT = 1; // The package is present and has changed at this CL
+ TOMBSTONE = 2; // The package got deleted at this CL
+ UNKNOWN = 3; // The database does not contain enough information to know
+ }
+
+ required string name = 1;
+ optional string repository = 2;
+ optional string build_file_path = 3;
+
+ // Default values
+ repeated string default_visibility_label = 1001;
+ optional bool default_obsolete = 1002;
+ optional bool default_testonly = 1003;
+ optional string default_deprecation = 1004;
+ optional string default_strict_java_deps = 1005;
+ repeated string default_copt = 1006;
+ optional string default_hdrs_check = 1007;
+ optional License default_license = 1008;
+ repeated string default_distrib = 1009;
+ optional bool default_visibility_set = 1010;
+
+ // Package-level data
+ repeated string default_setting = 2002;
+ repeated string subinclude_label = 2003;
+ repeated MakeVar make_variable = 2004;
+ repeated string file_system_dep = 2005;
+ optional bool depends_on_external_files = 2006;
+ optional bool dependencies_recorded = 2007;
+ optional bool contains_errors = 2008;
+ optional bool contains_temporary_errors = 2009;
+ repeated string skylark_label = 2010;
+
+ // Targets
+ repeated SourceFile source_file = 3001;
+ repeated PackageGroup package_group = 3002;
+ repeated Rule rule = 3003;
+
+ // Metadata
+ // Reason why the package could not be serialized.
+ optional string failure_reason = 4001;
+ optional State state = 4002;
+ repeated Event event = 4003;
+}
diff --git a/src/main/protobuf/bundlemerge.proto b/src/main/protobuf/bundlemerge.proto
new file mode 100644
index 0000000..b04eba3
--- /dev/null
+++ b/src/main/protobuf/bundlemerge.proto
@@ -0,0 +1,99 @@
+syntax = "proto2";
+
+package devtools.xcode;
+option java_outer_classname = "BundleMergeProtos";
+option java_package = "com.google.devtools.build.xcode.bundlemerge.proto";
+
+// Contains all the arguments necessary to drive the BundleMerge tool,
+// including the path to the output file and extra files to include in the
+// bundle.
+message Control {
+ // Paths to the plist files to merge into the final Plist.info file. These
+ // can be binary, XML, or ASCII format.
+ repeated string source_plist_file = 1;
+
+ // Path to the .ipa file to write. This is the final application bundle. Note
+ // this is ignored for nested bundles.
+ optional string out_file = 2;
+
+ // Which devices the app targets, which corresponds to the UIDeviceFamily
+ // setting. Should be one or more of the symbols in the
+ // com.google.devtools.build.xcode.common.TargetDeviceFamily enum (e.g.
+ // "IPAD", "IPHONE").
+ repeated string target_device_family = 3;
+
+ // One of the symbols in the com.google.devtools.build.xcode.common.Platform
+ // (e.g. "DEVICE", "SIMULATOR").
+ optional string platform = 4;
+
+ // The version of the SDK used to build the app.
+ optional string sdk_version = 5;
+
+ // Earliest iOS version on which the app will run.
+ optional string minimum_os_version = 6;
+
+ // The directory inside which all files are placed in the final bundle.
+ // For the top-most bundle, this is generally "Payload/{app_name}.app". If
+ // this bundle is nested, then this bundle_root is relative to to the parent's
+ // bundle_root.
+ optional string bundle_root = 7;
+
+ // All files to put in the bundle besides automatically-generated files such
+ // as Info.plist and PkgInfo. This should include the application binary.
+ repeated BundleFile bundle_file = 8;
+
+ repeated string merge_without_name_prefix_zip = 9 [deprecated=true];
+
+ // Zip files to merge with the final zip. Note that bundle_root is ignored
+ // when merging zips, so to place items in the bundle root, you should do one
+ // of the following:
+ // 1. make the merge_zips have entries that are named "{bundle_root}/foo"
+ // rather than just "foo"
+ // 2. set the root property on the MergeZip protobuf to {bundle_root}
+ // Note that the paths of these zips are always relative to the zip file root
+ // - they are not relative to the containing bundle.
+ repeated MergeZip merge_zip = 10;
+
+ // Variable substitutions to perform on property values in the merged
+ // .plist file.
+ repeated VariableSubstitution variable_substitution = 11;
+
+ // Bundles that are nested within this one. bundle_root in these bundles is
+ // relative to the containing bundle's bundle_root.
+ repeated Control nested_bundle = 12;
+
+ // Name of the executable for this bundle or unset if no such executable
+ // exists.
+ optional string executable_name = 13;
+}
+
+// Represents a zip file to merge with the final zip.
+message MergeZip {
+ // The prefix to prepend to every entry name before putting it in the final
+ // zip. For instance, "Payload/Foo.app/" (notice the final slash).
+ optional string entry_name_prefix = 1 [default = ""];
+
+ // The path to the source file to merge.
+ optional string source_path = 2;
+}
+
+message BundleFile {
+ // The path of the file to put in the bundle.
+ optional string source_file = 1;
+
+ // The path of the file in the bundle, relative to the bundle root.
+ optional string bundle_path = 2;
+
+ // The external file attribute field in the central directory record in the
+ // .zip file. If omitted, the ZipInputEntry.DEFAULT_EXTERNAL_FILE_ATTRIBUTE
+ // constant is used.
+ optional int32 external_file_attribute = 3;
+}
+
+message VariableSubstitution {
+ // The name of the varaible to substitute.
+ required string name = 1;
+
+ // The substitution value.
+ required string value = 2;
+}
diff --git a/src/main/protobuf/crosstool_config.proto b/src/main/protobuf/crosstool_config.proto
new file mode 100644
index 0000000..d95aa13
--- /dev/null
+++ b/src/main/protobuf/crosstool_config.proto
@@ -0,0 +1,352 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// File format for Blaze to configure Crosstool releases.
+
+syntax = "proto2";
+
+option java_package = "com.google.devtools.build.lib.view.config.crosstool";
+
+
+package com.google.devtools.build.lib.view.config.crosstool;
+
+// A description of a toolchain, which includes all the tools generally expected
+// to be available for building C/C++ targets, based on the GNU C compiler.
+//
+// System and cpu names are two overlapping concepts, which need to be both
+// supported at this time. The cpu name is the blaze command-line name for the
+// target system. The most common values are 'k8' and 'piii'. The system name is
+// a more generic identification of the executable system, based on the names
+// used by the GNU C compiler.
+//
+// Typically, the system name contains an identifier for the cpu (e.g. x86_64 or
+// alpha), an identifier for the machine (e.g. pc, or unknown), and an
+// identifier for the operating system (e.g. cygwin or linux-gnu). Typical
+// examples are 'x86_64-unknown-linux-gnu' and 'i686-unknown-cygwin'.
+//
+// The system name is used to determine if a given machine can execute a given
+// exectuable. In particular, it is used to check if the compilation products of
+// a toolchain can run on the host machine.
+message CToolchain {
+ // Optional flag. Some option lists have a corresponding optional flag list.
+ // These are appended at the end of the option list in case the corresponding
+ // default setting is set.
+ message OptionalFlag {
+ required string default_setting_name = 1;
+ repeated string flag = 2;
+ }
+
+ // A group of correlated flags. Supports parametrization via variable
+ // expansion.
+ // If a variable of list type is expanded, all flags in the flag group
+ // will be expanded for each element of the list.
+ // Only a single variable of list type may be referenced in a single flag
+ // group; the same variable may be referenced multiple times.
+ //
+ // For example:
+ // flag_group {
+ // flag: '-I'
+ // flag: '%{include_path}'
+ // }
+ // ... will get expanded to -I /to/path1 -I /to/path2 ... for each
+ // include_path /to/pathN.
+ //
+ // TODO(bazel-team): Write more elaborate documentation and add a link to it.
+ message FlagGroup {
+ repeated string flag = 1;
+ }
+
+ // A set of features; used to support logical 'and' when specifying feature
+ // requirements in FlagSet and Feature.
+ message FeatureSet {
+ repeated string feature = 1;
+ }
+
+ // A set of flags that are expanded in the command line for specific actions.
+ message FlagSet {
+ // The actions this flag set applies to; each flag set must specify at
+ // least one action.
+ repeated string action = 1;
+
+ // The flags applied via this flag set.
+ repeated FlagGroup flag_group = 2;
+
+ // A list of feature sets defining when this flag set gets applied. The
+ // flag set will be applied when any of the feature sets fully apply, that
+ // is, when all features of the feature set are enabled.
+ //
+ // If 'with_feature' is omitted, the flag set will be applied
+ // unconditionally for every action specified.
+ repeated FeatureSet with_feature = 3;
+ }
+
+ // Contains all flag specifications for one feature.
+ message Feature {
+ // The feature's name. Feature names are generally defined by Bazel; it is
+ // possible to introduce a feature without a change to Bazel by adding a
+ // 'feature' section to the toolchain and adding the corresponding string as
+ // feature in the BUILD file.
+ optional string name = 1;
+
+ // If the given feature is enabled, the flag sets will be applied for the
+ // actions in the modes that they are specified for.
+ repeated FlagSet flag_set = 2;
+
+ // A list of feature sets defining when this feature is supported by the
+ // toolchain. The feature is supported if any of the feature sets fully
+ // apply, that is, when all features of a feature set are enabled.
+ //
+ // If 'requires' is omitted, the feature is supported independently of which
+ // other features are enabled.
+ //
+ // Use this for example to filter flags depending on the build mode enabled
+ // (opt / fastbuild / dbg).
+ repeated FeatureSet requires = 3;
+
+ // A list of features that are automatically enabled when this feature is
+ // enabled. If any of the implied features cannot be enabled, this feature
+ // will (silently) not be enabled either.
+ repeated string implies = 4;
+
+ // A list of names this feature conflicts with.
+ // A feature cannot be enabled if:
+ // - 'provides' contains the name of a different feature that we want to
+ // enable.
+ // - 'provides' contains the same value as a 'provides' in a different
+ // feature that we want to enable.
+ //
+ // Use this in order to ensure that incompatible features cannot be
+ // accidentally activated at the same time, leading to hard to diagnose
+ // compiler errors.
+ repeated string provides = 5;
+ }
+
+ repeated Feature feature = 50;
+
+ // The unique identifier of the toolchain within the crosstool release. It
+ // must be possible to use this as a directory name in a path.
+ // It has to match the following regex: [a-zA-Z_][\.\- \w]*
+ required string toolchain_identifier = 1;
+
+ // A basic toolchain description.
+ required string host_system_name = 2;
+ required string target_system_name = 3;
+ required string target_cpu = 4;
+ required string target_libc = 5;
+ required string compiler = 6;
+
+ required string abi_version = 7;
+ required string abi_libc_version = 8;
+
+ // Tool locations. Relative paths are resolved relative to the configuration
+ // file directory.
+ repeated ToolPath tool_path = 9;
+
+ // Feature flags.
+ // TODO(bazel-team): Sink those into 'Feature' instances.
+ optional bool supports_gold_linker = 10 [default = false];
+ optional bool supports_thin_archives = 11 [default = false];
+ optional bool supports_start_end_lib = 28 [default = false];
+ optional bool supports_interface_shared_objects = 32 [default = false];
+ optional bool supports_embedded_runtimes = 40 [default = false];
+ // If specified, Blaze finds statically linked / dynamically linked runtime
+ // libraries in the declared crosstool filegroup. Otherwise, Blaze
+ // looks in "[static|dynamic]-runtime-libs-$TARGET_CPU".
+ optional string static_runtimes_filegroup = 45;
+ optional string dynamic_runtimes_filegroup = 46;
+ optional bool supports_incremental_linker = 41 [default = false];
+ // This should be true, if the toolchain supports the D flag to ar, which
+ // makes it output normalized archives that don't contain timestamps.
+ optional bool supports_normalizing_ar = 26 [default = false];
+ optional bool supports_fission = 43 [default = false];
+ optional bool needsPic = 12 [default = false];
+
+ // Compiler flags for C/C++/Asm compilation.
+ repeated string compiler_flag = 13;
+ repeated OptionalFlag optional_compiler_flag = 35;
+ // Additional compiler flags for C++ compilation.
+ repeated string cxx_flag = 14;
+ repeated OptionalFlag optional_cxx_flag = 36;
+ // Additional unfiltered compiler flags for C/C++/Asm compilation.
+ // These are not subject to nocopt filtering in cc_* rules.
+ repeated string unfiltered_cxx_flag = 25;
+ repeated OptionalFlag optional_unfiltered_cxx_flag = 37;
+ // Linker flags.
+ repeated string linker_flag = 15;
+ repeated OptionalFlag optional_linker_flag = 38;
+ // Additional linker flags when linking dynamic libraries.
+ repeated string dynamic_library_linker_flag = 27;
+ repeated OptionalFlag optional_dynamic_library_linker_flag = 39;
+ // Additional test-only linker flags.
+ repeated string test_only_linker_flag = 49;
+ // Objcopy flags for embedding files into binaries.
+ repeated string objcopy_embed_flag = 16;
+ // Ld flags for embedding files into binaries. This is used by filewrapper
+ // since it calls ld directly and needs to know what -m flag to pass.
+ repeated string ld_embed_flag = 23;
+ // Ar flags for combining object files into archives. If this is not set, it
+ // defaults to "rcsD".
+ repeated string ar_flag = 47;
+ // Ar flags for combining object files into archives when thin_archives is
+ // enabled. If this is not set, it defaults to "rcsDT".
+ repeated string ar_thin_archives_flag = 48;
+ // Additional compiler flags that are added for cc_plugin rules of type 'gcc'
+ repeated string gcc_plugin_compiler_flag = 34;
+
+ // Additional compiler and linker flags depending on the compilation mode.
+ repeated CompilationModeFlags compilation_mode_flags = 17;
+
+ // Additional compiler and linker flags depending on the LIPO mode.
+ repeated LipoModeFlags lipo_mode_flags = 44;
+
+ // Additional linker flags depending on the linking mode.
+ repeated LinkingModeFlags linking_mode_flags = 18;
+
+ // Plugin header directories for gcc and mao plugins. If none are set, the
+ // toolchain does not support plugins. Relative paths are resolved relative
+ // to the configuration file directory.
+ repeated string gcc_plugin_header_directory = 19;
+ repeated string mao_plugin_header_directory = 20;
+
+ // Make variables that are made accessible to rules.
+ repeated MakeVariable make_variable = 21;
+
+ // Built-in include directories for C++ compilation. These should be the exact
+ // paths used by the compiler, and are generally relative to the exec root.
+ // The paths used by the compiler can be determined by 'gcc -Wp,-v some.c'.
+ // We currently use the C++ paths also for C compilation, which is safe as
+ // long as there are no name clashes between C++ and C header files.
+ //
+ // Relative paths are resolved relative to the configuration file directory.
+ //
+ // If the compiler has --sysroot support, then these paths should use
+ // %sysroot% rather than the include path, and specify the sysroot attribute
+ // in order to give blaze the information necessary to make the correct
+ // replacements.
+ repeated string cxx_builtin_include_directory = 22;
+
+ // The built-in sysroot. If this attribute is not present, blaze does not
+ // allow using a different sysroot, i.e. through the --grte_top option. Also
+ // see the documentation above.
+ optional string builtin_sysroot = 24;
+
+ // The location and version of the default Python (in absence of
+ // --python_top and --python_version, respectively. The default
+ // --python_mode is always 'opt'.) For backward compatibility, if these
+ // attributes are not set, Blaze will use the crosstool v11-13 default
+ // values: "/usr/grte/v1" and "python2.4".
+ optional string default_python_top = 29;
+ optional string default_python_version = 30;
+ // Whether to preload swigdeps.so files in py_binaries and PAR files.
+ // This overrides the commandline flag.
+ optional bool python_preload_swigdeps = 42;
+
+ // The default GRTE to use. This should be a label, and gets the same
+ // treatment from Blaze as the --grte_top option. This setting is only used in
+ // the absence of an explicit --grte_top option. If unset, Blaze will not pass
+ // -sysroot by default. The local part must be 'everything', i.e.,
+ // '//some/label:everything'. There can only be one GRTE library per package,
+ // because the compiler expects the directory as a parameter of the -sysroot
+ // option.
+ // This may only be set to a non-empty value if builtin_sysroot is also set!
+ optional string default_grte_top = 31;
+
+ // Additional dependencies for Blaze-built .deb packages. All Debian packages
+ // that contain C++ binaries need to have the correct runtime
+ // libraries installed, and those depend on the crosstool version, which is
+ // why they are recorded here.
+ repeated string debian_extra_requires = 33;
+
+ // Next free id: 51
+}
+
+message ToolPath {
+ required string name = 1;
+ required string path = 2;
+}
+
+enum CompilationMode {
+ FASTBUILD = 1;
+ DBG = 2;
+ OPT = 3;
+ // This value is ignored and should not be used in new files.
+ COVERAGE = 4;
+}
+
+message CompilationModeFlags {
+ required CompilationMode mode = 1;
+ repeated string compiler_flag = 2;
+ repeated string cxx_flag = 3;
+ // Linker flags that are added when compiling in a certain mode.
+ repeated string linker_flag = 4;
+}
+
+enum LinkingMode {
+ FULLY_STATIC = 1;
+ MOSTLY_STATIC = 2;
+ DYNAMIC = 3;
+}
+
+message LinkingModeFlags {
+ required LinkingMode mode = 1;
+ repeated string linker_flag = 2;
+}
+
+enum LipoMode {
+ OFF = 1;
+ BINARY = 2;
+ // LIBRARY = 3; // RESERVED
+}
+
+message LipoModeFlags {
+ required LipoMode mode = 1;
+ repeated string compiler_flag = 2;
+ repeated string cxx_flag = 3;
+ repeated string linker_flag = 4;
+}
+
+message MakeVariable {
+ required string name = 1;
+ required string value = 2;
+}
+
+message DefaultCpuToolchain {
+ required string cpu = 1;
+ required string toolchain_identifier = 2;
+}
+
+// An entire crosstool release, containing the version number, a default target
+// cpu, a default toolchain for each supported cpu type, and a set of
+// toolchains.
+message CrosstoolRelease {
+ message DefaultSetting {
+ required string name = 1;
+ required bool default_value = 2;
+ }
+
+ // The major and minor version of the crosstool release.
+ required string major_version = 1;
+ required string minor_version = 2;
+
+ // The default cpu to use if none was specified.
+ required string default_target_cpu = 3;
+ // The default toolchain to use for each given cpu.
+ repeated DefaultCpuToolchain default_toolchain = 4;
+
+ // The default settings used in this release.
+ repeated DefaultSetting default_setting = 6;
+
+ // All the toolchains in this release.
+ repeated CToolchain toolchain = 5;
+}
diff --git a/src/main/protobuf/deps.proto b/src/main/protobuf/deps.proto
new file mode 100644
index 0000000..dee7949
--- /dev/null
+++ b/src/main/protobuf/deps.proto
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Definitions for dependency reports.
+
+syntax = "proto2";
+
+package blaze_deps;
+
+option java_package = "com.google.devtools.build.lib.view.proto";
+
+// A specific location within a source file.
+message SourceLocation {
+ required string path = 1;
+ optional int32 line = 2;
+ optional int32 column = 3;
+}
+
+message Dependency {
+
+ enum Kind {
+ // Direct dependency used explicitly in the source.
+ EXPLICIT = 0;
+ // Transitive dependency that is implicitly loaded and used by the compiler.
+ IMPLICIT = 1;
+ // Unused dependency.
+ UNUSED = 2;
+ }
+
+ // Path to the artifact representing this dependency.
+ required string path = 1;
+
+ // Dependency kind
+ required Kind kind = 2;
+
+ // Source file locations: compilers can pinpoint the uses of a dependency.
+ repeated SourceLocation location = 3;
+}
+
+// Top-level message found in .deps artifacts
+message Dependencies {
+ repeated Dependency dependency = 1;
+
+ // Name of the rule being analyzed.
+ optional string rule_label = 2;
+
+ // Whether the action was successful; even when compilation fails, partial
+ // dependency information can be useful.
+ optional bool success = 3;
+}
diff --git a/src/main/protobuf/extra_actions_base.proto b/src/main/protobuf/extra_actions_base.proto
new file mode 100644
index 0000000..bc3b69a
--- /dev/null
+++ b/src/main/protobuf/extra_actions_base.proto
@@ -0,0 +1,129 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// proto definitions for the blaze extra_action feature.
+
+syntax = "proto2";
+
+package blaze;
+
+option java_multiple_files = true;
+option java_package = "com.google.devtools.build.lib.actions.extra";
+
+// A list of extra actions and metadata for the print_action command.
+message ExtraActionSummary {
+ repeated DetailedExtraActionInfo action = 1;
+}
+
+// An individual action printed by the print_action command.
+message DetailedExtraActionInfo {
+ // If the given action was included in the output due to a request for a
+ // specific file, then this field contains the name of that file so that the
+ // caller can correctly associate the extra action with that file.
+ //
+ // The data in this message is currently not sufficient to run the action on a
+ // production machine, because not all necessary input files are identified,
+ // especially for C++.
+ //
+ // There is no easy way to fix this; we could require that all header files
+ // are declared and then add all of them here (which would be a huge superset
+ // of the files that are actually required), or we could run the include
+ // scanner and add those files here.
+ optional string triggering_file = 1;
+ // The actual action.
+ required ExtraActionInfo action = 2;
+}
+
+// Provides information to an extra_action on the original action it is
+// shadowing.
+message ExtraActionInfo {
+ extensions 1000 to max;
+
+ // The label of the ActionOwner of the shadowed action.
+ optional string owner = 1;
+ // An id uniquely describing the shadowed action at the ActionOwner level.
+ optional string id = 2;
+
+ // The mnemonic of the shadowed action. Used to distinguish actions with the
+ // same ActionType.
+ optional string mnemonic = 5;
+}
+
+message EnvironmentVariable {
+ required string name = 1;
+ required string value = 2;
+}
+
+// provides access to data that is specific to spawn actions.
+// Usually provided by actions using the "Spawn" & "Genrule" Mnemonics.
+message SpawnInfo {
+ extend ExtraActionInfo {
+ optional SpawnInfo spawn_info = 1003;
+ }
+
+ repeated string argument = 1;
+ repeated EnvironmentVariable variable = 2;
+ repeated string input_file = 4;
+ repeated string output_file = 5;
+}
+
+// Provides access to data that is specific to C++ compile actions.
+// Usually provided by actions using the "CppCompile" Mnemonic.
+message CppCompileInfo {
+ extend ExtraActionInfo {
+ optional CppCompileInfo cpp_compile_info = 1001;
+ }
+
+ optional string tool = 1;
+ repeated string compiler_option = 2;
+ optional string source_file = 3;
+ optional string output_file = 4;
+ // Due to header discovery, this won't include headers unless the build is
+ // actually performed. If set, this field will include the value of
+ // "source_file" in addition to the headers.
+ repeated string sources_and_headers = 5;
+}
+
+// Provides access to data that is specific to C++ link actions.
+// Usually provided by actions using the "CppLink" Mnemonic.
+message CppLinkInfo {
+ extend ExtraActionInfo {
+ optional CppLinkInfo cpp_link_info = 1002;
+ }
+
+ repeated string input_file = 1;
+ optional string output_file = 2;
+ optional string interface_output_file = 3;
+ optional string link_target_type = 4;
+ optional string link_staticness = 5;
+ repeated string link_stamp = 6;
+ repeated string build_info_header_artifact = 7;
+ repeated string link_opt = 8;
+}
+
+// Provides access to data that is specific to java compile actions.
+// Usually provided by actions using the "Javac" Mnemonic.
+message JavaCompileInfo {
+ extend ExtraActionInfo {
+ optional JavaCompileInfo java_compile_info = 1000;
+ }
+
+ optional string outputjar = 1;
+ repeated string classpath = 2;
+ repeated string sourcepath = 3;
+ repeated string source_file = 4;
+ repeated string javac_opt = 5;
+ repeated string processor = 6;
+ repeated string processorpath = 7;
+}
diff --git a/src/main/protobuf/test_status.proto b/src/main/protobuf/test_status.proto
new file mode 100644
index 0000000..485a3e1
--- /dev/null
+++ b/src/main/protobuf/test_status.proto
@@ -0,0 +1,110 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package blaze;
+
+option java_package = "com.google.devtools.build.lib.view.test";
+
+// Status data of test cases which failed (used only for printing test summary)
+enum FailedTestCasesStatus {
+ /** Information about every test case is available. */
+ FULL = 1;
+ /** Information about some test cases may be missing. */
+ PARTIAL = 2;
+ /** No information about individual test cases. */
+ NOT_AVAILABLE = 3;
+ /** This is an empty object still without data. */
+ EMPTY = 4;
+};
+
+// Detailed status data for a TestRunnerAction execution.
+enum BlazeTestStatus {
+ NO_STATUS = 0;
+ PASSED = 1;
+ FLAKY = 2;
+ TIMEOUT = 3;
+ FAILED = 4;
+ INCOMPLETE = 5;
+ REMOTE_FAILURE = 6;
+ FAILED_TO_BUILD = 7;
+ BLAZE_HALTED_BEFORE_TESTING = 8;
+};
+
+// TestCase contains detailed data about all tests (cases/suites/decorators)
+// ran, structured in a tree. This data will be later used to present the tests
+// by the web status server.
+message TestCase {
+ enum Type {
+ TEST_CASE = 0;
+ TEST_SUITE = 1;
+ TEST_DECORATOR = 2;
+ UNKNOWN = 3;
+ }
+
+ enum Status {
+ PASSED = 0;
+ FAILED = 1;
+ ERROR = 2;
+ }
+
+ repeated TestCase child = 1;
+ optional string name = 2;
+ optional string class_name = 3;
+ optional int64 run_duration_millis = 4;
+ optional string result = 5;
+ optional Type type = 6;
+ optional Status status = 7;
+ optional bool run = 8 [default = true];
+};
+
+// TestResultData holds the outcome data for a single test action (A
+// single test rule can result in multiple actions due to sharding and
+// runs_per_test settings.)
+message TestResultData {
+ // The following two fields are used for TestRunnerAction caching.
+ // This reflects the fact that failing tests are successful
+ // actions that might be cached, depending on option settings.
+ optional bool cachable = 1;
+ optional bool test_passed = 2;
+
+ // Following data is informational.
+ optional BlazeTestStatus status = 3 [default = NO_STATUS];
+ repeated string failed_logs = 4;
+ repeated string warning = 5;
+ optional bool has_coverage = 6;
+
+ // Returns if this was cached in remote execution.
+ optional bool remotely_cached = 7;
+
+ // Returns true if this was executed remotely
+ optional bool is_remote_strategy = 8;
+
+ // All associated test times (in ms).
+ repeated int64 test_times = 9;
+
+ // Passed log paths. Set if the test passed.
+ optional string passed_log = 10;
+
+ // Test times, without remote execution overhead (in ms).
+ repeated int64 test_process_times = 11;
+
+ // Total time in ms.
+ optional int64 run_duration_millis = 12;
+
+ // Additional build info
+ optional TestCase test_case = 13;
+ optional FailedTestCasesStatus failed_status = 14;
+};
diff --git a/src/main/protobuf/xcodegen.proto b/src/main/protobuf/xcodegen.proto
new file mode 100644
index 0000000..1d00fbc
--- /dev/null
+++ b/src/main/protobuf/xcodegen.proto
@@ -0,0 +1,164 @@
+syntax = "proto2";
+
+package build.xcode;
+option java_outer_classname = "XcodeGenProtos";
+option java_package = "com.google.devtools.build.xcode.xcodegen.proto";
+
+// Protos named "*Control" are protos that can be used to tell Xcodegen what to
+// do.
+
+// Contains all information needed to pass to Xcodegen for it to generate a
+// project file.
+message Control {
+ // Relative path to the project.pbxproj file. This value is used to construct
+ // relative paths in the xcodeproj file, as well as to specify where to write
+ // the resulting project file.
+ optional string pbxproj = 1;
+
+ // The targets to generate.
+ repeated TargetControl target = 2;
+
+ // Project-level build settings. These are shared with every target, unless
+ // an individual target overrides any of them.
+ repeated XcodeprojBuildSetting build_setting = 3;
+}
+
+// Information about dependency on a separate target.
+message DependencyControl {
+ // Label of the target in the project file that is depended upon.
+ optional string target_label = 1;
+
+ // Set to true if this dependency is that of a xctest target on its TEST_HOST.
+ // Considered 'false' if omitted.
+ optional bool test_host = 2;
+}
+
+// Contains information specific to a single target in the project file.
+// Next ID: 23
+message TargetControl {
+ // 'Name' of the target in the project file. This corresponds to the product
+ // name, and does not have to be unique.
+ optional string name = 1;
+
+ // Relative paths to source files to compile. Each source file is added to
+ // the sources build phase of the target, so any file accepted by Xcode is
+ // valid.
+ repeated string source_file = 2;
+
+ // Identical to source_file, but only Objective-C[++] source files are
+ // valid. These sources are compiled without ARC.
+ repeated string non_arc_source_file = 6;
+
+ // Relative paths to support files, such as BUILD, header, and non-compiled
+ // source files. These are included in the project so they can be opened in
+ // the project explorer view, but are not compiled into object files.
+ repeated string support_file = 3;
+
+ // Dependencies this target has on other targets.
+ repeated DependencyControl dependency = 4;
+
+ // (Used for objc_binary and objc_bundle_library targets) path to the
+ // -Info.plist file for the application.
+ optional string infoplist = 5;
+
+ // Label corresponding to the target. This is needed to determine a
+ // unique name for the target in the project.
+ optional string label = 7;
+
+ // Additional flags to pass to the [objective] c[++] compiler.
+ repeated string copt = 8;
+
+ // Additional flags to pass to the linker.
+ repeated string linkopt = 22;
+
+ // Paths to *.xcassets directories to include in the target as asset
+ // catalogs. For all targets, this causes the .xcassets directory to be
+ // included in the Project Navigator. For targets that can include them (e.g.
+ // objc_binary), this causes them to be included in the resources build
+ // phase.
+ repeated string xcassets_dir = 9;
+
+ // Miscellaneous build settings. Each setting is applied to all build
+ // configurations.
+ repeated XcodeprojBuildSetting build_setting = 10;
+
+ // Paths to strings or xib files. Xcodegen will determine automatically if
+ // they are localized (i.e. are in a directory named *.lproj) or not.
+ // TODO(bazel-team): Introduce a more powerful resource file protobuf, a
+ // list of which can be used to specify all resources.
+ repeated string general_resource_file = 11;
+
+ // SDK frameworks to link with this target. This should be every framework
+ // required by the targets in the transitive dependency tree. Xcodegen does
+ // not find these transitive dependencies automatically; they must be added
+ // to this repeated field.
+ repeated string sdk_framework = 12;
+
+ // Paths to non-SDK frameworks to link with this target, relative to the
+ // same path as every other path in the control file. This should be every
+ // framework required by the targets in the transitive dependency tree.
+ repeated string framework = 21;
+
+ // Path to all .xcdatamodel directories required. All directories inside
+ // a .xcdatamodeld directory are grouped into XCVersionGroups by the path of
+ // the .xcdatamodeld directory.
+ repeated string xcdatamodel = 13;
+
+ // Path to all .a libraries to link with this target. When used on static
+ // library targets, the library is not actually linked, but it will appear
+ // in the Xcode Project Navigator. These are considered to be pre-built
+ // libraries. In other words, they are NOT built by Xcode as a dependency
+ // before this target is built.
+ repeated string imported_library = 14;
+
+ // Path to directories to include as user header search paths. These are non-
+ // recursive. These should be specified here rather than in the build_setting
+ // field because the build_setting field requires paths to be specified
+ // relative to the container of the .xcodeproj directory, while this
+ // attribute is relative to the same path as every other path in the control
+ // file. These paths correspond to the "-iquote" arguments passed to the
+ // compiler.
+ repeated string user_header_search_path = 15;
+
+ // Path to directories to include as header search paths. These are non-
+ // recursive. These should be specified here rather than in the build_setting
+ // field because the build_setting field requires paths to be specified
+ // relative to the container of the .xcodeproj directory, while this
+ // attribute is relative to the same path as every other path in the control
+ // file. These paths correspond to the "-I" arguments passed to the
+ // compiler.
+ repeated string header_search_path = 16;
+
+ // GCC_PREFIX_HEADER path. Needs to be be specified here rather than in the
+ // build_setting field because the build_setting field requires paths to be
+ // specified relative to the container of the .xcodeproj directory, while
+ // this attribute is relative to the same path as every other path in the
+ // control file.
+ optional string pch_path = 20;
+
+ // Path to .bundle directories this target depends on. For static library
+ // targets, this only causes the bundle to appear in the Project Navigator.
+ // For application target, this also causes the bundle to be linked with the
+ // application.
+ repeated string bundle_import = 17;
+
+ // The product type, for instance "com.apple.product-type.bundle". If
+ // omitted, the presence of the infoplist field indicates the type:
+ // has infoplist: "com.apple.product-type.application"
+ // does not have infoplist: "com.apple.product-type.library.static"
+ optional string product_type = 18;
+
+ // SDK .dylib files to link with this target. Each name should not include the
+ // the path or the .dylib extension, e.g. "libz" to link in
+ // "SDKROOT/usr/lib/libz.dylib". For all targets, this causes the library to
+ // appear in the Project Navigator. For binary targets, this causes the
+ // library to be linked with the final binary.
+ repeated string sdk_dylib = 19;
+}
+
+// Represents the mapping of a build setting recognized by Xcode project files,
+// for instance ASSETCATALOG_COMPILER_APPICON_NAME.
+message XcodeprojBuildSetting {
+ optional string name = 1;
+ optional string value = 2;
+}
diff --git a/src/main/tools/BUILD b/src/main/tools/BUILD
new file mode 100644
index 0000000..0e73eda
--- /dev/null
+++ b/src/main/tools/BUILD
@@ -0,0 +1,23 @@
+package(default_visibility = ["//src:__subpackages__"])
+
+cc_binary(
+ name = "process-wrapper",
+ srcs = ["process-wrapper.c"],
+ copts = ["-std=c99"],
+)
+
+cc_binary(
+ name = "build-runfiles",
+ srcs = ["build-runfiles.cc"],
+)
+
+cc_binary(
+ name = "namespace-sandbox",
+ srcs = select({
+ "//src:darwin": ["namespace-sandbox-dummy.c"],
+ "//conditions:default": ["namespace-sandbox.c"],
+ }),
+ copts = ["-std=c99"],
+)
+
+exports_files(["build_interface_so"])
diff --git a/src/main/tools/build-runfiles.cc b/src/main/tools/build-runfiles.cc
new file mode 100644
index 0000000..e0b8f57
--- /dev/null
+++ b/src/main/tools/build-runfiles.cc
@@ -0,0 +1,432 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This program creates a "runfiles tree" from a "runfiles manifest".
+//
+// The command line arguments are an input manifest INPUT and an output
+// directory RUNFILES. First, the files in the RUNFILES directory are scanned
+// and any extraneous ones are removed. Second, any missing files are created.
+// Finally, a copy of the input manifest is written to RUNFILES/MANIFEST.
+//
+// The input manifest consists of lines, each containing a relative path within
+// the runfiles, a space, and an optional absolute path. If this second path
+// is present, a symlink is created pointing to it; otherwise an empty file is
+// created.
+//
+// Given the line
+// <workspace root>/output/path /real/path
+// we will create directories
+// RUNFILES/<workspace root>
+// RUNFILES/<workspace root>/output
+// a symlink
+// RUNFILES/<workspace root>/output/path -> /real/path
+// and the output manifest will contain a line
+// <workspace root>/output/path /real/path
+//
+// If --use_metadata is supplied, every other line is treated as opaque
+// metadata, and is ignored here.
+//
+// All output paths must be relative and generally (but not always) begin with
+// <workspace root>. No output path may be equal to another. No output path may
+// be a path prefix of another.
+
+#define _FILE_OFFSET_BITS 64
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <map>
+#include <string>
+
+// program_invocation_short_name is not portable.
+static const char *argv0;
+
+const char *input_filename;
+const char *output_base_dir;
+
+#define LOG() { \
+ fprintf(stderr, "%s (args %s %s): ", \
+ argv0, input_filename, output_base_dir); \
+}
+
+#define DIE(args...) { \
+ LOG(); \
+ fprintf(stderr, args); \
+ fprintf(stderr, "\n"); \
+ exit(1); \
+}
+
+#define PDIE(args...) { \
+ int saved_errno = errno; \
+ LOG(); \
+ fprintf(stderr, args); \
+ fprintf(stderr, ": %s [%d]\n", strerror(saved_errno), saved_errno); \
+ exit(1); \
+}
+
+enum FileType {
+ FILE_TYPE_REGULAR,
+ FILE_TYPE_DIRECTORY,
+ FILE_TYPE_SYMLINK
+};
+
+struct FileInfo {
+ FileType type;
+ std::string symlink_target;
+
+ bool operator==(const FileInfo &other) const {
+ return type == other.type && symlink_target == other.symlink_target;
+ }
+
+ bool operator!=(const FileInfo &other) const {
+ return !(*this == other);
+ }
+};
+
+typedef std::map<std::string, FileInfo> FileInfoMap;
+
+class RunfilesCreator {
+ public:
+ explicit RunfilesCreator(const std::string &output_base)
+ : output_base_(output_base),
+ output_filename_("MANIFEST"),
+ temp_filename_(output_filename_ + ".tmp") {
+ SetupOutputBase();
+ if (chdir(output_base_.c_str()) != 0) {
+ PDIE("chdir '%s'", output_base_.c_str());
+ }
+ }
+
+ void ReadManifest(const std::string &manifest_file, bool allow_relative,
+ bool use_metadata) {
+ FILE *outfile = fopen(temp_filename_.c_str(), "w");
+ if (!outfile) {
+ PDIE("opening '%s/%s' for writing", output_base_.c_str(),
+ temp_filename_.c_str());
+ }
+ FILE *infile = fopen(manifest_file.c_str(), "r");
+ if (!infile) {
+ PDIE("opening '%s' for reading", manifest_file.c_str());
+ }
+
+ // read input manifest
+ int lineno = 0;
+ char buf[3 * PATH_MAX];
+ while (fgets(buf, sizeof buf, infile)) {
+ // copy line to output manifest
+ if (fputs(buf, outfile) == EOF) {
+ PDIE("writing to '%s/%s'", output_base_.c_str(),
+ temp_filename_.c_str());
+ }
+
+ // parse line
+ ++lineno;
+ // Skip metadata lines. They are used solely for
+ // dependency checking.
+ if (use_metadata && lineno % 2 == 0) continue;
+
+ int n = strlen(buf)-1;
+ if (!n || buf[n] != '\n') {
+ DIE("missing terminator at line %d: '%s'\n", lineno, buf);
+ }
+ buf[n] = '\0';
+ if (buf[0] == '/') {
+ DIE("paths must not be absolute: line %d: '%s'\n", lineno, buf);
+ }
+ const char *s = strchr(buf, ' ');
+ if (!s) {
+ DIE("missing field delimiter at line %d: '%s'\n", lineno, buf);
+ } else if (strchr(s+1, ' ')) {
+ DIE("link or target filename contains space on line %d: '%s'\n", lineno, buf);
+ }
+ std::string link(buf, s-buf);
+ const char *target = s+1;
+ if (!allow_relative && target[0] != '\0' && target[0] != '/'
+ && target[1] != ':') { // Match Windows paths, e.g. C:\foo or C:/foo.
+ DIE("expected absolute path at line %d: '%s'\n", lineno, buf);
+ }
+
+ FileInfo *info = &manifest_[link];
+ if (target[0] == '\0') {
+ // No target means an empty file.
+ info->type = FILE_TYPE_REGULAR;
+ } else {
+ info->type = FILE_TYPE_SYMLINK;
+ info->symlink_target = target;
+ }
+
+ FileInfo parent_info;
+ parent_info.type = FILE_TYPE_DIRECTORY;
+
+ while (true) {
+ int k = link.rfind('/');
+ if (k < 0) break;
+ link.erase(k, std::string::npos);
+ if (!manifest_.insert(std::make_pair(link, parent_info)).second) break;
+ }
+ }
+ if (fclose(outfile) != 0) {
+ PDIE("writing to '%s/%s'", output_base_.c_str(),
+ temp_filename_.c_str());
+ }
+ fclose(infile);
+
+ // Don't delete the temp manifest file.
+ manifest_[temp_filename_].type = FILE_TYPE_REGULAR;
+ }
+
+ void CreateRunfiles() {
+ if (unlink(output_filename_.c_str()) != 0 && errno != ENOENT) {
+ PDIE("removing previous file at '%s/%s'", output_base_.c_str(),
+ output_filename_.c_str());
+ }
+
+ ScanTreeAndPrune(".");
+ CreateFiles();
+
+ // rename output file into place
+ if (rename(temp_filename_.c_str(), output_filename_.c_str()) != 0) {
+ PDIE("renaming '%s/%s' to '%s/%s'",
+ output_base_.c_str(), temp_filename_.c_str(),
+ output_base_.c_str(), output_filename_.c_str());
+ }
+ }
+
+ private:
+ void SetupOutputBase() {
+ struct stat st;
+ if (stat(output_base_.c_str(), &st) != 0) {
+ // Technically, this will cause problems if the user's umask contains
+ // 0200, but we don't care. Anyone who does that deserves what's coming.
+ if (mkdir(output_base_.c_str(), 0777) != 0) {
+ PDIE("creating directory '%s'", output_base_.c_str());
+ }
+ } else {
+ EnsureDirReadAndWritePerms(output_base_);
+ }
+ }
+
+ void ScanTreeAndPrune(const std::string &path) {
+ // A note on non-empty files:
+ // We don't distinguish between empty and non-empty files. That is, if
+ // there's a file that has contents, we don't truncate it here, even though
+ // the manifest supports creation of empty files, only. Given that
+ // .runfiles are *supposed* to be immutable, this shouldn't be a problem.
+ EnsureDirReadAndWritePerms(path);
+
+ struct dirent *entry;
+ DIR *dh = opendir(path.c_str());
+ if (!dh) {
+ PDIE("opendir '%s'", path.c_str());
+ }
+
+ errno = 0;
+ const std::string prefix = (path == "." ? "" : path + "/");
+ while ((entry = readdir(dh)) != NULL) {
+ if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue;
+
+ std::string entry_path = prefix + entry->d_name;
+ FileInfo actual_info;
+ actual_info.type = DentryToFileType(entry_path, entry->d_type);
+
+ if (actual_info.type == FILE_TYPE_SYMLINK) {
+ ReadLinkOrDie(entry_path, &actual_info.symlink_target);
+ }
+
+ FileInfoMap::iterator expected_it = manifest_.find(entry_path);
+ if (expected_it == manifest_.end() ||
+ expected_it->second != actual_info) {
+ DelTree(entry_path, actual_info.type);
+ } else {
+ manifest_.erase(expected_it);
+ if (actual_info.type == FILE_TYPE_DIRECTORY) {
+ ScanTreeAndPrune(entry_path);
+ }
+ }
+
+ errno = 0;
+ }
+ if (errno != 0) {
+ PDIE("reading directory '%s'", path.c_str());
+ }
+ closedir(dh);
+ }
+
+ void CreateFiles() {
+ for (FileInfoMap::const_iterator it = manifest_.begin();
+ it != manifest_.end(); ++it) {
+ const std::string &path = it->first;
+ switch (it->second.type) {
+ case FILE_TYPE_DIRECTORY:
+ if (mkdir(path.c_str(), 0777) != 0) {
+ PDIE("mkdir '%s'", path.c_str());
+ }
+ break;
+ case FILE_TYPE_REGULAR:
+ {
+ int fd = open(path.c_str(), O_CREAT|O_EXCL|O_WRONLY, 0555);
+ if (fd < 0) {
+ PDIE("creating empty file '%s'", path.c_str());
+ }
+ close(fd);
+ }
+ break;
+ case FILE_TYPE_SYMLINK:
+ if (symlink(it->second.symlink_target.c_str(), path.c_str()) != 0) {
+ PDIE("symlinking '%s' -> '%s'", path.c_str(),
+ it->second.symlink_target.c_str());
+ }
+ break;
+ }
+ }
+ }
+
+ FileType DentryToFileType(const std::string &path, char d_type) {
+ if (d_type == DT_UNKNOWN) {
+ struct stat st;
+ LStatOrDie(path, &st);
+ if (S_ISDIR(st.st_mode)) {
+ return FILE_TYPE_DIRECTORY;
+ } else if (S_ISLNK(st.st_mode)) {
+ return FILE_TYPE_SYMLINK;
+ } else {
+ return FILE_TYPE_REGULAR;
+ }
+ } else if (d_type == DT_DIR) {
+ return FILE_TYPE_DIRECTORY;
+ } else if (d_type == DT_LNK) {
+ return FILE_TYPE_SYMLINK;
+ } else {
+ return FILE_TYPE_REGULAR;
+ }
+ }
+
+ void LStatOrDie(const std::string &path, struct stat *st) {
+ if (lstat(path.c_str(), st) != 0) {
+ PDIE("stating file '%s'", path.c_str());
+ }
+ }
+
+ void ReadLinkOrDie(const std::string &path, std::string *output) {
+ char readlink_buffer[PATH_MAX];
+ int sz = readlink(path.c_str(), readlink_buffer, sizeof(readlink_buffer));
+ if (sz < 0) {
+ PDIE("reading symlink '%s'", path.c_str());
+ }
+ // readlink returns a non-null terminated string.
+ std::string(readlink_buffer, sz).swap(*output);
+ }
+
+ void EnsureDirReadAndWritePerms(const std::string &path) {
+ const int kMode = 0700;
+ struct stat st;
+ LStatOrDie(path, &st);
+ if ((st.st_mode & kMode) != kMode) {
+ int new_mode = (st.st_mode | kMode) & ALLPERMS;
+ if (chmod(path.c_str(), new_mode) != 0) {
+ PDIE("chmod '%s'", path.c_str());
+ }
+ }
+ }
+
+ void DelTree(const std::string &path, FileType file_type) {
+ if (file_type != FILE_TYPE_DIRECTORY) {
+ if (unlink(path.c_str()) != 0) {
+ PDIE("unlinking '%s'", path.c_str());
+ }
+ return;
+ }
+
+ EnsureDirReadAndWritePerms(path);
+
+ struct dirent *entry;
+ DIR *dh = opendir(path.c_str());
+ if (!dh) {
+ PDIE("opendir '%s'", path.c_str());
+ }
+ errno = 0;
+ while ((entry = readdir(dh)) != NULL) {
+ if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue;
+ const std::string entry_path = path + '/' + entry->d_name;
+ FileType entry_file_type = DentryToFileType(entry_path, entry->d_type);
+ DelTree(entry_path, entry_file_type);
+ errno = 0;
+ }
+ if (errno != 0) {
+ PDIE("readdir '%s'", path.c_str());
+ }
+ closedir(dh);
+ if (rmdir(path.c_str()) != 0) {
+ PDIE("rmdir '%s'", path.c_str());
+ }
+ }
+
+ private:
+ std::string output_base_;
+ std::string output_filename_;
+ std::string temp_filename_;
+
+ FileInfoMap manifest_;
+};
+
+int main(int argc, char **argv) {
+ argv0 = argv[0];
+
+ argc--; argv++;
+ bool allow_relative = false;
+ bool use_metadata = false;
+
+ while (argc >= 1) {
+ if (strcmp(argv[0], "--allow_relative") == 0) {
+ allow_relative = true;
+ argc--; argv++;
+ } else if (strcmp(argv[0], "--use_metadata") == 0) {
+ use_metadata = true;
+ argc--; argv++;
+ } else {
+ break;
+ }
+ }
+
+ if (argc != 2) {
+ fprintf(stderr, "usage: %s [--allow_relative] INPUT RUNFILES\n",
+ argv0);
+ return 1;
+ }
+
+ input_filename = argv[0];
+ output_base_dir = argv[1];
+
+ std::string manifest_file = input_filename;
+ if (input_filename[0] != '/') {
+ char cwd_buf[PATH_MAX];
+ if (getcwd(cwd_buf, sizeof(cwd_buf)) == NULL) {
+ PDIE("getcwd failed");
+ }
+ manifest_file = std::string(cwd_buf) + '/' + manifest_file;
+ }
+
+ RunfilesCreator runfiles_creator(output_base_dir);
+ runfiles_creator.ReadManifest(manifest_file, allow_relative, use_metadata);
+ runfiles_creator.CreateRunfiles();
+
+ return 0;
+}
diff --git a/src/main/tools/build_interface_so b/src/main/tools/build_interface_so
new file mode 100755
index 0000000..626e707
--- /dev/null
+++ b/src/main/tools/build_interface_so
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ $# != 2 ]]; then
+ echo "Usage: $0 <so> <interface so>" 1>&2
+ exit 1
+fi
+
+exec cp $1 $2
diff --git a/src/main/tools/namespace-sandbox-dummy.c b/src/main/tools/namespace-sandbox-dummy.c
new file mode 100644
index 0000000..dd9f47f
--- /dev/null
+++ b/src/main/tools/namespace-sandbox-dummy.c
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a dummy file to compile on platforms where namespace sandboxing
+// doesn't work (ie. other than Linux). We need this for main/tools/BUILD file
+// - we can't restrict visibility of namespace-sandbox based on platform;
+// instead bazel build main/tools:namespace-sandbox is a no-op on non supported
+// platforms (if we didn't have this file, it would fail with a non-informative
+// message)
+
+int main() {
+ return 0;
+}
diff --git a/src/main/tools/namespace-sandbox.c b/src/main/tools/namespace-sandbox.c
new file mode 100644
index 0000000..08e5fc2
--- /dev/null
+++ b/src/main/tools/namespace-sandbox.c
@@ -0,0 +1,323 @@
+#define _GNU_SOURCE
+
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <limits.h>
+#include <linux/capability.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+static int global_debug = 0;
+static int global_cpid; // Returned by fork()
+
+#define PRINT_DEBUG(...) do { if (global_debug) {fprintf(stderr, "sandbox.c: " __VA_ARGS__);}} while(0)
+
+#define CHECK_CALL(x) if ((x) == -1) { perror(#x); exit(1); }
+#define CHECK_NOT_NULL(x) if (x == NULL) { perror(#x); exit(1); }
+#define DIE() do { fprintf(stderr, "Error in %d\n", __LINE__); exit(-1); } while(0);
+
+int kChildrenCleanupDelay = 1;
+
+void Usage() {
+ fprintf(stderr,
+ "Usage: ./sandbox [-R sandbox-root] [-m mount] -C command arg1\n"
+ "Mandatory arguments:\n"
+ " -C command to run inside sandbox, followed by arguments\n"
+ " -S directory which will become the root of the sandbox\n"
+ "\n"
+ "Optional arguments:\n"
+ " -t absolute path to bazel tools directory\n"
+ " -T timeout after which sandbox will be terminated\n"
+ " -m system directory to mount inside the sandbox\n"
+ " Multiple directories can be specified and each of them will\n"
+ " be mount as readonly\n"
+ " -D if set, debug info will be printed\n");
+ exit(1);
+}
+
+void PropagateSignals();
+void EnableAlarm();
+void SetupSlashDev();
+
+static volatile sig_atomic_t global_signal_received = 0;
+
+int main(int argc, char *argv[]) {
+ char *include_prefix = NULL;
+ char *sandbox_root = NULL;
+ char *tools = NULL;
+ char **mounts = malloc(argc * sizeof(char*));
+ char **includes = malloc(argc * sizeof(char*));
+ int num_mounts = 0;
+ int num_includes = 0;
+ int iArg = 0;
+ int uid = getuid();
+ int gid = getgid();
+ int timeout = 0;
+
+ for (iArg = 1; iArg < argc - 1; iArg++) {
+ if (strlen(argv[iArg]) != 2) {
+ Usage();
+ }
+ if (argv[iArg][0] != '-') {
+ Usage();
+ }
+ switch (argv[iArg][1]) {
+ case 'S':
+ if (sandbox_root == NULL) {
+ sandbox_root = argv[++iArg];
+ } else {
+ fprintf(stderr,
+ "Multiple sandbox roots (-S) specified (expected one).\n");
+ Usage();
+ }
+ break;
+ case 'm':
+ mounts[num_mounts++] = argv[++iArg];
+ break;
+ case 'D':
+ global_debug = 1;
+ break;
+ case 'T':
+ sscanf(argv[iArg], "%d", &timeout);
+ break;
+ case 'N':
+ include_prefix = argv[++iArg];
+ break;
+ case 'n':
+ includes[num_includes++] = argv[++iArg];
+ break;
+ case 'C':
+ iArg++;
+ goto parsing_finished;
+ case 't':
+ tools = argv[++iArg];
+ break;
+ default:
+ fprintf(stderr, "Unrecognized argument: %s\n", argv[iArg]);
+ Usage();
+ }
+ }
+
+parsing_finished:
+ if (iArg == argc) {
+ fprintf(stderr, "No command specified.\n");
+ Usage();
+ }
+ if (timeout < 0) {
+ fprintf(stderr, "Invalid timeout (-T) value: %d", timeout);
+ Usage();
+ }
+
+ // parsed all arguments, now prepare sandbox
+
+ PRINT_DEBUG("%s\n", sandbox_root);
+ // create new namespaces in which this process and its children will live
+ CHECK_CALL(unshare(CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC | CLONE_NEWUSER));
+ CHECK_CALL(mount("none", "/", NULL, MS_REC | MS_PRIVATE, NULL));
+ // mount sandbox and go there
+ CHECK_CALL(mount(sandbox_root, sandbox_root, NULL, MS_BIND | MS_NOSUID, NULL));
+ CHECK_CALL(chdir(sandbox_root));
+
+ SetupSlashDev();
+ // mount blaze specific directories - tools/ and build-runfiles/
+ if (tools != NULL) {
+ PRINT_DEBUG("tools: %s\n", tools);
+ CHECK_CALL(mkdir("tools", 0755));
+ CHECK_CALL(mount(tools, "tools", NULL, MS_BIND | MS_RDONLY, NULL));
+ }
+
+ // mounts passed in argv; those are mostly dirs for shared libs
+ for (int i = 0; i < num_mounts; i++) {
+ CHECK_CALL(mount(mounts[i], mounts[i] + 1, NULL, MS_BIND | MS_RDONLY, NULL));
+ }
+
+ // c++ compilation
+ // headers go in separate directory
+ if (include_prefix != NULL) {
+ CHECK_CALL(chdir(include_prefix));
+ for (int i = 0; i < num_includes; i++) {
+ // TODO(bazel-team) sometimes list of -iquote given by bazel contains
+ // invalid (non-existing) entries, ideally we would like not to have them
+ PRINT_DEBUG("include: %s\n", includes[i]);
+ if (mount(includes[i], includes[i] + 1 , NULL, MS_BIND, NULL) > -1) {
+ continue;
+ }
+ if (errno == ENOENT) {
+ continue;
+ }
+ CHECK_CALL(-1);
+ }
+ CHECK_CALL(chdir(".."));
+ }
+
+ // set group and user mapping from outer namespace to inner:
+ // no changes in the parent, be root in the child
+ int uid_fd, gid_fd;
+ char uid_mapping[64], gid_mapping[64];
+ sprintf(uid_mapping, "0 %d 1\n", uid);
+ sprintf(gid_mapping, "0 %d 1\n", gid);
+ uid_fd = open("/proc/self/uid_map", O_WRONLY);
+ CHECK_CALL(uid_fd);
+ CHECK_CALL(write(uid_fd, uid_mapping, strlen(uid_mapping)));
+ CHECK_CALL(close(uid_fd));
+
+ gid_fd = open("/proc/self/gid_map", O_WRONLY);
+ CHECK_CALL(gid_fd);
+ CHECK_CALL(write(gid_fd, gid_mapping, strlen(gid_mapping)));
+ CHECK_CALL(close(gid_fd));
+
+ CHECK_CALL(setresuid(0, 0, 0));
+ CHECK_CALL(setresgid(0, 0, 0));
+
+ CHECK_CALL(mkdir("proc", 0755));
+ CHECK_CALL(mount("/proc", "proc", NULL, MS_REC | MS_BIND, NULL));
+ // make sandbox actually hermetic:
+ // move the real root to old_root, then detach it
+ char old_root[16] = "old-root-XXXXXX";
+ CHECK_NOT_NULL(mkdtemp(old_root));
+ // pivot_root has no wrapper in libc, so we need syscall()
+ CHECK_CALL(syscall(SYS_pivot_root, ".", old_root));
+ CHECK_CALL(chroot("."));
+ CHECK_CALL(umount2(old_root, MNT_DETACH));
+ CHECK_CALL(rmdir(old_root));
+
+ free(mounts);
+ free(includes);
+
+ for (int i = iArg; i < argc; i += 1) {
+ PRINT_DEBUG("arg: %s\n", argv[i]);
+ }
+
+ // spawn child and wait until it finishes
+ global_cpid = fork();
+ if (global_cpid == 0) {
+ CHECK_CALL(setpgid(0, 0));
+ // if the execvp below fails with "No such file or directory" it means that:
+ // a) the binary is not in the sandbox (which means it wasn't included in
+ // the inputs)
+ // b) the binary uses shared library which is not inside sandbox - you can
+ // check for that by running "ldd ./a.out" (by default directories
+ // starting with /lib* and /usr/lib* should be there)
+ // c) the binary uses elf interpreter which is not inside sandbox - you can
+ // check for that by running "readelf -a a.out | grep interpreter" (the
+ // sandbox code assumes that it is either in /lib*/ or /usr/lib*/)
+ CHECK_CALL(execvp(argv[iArg], argv + iArg));
+ PRINT_DEBUG("Exec failed near %s:%d\n", __FILE__, __LINE__);
+ exit(1);
+ } else {
+ // PARENT
+ // make sure that all signals propagate to children (mostly useful to kill
+ // entire sandbox)
+ PropagateSignals();
+ // after given timeout, kill children
+ EnableAlarm(timeout);
+ int status = 0;
+ while (1) {
+ PRINT_DEBUG("Waiting for the child...\n");
+ pid_t pid = wait(&status);
+ if (global_signal_received) {
+ PRINT_DEBUG("Received signal: %s\n", strsignal(global_signal_received));
+ CHECK_CALL(killpg(global_cpid, global_signal_received));
+ // give children some time for cleanup before they terminate
+ sleep(kChildrenCleanupDelay);
+ CHECK_CALL(killpg(global_cpid, SIGKILL));
+ exit(128 | global_signal_received);
+ }
+ if (errno == EINTR) {
+ continue;
+ }
+ if (pid < 0) {
+ perror("Wait failed:");
+ exit(1);
+ }
+ if (WIFEXITED(status)) {
+ PRINT_DEBUG("Child exited with status: %d\n", WEXITSTATUS(status));
+ exit(WEXITSTATUS(status));
+ }
+ if (WIFSIGNALED(status)) {
+ PRINT_DEBUG("Child terminated by a signal: %d\n", WTERMSIG(status));
+ exit(WEXITSTATUS(status));
+ }
+ if (WIFSTOPPED(status)) {
+ PRINT_DEBUG("Child stopped by a signal: %d\n", WSTOPSIG(status));
+ }
+ }
+ }
+
+ return 0;
+}
+
+void SignalHandler(int signum, siginfo_t *info, void *uctxt) {
+ global_signal_received = signum;
+}
+
+void PropagateSignals() {
+ // propagate some signals received by the parent to processes in sandbox, so
+ // that it's easier to terminate entire sandbox
+ struct sigaction action = {};
+ action.sa_flags = SA_SIGINFO;
+ action.sa_sigaction = SignalHandler;
+
+ // handle all signals that could terminate the process
+ int signals[] = {SIGHUP, SIGINT, SIGKILL, SIGPIPE, SIGALRM, SIGTERM, SIGPOLL,
+ SIGPROF, SIGVTALRM,
+ // signals below produce core dump by default, however at the moment we'll
+ // just terminate
+ SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGBUS, SIGSYS, SIGTRAP, SIGXCPU,
+ SIGXFSZ, -1};
+ for (int *p = signals; *p != -1; p++) {
+ sigaction(*p, &action, NULL);
+ }
+}
+
+void SetupSlashDev() {
+ CHECK_CALL(mkdir("dev", 0755));
+ const char *devs[] = {
+ "/dev/null",
+ "/dev/random",
+ "/dev/urandom",
+ "/dev/zero",
+ NULL
+ };
+ for (int i = 0; devs[i] != NULL; i++) {
+ // open+close to create the file, which will become mount point for actual
+ // device
+ int handle = open(devs[i] + 1, O_CREAT | O_RDONLY, 0644);
+ CHECK_CALL(handle);
+ CHECK_CALL(close(handle));
+ CHECK_CALL(mount(devs[i], devs[i] + 1, NULL, MS_BIND, NULL));
+ }
+}
+
+void EnableAlarm(int timeout) {
+ if (timeout <= 0) return;
+
+ struct itimerval timer = {};
+ timer.it_value.tv_sec = (long) timeout;
+ CHECK_CALL(setitimer(ITIMER_REAL, &timer, NULL));
+}
diff --git a/src/main/tools/process-wrapper.c b/src/main/tools/process-wrapper.c
new file mode 100644
index 0000000..5aff63c
--- /dev/null
+++ b/src/main/tools/process-wrapper.c
@@ -0,0 +1,228 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// process-wrapper runs a subprocess with a given timeout (optional),
+// redirecting stdout and stderr to given files. Upon exit, whether
+// from normal termination or timeout, the subprocess (and any of its children)
+// is killed.
+//
+// The exit status of this program is whatever the child process returned,
+// unless process-wrapper receives a signal. ie, on SIGTERM this program will
+// die with raise(SIGTERM) even if the child process handles SIGTERM with
+// exit(0).
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <fcntl.h>
+#include <math.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+// Not in headers on OSX.
+extern char **environ;
+
+static int global_pid; // Returned from fork().
+static int global_signal = -1;
+static double global_kill_delay = 0.0;
+
+#define DIE(args...) { \
+ fprintf(stderr, args); \
+ fprintf(stderr, " --- "); \
+ perror(NULL); \
+ fprintf(stderr, "\n"); \
+ exit(EXIT_FAILURE); \
+}
+
+#define CHECK_CALL(x) if (x != 0) { perror(#x); exit(1); }
+
+// Make sure the process and all subprocesses are killed.
+static void KillEverything(int pgrp) {
+ kill(-pgrp, SIGTERM);
+
+ // Round up fractional seconds in this polling implementation.
+ int kill_delay = (int)(global_kill_delay+0.999) ;
+ // If the process is still alive, give it some time to die gracefully.
+ while (kill(-pgrp, 0) == 0 && kill_delay-- > 0) {
+ sleep(1);
+ }
+
+ kill(-pgrp, SIGKILL);
+}
+
+// Called when timeout or Signal occurs.
+static void OnSignal(int sig) {
+ global_signal = sig;
+ if (sig == SIGALRM) {
+ // SIGALRM represents a timeout, so we should give the process a bit of
+ // time to die gracefully if it needs it.
+ KillEverything(global_pid);
+ } else {
+ // Signals should kill the process quickly, as it's typically blocking
+ // the return of the prompt after a user hits "Ctrl-C".
+ kill(-global_pid, SIGKILL);
+ }
+}
+
+// Set up a signal handler which kills all subprocesses when the
+// given signal is triggered.
+static void InstallSignalHandler(int sig) {
+ struct sigaction sa = {};
+
+ sa.sa_handler = OnSignal;
+ sigemptyset(&sa.sa_mask);
+ CHECK_CALL(sigaction(sig, &sa, NULL));
+}
+
+// Revert signal handler to default.
+static void UnHandle(int sig) {
+ struct sigaction sa = {};
+ sa.sa_handler = SIG_DFL;
+ sigemptyset(&sa.sa_mask);
+ CHECK_CALL(sigaction(sig, &sa, NULL));
+}
+
+// Enable the given timeout, or no-op if the timeout is non-positive.
+static void EnableAlarm(double timeout) {
+ if (timeout <= 0) return;
+
+ struct itimerval timer = {};
+ timer.it_interval.tv_sec = 0;
+ timer.it_interval.tv_usec = 0;
+
+ double int_val, fraction_val;
+ fraction_val = modf(timeout, &int_val);
+ timer.it_value.tv_sec = (long) int_val;
+ timer.it_value.tv_usec = (long) (fraction_val * 1e6);
+ CHECK_CALL(setitimer(ITIMER_REAL, &timer, NULL));
+}
+
+static void ClearSignalMask() {
+ // Use an empty signal mask and default signal handlers in the
+ // subprocess.
+ sigset_t sset;
+ sigemptyset(&sset);
+ sigprocmask(SIG_SETMASK, &sset, NULL);
+ for (int i = 1; i < NSIG; ++i) {
+ if (i == SIGKILL || i == SIGSTOP) continue;
+
+ struct sigaction sa = {};
+ sa.sa_handler = SIG_DFL;
+ sigemptyset(&sa.sa_mask);
+ sigaction(i, &sa, NULL);
+ }
+}
+
+static int WaitChild(pid_t pid, const char *name) {
+ int err = 0;
+ int status = 0;
+ do {
+ err = waitpid(pid, &status, 0);
+ } while (err == -1 && errno == EINTR);
+
+ if (err == -1) {
+ DIE("wait on %s (pid %d) failed", name, pid);
+ }
+ return status;
+}
+
+// Usage: process-wrapper
+// <timeout_sec> <kill_delay_sec> <stdout file> <stderr file>
+// [cmdline]
+int main(int argc, char *argv[]) {
+ if (argc <= 5) {
+ DIE("Not enough cmd line arguments to process-wrapper");
+ }
+
+ // Parse the cmdline args to get the timeout and redirect files.
+ argv++;
+ double timeout;
+ if (sscanf(*argv++, "%lf", &timeout) != 1) {
+ DIE("timeout_sec is not a real number.");
+ }
+ if (sscanf(*argv++, "%lf", &global_kill_delay) != 1) {
+ DIE("kill_delay_sec is not a real number.");
+ }
+ char *stdout_path = *argv++;
+ char *stderr_path = *argv++;
+
+ if (strcmp(stdout_path, "-")) {
+ // Redirect stdout and stderr.
+ int fd_out = open(stdout_path, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+ if (fd_out == -1) {
+ DIE("Could not open %s for stdout", stdout_path);
+ }
+ if (dup2(fd_out, STDOUT_FILENO) == -1) {
+ DIE("dup2 failed for stdout");
+ }
+ CHECK_CALL(close(fd_out));
+ }
+
+ if (strcmp(stderr_path, "-")) {
+ int fd_err = open(stderr_path, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+ if (fd_err == -1) {
+ DIE("Could not open %s for stderr", stderr_path);
+ }
+ if (dup2(fd_err, STDERR_FILENO) == -1) {
+ DIE("dup2 failed for stderr");
+ }
+ CHECK_CALL(close(fd_err));
+ }
+
+ global_pid = fork();
+ if (global_pid < 0) {
+ DIE("Fork failed");
+ } else if (global_pid == 0) {
+ // In child.
+ if (setsid() == -1) {
+ DIE("Could not setsid from child");
+ }
+ ClearSignalMask();
+ // Force umask to include read and execute for everyone, to make
+ // output permissions predictable.
+ umask(022);
+
+ execvp(argv[0], argv); // Does not return.
+ DIE("execvpe %s failed", argv[0]);
+ } else {
+ // In parent.
+ InstallSignalHandler(SIGALRM);
+ InstallSignalHandler(SIGTERM);
+ InstallSignalHandler(SIGINT);
+ EnableAlarm(timeout);
+
+ int status = WaitChild(global_pid, argv[0]);
+
+ // The child is done, but may have grandchildren.
+ kill(-global_pid, SIGKILL);
+ if (global_signal > 0) {
+ // Don't trust the exit code if we got a timeout or signal.
+ UnHandle(global_signal);
+ raise(global_signal);
+ } else if (WIFEXITED(status)) {
+ exit(WEXITSTATUS(status));
+ } else {
+ int sig = WTERMSIG(status);
+ UnHandle(sig);
+ raise(sig);
+ }
+ }
+}
diff --git a/src/objc_tools/actooloribtoolzip/README b/src/objc_tools/actooloribtoolzip/README
new file mode 100644
index 0000000..7779de5
--- /dev/null
+++ b/src/objc_tools/actooloribtoolzip/README
@@ -0,0 +1,5 @@
+actooloribtoolzip runs actool or ibtool, which compiles asset catalog files and
+compiles storyboards, respectively, and zips up the output, because actool and
+ibtool return an unpredictable number of output files.
+
+actool and ibtool only run on Darwin, so actooloribtoolzip only runs on Darwin.
diff --git a/src/objc_tools/actooloribtoolzip/java/com/google/devtools/build/xcode/actooloribtoolzip/ActoolOrIbtoolZip.java b/src/objc_tools/actooloribtoolzip/java/com/google/devtools/build/xcode/actooloribtoolzip/ActoolOrIbtoolZip.java
new file mode 100644
index 0000000..3b41581
--- /dev/null
+++ b/src/objc_tools/actooloribtoolzip/java/com/google/devtools/build/xcode/actooloribtoolzip/ActoolOrIbtoolZip.java
@@ -0,0 +1,79 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.actooloribtoolzip;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.xcode.zippingoutput.Arguments;
+import com.google.devtools.build.xcode.zippingoutput.Wrapper;
+import com.google.devtools.build.xcode.zippingoutput.Wrappers;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * A tool which wraps actool or ibtool, by running actool/ibtool and zipping its output. See the
+ * JavaDoc for {@link Wrapper} for more information.
+ */
+public class ActoolOrIbtoolZip implements Wrapper {
+
+ private static final Function<String, String> CANONICAL_PATH =
+ new Function<String, String>() {
+ @Override
+ public String apply(String path) {
+ File file = new File(path);
+ if (file.exists()) {
+ try {
+ return file.getCanonicalPath();
+ } catch (IOException e) {
+ // Pass through to return raw path
+ }
+ }
+ return path;
+ }
+ };
+
+ @Override
+ public String name() {
+ return "ActoolOrIbtoolZip";
+ }
+
+ @Override
+ public String subtoolName() {
+ return "actool/ibtool";
+ }
+
+ @Override
+ public Iterable<String> subCommand(Arguments args, String outputDirectory) {
+ return new ImmutableList.Builder<String>()
+ .add(args.subtoolCmd())
+ .add("--compile")
+ .add(outputDirectory)
+ // actool munges paths in some way which doesn't work if one of the directories in the path
+ // is a symlink.
+ .addAll(Iterables.transform(args.subtoolExtraArgs(), CANONICAL_PATH))
+ .build();
+ }
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ Wrappers.execute(args, new ActoolOrIbtoolZip());
+ }
+
+ @Override
+ public boolean outputDirectoryMustExist() {
+ return true;
+ }
+}
diff --git a/src/objc_tools/bundlemerge/README b/src/objc_tools/bundlemerge/README
new file mode 100644
index 0000000..1701f81
--- /dev/null
+++ b/src/objc_tools/bundlemerge/README
@@ -0,0 +1,2 @@
+bundlemerge merges the assorted files making up an iOS application into a final
+IPA bundle.
diff --git a/src/objc_tools/bundlemerge/java/com/google/devtools/build/xcode/bundlemerge/BundleMerge.java b/src/objc_tools/bundlemerge/java/com/google/devtools/build/xcode/bundlemerge/BundleMerge.java
new file mode 100644
index 0000000..daa93a4
--- /dev/null
+++ b/src/objc_tools/bundlemerge/java/com/google/devtools/build/xcode/bundlemerge/BundleMerge.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.bundlemerge;
+
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.Control;
+import com.google.devtools.build.xcode.plmerge.PlistMerging.ValidationException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+
+/**
+ * Command-line entry point for {@link BundleMerging}. functionality. The only argument passed to
+ * this command-line utility should be a control file which contains a binary serialization of the
+ * {@link Control} protocol buffer.
+ */
+public class BundleMerge {
+ private BundleMerge() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 1) {
+ System.err.println("Expect exactly one argument: path to control file");
+ System.exit(1);
+ }
+ FileSystem fileSystem = FileSystems.getDefault();
+ Control control;
+ try (InputStream in = Files.newInputStream(fileSystem.getPath(args[0]))) {
+ control = Control.parseFrom(in);
+ }
+ try {
+ BundleMerging.merge(fileSystem, control);
+ } catch (ValidationException e) {
+ // Don't print stack traces for validation errors.
+ System.err.print("\nBundle merge failed: " + e.getMessage() + "\n");
+ System.exit(1);
+ }
+ }
+}
diff --git a/src/objc_tools/bundlemerge/java/com/google/devtools/build/xcode/bundlemerge/BundleMerging.java b/src/objc_tools/bundlemerge/java/com/google/devtools/build/xcode/bundlemerge/BundleMerging.java
new file mode 100644
index 0000000..3a061db
--- /dev/null
+++ b/src/objc_tools/bundlemerge/java/com/google/devtools/build/xcode/bundlemerge/BundleMerging.java
@@ -0,0 +1,216 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.bundlemerge;
+
+import static com.google.devtools.build.singlejar.ZipCombiner.DOS_EPOCH;
+import static com.google.devtools.build.singlejar.ZipCombiner.OutputMode.FORCE_DEFLATE;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.singlejar.ZipCombiner;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.BundleFile;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.Control;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.MergeZip;
+import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.VariableSubstitution;
+import com.google.devtools.build.xcode.common.Platform;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+import com.google.devtools.build.xcode.plmerge.KeysToRemoveIfEmptyString;
+import com.google.devtools.build.xcode.plmerge.PlistMerging;
+import com.google.devtools.build.xcode.zip.ZipInputEntry;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import javax.annotation.CheckReturnValue;
+
+/**
+ * Implementation of the final steps to create an iOS application bundle.
+ *
+ * TODO(bazel-team): Add asset catalog compilation and bundling to this logic.
+ */
+public final class BundleMerging {
+ @VisibleForTesting final FileSystem fileSystem;
+ @VisibleForTesting final Path outputZip;
+ @VisibleForTesting final ImmutableList<ZipInputEntry> inputs;
+ @VisibleForTesting final ImmutableList<MergeZip> mergeZips;
+
+ /**
+ * We can instantiate this class for testing purposes. For typical uses, just use
+ * {@link #merge(FileSystem, Control)}.
+ */
+ private BundleMerging(FileSystem fileSystem,
+ Path outputZip, ImmutableList<ZipInputEntry> inputs, ImmutableList<MergeZip> mergeZips) {
+ this.fileSystem = Preconditions.checkNotNull(fileSystem);
+ this.outputZip = Preconditions.checkNotNull(outputZip);
+ this.inputs = Preconditions.checkNotNull(inputs);
+ this.mergeZips = Preconditions.checkNotNull(mergeZips);
+ }
+
+ /**
+ * Joins two paths to be used in a zip file. The {@code right} part of the path must be relative.
+ * The {@code left} part could or could not have a trailing slash. These paths are used in .ipa
+ * (.zip) files, which must use forward slashes, so they are hard-coded here.
+ * <p>
+ * TODO(bazel-team): This is messy. See if we can use some common joining function that handles
+ * empty paths and doesn't automatically inherit the path conventions of the host platform.
+ */
+ private static String joinPath(String left, String right) {
+ Preconditions.checkArgument(!right.startsWith("/"), "'right' must be relative: %s", right);
+ if (left.isEmpty() || right.isEmpty() || left.endsWith("/")) {
+ return left + right;
+ } else {
+ return left + "/" + right;
+ }
+ }
+
+ private static final String INFOPLIST_FILENAME = "Info.plist";
+ private static final String PKGINFO_FILENAME = "PkgInfo";
+
+ /**
+ * Adds merge artifacts from the given {@code control} into builders that collect merge zips and
+ * individual files. {@code bundleRoot} is prepended to each path, except the paths in the merge
+ * zips.
+ */
+ private static void mergeInto(
+ Path tempDir, FileSystem fileSystem, Control control, String bundleRoot,
+ ImmutableList.Builder<ZipInputEntry> packagedFilesBuilder,
+ ImmutableList.Builder<MergeZip> mergeZipsBuilder, boolean includePkgInfo) throws IOException {
+ Path tempMergedPlist = Files.createTempFile(tempDir, null, INFOPLIST_FILENAME);
+ Path tempPkgInfo = Files.createTempFile(tempDir, null, PKGINFO_FILENAME);
+
+ // Generate the Info.plist and PkgInfo files to include in the app bundle.
+ ImmutableList.Builder<Path> sourcePlistFilesBuilder = new ImmutableList.Builder<>();
+ for (String sourcePlist : control.getSourcePlistFileList()) {
+ sourcePlistFilesBuilder.add(fileSystem.getPath(sourcePlist));
+ }
+ ImmutableList<Path> sourcePlistFiles = sourcePlistFilesBuilder.build();
+ ImmutableMap.Builder<String, String> substitutionMap = ImmutableMap.builder();
+ for (VariableSubstitution substitution : control.getVariableSubstitutionList()) {
+ substitutionMap.put(substitution.getName(), substitution.getValue());
+ }
+ PlistMerging plistMerging = PlistMerging.from(
+ sourcePlistFiles,
+ PlistMerging.automaticEntries(
+ TargetDeviceFamily.fromBundleMergeNames(control.getTargetDeviceFamilyList()),
+ Platform.valueOf(control.getPlatform()),
+ control.getSdkVersion(),
+ control.getMinimumOsVersion()),
+ substitutionMap.build(),
+ new KeysToRemoveIfEmptyString("CFBundleIconFile", "NSPrincipalClass"));
+ if (control.hasExecutableName()) {
+ plistMerging.setExecutableName(control.getExecutableName());
+ }
+ plistMerging.write(tempMergedPlist, tempPkgInfo);
+
+
+ bundleRoot = joinPath(bundleRoot, control.getBundleRoot());
+
+ // Add files to zip configuration which creates the final application bundle.
+ packagedFilesBuilder
+ .add(new ZipInputEntry(tempMergedPlist, joinPath(bundleRoot, INFOPLIST_FILENAME)));
+ if (includePkgInfo) {
+ packagedFilesBuilder
+ .add(new ZipInputEntry(tempPkgInfo, joinPath(bundleRoot, PKGINFO_FILENAME)));
+ }
+ for (BundleFile bundleFile : control.getBundleFileList()) {
+ int externalFileAttribute = bundleFile.hasExternalFileAttribute()
+ ? bundleFile.getExternalFileAttribute() : ZipInputEntry.DEFAULT_EXTERNAL_FILE_ATTRIBUTE;
+ packagedFilesBuilder.add(
+ new ZipInputEntry(
+ fileSystem.getPath(bundleFile.getSourceFile()),
+ joinPath(bundleRoot, bundleFile.getBundlePath()),
+ externalFileAttribute));
+ }
+
+ for (String mergeZip : control.getMergeWithoutNamePrefixZipList()) {
+ mergeZipsBuilder.add(MergeZip.newBuilder()
+ .setSourcePath(mergeZip)
+ .build());
+ }
+ mergeZipsBuilder.addAll(control.getMergeZipList());
+
+ for (Control nestedControl : control.getNestedBundleList()) {
+ mergeInto(tempDir, fileSystem, nestedControl, bundleRoot, packagedFilesBuilder,
+ mergeZipsBuilder, /*includePkgInfo=*/false);
+ }
+ }
+
+ /**
+ * Returns a zipper configuration that can be executed to create the application bundle.
+ */
+ @CheckReturnValue
+ @VisibleForTesting
+ static BundleMerging merging(Path tempDir, FileSystem fileSystem, Control control)
+ throws IOException {
+ ImmutableList.Builder<MergeZip> mergeZipsBuilder = new ImmutableList.Builder<>();
+ ImmutableList.Builder<ZipInputEntry> packagedFilesBuilder =
+ new ImmutableList.Builder<ZipInputEntry>();
+
+ mergeInto(tempDir, fileSystem, control, /*bundleRoot=*/"", packagedFilesBuilder,
+ mergeZipsBuilder, /*includePkgInfo=*/true);
+
+ return new BundleMerging(fileSystem, fileSystem.getPath(control.getOutFile()),
+ packagedFilesBuilder.build(), mergeZipsBuilder.build());
+ }
+
+ /**
+ * Copies all entries from the source zip into a destination zip using the given combiner. The
+ * contents of the source zip can appear to be in a sub-directory of the destination zip by
+ * passing a non-empty string for the entry names prefix with a trailing '/'.
+ */
+ private void addEntriesFromOtherZip(ZipCombiner combiner, Path sourceZip, String entryNamesPrefix)
+ throws IOException {
+ try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(sourceZip))) {
+ while (true) {
+ ZipEntry zipInEntry = zipIn.getNextEntry();
+ if (zipInEntry == null) {
+ break;
+ }
+ // TODO(bazel-team): preserve the external file attribute field in the source zip entry.
+ combiner.addFile(entryNamesPrefix + zipInEntry.getName(), DOS_EPOCH, zipIn,
+ ZipInputEntry.DEFAULT_DIRECTORY_ENTRY_INFO);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void execute() throws IOException {
+ try (OutputStream out = Files.newOutputStream(outputZip);
+ ZipCombiner combiner = new ZipCombiner(FORCE_DEFLATE, out)) {
+ ZipInputEntry.addAll(combiner, inputs);
+ for (MergeZip mergeZip : mergeZips) {
+ addEntriesFromOtherZip(
+ combiner, fileSystem.getPath(mergeZip.getSourcePath()), mergeZip.getEntryNamePrefix());
+ }
+ }
+ }
+
+ /**
+ * Creates an {@code .ipa} file for an iOS application.
+ * @param fileSystem used to resolve paths specified in {@code control}
+ * @param control specifies the locations of input and output files and other parameters used to
+ * create the final {@code .ipa} file.
+ */
+ public static void merge(FileSystem fileSystem, Control control) throws IOException {
+ merging(Files.createTempDirectory("mergebundle"), fileSystem, control).execute();
+ }
+}
diff --git a/src/objc_tools/momczip/README b/src/objc_tools/momczip/README
new file mode 100644
index 0000000..1c87f43
--- /dev/null
+++ b/src/objc_tools/momczip/README
@@ -0,0 +1,2 @@
+momczip invokes momc ("Managed object model compiler") and zips up the output,
+as the number of output files is unpredictable.
diff --git a/src/objc_tools/momczip/java/com/google/devtools/build/xcode/momczip/MomcZip.java b/src/objc_tools/momczip/java/com/google/devtools/build/xcode/momczip/MomcZip.java
new file mode 100644
index 0000000..8cd6589
--- /dev/null
+++ b/src/objc_tools/momczip/java/com/google/devtools/build/xcode/momczip/MomcZip.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.momczip;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.zippingoutput.Arguments;
+import com.google.devtools.build.xcode.zippingoutput.Wrapper;
+import com.google.devtools.build.xcode.zippingoutput.Wrappers;
+
+import java.io.IOException;
+
+/**
+ * A tool which wraps momc, by running momc and zipping its output. See the JavaDoc for
+ * {@link Wrapper} for more information.
+ */
+public class MomcZip implements Wrapper {
+ @Override
+ public String name() {
+ return "MomcZip";
+ }
+
+ @Override
+ public String subtoolName() {
+ return "momc";
+ }
+
+ @Override
+ public Iterable<String> subCommand(Arguments args, String outputDirectory) {
+ return new ImmutableList.Builder<String>()
+ .add(args.subtoolCmd())
+ .addAll(args.subtoolExtraArgs())
+ .add(outputDirectory)
+ .build();
+ }
+
+ public static void main(String[] args) throws IOException, InterruptedException {
+ Wrappers.execute(args, new MomcZip());
+ }
+
+ @Override
+ public boolean outputDirectoryMustExist() {
+ return false;
+ }
+}
diff --git a/src/objc_tools/plmerge/README b/src/objc_tools/plmerge/README
new file mode 100644
index 0000000..972409f
--- /dev/null
+++ b/src/objc_tools/plmerge/README
@@ -0,0 +1,2 @@
+plmerge merges plist files into one. It is similar to Xcode's built in
+builtin-infoPlistUtility.
diff --git a/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/KeysToRemoveIfEmptyString.java b/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/KeysToRemoveIfEmptyString.java
new file mode 100644
index 0000000..5f4662f
--- /dev/null
+++ b/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/KeysToRemoveIfEmptyString.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.plmerge;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Iterator;
+
+/**
+ * A glorified {@link Iterable} which contains keys which should be automatically removed from the
+ * final plist if they are empty strings.
+ */
+public final class KeysToRemoveIfEmptyString implements Iterable<String> {
+ private final Iterable<String> keyNames;
+
+ public KeysToRemoveIfEmptyString(String... keyNames) {
+ this.keyNames = ImmutableList.copyOf(keyNames);
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return keyNames.iterator();
+ }
+}
+
diff --git a/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/PlMerge.java b/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/PlMerge.java
new file mode 100644
index 0000000..e61f2b9
--- /dev/null
+++ b/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/PlMerge.java
@@ -0,0 +1,84 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.plmerge;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import com.dd.plist.NSObject;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Entry point for the {@code plmerge} tool, which merges the data from one or more plists into a
+ * single binary plist. This tool's functionality is similar to that of the
+ * {@code builtin-infoPlistUtility} in Xcode.
+ */
+public class PlMerge {
+ /**
+ * Options for {@link PlMerge}.
+ */
+ public static class PlMergeOptions extends OptionsBase {
+ @Option(
+ name = "source_file",
+ help = "Paths to the plist files to merge. These can be binary, XML, or ASCII format. "
+ + "Repeat this flag to specify multiple files. Required.",
+ allowMultiple = true,
+ defaultValue = "null")
+ public List<String> sourceFiles;
+
+ @Option(
+ name = "out_file",
+ help = "Path to the output file. Required.",
+ defaultValue = "null")
+ public String outFile;
+ }
+
+ public static void main(String[] args) throws IOException, OptionsParsingException {
+ OptionsParser parser = OptionsParser.newOptionsParser(PlMergeOptions.class);
+ parser.parse(args);
+ PlMergeOptions options = parser.getOptions(PlMergeOptions.class);
+ if (options.sourceFiles.isEmpty()) {
+ missingArg("At least one --source_file");
+ }
+ if (options.outFile == null) {
+ missingArg("--out_file");
+ }
+ FileSystem fileSystem = FileSystems.getDefault();
+
+ List<Path> sourceFilePaths = new ArrayList<>();
+ for (String sourceFile : options.sourceFiles) {
+ sourceFilePaths.add(fileSystem.getPath(sourceFile));
+ }
+
+ PlistMerging merging = PlistMerging.from(sourceFilePaths, ImmutableMap.<String, NSObject>of(),
+ ImmutableMap.<String, String>of(), new KeysToRemoveIfEmptyString());
+ merging.writePlist(fileSystem.getPath(options.outFile));
+ }
+
+ private static void missingArg(String flag) {
+ throw new IllegalArgumentException(flag + " is required:\n"
+ + Options.getUsage(PlMergeOptions.class));
+ }
+}
diff --git a/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/PlistMerging.java b/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/PlistMerging.java
new file mode 100644
index 0000000..dba8f84
--- /dev/null
+++ b/src/objc_tools/plmerge/java/com/google/devtools/build/xcode/plmerge/PlistMerging.java
@@ -0,0 +1,282 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.plmerge;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.ByteSource;
+import com.google.devtools.build.xcode.common.Platform;
+import com.google.devtools.build.xcode.common.TargetDeviceFamily;
+import com.google.devtools.build.xcode.util.Equaling;
+import com.google.devtools.build.xcode.util.Intersection;
+import com.google.devtools.build.xcode.util.Mapping;
+import com.google.devtools.build.xcode.util.Value;
+
+import com.dd.plist.BinaryPropertyListWriter;
+import com.dd.plist.NSArray;
+import com.dd.plist.NSDictionary;
+import com.dd.plist.NSObject;
+import com.dd.plist.NSString;
+import com.dd.plist.PropertyListFormatException;
+import com.dd.plist.PropertyListParser;
+
+import org.xml.sax.SAXException;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+/**
+ * Utility code for merging project files.
+ */
+public class PlistMerging extends Value<PlistMerging> {
+
+ /**
+ * Exception type thrown when validation of the plist file fails.
+ */
+ public static class ValidationException extends RuntimeException {
+ ValidationException(String message) {
+ super(message);
+ }
+ }
+
+ private final NSDictionary merged;
+
+ @VisibleForTesting
+ PlistMerging(NSDictionary merged) {
+ super(merged);
+ this.merged = merged;
+ }
+
+ /**
+ * Merges several plist files into a single {@code NSDictionary}. Each file should be a plist (of
+ * one of these formats: ASCII, Binary, or XML) that contains an NSDictionary.
+ */
+ @VisibleForTesting
+ static NSDictionary merge(Iterable<? extends Path> sourceFilePaths) throws IOException {
+ NSDictionary result = new NSDictionary();
+ for (Path sourceFilePath : sourceFilePaths) {
+ result.putAll(readPlistFile(sourceFilePath));
+ }
+ return result;
+ }
+
+ public static NSDictionary readPlistFile(final Path sourceFilePath) throws IOException {
+ ByteSource rawBytes = new Utf8BomSkippingByteSource(sourceFilePath);
+
+ try {
+ try (InputStream in = rawBytes.openStream()) {
+ return (NSDictionary) PropertyListParser.parse(in);
+ } catch (PropertyListFormatException | ParseException e) {
+ // If we failed to parse, the plist may implicitly be a map. To handle this, wrap the plist
+ // with {}.
+ // TODO(bazel-team): Do this in a cleaner way.
+ ByteSource concatenated = ByteSource.concat(
+ ByteSource.wrap(new byte[] {'{'}),
+ rawBytes,
+ ByteSource.wrap(new byte[] {'}'}));
+ try (InputStream in = concatenated.openStream()) {
+ return (NSDictionary) PropertyListParser.parse(in);
+ }
+ }
+ } catch (PropertyListFormatException | ParseException | ParserConfigurationException
+ | SAXException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Writes the results of a merge operation to a plist file.
+ * @param plistPath the path of the plist to write in binary format
+ */
+ public void writePlist(Path plistPath) throws IOException {
+ try (OutputStream out = Files.newOutputStream(plistPath)) {
+ BinaryPropertyListWriter.write(out, merged);
+ }
+ }
+
+ /**
+ * Writes a PkgInfo file based on certain keys in the merged plist.
+ * @param pkgInfoPath the path of the PkgInfo file to write. In many iOS apps, this file just
+ * contains the raw string {@code APPL????}.
+ */
+ public void writePkgInfo(Path pkgInfoPath) throws IOException {
+ String pkgInfo =
+ Mapping.of(merged, "CFBundlePackageType").or(NSObject.wrap("APPL")).toString()
+ + Mapping.of(merged, "CFBundleSignature").or(NSObject.wrap("????")).toString();
+ Files.write(pkgInfoPath, pkgInfo.getBytes(StandardCharsets.UTF_8));
+ }
+
+ /** Invokes {@link #writePlist(Path)} and {@link #writePkgInfo(Path)}. */
+ public void write(Path plistPath, Path pkgInfoPath) throws IOException {
+ writePlist(plistPath);
+ writePkgInfo(pkgInfoPath);
+ }
+
+ /**
+ * Returns a map containing entries that should be added to the merged plist. These are usually
+ * generated by Xcode automatically during the build process.
+ */
+ public static Map<String, NSObject> automaticEntries(
+ Iterable<TargetDeviceFamily> targetedDeviceFamily, Platform platform, String sdkVersion,
+ String minimumOsVersion) {
+ ImmutableMap.Builder<String, NSObject> result = new ImmutableMap.Builder<>();
+ List<Integer> uiDeviceFamily =
+ Mapping.of(
+ TargetDeviceFamily.UI_DEVICE_FAMILY_VALUES,
+ ImmutableSet.copyOf(targetedDeviceFamily))
+ .get();
+ result.put("UIDeviceFamily", NSObject.wrap(uiDeviceFamily.toArray()));
+ result.put("DTPlatformName", NSObject.wrap(platform.getLowerCaseNameInPlist()));
+ result.put("DTSDKName", NSObject.wrap(platform.getLowerCaseNameInPlist() + sdkVersion));
+ result.put("CFBundleSupportedPlatforms", new NSArray(NSObject.wrap(platform.getNameInPlist())));
+
+ if (platform == Platform.DEVICE) {
+ // TODO(bazel-team): Figure out if there are more appropriate values to put here, or if any
+ // can be omitted. These have been copied from a plist file generated by Xcode for a device
+ // build.
+ result.put("DTCompiler", NSObject.wrap("com.apple.compilers.llvm.clang.1_0"));
+ result.put("BuildMachineOSBuild", NSObject.wrap("13D65"));
+ result.put("DTPlatformBuild", NSObject.wrap("11B508"));
+ result.put("DTSDKBuild", NSObject.wrap("11B508"));
+ result.put("DTXcode", NSObject.wrap("0502"));
+ result.put("DTXcodeBuild", NSObject.wrap("5A3005"));
+ result.put("DTPlatformVersion", NSObject.wrap(sdkVersion));
+ result.put("MinimumOSVersion", NSObject.wrap(minimumOsVersion));
+ }
+ return result.build();
+ }
+
+ /**
+ * Generates final merged Plist file and PkgInfo file in the specified locations, and includes the
+ * "automatic" entries in the Plist.
+ */
+ public static PlistMerging from(List<Path> sourceFiles, Map<String, NSObject> automaticEntries,
+ Map<String, String> substitutions, KeysToRemoveIfEmptyString keysToRemoveIfEmptyString)
+ throws IOException {
+ NSDictionary merged = PlistMerging.merge(sourceFiles);
+
+ Set<String> conflictingEntries = Intersection.of(automaticEntries.keySet(), merged.keySet());
+ Preconditions.checkArgument(conflictingEntries.isEmpty(),
+ "The following plist entries are generated automatically, but are present in one of the "
+ + "input lists: %s", conflictingEntries);
+ merged.putAll(automaticEntries);
+
+ for (Map.Entry<String, NSObject> entry : merged.entrySet()) {
+ if (entry.getValue().toJavaObject() instanceof String) {
+ String newValue = substituteEnvironmentVariable(
+ substitutions, (String) entry.getValue().toJavaObject());
+ merged.put(entry.getKey(), newValue);
+ }
+ }
+
+ for (String key : keysToRemoveIfEmptyString) {
+ if (Equaling.of(Mapping.of(merged, key), Optional.<NSObject>of(new NSString("")))) {
+ merged.remove(key);
+ }
+ }
+
+ return new PlistMerging(merged);
+ }
+
+ // Assume that if an RFC 1034 format string is specified, the value is RFC 1034 compliant.
+ private static String substituteEnvironmentVariable(
+ Map<String, String> substitutions, String string) {
+ // The substitution is *not* performed recursively.
+ for (Map.Entry<String, String> variable : substitutions.entrySet()) {
+ for (String variableNameWithFormatString : withFormatStrings(variable.getKey())) {
+ string = string
+ .replace("${" + variableNameWithFormatString + "}", variable.getValue())
+ .replace("$(" + variableNameWithFormatString + ")", variable.getValue());
+ }
+ }
+
+ return string;
+ }
+
+ private static ImmutableSet<String> withFormatStrings(String variableName) {
+ return ImmutableSet.of(variableName, variableName + ":rfc1034identifier");
+ }
+
+ @VisibleForTesting
+ NSDictionary asDictionary() {
+ return merged;
+ }
+
+ /**
+ * Sets the given executable name on this merged plist in the {@code CFBundleExecutable}
+ * attribute.
+ *
+ * @param executableName name of the bundle executable
+ * @return this plist merging
+ * @throws ValidationException if the plist already contains an incompatible
+ * {@code CFBundleExecutable} entry
+ */
+ public PlistMerging setExecutableName(String executableName) {
+ NSString bundleExecutable = (NSString) merged.get("CFBundleExecutable");
+
+ if (bundleExecutable == null) {
+ merged.put("CFBundleExecutable", executableName);
+ } else if (!executableName.equals(bundleExecutable.getContent())) {
+ throw new ValidationException(String.format(
+ "Blaze generated the executable %s but the Plist CFBundleExecutable is %s",
+ executableName, bundleExecutable));
+ }
+
+ return this;
+ }
+
+ private static class Utf8BomSkippingByteSource extends ByteSource {
+
+ private static final byte[] UTF8_BOM =
+ new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF };
+
+ private final Path path;
+
+ public Utf8BomSkippingByteSource(Path path) {
+ this.path = path;
+ }
+
+ @Override
+ public InputStream openStream() throws IOException {
+ InputStream stream = new BufferedInputStream(Files.newInputStream(path));
+ stream.mark(UTF8_BOM.length);
+ byte[] buffer = new byte[UTF8_BOM.length];
+ int read = stream.read(buffer);
+ stream.reset();
+ buffer = Arrays.copyOf(buffer, read);
+
+ if (UTF8_BOM.length == read && Arrays.equals(buffer, UTF8_BOM)) {
+ stream.skip(UTF8_BOM.length);
+ }
+
+ return stream;
+ }
+ }
+}
diff --git a/src/objc_tools/xcodegen/README b/src/objc_tools/xcodegen/README
new file mode 100644
index 0000000..c91dea7
--- /dev/null
+++ b/src/objc_tools/xcodegen/README
@@ -0,0 +1,2 @@
+xcodegen generates xcode project files for metadata about a given set of
+targets.
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/AggregateKey.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/AggregateKey.java
new file mode 100644
index 0000000..d4e2e24
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/AggregateKey.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Optional;
+import com.google.devtools.build.xcode.util.Value;
+
+import java.nio.file.Path;
+
+/**
+ * Data used to as a key when assigning items to aggregate references
+ * (see {@link AggregateReferenceType}). All items sharing a single key value should belong to the
+ * same group.
+ * <p>
+ * Note that the information in the grouping key is also the information that is used to create a
+ * new group, minus the actual list of files in the group. See
+ * {@link AggregateReferenceType#create(AggregateKey, Iterable)}.
+ */
+public class AggregateKey extends Value<AggregateKey> {
+ private final Optional<String> name;
+ private final Optional<Path> path;
+
+ public AggregateKey(Optional<String> name, Optional<Path> path) {
+ super(name, path);
+ this.name = name;
+ this.path = path;
+ }
+
+ public Optional<String> name() {
+ return name;
+ }
+
+ public Optional<Path> path() {
+ return path;
+ }
+
+ /**
+ * Returns an instance that indicates any item assigned to it should not belong to a group.
+ */
+ public static AggregateKey standalone() {
+ return new AggregateKey(Optional.<String>absent(), Optional.<Path>absent());
+ }
+
+ /**
+ * Indicates that this grouping key means an item should not belong to any group.
+ */
+ public boolean isStandalone() {
+ return !path().isPresent() && !name().isPresent();
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/AggregateReferenceType.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/AggregateReferenceType.java
new file mode 100644
index 0000000..3dc9787
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/AggregateReferenceType.java
@@ -0,0 +1,129 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.SetMultimap;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXVariantGroup;
+import com.facebook.buck.apple.xcode.xcodeproj.XCVersionGroup;
+
+import java.nio.file.Path;
+
+/**
+ * An aggregate reference is a kind of PBXReference that contains one or more files, grouped by some
+ * criteria, and appearing as a group in the Xcode project navigator, and often handled as a single
+ * file during the build phase and in other situations.
+ */
+public enum AggregateReferenceType {
+ /**
+ * A group which contains multiple .xcdatamodel directories where each is a different version of
+ * the same schema. We may have to support other files besides .xcdatamodel in the future.
+ * Instances of this group are represented by {@link XCVersionGroup} and are grouped by the
+ * relative path of the containing .xcdatamodeld directory.
+ */
+ XCVersionGroup {
+ @Override
+ public PBXReference create(AggregateKey key, Iterable<PBXFileReference> children) {
+ XCVersionGroup result = new XCVersionGroup(
+ key.name().orNull(),
+ key.path().isPresent() ? key.path().get().toString() : null,
+ SourceTree.GROUP);
+ Iterables.addAll(result.getChildren(), children);
+ return result;
+ }
+
+ @Override
+ public AggregateKey aggregateKey(Path path) {
+ Path parent = path.getParent();
+ if (parent.getFileName().toString().endsWith(".xcdatamodeld")) {
+ return new AggregateKey(
+ Optional.of(parent.getFileName().toString()), Optional.of(parent));
+ } else {
+ return AggregateKey.standalone();
+ }
+ }
+
+ @Override
+ public Path pathInAggregate(Path path) {
+ return path.getFileName();
+ }
+ },
+
+ /**
+ * A group which contains the same content in multiple languages, each language belonging to a
+ * different file. Instances of this group are represented by {@link PBXVariantGroup} and are
+ * grouped by the base name of the file (e.g. "foo" in "/usr/bar/foo").
+ */
+ PBXVariantGroup {
+ @Override
+ public PBXReference create(AggregateKey key, Iterable<PBXFileReference> children) {
+ PBXVariantGroup result = new PBXVariantGroup(
+ key.name().orNull(),
+ key.path().isPresent() ? key.path().get().toString() : null,
+ SourceTree.GROUP);
+ Iterables.addAll(result.getChildren(), children);
+ return result;
+ }
+
+ @Override
+ public AggregateKey aggregateKey(Path path) {
+ if (Resources.languageOfLprojDir(path).isPresent()) {
+ return new AggregateKey(
+ Optional.of(path.getFileName().toString()), Optional.<Path>absent());
+ } else {
+ return AggregateKey.standalone();
+ }
+ }
+
+ @Override
+ public Path pathInAggregate(Path path) {
+ return path;
+ }
+ };
+
+ /**
+ * Creates a new instance of this group with the group information and children.
+ */
+ public abstract PBXReference create(AggregateKey key, Iterable<PBXFileReference> children);
+
+ /**
+ * Returns the value by which this item should be grouped. All items sharing the same key should
+ * belong to the same group. An {@link AggregateKey#standalone()} return here indicates that the
+ * item should not belong to a group and should be built and treated as a standalone file.
+ */
+ public abstract AggregateKey aggregateKey(Path path);
+
+ public abstract Path pathInAggregate(Path path);
+
+ /**
+ * Groups a sequence of items according to their {@link #aggregateKey(Path)}.
+ */
+ public SetMultimap<AggregateKey, Path> aggregates(Iterable<Path> paths) {
+ ImmutableSetMultimap.Builder<AggregateKey, Path> result =
+ new ImmutableSetMultimap.Builder<>();
+ for (Path path : paths) {
+ AggregateKey key = aggregateKey(path);
+ Path referencePath = key.isStandalone() ? path : pathInAggregate(path);
+ result.put(key, referencePath);
+ }
+ return result.build();
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/CurrentVersionSetter.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/CurrentVersionSetter.java
new file mode 100644
index 0000000..a14c7b0
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/CurrentVersionSetter.java
@@ -0,0 +1,92 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.xcode.plmerge.PlistMerging;
+
+import com.dd.plist.NSDictionary;
+import com.dd.plist.NSString;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+import com.facebook.buck.apple.xcode.xcodeproj.XCVersionGroup;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Processes the {@link XCVersionGroup} instances in a sequence of {@link PBXReference}s to have
+ * the {@code currentVersion} field set properly according to the {@code .xccurrentversion} file, if
+ * available.
+ *
+ * <p>This will NOT set the current version for any group where one of the following is true:
+ * <ul>
+ * <li>the sourceTree of the group is not GROUP (meaning it is not relative to the workspace root)
+ * <li>the {@code .xccurrentversion} file does not exist or is not accessible
+ * <li>the plist in the file does not contain the correct entry
+ * <li>the {@link XCVersionGroup} does not have a child that matches the version in the
+ * {@code .xccurrentversion} file
+ * </ul>
+ */
+public final class CurrentVersionSetter implements PbxReferencesProcessor {
+ private final Path workspaceRoot;
+
+ public CurrentVersionSetter(Path workspaceRoot) {
+ this.workspaceRoot = Preconditions.checkNotNull(workspaceRoot);
+ }
+
+ @Override
+ public Iterable<PBXReference> process(Iterable<PBXReference> references) {
+ for (PBXReference reference : references) {
+ if ((reference instanceof XCVersionGroup) && (reference.getPath() != null)) {
+ trySetCurrentVersion((XCVersionGroup) reference);
+ }
+ }
+ return references;
+ }
+
+ private void trySetCurrentVersion(XCVersionGroup group) {
+ if (group.getSourceTree() != SourceTree.GROUP) {
+ return;
+ }
+
+ Path groupPath = workspaceRoot.resolve(group.getPath());
+ Path currentVersionPlist = groupPath.resolve(".xccurrentversion");
+ if (!Files.isReadable(currentVersionPlist)) {
+ return;
+ }
+
+ NSDictionary plist;
+ try {
+ plist = PlistMerging.readPlistFile(currentVersionPlist);
+ } catch (IOException e) {
+ return;
+ }
+ NSString currentVersion = (NSString) plist.get("_XCCurrentVersionName");
+ if (currentVersion == null) {
+ return;
+ }
+
+ for (PBXFileReference child : group.getChildren()) {
+ if (child.getName().equals(currentVersion.getContent())) {
+ group.setCurrentVersion(Optional.of(child));
+ return;
+ }
+ }
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/FileReference.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/FileReference.java
new file mode 100644
index 0000000..99522a7
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/FileReference.java
@@ -0,0 +1,98 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.xcode.util.Value;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+
+import java.io.File;
+
+/**
+ * Contains data similar to {@link PBXFileReference}, but is actually a value type, with working
+ * {@link #equals(Object)} and {@link #hashCode()} methods.
+ * <p>
+ * TODO(bazel-team): Consider just adding equals and hashCode methods to PBXFileReference. This may
+ * not be as straight-forward as it sounds, since the Xcodeproj serialization logic and all related
+ * classes are based on identity equality semantics.
+ */
+public class FileReference extends Value<FileReference> {
+ private final String name;
+ private final Optional<String> path;
+ private final SourceTree sourceTree;
+ private final Optional<String> explicitFileType;
+
+ @VisibleForTesting
+ FileReference(
+ String name,
+ Optional<String> path,
+ SourceTree sourceTree,
+ Optional<String> explicitFileType) {
+ super(name, path, sourceTree, explicitFileType);
+ this.name = name;
+ this.path = path;
+ this.sourceTree = sourceTree;
+ this.explicitFileType = explicitFileType;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public Optional<String> path() {
+ return path;
+ }
+
+ public SourceTree sourceTree() {
+ return sourceTree;
+ }
+
+ public Optional<String> explicitFileType() {
+ return explicitFileType;
+ }
+
+ /**
+ * Returns an instance whose name is the base name of the path.
+ */
+ public static FileReference of(String path, SourceTree sourceTree) {
+ return new FileReference(
+ new File(path).getName(),
+ Optional.of(path),
+ sourceTree,
+ Optional.<String>absent());
+ }
+
+ /**
+ * Returns an instance with a path and without an {@link #explicitFileType()}.
+ */
+ public static FileReference of(String name, String path, SourceTree sourceTree) {
+ return new FileReference(
+ name, Optional.of(path), sourceTree, Optional.<String>absent());
+ }
+
+ /**
+ * Returns an instance equivalent to this one, but with {@link #explicitFileType()} set to the
+ * given value. This instance should not already have a value set for {@link #explicitFileType()}.
+ */
+ public FileReference withExplicitFileType(String explicitFileType) {
+ Preconditions.checkState(!explicitFileType().isPresent(),
+ "should not already have explicitFileType: %s", this);
+ return new FileReference(name(), path(), sourceTree(), Optional.of(explicitFileType));
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/HasProjectNavigatorFiles.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/HasProjectNavigatorFiles.java
new file mode 100644
index 0000000..0f2a4bf
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/HasProjectNavigatorFiles.java
@@ -0,0 +1,28 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+
+/**
+ * An object that knows some references that should be added to the main group so they appear in the
+ * project navigator view in Xcode.
+ */
+public interface HasProjectNavigatorFiles {
+ /**
+ * Returns all references known by this object that should be added to the main group.
+ */
+ public Iterable<PBXReference> mainGroupReferences();
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LibraryObjects.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LibraryObjects.java
new file mode 100644
index 0000000..f751d9d
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LibraryObjects.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXBuildFile;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFrameworksBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+
+/**
+ * Collector that gathers references to libraries and frameworks when generating {@linkplain
+ * PBXFrameworksBuildPhase framework build phases} for later display in XCode.
+ *
+ * <p>Use this class to {@link #newBuildPhase() generate} a framework build phase for each target,
+ * by adding things that are linked with the final binary:
+ * {@link BuildPhaseBuilder#addFramework frameworks} ("v1_6/GoogleMaps.framework"),
+ * {@link BuildPhaseBuilder#addSdkFramework SDK frameworks} ("XCTest.framework",
+ * "Foundation.framework") or {@link BuildPhaseBuilder#addDylib dylibs} ("libz.dylib"). Anything
+ * added here will also be returned by {@link #mainGroupReferences}.
+ *
+ * <p>File references used by this class are de-duplicated against a global cache.
+ */
+public final class LibraryObjects implements HasProjectNavigatorFiles {
+
+ @VisibleForTesting static final String FRAMEWORK_FILE_TYPE = "wrapper.framework";
+ @VisibleForTesting static final String DYLIB_FILE_TYPE = "compiled.mach-o.dylib";
+
+ private final LinkedHashMap<FileReference, PBXReference> fileToMainGroupReferences =
+ new LinkedHashMap<>();
+ private final PBXFileReferences fileReferenceCache;
+
+ /**
+ * @param fileReferenceCache global file reference repository used to avoid creating the same
+ * file reference twice
+ */
+ public LibraryObjects(PBXFileReferences fileReferenceCache) {
+ this.fileReferenceCache = checkNotNull(fileReferenceCache);
+ }
+
+ /**
+ * Builder that assembles information required to generate a {@link PBXFrameworksBuildPhase}.
+ */
+ public final class BuildPhaseBuilder {
+
+ private final LinkedHashSet<FileReference> fileReferences = new LinkedHashSet<>();
+
+ private BuildPhaseBuilder() {} // Don't allow instantiation from outside the enclosing class.
+
+ /**
+ * Creates a new dylib library based on the passed name.
+ *
+ * @param name simple dylib without ".dylib" suffix, e.g. "libz"
+ */
+ public BuildPhaseBuilder addDylib(String name) {
+ FileReference reference =
+ FileReference.of(String.format("usr/lib/%s.dylib", name), SourceTree.SDKROOT)
+ .withExplicitFileType(DYLIB_FILE_TYPE);
+ fileReferences.add(reference);
+ return this;
+ }
+
+ /**
+ * Creates a new SDK framework based on the passed name.
+ *
+ * @param name simple framework name without ".framework" suffix, e.g. "Foundation"
+ */
+ public BuildPhaseBuilder addSdkFramework(String name) {
+ String location = String.format("System/Library/Frameworks/%s.framework", name);
+ FileReference reference =
+ FileReference.of(location, SourceTree.SDKROOT).withExplicitFileType(FRAMEWORK_FILE_TYPE);
+ fileReferences.add(reference);
+ return this;
+ }
+
+ /**
+ * Creates a new (non-SDK) framework based on the given path.
+ *
+ * @param execPath path to the framework's folder, relative to the xcodeproject's path root,
+ * e.g. "v1_6/GoogleMaps.framework"
+ */
+ public BuildPhaseBuilder addFramework(String execPath) {
+ FileReference reference =
+ FileReference.of(execPath, SourceTree.GROUP).withExplicitFileType(FRAMEWORK_FILE_TYPE);
+ fileReferences.add(reference);
+ return this;
+ }
+
+ /**
+ * Returns a new build phase containing the added libraries.
+ */
+ public PBXFrameworksBuildPhase build() {
+ PBXFrameworksBuildPhase buildPhase = new PBXFrameworksBuildPhase();
+ for (FileReference reference : fileReferences) {
+ PBXFileReference fileRef = fileReferenceCache.get(reference);
+ buildPhase.getFiles().add(new PBXBuildFile(fileRef));
+ fileToMainGroupReferences.put(reference, fileRef);
+ }
+ return buildPhase;
+ }
+ }
+
+ /**
+ * Returns a builder for a new build phase.
+ */
+ public BuildPhaseBuilder newBuildPhase() {
+ return new BuildPhaseBuilder();
+ }
+
+ @Override
+ public Iterable<PBXReference> mainGroupReferences() {
+ return fileToMainGroupReferences.values();
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LocalPBXContainerItemProxy.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LocalPBXContainerItemProxy.java
new file mode 100644
index 0000000..5089942
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LocalPBXContainerItemProxy.java
@@ -0,0 +1,62 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Preconditions;
+
+import com.facebook.buck.apple.xcode.XcodeprojSerializer;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXContainerItemProxy;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXObject;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXProject;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+
+/**
+ * Represents a PBXContainerItemProxy object that does not reference a remote (other project file)
+ * object.
+ * <p>
+ * TODO(bazel-team): Upstream this to Buck.
+ */
+public class LocalPBXContainerItemProxy extends PBXContainerItemProxy {
+ private static final PBXFileReference DUMMY_FILE_REFERENCE =
+ new PBXFileReference("", "", SourceTree.ABSOLUTE);
+
+ private final PBXProject containerPortalAsProject;
+ private final PBXObject remoteGlobalIdHolder;
+
+ public LocalPBXContainerItemProxy(
+ PBXProject containerPortalAsProject, PBXObject remoteGlobalIdHolder, ProxyType proxyType) {
+ super(DUMMY_FILE_REFERENCE, "" /* remoteGlobalIDString */, proxyType);
+ this.containerPortalAsProject = Preconditions.checkNotNull(containerPortalAsProject);
+ this.remoteGlobalIdHolder = Preconditions.checkNotNull(remoteGlobalIdHolder);
+ }
+
+ public PBXProject getContainerPortalAsProject() {
+ return containerPortalAsProject;
+ }
+
+ public PBXObject getRemoteGlobalIdHolder() {
+ return remoteGlobalIdHolder;
+ }
+
+ @Override
+ public void serializeInto(XcodeprojSerializer s) {
+ // Note we don't call super.serializeInto because we don't want the DUMMY_FILE_REFERENCE to
+ // get saved to the project file (even though it is probably harmless).
+ s.addField("containerPortal", containerPortalAsProject.getGlobalID());
+ s.addField("remoteGlobalIDString", remoteGlobalIdHolder);
+ s.addField("proxyType", getProxyType().getIntValue());
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LocalPBXTargetDependency.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LocalPBXTargetDependency.java
new file mode 100644
index 0000000..deb9817
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/LocalPBXTargetDependency.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.facebook.buck.apple.xcode.XcodeprojSerializer;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXTargetDependency;
+
+/**
+ * A target dependency that is not remote, which is similar to the normal Buck PBXTargetDependency,
+ * but includes a {@code target} field.
+ * <p>
+ * TODO(bazel-team): Upstream this to Buck.
+ */
+public class LocalPBXTargetDependency extends PBXTargetDependency {
+ public LocalPBXTargetDependency(LocalPBXContainerItemProxy targetProxy) {
+ super(targetProxy);
+ }
+
+ @Override
+ public void serializeInto(XcodeprojSerializer s) {
+ super.serializeInto(s);
+ s.addField("target", getTargetProxy().getRemoteGlobalIDString());
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PBXBuildFiles.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PBXBuildFiles.java
new file mode 100644
index 0000000..ff4e718
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PBXBuildFiles.java
@@ -0,0 +1,175 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.SetMultimap;
+import com.google.devtools.build.xcode.util.Mapping;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXBuildFile;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXVariantGroup;
+import com.facebook.buck.apple.xcode.xcodeproj.XCVersionGroup;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A kind of cache which makes it easier to collect and manage PBXBuildFile and PBXReference
+ * objects. It knows how to create new PBXBuildFile, PBXVariantGroup, and PBXFileReference objects
+ * from {@link Path} objects and sequences thereof.
+ * <p>
+ * A PBXFileReference specifies a path to a file and its <em>name</em>. The name is confusingly
+ * defined as the real file name for non-localized files (e.g. "foo" in "bar/foo"), and the language
+ * name for localized files (e.g. "en" in "bar/en.lproj/foo").
+ * <p>
+ * A PBXVariantGroup is a set of PBXFileReferences with the same file name (the virtual name). Each
+ * file is in some directory named *.lproj. For instance, the following files would belong to the
+ * same PBXVariantGroup:
+ *
+ * <ul>
+ * <li>foo1/en.lproj/file.strings
+ * <li>foo2/ru.lproj/file.strings
+ * </ul>
+ *
+ * Where the virtual name is "file.strings". Note that because of the way PBXVariantGroups are named
+ * and specified in .xcodeproj files, it is possible Xcode does or will use it for groups not
+ * defined by localization, but we currently ignore that possibility.
+ * <p>
+ * A PBXBuildFile is the simplest object - it is simply a reference to a PBXReference, which can be
+ * either a PBXFileReference or PBXVariantGroup. The fact that PBXFileReference and PBXVariantGroup
+ * are considered kinds of PBXReference is reflected in the Java inheritance hierarchy for the
+ * classes that model these Xcode objects.
+ * <p>
+ * The PBXBuildFile is the object referred to in the build phases of targets, so it is seen as a
+ * buildable or compilable unit. The PBXFileReference and PBXVariantGroup objects are referred to
+ * by the PBXGroups, which define what is visible in the Project Navigator view in Xcode.
+ * <p>
+ * This class assumes that all paths given through the public API are specified relative to the
+ * Xcodegen root, and creates PBXFileReferences that should be added to a group whose path is the
+ * root.
+ * TODO(bazel-team): Make this an immutable type, of which multiple instances are created as new
+ * references and build files are added. The current API is side-effect-based and confusing.
+ */
+final class PBXBuildFiles implements HasProjectNavigatorFiles {
+ /**
+ * Map from Paths to the PBXBuildFile that encompasses all and only those Paths. Because the
+ * {@link PBXBuildFile}s in this map encompass multiple files, their
+ * {@link PBXBuildFile#getFileRef()} value is a {@link PBXVariantGroup} or {@link XCVersionGroup}.
+ * <p>
+ * Note that this Map reflects the intention of the API, namely that {"a"} does not map to the
+ * same thing as {"a", "b"}, and you cannot get a build file with only one of the corresponding
+ * files - you need the whole set.
+ */
+ private Map<ImmutableSet<Path>, PBXBuildFile> aggregateBuildFiles;
+
+ private Map<FileReference, PBXBuildFile> standaloneBuildFiles;
+ private PBXFileReferences pbxReferences;
+ private List<PBXReference> mainGroupReferences;
+
+ public PBXBuildFiles(PBXFileReferences pbxFileReferences) {
+ this.aggregateBuildFiles = new HashMap<>();
+ this.standaloneBuildFiles = new HashMap<>();
+ this.pbxReferences = Preconditions.checkNotNull(pbxFileReferences);
+ this.mainGroupReferences = new ArrayList<>();
+ }
+
+ private PBXBuildFile aggregateBuildFile(ImmutableSet<Path> paths, PBXReference reference) {
+ Preconditions.checkArgument(!paths.isEmpty(), "paths must be non-empty");
+ for (PBXBuildFile cached : Mapping.of(aggregateBuildFiles, paths).asSet()) {
+ return cached;
+ }
+ PBXBuildFile buildFile = new PBXBuildFile(reference);
+ mainGroupReferences.add(reference);
+ aggregateBuildFiles.put(paths, buildFile);
+ return buildFile;
+ }
+
+ /**
+ * Returns new or cached instances of PBXBuildFiles corresponding to files that may or may not
+ * belong to an aggregate reference (see {@link AggregateReferenceType}). Files specified by the
+ * {@code paths} argument are grouped into individual PBXBuildFiles using the given
+ * {@link AggregateReferenceType}. Files that are standalone are not put in an aggregate
+ * reference, but are put in a standalone PBXBuildFile in the returned sequence.
+ */
+ public Iterable<PBXBuildFile> get(AggregateReferenceType type, Iterable<Path> paths) {
+ ImmutableList.Builder<PBXBuildFile> result = new ImmutableList.Builder<>();
+ SetMultimap<AggregateKey, Path> keyedPaths = type.aggregates(paths);
+ for (Map.Entry<AggregateKey, Collection<Path>> aggregation : keyedPaths.asMap().entrySet()) {
+ if (!aggregation.getKey().isStandalone()) {
+ ImmutableSet<Path> itemPaths = ImmutableSet.copyOf(aggregation.getValue());
+ result.add(aggregateBuildFile(
+ itemPaths, type.create(aggregation.getKey(), fileReferences(itemPaths))));
+ }
+ }
+ for (Path generalResource : keyedPaths.get(AggregateKey.standalone())) {
+ result.add(getStandalone(FileReference.of(generalResource.toString(), SourceTree.GROUP)));
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Returns a new or cached instance of a PBXBuildFile for a file that is not part of a variant
+ * group.
+ */
+ public PBXBuildFile getStandalone(FileReference file) {
+ for (PBXBuildFile cached : Mapping.of(standaloneBuildFiles, file).asSet()) {
+ return cached;
+ }
+ PBXBuildFile buildFile = new PBXBuildFile(pbxReferences.get(file));
+ mainGroupReferences.add(pbxReferences.get(file));
+ standaloneBuildFiles.put(file, buildFile);
+ return buildFile;
+ }
+
+ /**
+ * Applies {@link #fileReference(Path)} to each item in the sequence.
+ */
+ private final Iterable<PBXFileReference> fileReferences(Iterable<Path> paths) {
+ ImmutableList.Builder<PBXFileReference> result = new ImmutableList.Builder<>();
+ for (Path path : paths) {
+ result.add(fileReference(path));
+ }
+ return result.build();
+ }
+
+ /**
+ * Returns a new or cached PBXFileReference for the given file. The name of the reference depends
+ * on whether the file is in a localized (*.lproj) directory. If it is localized, then the name
+ * of the reference is the name of the language (the text before ".lproj"). Otherwise, the name is
+ * the same as the file name (e.g. Localizable.strings). This is confusing, but it is how Xcode
+ * creates PBXFileReferences.
+ */
+ private PBXFileReference fileReference(Path path) {
+ Optional<String> language = Resources.languageOfLprojDir(path);
+ String name = language.isPresent() ? language.get() : path.getFileName().toString();
+ return pbxReferences.get(FileReference.of(name, path.toString(), SourceTree.GROUP));
+ }
+
+ @Override
+ public Iterable<PBXReference> mainGroupReferences() {
+ return mainGroupReferences;
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PBXFileReferences.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PBXFileReferences.java
new file mode 100644
index 0000000..afc306b
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PBXFileReferences.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.devtools.build.xcode.util.Mapping;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A factory for {@link PBXFileReference}s. {@link PBXFileReference}s are inherently non-value
+ * types, in that each created instance appears in the serialized Xcodeproj file, even if some of
+ * the instances are equivalent. This serves as a cache so that a value type ({@link FileReference})
+ * can be converted to a canonical, cached {@link PBXFileReference} which is equivalent to it.
+ */
+final class PBXFileReferences {
+ private final Map<FileReference, PBXFileReference> cache = new HashMap<>();
+
+ /**
+ * Supplies a reference, containing values for fields specified in {@code reference}.
+ */
+ public PBXFileReference get(FileReference reference) {
+ for (PBXFileReference existing : Mapping.of(cache, reference).asSet()) {
+ return existing;
+ }
+
+ PBXFileReference result = new PBXFileReference(
+ reference.name(), reference.path().orNull(), reference.sourceTree());
+ result.setExplicitFileType(reference.explicitFileType());
+
+ cache.put(reference, result);
+ return result;
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PbxReferencesGrouper.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PbxReferencesGrouper.java
new file mode 100644
index 0000000..9190133
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PbxReferencesGrouper.java
@@ -0,0 +1,173 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.xcode.util.Containing;
+import com.google.devtools.build.xcode.util.Equaling;
+import com.google.devtools.build.xcode.util.Mapping;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXGroup;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXVariantGroup;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A {@link PBXReference} processor to group self-contained PBXReferences into PBXGroups. Grouping
+ * is done to make it easier to navigate the files of the project in Xcode's Project Navigator.
+ *
+ * <p>A <em>self-contained</em> reference is one that is not a member of a PBXVariantGroup or other
+ * aggregate group, although a self-contained reference may contain such a reference as a child.
+ *
+ * <p>This implementation arranges the {@code PBXFileReference}s into a hierarchy of
+ * {@code PBXGroup}s that mirrors the actual location of the files on disk.
+ *
+ * <p>When using this grouper, the top-level items are the following:
+ * <ul>
+ * <li>BUILT_PRODUCTS_DIR - a group containing items in the SourceRoot of this name
+ * <li>SDKROOT - a group containing items that are part of the Xcode install, such as SDK
+ * frameworks
+ * <li>workspace_root - a group containing items within the root of the workspace of the client
+ * <li>miscellaneous - anything that does not belong in one of the above groups is placed directly
+ * in the main group.
+ * </ul>
+ */
+public class PbxReferencesGrouper implements PbxReferencesProcessor {
+ private final FileSystem fileSystem;
+
+ public PbxReferencesGrouper(FileSystem fileSystem) {
+ this.fileSystem = Preconditions.checkNotNull(fileSystem, "fileSystem");
+ }
+
+ /**
+ * Converts a {@code String} to a {@code Path} using this instance's file system.
+ */
+ private Path path(String pathString) {
+ return RelativePaths.fromString(fileSystem, pathString);
+ }
+
+ /**
+ * Returns the deepest directory that contains both paths.
+ */
+ private Path deepestCommonContainer(Path path1, Path path2) {
+ Path container = path("");
+ int nameIndex = 0;
+ while ((nameIndex < Math.min(path1.getNameCount(), path2.getNameCount()))
+ && Equaling.of(path1.getName(nameIndex), path2.getName(nameIndex))) {
+ container = container.resolve(path1.getName(nameIndex));
+ nameIndex++;
+ }
+ return container;
+ }
+
+ /**
+ * Returns the parent of the given path. This is similar to {@link Path#getParent()}, but is
+ * nullable-phobic. {@link Path#getParent()} considers the root of the filesystem to be the null
+ * Path. This method uses {@code path("")} for the root. This is also how the implementation of
+ * {@link PbxReferencesGrouper} expresses <em>root</em> in general.
+ */
+ private Path parent(Path path) {
+ return (path.getNameCount() == 1) ? path("") : path.getParent();
+ }
+
+ /**
+ * The directory of the PBXGroup that will contain the given reference. For most references, this
+ * is just the actual parent directory. For {@code PBXVariantGroup}s, whose children are not
+ * guaranteed to be in any common directory except the client root, this returns the deepest
+ * common container of each child in the group.
+ */
+ private Path dirOfContainingPbxGroup(PBXReference reference) {
+ if (reference instanceof PBXVariantGroup) {
+ PBXVariantGroup variantGroup = (PBXVariantGroup) reference;
+ Path path = Paths.get(variantGroup.getChildren().get(0).getPath());
+ for (PBXReference child : variantGroup.getChildren()) {
+ path = deepestCommonContainer(path, path(child.getPath()));
+ }
+ return path;
+ } else {
+ return parent(path(reference.getPath()));
+ }
+ }
+
+ /**
+ * Contains the populated PBXGroups for a certain source tree.
+ */
+ private class Groups {
+ /**
+ * Map of paths to the PBXGroup that is used to contain all files and groups in that path.
+ */
+ final Map<Path, PBXGroup> groupCache;
+
+ Groups(String rootGroupName, SourceTree sourceTree) {
+ groupCache = new HashMap<>();
+ groupCache.put(path(""), new PBXGroup(rootGroupName, "" /* path */, sourceTree));
+ }
+
+ PBXGroup rootGroup() {
+ return Mapping.of(groupCache, path("")).get();
+ }
+
+ void add(Path dirOfContainingPbxGroup, PBXReference reference) {
+ for (PBXGroup container : Mapping.of(groupCache, dirOfContainingPbxGroup).asSet()) {
+ container.getChildren().add(reference);
+ return;
+ }
+ PBXGroup newGroup = new PBXGroup(dirOfContainingPbxGroup.getFileName().toString(),
+ null /* path */, SourceTree.GROUP);
+ newGroup.getChildren().add(reference);
+ add(parent(dirOfContainingPbxGroup), newGroup);
+ groupCache.put(dirOfContainingPbxGroup, newGroup);
+ }
+ }
+
+ @Override
+ public Iterable<PBXReference> process(Iterable<PBXReference> references) {
+ Map<SourceTree, Groups> groupsBySourceTree = ImmutableMap.of(
+ SourceTree.GROUP, new Groups("workspace_root", SourceTree.GROUP),
+ SourceTree.SDKROOT, new Groups("SDKROOT", SourceTree.SDKROOT),
+ SourceTree.BUILT_PRODUCTS_DIR,
+ new Groups("BUILT_PRODUCTS_DIR", SourceTree.BUILT_PRODUCTS_DIR));
+ ImmutableList.Builder<PBXReference> result = new ImmutableList.Builder<>();
+
+ for (PBXReference reference : references) {
+ if (Containing.key(groupsBySourceTree, reference.getSourceTree())) {
+ Path containingDir = dirOfContainingPbxGroup(reference);
+ Mapping.of(groupsBySourceTree, reference.getSourceTree())
+ .get()
+ .add(containingDir, reference);
+ } else {
+ // The reference is not inside any expected source tree, so don't try anything clever. Just
+ // add it to the main group directly (not in a nested PBXGroup).
+ result.add(reference);
+ }
+ }
+
+ for (Groups groupsRoot : groupsBySourceTree.values()) {
+ if (!groupsRoot.rootGroup().getChildren().isEmpty()) {
+ result.add(groupsRoot.rootGroup());
+ }
+ }
+
+ return result.build();
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PbxReferencesProcessor.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PbxReferencesProcessor.java
new file mode 100644
index 0000000..6632237
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/PbxReferencesProcessor.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+
+/**
+ * Processes a sequence of self-contained references in some manner before they are added to a
+ * project.
+ *
+ * <p>A <em>self-contained</em> reference is one that is not a member of a PBXVariantGroup or other
+ * aggregate group, although a self-contained reference may contain such a reference as a child.
+ */
+public interface PbxReferencesProcessor {
+ /**
+ * Processes the references of the main group to generate another sequence of references, which
+ * may or may not reuse the input references.
+ *
+ * <p>The on-disk path of the main group is assumed to point to workspace root. This is important
+ * when the {@code sourceRoot} property of the references in the input or output arguments are
+ * {@link SourceTree#GROUP}.
+ */
+ Iterable<PBXReference> process(Iterable<PBXReference> references);
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/RelativePaths.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/RelativePaths.java
new file mode 100644
index 0000000..2c42c9b
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/RelativePaths.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableList;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * Utility methods for working with relative {@link Path} objects.
+ */
+class RelativePaths {
+ private RelativePaths() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Converts the given string to a {@code Path}, confirming it is relative.
+ */
+ static Path fromString(FileSystem fileSystem, String pathString) {
+ Path path = fileSystem.getPath(pathString);
+ checkArgument(!path.isAbsolute(), "Expected relative path but got: %s", path);
+ return path;
+ }
+
+ /**
+ * Converts each item in {@code pathStrings} using {@link #fromString(FileSystem, String)}.
+ */
+ static List<Path> fromStrings(FileSystem fileSystem, Iterable<String> pathStrings) {
+ ImmutableList.Builder<Path> result = new ImmutableList.Builder<>();
+ for (String pathString : pathStrings) {
+ result.add(fromString(fileSystem, pathString));
+ }
+ return result.build();
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/Resources.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/Resources.java
new file mode 100644
index 0000000..170d39e
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/Resources.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.xcode.util.Equaling;
+import com.google.devtools.build.xcode.util.Value;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXBuildFile;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXResourcesBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXTarget.ProductType;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Contains information about resources in an Xcode project.
+ */
+public class Resources extends Value<Resources> {
+
+ private final ImmutableSetMultimap<TargetControl, PBXBuildFile> buildFiles;
+
+ private Resources(ImmutableSetMultimap<TargetControl, PBXBuildFile> buildFiles) {
+ super(buildFiles);
+ this.buildFiles = buildFiles;
+ }
+
+ /**
+ * Build files that should be added to the PBXResourcesBuildPhase for the given target.
+ */
+ public ImmutableSetMultimap<TargetControl, PBXBuildFile> buildFiles() {
+ return buildFiles;
+ }
+
+ /**
+ * Returns the PBXResourcesBuildPhase for the given target, if applicable. It will return an
+ * absent {@code Optional} if the target is a library or there are no resources to compile.
+ */
+ public PBXResourcesBuildPhase resourcesBuildPhase(TargetControl targetControl) {
+ PBXResourcesBuildPhase resourcesPhase = new PBXResourcesBuildPhase();
+ resourcesPhase.getFiles().addAll(buildFiles().get(targetControl));
+ return resourcesPhase;
+ }
+
+ public static Optional<String> languageOfLprojDir(Path child) {
+ Path parent = child.getParent();
+ if (parent == null) {
+ return Optional.absent();
+ }
+ String dirName = parent.getFileName().toString();
+ String lprojSuffix = ".lproj";
+ if (dirName.endsWith(lprojSuffix)) {
+ return Optional.of(dirName.substring(0, dirName.length() - lprojSuffix.length()));
+ } else {
+ return Optional.absent();
+ }
+ }
+
+ public static Resources fromTargetControls(
+ FileSystem fileSystem, PBXBuildFiles pbxBuildFiles, Iterable<TargetControl> targetControls) {
+ ImmutableSetMultimap.Builder<TargetControl, PBXBuildFile> buildFiles =
+ new ImmutableSetMultimap.Builder<>();
+
+ for (TargetControl targetControl : targetControls) {
+ List<PBXBuildFile> targetBuildFiles = new ArrayList<>();
+
+ Iterable<String> simpleImports =
+ Iterables.concat(targetControl.getXcassetsDirList(), targetControl.getBundleImportList());
+ // Add .bundle, .xcassets directories to the Project Navigator so they are visible from within
+ // Xcode.
+ // Bundle imports are handled very similarly to asset catalogs, so we just add them with the
+ // same logic. Xcode's automatic file type detection logic is smart enough to see it is a
+ // bundle and link it properly, and add the {@code lastKnownFileType} property.
+ for (String simpleImport : simpleImports) {
+ targetBuildFiles.add(
+ pbxBuildFiles.getStandalone(FileReference.of(simpleImport, SourceTree.GROUP)));
+ }
+
+ Iterables.addAll(
+ targetBuildFiles,
+ pbxBuildFiles.get(
+ AggregateReferenceType.PBXVariantGroup,
+ RelativePaths.fromStrings(fileSystem, targetControl.getGeneralResourceFileList())));
+
+ // If this target is a binary, save the build files. Otherwise, we don't need them. The file
+ // references we generated with fileObjects will be added to the main group later.
+ if (!Equaling.of(
+ ProductType.STATIC_LIBRARY, XcodeprojGeneration.productType(targetControl))) {
+ buildFiles.putAll(targetControl, targetBuildFiles);
+ }
+ }
+
+ return new Resources(buildFiles.build());
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/SourceFile.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/SourceFile.java
new file mode 100644
index 0000000..c89427c
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/SourceFile.java
@@ -0,0 +1,77 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.util.Value;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+
+/**
+ * Source file path with information on how to build it.
+ */
+public class SourceFile extends Value<SourceFile> {
+ /** Indicates how a source file is built or not built. */
+ public enum BuildType {
+ NO_BUILD, BUILD, NON_ARC_BUILD;
+ }
+
+ private final BuildType buildType;
+ private final Path path;
+
+ private SourceFile(BuildType buildType, Path path) {
+ super(buildType, path);
+ this.buildType = buildType;
+ this.path = path;
+ }
+
+ public BuildType buildType() {
+ return buildType;
+ }
+
+ public Path path() {
+ return path;
+ }
+
+ /**
+ * Returns information on all source files in a target. In particular, this includes:
+ * <ul>
+ * <li>arc-compiled source files
+ * <li>non-arc-compiled source files
+ * <li>support files, such as BUILD and header files
+ * <li>Info.plist file
+ * </ul>
+ */
+ public static Iterable<SourceFile> allSourceFiles(FileSystem fileSystem, TargetControl control) {
+ ImmutableList.Builder<SourceFile> result = new ImmutableList.Builder<>();
+ for (Path plainSource : RelativePaths.fromStrings(fileSystem, control.getSourceFileList())) {
+ result.add(new SourceFile(BuildType.BUILD, plainSource));
+ }
+ for (Path nonArcSource
+ : RelativePaths.fromStrings(fileSystem, control.getNonArcSourceFileList())) {
+ result.add(new SourceFile(BuildType.NON_ARC_BUILD, nonArcSource));
+ }
+ for (Path supportSource : RelativePaths.fromStrings(fileSystem, control.getSupportFileList())) {
+ result.add(new SourceFile(BuildType.NO_BUILD, supportSource));
+ }
+ if (control.hasInfoplist()) {
+ result.add(new SourceFile(
+ BuildType.NO_BUILD, RelativePaths.fromString(fileSystem, control.getInfoplist())));
+ }
+ return result.build();
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/Xcdatamodels.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/Xcdatamodels.java
new file mode 100644
index 0000000..a18f3dd
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/Xcdatamodels.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.devtools.build.xcode.util.Equaling;
+import com.google.devtools.build.xcode.util.Value;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXBuildFile;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXTarget.ProductType;
+
+import java.nio.file.FileSystem;
+
+/**
+ * Contains information about .xcdatamodel directories in an Xcode project.
+ */
+public class Xcdatamodels extends Value<Xcdatamodels> {
+
+ private final ImmutableSetMultimap<TargetControl, PBXBuildFile> buildFiles;
+
+ private Xcdatamodels(ImmutableSetMultimap<TargetControl, PBXBuildFile> buildFiles) {
+ super(buildFiles);
+ this.buildFiles = buildFiles;
+ }
+
+ /**
+ * Map of each build file that should be added to the sources build phase for each target, given
+ * the target's control data.
+ */
+ public ImmutableSetMultimap<TargetControl, PBXBuildFile> buildFiles() {
+ return buildFiles;
+ }
+
+ public static Xcdatamodels fromTargetControls(
+ FileSystem fileSystem, PBXBuildFiles pbxBuildFiles, Iterable<TargetControl> targetControls) {
+ ImmutableSetMultimap.Builder<TargetControl, PBXBuildFile> targetLabelToBuildFiles =
+ new ImmutableSetMultimap.Builder<>();
+ for (TargetControl targetControl : targetControls) {
+ Iterable<PBXBuildFile> targetBuildFiles =
+ pbxBuildFiles.get(
+ AggregateReferenceType.XCVersionGroup,
+ RelativePaths.fromStrings(fileSystem, targetControl.getXcdatamodelList()));
+
+ // If this target is not a static library, save the build files. If it's a static lib, we
+ // don't need them. The file references we generated with fileObjects will be added to the
+ // main group later.
+ if (!Equaling.of(
+ ProductType.STATIC_LIBRARY, XcodeprojGeneration.productType(targetControl))) {
+ targetLabelToBuildFiles.putAll(targetControl, targetBuildFiles);
+ }
+ }
+ return new Xcdatamodels(targetLabelToBuildFiles.build());
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/XcodeGen.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/XcodeGen.java
new file mode 100644
index 0000000..2c5bca6
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/XcodeGen.java
@@ -0,0 +1,99 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.Control;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXProject;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Entry-point for the command-line Xcode project generator.
+ */
+public class XcodeGen {
+ /**
+ * Options for {@link XcodeGen}.
+ */
+ public static class XcodeGenOptions extends OptionsBase {
+ @Option(
+ name = "control",
+ help = "Path to a control file, which contains only a binary serialized instance of "
+ + "the Control protocol buffer. Required.",
+ defaultValue = "null")
+ public String control;
+ }
+
+ public static void main(String[] args) throws IOException, OptionsParsingException {
+ OptionsParser parser = OptionsParser.newOptionsParser(XcodeGenOptions.class);
+ parser.parse(args);
+ XcodeGenOptions options = parser.getOptions(XcodeGenOptions.class);
+ if (options.control == null) {
+ throw new IllegalArgumentException("--control must be specified\n"
+ + Options.getUsage(XcodeGenOptions.class));
+ }
+ FileSystem fileSystem = FileSystems.getDefault();
+
+ Control controlPb;
+ try (InputStream in = Files.newInputStream(fileSystem.getPath(options.control))) {
+ controlPb = Control.parseFrom(in);
+ }
+ Path pbxprojPath = fileSystem.getPath(controlPb.getPbxproj());
+
+ Path symlinkToInsideWorkspace = fileSystem.getPath("tools/objc/precomp_xcodegen_deploy.jar");
+ Path workspaceRoot;
+ if (!Files.exists(symlinkToInsideWorkspace)) {
+ workspaceRoot = XcodeprojGeneration.relativeWorkspaceRoot(pbxprojPath);
+ } else {
+ // Get the absolute path to the workspace root.
+
+ // TODO(bazel-team): Remove this hack, possibly by converting Xcodegen to be run with
+ // "bazel run" and using RUNFILES to get the workspace root. For now, this is needed to work
+ // around Xcode's handling of symlinks not playing nicely with how Bazel stores output
+ // artifacts in /private/var/tmp. This means a relative path from .xcodeproj in bazel-out to
+ // the workspace root in .xcodeproj will not work properly at certain times during
+ // Xcode/xcodebuild execution.
+ workspaceRoot = symlinkToInsideWorkspace
+ .toRealPath()
+ .resolve("../../..")
+ .normalize();
+ }
+
+ try (OutputStream out = Files.newOutputStream(pbxprojPath)) {
+ // This workspace root here is relative to the PWD, so that the .xccurrentversion
+ // files can actually be read. The other workspaceRoot is relative to the .xcodeproj
+ // root or is absolute.
+ Path relativeWorkspaceRoot = fileSystem.getPath(".");
+ PBXProject project = XcodeprojGeneration.xcodeproj(
+ workspaceRoot, controlPb,
+ ImmutableList.of(
+ new CurrentVersionSetter(relativeWorkspaceRoot),
+ new PbxReferencesGrouper(fileSystem)));
+ XcodeprojGeneration.write(out, project);
+ }
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/XcodeprojGeneration.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/XcodeprojGeneration.java
new file mode 100644
index 0000000..8a108f5
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/XcodeprojGeneration.java
@@ -0,0 +1,535 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.devtools.build.xcode.common.BuildOptionsUtil.DEFAULT_OPTIONS_NAME;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.xcode.common.XcodeprojPath;
+import com.google.devtools.build.xcode.util.Containing;
+import com.google.devtools.build.xcode.util.Equaling;
+import com.google.devtools.build.xcode.util.Interspersing;
+import com.google.devtools.build.xcode.util.Mapping;
+import com.google.devtools.build.xcode.xcodegen.LibraryObjects.BuildPhaseBuilder;
+import com.google.devtools.build.xcode.xcodegen.SourceFile.BuildType;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.Control;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.DependencyControl;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl;
+import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting;
+
+import com.dd.plist.NSArray;
+import com.dd.plist.NSDictionary;
+import com.dd.plist.NSObject;
+import com.dd.plist.NSString;
+import com.facebook.buck.apple.xcode.GidGenerator;
+import com.facebook.buck.apple.xcode.XcodeprojSerializer;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXBuildFile;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXContainerItemProxy.ProxyType;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXCopyFilesBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFrameworksBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXNativeTarget;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXProject;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference.SourceTree;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXResourcesBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXSourcesBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXTarget.ProductType;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXTargetDependency;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Utility code for generating Xcode project files.
+ */
+public class XcodeprojGeneration {
+ public static final String FILE_TYPE_ARCHIVE_LIBRARY = "archive.ar";
+ public static final String FILE_TYPE_WRAPPER_APPLICATION = "wrapper.application";
+ public static final String FILE_TYPE_WRAPPER_BUNDLE = "wrapper.cfbundle";
+ public static final String FILE_TYPE_APP_EXTENSION = "wrapper.app-extension";
+
+ @VisibleForTesting
+ static final String APP_NEEDS_SOURCE_ERROR =
+ "Due to limitations in Xcode, application projects must have at least one source file.";
+
+ private XcodeprojGeneration() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Determines the relative path to the workspace root from the path of the project.pbxproj output
+ * file. An absolute path is preferred if available.
+ */
+ static Path relativeWorkspaceRoot(Path pbxproj) {
+ int levelsToExecRoot = pbxproj.getParent().getParent().getNameCount();
+ return pbxproj.getFileSystem().getPath(Joiner
+ .on('/')
+ .join(Collections.nCopies(levelsToExecRoot, "..")));
+ }
+
+ /**
+ * Writes a project to an {@code OutputStream} in the correct encoding.
+ */
+ public static void write(OutputStream out, PBXProject project) throws IOException {
+ XcodeprojSerializer ser = new XcodeprojSerializer(
+ new GidGenerator(ImmutableSet.<String>of()), project);
+ Writer outWriter = new OutputStreamWriter(out, StandardCharsets.UTF_8);
+ // toXMLPropertyList includes an XML encoding specification (UTF-8), which we specify above.
+ // Standard Xcodeproj files use the toASCIIPropertyList format, but Xcode will rewrite
+ // XML-encoded project files automatically when first opening them. We use XML to prevent
+ // encoding issues, since toASCIIPropertyList does not include the UTF-8 encoding comment, and
+ // Xcode by default apparently uses MacRoman.
+ // This encoding concern is probably why Buck also generates XML project files as well.
+ outWriter.write(ser.toPlist().toXMLPropertyList());
+ outWriter.flush();
+ }
+
+ private static final EnumSet<ProductType> SUPPORTED_PRODUCT_TYPES = EnumSet.of(
+ ProductType.STATIC_LIBRARY,
+ ProductType.APPLICATION,
+ ProductType.BUNDLE,
+ ProductType.UNIT_TEST,
+ ProductType.APP_EXTENSION);
+
+ private static final EnumSet<ProductType> PRODUCT_TYPES_THAT_HAVE_A_BINARY = EnumSet.of(
+ ProductType.APPLICATION,
+ ProductType.BUNDLE,
+ ProductType.UNIT_TEST,
+ ProductType.APP_EXTENSION);
+
+ /**
+ * Detects the product type of the given target based on multiple fields in {@code targetControl}.
+ * {@code productType} is set as a field on {@code PBXNativeTarget} objects in Xcode project
+ * files, and we support three values: {@link ProductType#APPLICATION},
+ * {@link ProductType#STATIC_LIBRARY}, and {@link ProductType#BUNDLE}. The product type is not
+ * only what xcodegen sets the {@code productType} field to - it also dictates what can be built
+ * with this target (e.g. a library cannot be built with resources), what build phase it should be
+ * added to of its dependers, and the name and shape of its build output.
+ */
+ public static ProductType productType(TargetControl targetControl) {
+ if (targetControl.hasProductType()) {
+ for (ProductType supportedType : SUPPORTED_PRODUCT_TYPES) {
+ if (targetControl.getProductType().equals(supportedType.identifier)) {
+ return supportedType;
+ }
+ }
+ throw new IllegalArgumentException(
+ "Unsupported product type: " + targetControl.getProductType());
+ }
+
+ return targetControl.hasInfoplist() ? ProductType.APPLICATION : ProductType.STATIC_LIBRARY;
+ }
+
+ private static String productName(TargetControl targetControl) {
+ if (Equaling.of(ProductType.STATIC_LIBRARY, productType(targetControl))) {
+ // The product names for static libraries must be unique since the final
+ // binary is linked with "clang -l${LIBRARY_PRODUCT_NAME}" for each static library.
+ // Unlike other product types, a full application may have dozens of static libraries,
+ // so rather than just use the target name, we use the full label to generate the product
+ // name.
+ return labelToXcodeTargetName(targetControl.getLabel());
+ } else {
+ return targetControl.getName();
+ }
+ }
+
+ /**
+ * Returns the file reference corresponding to the {@code productReference} of the given target.
+ * The {@code productReference} is the build output of a target, and its name and file type
+ * (stored in the {@link FileReference}) change based on the product type.
+ */
+ private static FileReference productReference(TargetControl targetControl) {
+ ProductType type = productType(targetControl);
+ String productName = productName(targetControl);
+
+ switch (type) {
+ case APPLICATION:
+ return FileReference.of(String.format("%s.app", productName), SourceTree.BUILT_PRODUCTS_DIR)
+ .withExplicitFileType(FILE_TYPE_WRAPPER_APPLICATION);
+ case STATIC_LIBRARY:
+ return FileReference.of(
+ String.format("lib%s.a", productName), SourceTree.BUILT_PRODUCTS_DIR)
+ .withExplicitFileType(FILE_TYPE_ARCHIVE_LIBRARY);
+ case BUNDLE:
+ return FileReference.of(
+ String.format("%s.bundle", productName), SourceTree.BUILT_PRODUCTS_DIR)
+ .withExplicitFileType(FILE_TYPE_WRAPPER_BUNDLE);
+ case UNIT_TEST:
+ return FileReference.of(
+ String.format("%s.xctest", productName), SourceTree.BUILT_PRODUCTS_DIR)
+ .withExplicitFileType(FILE_TYPE_WRAPPER_BUNDLE);
+ case APP_EXTENSION:
+ return FileReference.of(
+ String.format("%s.appex", productName), SourceTree.BUILT_PRODUCTS_DIR)
+ .withExplicitFileType(FILE_TYPE_APP_EXTENSION);
+ default:
+ throw new IllegalArgumentException("unknown: " + type);
+ }
+ }
+
+ private static class TargetInfo {
+ final TargetControl control;
+ final PBXNativeTarget nativeTarget;
+ final PBXFrameworksBuildPhase frameworksPhase;
+ final PBXResourcesBuildPhase resourcesPhase;
+ final PBXBuildFile productBuildFile;
+ final PBXTargetDependency targetDependency;
+ final NSDictionary buildConfig;
+
+ TargetInfo(TargetControl control,
+ PBXNativeTarget nativeTarget,
+ PBXFrameworksBuildPhase frameworksPhase,
+ PBXResourcesBuildPhase resourcesPhase,
+ PBXBuildFile productBuildFile,
+ PBXTargetDependency targetDependency,
+ NSDictionary buildConfig) {
+ this.control = control;
+ this.nativeTarget = nativeTarget;
+ this.frameworksPhase = frameworksPhase;
+ this.resourcesPhase = resourcesPhase;
+ this.productBuildFile = productBuildFile;
+ this.targetDependency = targetDependency;
+ this.buildConfig = buildConfig;
+ }
+
+ /**
+ * Returns the path to the built, statically-linked binary for this target. The path contains
+ * build-setting variables and may be used in a build setting such as {@code TEST_HOST}.
+ *
+ * <p>One example return value is {@code $(BUILT_PRODUCTS_DIR)/Foo.app/Foo}.
+ */
+ String staticallyLinkedBinary() {
+ ProductType type = productType(control);
+ Preconditions.checkArgument(
+ Containing.item(PRODUCT_TYPES_THAT_HAVE_A_BINARY, type),
+ "This product type (%s) is not known to have a binary.", type);
+ FileReference productReference = productReference(control);
+ return String.format("$(%s)/%s/%s",
+ productReference.sourceTree().name(),
+ productReference.path().or(productReference.name()),
+ control.getName());
+ }
+
+ /**
+ * Adds the given dependency to the list of dependencies, the
+ * appropriate build phase if applicable, and the appropriate build setting values if
+ * applicable, of this target.
+ */
+ void addDependencyInfo(
+ DependencyControl dependencyControl, Map<String, TargetInfo> targetInfoByLabel) {
+ TargetInfo dependencyInfo =
+ Mapping.of(targetInfoByLabel, dependencyControl.getTargetLabel()).get();
+ if (dependencyControl.getTestHost()) {
+ buildConfig.put("TEST_HOST", dependencyInfo.staticallyLinkedBinary());
+ buildConfig.put("BUNDLE_LOADER", dependencyInfo.staticallyLinkedBinary());
+ } else if (productType(dependencyInfo.control) == ProductType.BUNDLE) {
+ resourcesPhase.getFiles().add(dependencyInfo.productBuildFile);
+ } else if (productType(dependencyInfo.control) == ProductType.APP_EXTENSION) {
+ PBXCopyFilesBuildPhase copyFilesPhase = new PBXCopyFilesBuildPhase(
+ PBXCopyFilesBuildPhase.Destination.PLUGINS, /*path=*/"");
+ copyFilesPhase.getFiles().add(dependencyInfo.productBuildFile);
+ nativeTarget.getBuildPhases().add(copyFilesPhase);
+ } else {
+ frameworksPhase.getFiles().add(dependencyInfo.productBuildFile);
+ }
+ nativeTarget.getDependencies().add(dependencyInfo.targetDependency);
+ }
+ }
+
+ private static String labelToXcodeTargetName(String label) {
+ String pathFromWorkspaceRoot = label.replace("//", "").replace(':', '/');
+ List<String> components = Splitter.on('/').splitToList(pathFromWorkspaceRoot);
+ return Joiner.on('_').join(Lists.reverse(components));
+ }
+
+ private static NSDictionary nonArcCompileSettings() {
+ NSDictionary result = new NSDictionary();
+ result.put("COMPILER_FLAGS", "-fno-objc-arc");
+ return result;
+ }
+
+ private static boolean hasAtLeastOneCompilableSource(TargetControl control) {
+ return (control.getSourceFileCount() != 0) || (control.getNonArcSourceFileCount() != 0);
+ }
+
+ private static <E> Iterable<E> plus(Iterable<E> before, E... rest) {
+ return Iterables.concat(before, ImmutableList.copyOf(rest));
+ }
+
+ /**
+ * Returns the final header search paths to be placed in a build configuration.
+ */
+ private static NSArray headerSearchPaths(Iterable<String> paths) {
+ ImmutableList.Builder<String> result = new ImmutableList.Builder<>();
+ for (String path : paths) {
+ // TODO(bazel-team): Remove this hack once the released version of Bazel is prepending
+ // "$(WORKSPACE_ROOT)/" to every "source rooted" path.
+ if (!path.startsWith("$")) {
+ path = "$(WORKSPACE_ROOT)/" + path;
+ }
+ result.add(path);
+ }
+ return (NSArray) NSObject.wrap(result.build());
+ }
+
+ /**
+ * Returns the {@code FRAMEWORK_SEARCH_PATHS} array for a target's build config given the list of
+ * {@code .framework} directory paths.
+ */
+ private static NSArray frameworkSearchPaths(Iterable<String> frameworks) {
+ ImmutableSet.Builder<NSString> result = new ImmutableSet.Builder<>();
+ for (String framework : frameworks) {
+ result.add(new NSString("$(WORKSPACE_ROOT)/" + Paths.get(framework).getParent()));
+ }
+ // This is needed by XcTest targets (and others, just in case) for SenTestingKit.framework.
+ result.add(new NSString("$(SDKROOT)/Developer/Library/Frameworks"));
+ // This is needed by non-XcTest targets that use XcTest.framework, for instance for test
+ // utility libraries packaged as an objc_library.
+ result.add(new NSString("$(PLATFORM_DIR)/Developer/Library/Frameworks"));
+
+ return (NSArray) NSObject.wrap(result.build().asList());
+ }
+
+ private static PBXFrameworksBuildPhase buildLibraryInfo(
+ LibraryObjects libraryObjects, TargetControl target) {
+ BuildPhaseBuilder builder = libraryObjects.newBuildPhase();
+ for (String dylib : target.getSdkDylibList()) {
+ builder.addDylib(dylib);
+ }
+ for (String sdkFramework : target.getSdkFrameworkList()) {
+ builder.addSdkFramework(sdkFramework);
+ }
+ for (String framework : target.getFrameworkList()) {
+ builder.addFramework(framework);
+ }
+ return builder.build();
+ }
+
+ private static ImmutableList<String> otherLdflags(TargetControl targetControl) {
+ Iterable<String> givenFlags = targetControl.getLinkoptList();
+ ImmutableList.Builder<String> flags = new ImmutableList.Builder<>();
+ flags.addAll(givenFlags);
+ if (!Equaling.of(ProductType.STATIC_LIBRARY, productType(targetControl))) {
+ flags.addAll(Interspersing.prependEach(
+ "$(WORKSPACE_ROOT)/", targetControl.getImportedLibraryList()));
+ }
+ return flags.build();
+ }
+
+ /** Generates a project file. */
+ public static PBXProject xcodeproj(Path workspaceRoot, Control control,
+ Iterable<PbxReferencesProcessor> postProcessors) {
+ checkArgument(control.hasPbxproj(), "Must set pbxproj field on control proto.");
+ FileSystem fileSystem = workspaceRoot.getFileSystem();
+
+ XcodeprojPath<Path> outputPath = XcodeprojPath.converter().fromPath(
+ RelativePaths.fromString(fileSystem, control.getPbxproj()));
+
+ NSDictionary projBuildConfigMap = new NSDictionary();
+ projBuildConfigMap.put("ARCHS", new NSArray(
+ new NSString("arm7"), new NSString("arm7s"), new NSString("arm64")));
+ projBuildConfigMap.put("CLANG_ENABLE_OBJC_ARC", "YES");
+ projBuildConfigMap.put("SDKROOT", "iphoneos");
+ projBuildConfigMap.put("IPHONEOS_DEPLOYMENT_TARGET", "7.0");
+ projBuildConfigMap.put("GCC_VERSION", "com.apple.compilers.llvm.clang.1_0");
+ projBuildConfigMap.put("CODE_SIGN_IDENTITY[sdk=iphoneos*]", "iPhone Developer");
+
+ for (XcodeprojBuildSetting projectSetting : control.getBuildSettingList()) {
+ projBuildConfigMap.put(projectSetting.getName(), projectSetting.getValue());
+ }
+
+ PBXProject project = new PBXProject(outputPath.getProjectName());
+ project.getMainGroup().setPath(workspaceRoot.toString());
+ try {
+ project
+ .getBuildConfigurationList()
+ .getBuildConfigurationsByName()
+ .get(DEFAULT_OPTIONS_NAME)
+ .setBuildSettings(projBuildConfigMap);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+
+ Map<String, TargetInfo> targetInfoByLabel = new HashMap<>();
+
+ PBXFileReferences fileReferences = new PBXFileReferences();
+ LibraryObjects libraryObjects = new LibraryObjects(fileReferences);
+ PBXBuildFiles pbxBuildFiles = new PBXBuildFiles(fileReferences);
+ Resources resources =
+ Resources.fromTargetControls(fileSystem, pbxBuildFiles, control.getTargetList());
+ Xcdatamodels xcdatamodels =
+ Xcdatamodels.fromTargetControls(fileSystem, pbxBuildFiles, control.getTargetList());
+ // We use a hash set for the Project Navigator files so that the same PBXFileReference does not
+ // get added twice. Because PBXFileReference uses equality-by-identity semantics, this requires
+ // the PBXFileReferences cache to properly return the same reference for functionally-equivalent
+ // files.
+ Set<PBXReference> projectNavigatorFiles = new LinkedHashSet<>();
+ for (TargetControl targetControl : control.getTargetList()) {
+ checkArgument(targetControl.hasName(), "TargetControl requires a name: %s", targetControl);
+ checkArgument(targetControl.hasLabel(), "TargetControl requires a label: %s", targetControl);
+
+ ProductType productType = productType(targetControl);
+ Preconditions.checkArgument(
+ (productType != ProductType.APPLICATION) || hasAtLeastOneCompilableSource(targetControl),
+ APP_NEEDS_SOURCE_ERROR);
+ PBXSourcesBuildPhase sourcesBuildPhase = new PBXSourcesBuildPhase();
+
+ for (SourceFile source : SourceFile.allSourceFiles(fileSystem, targetControl)) {
+ PBXFileReference fileRef =
+ fileReferences.get(FileReference.of(source.path().toString(), SourceTree.GROUP));
+ projectNavigatorFiles.add(fileRef);
+ if (Equaling.of(source.buildType(), BuildType.NO_BUILD)) {
+ continue;
+ }
+ PBXBuildFile buildFile = new PBXBuildFile(fileRef);
+ if (Equaling.of(source.buildType(), BuildType.NON_ARC_BUILD)) {
+ buildFile.setSettings(Optional.of(nonArcCompileSettings()));
+ }
+ sourcesBuildPhase.getFiles().add(buildFile);
+ }
+ sourcesBuildPhase.getFiles().addAll(xcdatamodels.buildFiles().get(targetControl));
+
+ PBXFileReference productReference = fileReferences.get(productReference(targetControl));
+ projectNavigatorFiles.add(productReference);
+
+ NSDictionary targetBuildConfigMap = new NSDictionary();
+ // TODO(bazel-team): Stop adding the workspace root automatically once the
+ // released version of Bazel starts passing it.
+ targetBuildConfigMap.put("USER_HEADER_SEARCH_PATHS",
+ headerSearchPaths(
+ plus(targetControl.getUserHeaderSearchPathList(), "$(WORKSPACE_ROOT)")));
+ targetBuildConfigMap.put("HEADER_SEARCH_PATHS",
+ headerSearchPaths(
+ plus(targetControl.getHeaderSearchPathList(), "$(inherited)")));
+ targetBuildConfigMap.put("FRAMEWORK_SEARCH_PATHS",
+ frameworkSearchPaths(targetControl.getFrameworkList()));
+
+ targetBuildConfigMap.put("WORKSPACE_ROOT", workspaceRoot.toString());
+
+ if (targetControl.hasPchPath()) {
+ targetBuildConfigMap.put(
+ "GCC_PREFIX_HEADER", "$(WORKSPACE_ROOT)/" + targetControl.getPchPath());
+ }
+
+ targetBuildConfigMap.put("PRODUCT_NAME", productName(targetControl));
+ if (targetControl.hasInfoplist()) {
+ targetBuildConfigMap.put(
+ "INFOPLIST_FILE", "$(WORKSPACE_ROOT)/" + targetControl.getInfoplist());
+ }
+
+ if (targetControl.getCoptCount() > 0) {
+ targetBuildConfigMap.put("OTHER_CFLAGS", NSObject.wrap(targetControl.getCoptList()));
+ }
+ targetBuildConfigMap.put("OTHER_LDFLAGS", NSObject.wrap(otherLdflags(targetControl)));
+ for (XcodeprojBuildSetting setting : targetControl.getBuildSettingList()) {
+ String name = setting.getName();
+ String value = setting.getValue();
+ // TODO(bazel-team): Remove this hack after next Bazel release.
+ if (name.equals("CODE_SIGN_ENTITLEMENTS") && !value.startsWith("$")) {
+ value = "$(WORKSPACE_ROOT)/" + value;
+ }
+ targetBuildConfigMap.put(name, value);
+ }
+
+ PBXNativeTarget target = new PBXNativeTarget(
+ labelToXcodeTargetName(targetControl.getLabel()), productType);
+ try {
+ target
+ .getBuildConfigurationList()
+ .getBuildConfigurationsByName()
+ .get(DEFAULT_OPTIONS_NAME)
+ .setBuildSettings(targetBuildConfigMap);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ target.setProductReference(productReference);
+
+ PBXFrameworksBuildPhase frameworksPhase = buildLibraryInfo(libraryObjects, targetControl);
+ PBXResourcesBuildPhase resourcesPhase = resources.resourcesBuildPhase(targetControl);
+
+ for (String importedArchive : targetControl.getImportedLibraryList()) {
+ PBXFileReference fileReference = fileReferences.get(
+ FileReference.of(importedArchive, SourceTree.GROUP)
+ .withExplicitFileType(FILE_TYPE_ARCHIVE_LIBRARY));
+ projectNavigatorFiles.add(fileReference);
+ }
+
+ project.getTargets().add(target);
+
+ target.getBuildPhases().add(frameworksPhase);
+ target.getBuildPhases().add(sourcesBuildPhase);
+ target.getBuildPhases().add(resourcesPhase);
+
+ checkState(!Mapping.of(targetInfoByLabel, targetControl.getLabel()).isPresent(),
+ "Mapping already exists for target with label %s in map: %s",
+ targetControl.getLabel(), targetInfoByLabel);
+ targetInfoByLabel.put(
+ targetControl.getLabel(),
+ new TargetInfo(
+ targetControl,
+ target,
+ frameworksPhase,
+ resourcesPhase,
+ new PBXBuildFile(productReference),
+ new LocalPBXTargetDependency(
+ new LocalPBXContainerItemProxy(
+ project, target, ProxyType.TARGET_REFERENCE)),
+ targetBuildConfigMap));
+ }
+
+ for (HasProjectNavigatorFiles references : ImmutableList.of(pbxBuildFiles, libraryObjects)) {
+ Iterables.addAll(projectNavigatorFiles, references.mainGroupReferences());
+ }
+
+ Iterable<PBXReference> processedProjectFiles = projectNavigatorFiles;
+ for (PbxReferencesProcessor postProcessor : postProcessors) {
+ processedProjectFiles = postProcessor.process(processedProjectFiles);
+ }
+
+ Iterables.addAll(project.getMainGroup().getChildren(), processedProjectFiles);
+ for (TargetInfo targetInfo : targetInfoByLabel.values()) {
+ for (DependencyControl dependency : targetInfo.control.getDependencyList()) {
+ targetInfo.addDependencyInfo(dependency, targetInfoByLabel);
+ }
+ }
+
+ return project;
+ }
+}
diff --git a/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/testing/PbxTypes.java b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/testing/PbxTypes.java
new file mode 100644
index 0000000..8d0a2ee
--- /dev/null
+++ b/src/objc_tools/xcodegen/java/com/google/devtools/build/xcode/xcodegen/testing/PbxTypes.java
@@ -0,0 +1,100 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.xcodegen.testing;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.xcodegen.FileReference;
+
+import com.facebook.buck.apple.xcode.xcodeproj.PBXBuildFile;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFileReference;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXFrameworksBuildPhase;
+import com.facebook.buck.apple.xcode.xcodeproj.PBXReference;
+
+/**
+ * Collection of static utility methods for PBX type transformations, either to other
+ * representations of the same or our domain objects.
+ */
+public class PbxTypes {
+
+ private PbxTypes() {}
+
+ /**
+ * Returns all file references the {@code phase} depends on in
+ * {@link PBXFrameworksBuildPhase#getFiles()}.
+ */
+ public static ImmutableList<FileReference> fileReferences(PBXFrameworksBuildPhase phase) {
+ return fileReferences(pbxFileReferences(phase));
+ }
+
+ /**
+ * Transforms the given list of PBX references to file references.
+ */
+ public static ImmutableList<FileReference> fileReferences(
+ Iterable<? extends PBXReference> references) {
+ ImmutableList.Builder<FileReference> fileReferences = ImmutableList.builder();
+ for (PBXReference reference : references) {
+ fileReferences.add(fileReference(reference));
+ }
+ return fileReferences.build();
+ }
+
+ /**
+ * Extracts the list of PBX references {@code phase} depends on through
+ * {@link PBXFrameworksBuildPhase#getFiles()}.
+ */
+ public static ImmutableList<PBXReference> pbxFileReferences(PBXFrameworksBuildPhase phase) {
+ ImmutableList.Builder<PBXReference> phaseFileReferences = ImmutableList.builder();
+ for (PBXBuildFile buildFile : phase.getFiles()) {
+ phaseFileReferences.add(buildFile.getFileRef());
+ }
+ return phaseFileReferences.build();
+ }
+
+ /**
+ * Converts a PBX file reference to its domain equivalent.
+ */
+ public static FileReference fileReference(PBXReference reference) {
+ FileReference fileReference =
+ FileReference.of(reference.getName(), reference.getPath(), reference.getSourceTree());
+ if (reference instanceof PBXFileReference) {
+ Optional<String> explicitFileType = ((PBXFileReference) reference).getExplicitFileType();
+ if (explicitFileType.isPresent()) {
+ return fileReference.withExplicitFileType(explicitFileType.get());
+ }
+ }
+ return fileReference;
+ }
+
+ /**
+ * Returns the string representation of all references the {@code phase} depends on in
+ * {@link PBXFrameworksBuildPhase#getFiles()}.
+ *
+ */
+ public static ImmutableList<String> referencePaths(PBXFrameworksBuildPhase phase) {
+ return paths(pbxFileReferences(phase));
+ }
+
+ /**
+ * Transforms the given list of references into their string path representations.
+ */
+ public static ImmutableList<String> paths(Iterable<? extends PBXReference> references) {
+ ImmutableList.Builder<String> paths = ImmutableList.builder();
+ for (PBXReference reference : references) {
+ paths.add(reference.getPath());
+ }
+ return paths.build();
+ }
+}
diff --git a/src/test/java/BUILD b/src/test/java/BUILD
new file mode 100644
index 0000000..03b1481
--- /dev/null
+++ b/src/test/java/BUILD
@@ -0,0 +1,168 @@
+java_library(
+ name = "testutil",
+ srcs = glob(["com/google/devtools/build/lib/testutil/*.java"]),
+ visibility = ["//visibility:public"],
+ deps = [
+ "//src/main/java:bazel-core",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_test(
+ name = "skyframe_test",
+ srcs = glob([
+ "com/google/devtools/build/skyframe/*.java",
+ ]),
+ args = ["com.google.devtools.build.skyframe.AllTests"],
+ deps = [
+ ":testutil",
+ "//src/main/java:bazel-core",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:jsr305",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_test(
+ name = "options_test",
+ srcs = glob([
+ "com/google/devtools/common/options/*.java",
+ ]),
+ args = ["com.google.devtools.common.options.AllTests"],
+ deps = [
+ ":testutil",
+ "//src/main/java:bazel-core",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+test_prefix = "com/google/devtools/build/lib"
+
+java_library(
+ name = "foundations_testutil",
+ srcs = glob([
+ "%s/vfs/util/*.java" % test_prefix,
+ ]),
+ deps = [
+ ":testutil",
+ "//src/main/java:bazel-core",
+ "//src/main/java:shell",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_library(
+ name = "test_runner",
+ srcs = [test_prefix + "/AllTests.java"],
+ deps = [
+ ":testutil",
+ "//third_party:junit4",
+ ],
+)
+
+java_test(
+ name = "foundations_test",
+ srcs = glob(
+ ["%s/%s" % (test_prefix, p) for p in [
+ "concurrent/*.java",
+ "collect/*.java",
+ "collect/nestedset/*.java",
+ "events/*.java",
+ "testutiltests/*.java",
+ "unix/*.java",
+ "util/*.java",
+ "util/io/*.java",
+ "vfs/*.java",
+ "vfs/inmemoryfs/*.java",
+ ]],
+ # java_rules_oss doesn't support resource loading with
+ # qualified paths.
+ exclude = [
+ test_prefix + f
+ for f in [
+ "/util/DependencySetWindowsTest.java",
+ "/util/ResourceFileLoaderTest.java",
+ "/vfs/PathFragmentWindowsTest.java",
+ "/vfs/PathWindowsTest.java",
+ ]
+ ],
+ ),
+ args = ["com.google.devtools.build.lib.AllTests"],
+ data = glob([test_prefix + "/vfs/*.zip"]) + [
+ "//src/main/native:libunix.dylib",
+ "//src/main/native:libunix.so",
+ ],
+ deps = [
+ ":foundations_testutil",
+ ":test_runner",
+ ":testutil",
+ "//src/main/java:bazel-core",
+ "//src/main/java:shell",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_test(
+ name = "windows_test",
+ srcs = glob(["%s/%s" % (test_prefix, p) for p in [
+ "util/DependencySetWindowsTest.java",
+ "vfs/PathFragmentWindowsTest.java",
+ "vfs/PathWindowsTest.java",
+ ]]),
+ args = [
+ "com.google.devtools.build.lib.AllTests",
+ ],
+ data = [
+ "//src/main/native:libunix.dylib",
+ "//src/main/native:libunix.so",
+ ],
+ jvm_flags = ["-Dblaze.os=Windows"],
+ deps = [
+ ":foundations_testutil",
+ ":test_runner",
+ ":testutil",
+ "//src/main/java:bazel-core",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_test(
+ name = "actions_test",
+ srcs = glob([
+ "com/google/devtools/build/lib/actions/**/*.java",
+ ]),
+ args = ["com.google.devtools.build.lib.AllTests"],
+ data = [
+ "//src/main/native:libunix.dylib",
+ "//src/main/native:libunix.so",
+ ],
+ deps = [
+ ":foundations_testutil",
+ ":test_runner",
+ ":testutil",
+ "//src/main/java:bazel-core",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:jsr305",
+ "//third_party:junit4",
+ "//third_party:mockito",
+ "//third_party:truth",
+ ],
+)
diff --git a/src/test/java/com/google/devtools/build/lib/AllTests.java b/src/test/java/com/google/devtools/build/lib/AllTests.java
new file mode 100644
index 0000000..6aa19f1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/AllTests.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib;
+
+import com.google.devtools.build.lib.testutil.ClasspathSuite;
+
+import org.junit.runner.RunWith;
+
+/**
+ * Test suite for options parsing framework.
+ */
+@RunWith(ClasspathSuite.class)
+public class AllTests {
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporterTest.java b/src/test/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporterTest.java
new file mode 100644
index 0000000..8af46e4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporterTest.java
@@ -0,0 +1,293 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.events.EventCollector;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.Clock;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test for the {@link ActionExecutionStatusReporter} class.
+ */
+@RunWith(JUnit4.class)
+public class ActionExecutionStatusReporterTest {
+ private static final class MockClock implements Clock {
+ private long millis = 0;
+
+ public void advance() {
+ advanceBy(1000);
+ }
+
+ public void advanceBy(long millis) {
+ Preconditions.checkArgument(millis > 0);
+ this.millis += millis;
+ }
+
+ @Override
+ public long currentTimeMillis() {
+ return millis;
+ }
+
+ @Override
+ public long nanoTime() {
+ // There's no reason to use a nanosecond-precision for a mock clock.
+ return millis * 1000000L;
+ }
+ }
+
+ private EventCollector collector;
+ private ActionExecutionStatusReporter statusReporter;
+ private EventBus eventBus;
+ private MockClock clock = new MockClock();
+
+ private Action mockAction(String progressMessage) { return mockAction(progressMessage, false); }
+
+ private Action mockAction(String progressMessage, boolean remote) {
+ Action action = Mockito.mock(Action.class);
+ when(action.describeStrategy(null)).thenReturn(remote ? "remote" : "something else");
+ when(action.getProgressMessage()).thenReturn(progressMessage);
+ if (progressMessage == null) {
+ when(action.prettyPrint()).thenReturn("default message");
+ }
+ return action;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ collector = new EventCollector(EventKind.ALL_EVENTS);
+ Reporter reporter = new Reporter();
+ reporter.addHandler(collector);
+ statusReporter = ActionExecutionStatusReporter.create(reporter, clock);
+ eventBus = new EventBus();
+ eventBus.register(statusReporter);
+ }
+
+ private void verifyNoOutput() {
+ collector.clear();
+ statusReporter.showCurrentlyExecutingActions("");
+ assertEquals(0, collector.count());
+ }
+
+ private void verifyOutput(String... lines) throws Exception {
+ collector.clear();
+ statusReporter.showCurrentlyExecutingActions("");
+ assertThat(Splitter.on("\n").omitEmptyStrings().trimResults().split(
+ Iterables.getOnlyElement(collector).getMessage().replaceAll(" +", " ")))
+ .containsExactlyElementsIn(Arrays.asList(lines)).inOrder();
+ }
+
+ private void verifyWarningOutput(String... lines) throws Exception {
+ collector.clear();
+ statusReporter.warnAboutCurrentlyExecutingActions();
+ assertThat(Splitter.on("\n").omitEmptyStrings().trimResults().split(
+ Iterables.getOnlyElement(collector).getMessage().replaceAll(" +", " ")))
+ .containsExactlyElementsIn(Arrays.asList(lines)).inOrder();
+ }
+
+ @Test
+ public void testCategories() throws Exception {
+ verifyNoOutput();
+ verifyWarningOutput("There are no active jobs - stopping the build");
+ setPreparing(mockAction("action1"));
+ clock.advance();
+ verifyWarningOutput("Still waiting for unfinished jobs");
+ setScheduling(mockAction("action2"));
+ clock.advance();
+ setRunning(mockAction("action3", true));
+ clock.advance();
+ setRunning(mockAction("action4", false));
+ verifyOutput("Still waiting for 4 jobs to complete:",
+ "Preparing:", "action1, 3 s",
+ "Running (remote):", "action3, 1 s",
+ "Running (something else):", "action4, 0 s",
+ "Scheduling:", "action2, 2 s");
+ verifyWarningOutput("Still waiting for 3 jobs to complete:",
+ "Running (remote):", "action3, 1 s",
+ "Running (something else):", "action4, 0 s",
+ "Scheduling:", "action2, 2 s",
+ "Build will be stopped after these tasks terminate");
+ }
+
+ @Test
+ public void testSingleAction() throws Exception {
+ Action action = mockAction("action1", true);
+ verifyNoOutput();
+ setPreparing(action);
+ clock.advanceBy(1200);
+ verifyOutput("Still waiting for 1 job to complete:", "Preparing:", "action1, 1 s");
+ clock.advanceBy(5000);
+
+ setScheduling(action);
+ clock.advanceBy(1200);
+ // Only started *scheduling* 1200 ms ago, not 6200 ms ago.
+ verifyOutput("Still waiting for 1 job to complete:", "Scheduling:", "action1, 1 s");
+ setRunning(action);
+ clock.advanceBy(3000);
+ // Only started *running* 3000 ms ago, not 4200 ms ago.
+ verifyOutput("Still waiting for 1 job to complete:", "Running (remote):", "action1, 3 s");
+ statusReporter.remove(action);
+ verifyNoOutput();
+ }
+
+ @Test
+ public void testDynamicUpdate() throws Exception {
+ Action action = mockAction("action1", true);
+ verifyNoOutput();
+ setPreparing(action);
+ clock.advance();
+ verifyOutput("Still waiting for 1 job to complete:", "Preparing:", "action1, 1 s");
+ setScheduling(action);
+ clock.advance();
+ verifyOutput("Still waiting for 1 job to complete:", "Scheduling:", "action1, 1 s");
+ setRunning(action);
+ clock.advance();
+ verifyOutput("Still waiting for 1 job to complete:", "Running (remote):", "action1, 1 s");
+ clock.advance();
+
+ eventBus.post(ActionStatusMessage.analysisStrategy(action));
+ // Locality strategy was changed, so timer was reset to 0 s.
+ verifyOutput("Still waiting for 1 job to complete:", "Analyzing:", "action1, 0 s");
+ statusReporter.remove(action);
+ verifyNoOutput();
+ }
+
+ @Test
+ public void testGroups() throws Exception {
+ verifyNoOutput();
+ List<Action> actions = ImmutableList.of(
+ mockAction("remote1", true), mockAction("remote2", true), mockAction("remote3", true),
+ mockAction("local1", false), mockAction("local2", false), mockAction("local3", false));
+
+ for (Action a : actions) {
+ setScheduling(a);
+ clock.advance();
+ }
+
+ verifyOutput("Still waiting for 6 jobs to complete:",
+ "Scheduling:",
+ "remote1, 6 s", "remote2, 5 s", "remote3, 4 s",
+ "local1, 3 s", "local2, 2 s", "local3, 1 s");
+
+ for (Action a : actions) {
+ setRunning(a);
+ clock.advanceBy(2000);
+ }
+
+ // Timers got reset because now they are no longer scheduling but running.
+ verifyOutput("Still waiting for 6 jobs to complete:",
+ "Running (remote):", "remote1, 12 s", "remote2, 10 s", "remote3, 8 s",
+ "Running (something else):", "local1, 6 s", "local2, 4 s", "local3, 2 s");
+
+ statusReporter.remove(actions.get(0));
+ verifyOutput("Still waiting for 5 jobs to complete:",
+ "Running (remote):", "remote2, 10 s", "remote3, 8 s",
+ "Running (something else):", "local1, 6 s", "local2, 4 s", "local3, 2 s");
+ }
+
+ @Test
+ public void testTruncation() throws Exception {
+ verifyNoOutput();
+ List<Action> actions = new ArrayList<>();
+ for (int i = 1; i <= 100; i++) {
+ Action a = mockAction("a" + i);
+ actions.add(a);
+ setScheduling(a);
+ clock.advance();
+ }
+ verifyOutput("Still waiting for 100 jobs to complete:", "Scheduling:",
+ "a1, 100 s", "a2, 99 s", "a3, 98 s", "a4, 97 s", "a5, 96 s",
+ "a6, 95 s", "a7, 94 s", "a8, 93 s", "a9, 92 s", "... 91 more jobs");
+
+ for (int i = 0; i < 5; i++) {
+ setRunning(actions.get(i));
+ clock.advance();
+ }
+ verifyOutput("Still waiting for 100 jobs to complete:",
+ "Running (something else):", "a1, 5 s", "a2, 4 s", "a3, 3 s", "a4, 2 s", "a5, 1 s",
+ "Scheduling:", "a6, 100 s", "a7, 99 s", "a8, 98 s", "a9, 97 s", "a10, 96 s",
+ "a11, 95 s", "a12, 94 s", "a13, 93 s", "a14, 92 s", "... 86 more jobs");
+ }
+
+ @Test
+ public void testOrdering() throws Exception {
+ verifyNoOutput();
+ setScheduling(mockAction("a1"));
+ clock.advance();
+ setPreparing(mockAction("b1"));
+ clock.advance();
+ setPreparing(mockAction("b2"));
+ clock.advance();
+ setScheduling(mockAction("a2"));
+ clock.advance();
+ verifyOutput("Still waiting for 4 jobs to complete:",
+ "Preparing:", "b1, 3 s", "b2, 2 s",
+ "Scheduling:", "a1, 4 s", "a2, 1 s");
+ }
+
+ @Test
+ public void testNoProgressMessage() throws Exception {
+ verifyNoOutput();
+ setScheduling(mockAction(null));
+ verifyOutput("Still waiting for 1 job to complete:", "Scheduling:", "default message, 0 s");
+ }
+
+ @Test
+ public void testWaitTimeCalculation() throws Exception {
+ // --progress_report_interval=0
+ assertEquals(10, ActionExecutionStatusReporter.getWaitTime(0, 0));
+ assertEquals(30, ActionExecutionStatusReporter.getWaitTime(0, 10));
+ assertEquals(60, ActionExecutionStatusReporter.getWaitTime(0, 30));
+ assertEquals(60, ActionExecutionStatusReporter.getWaitTime(0, 60));
+
+ // --progress_report_interval=42
+ assertEquals(42, ActionExecutionStatusReporter.getWaitTime(42, 0));
+ assertEquals(42, ActionExecutionStatusReporter.getWaitTime(42, 42));
+
+ // --progress_report_interval=30 (looks like one of the default timeout stages)
+ assertEquals(30, ActionExecutionStatusReporter.getWaitTime(30, 0));
+ assertEquals(30, ActionExecutionStatusReporter.getWaitTime(30, 30));
+ }
+
+ private void setScheduling(ActionMetadata action) {
+ eventBus.post(ActionStatusMessage.schedulingStrategy(action));
+ }
+
+ private void setPreparing(ActionMetadata action) {
+ eventBus.post(ActionStatusMessage.preparingStrategy(action));
+ }
+
+ private void setRunning(ActionMetadata action) {
+ eventBus.post(ActionStatusMessage.runningStrategy(action));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java b/src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java
new file mode 100644
index 0000000..71cf9c4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java
@@ -0,0 +1,246 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER;
+import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ARTIFACT_OWNER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Tests {@link ArtifactFactory}. Also see {@link ArtifactTest} for a test
+ * of individual artifacts.
+ */
+@RunWith(JUnit4.class)
+public class ArtifactFactoryTest {
+
+ private FsApparatus scratch = FsApparatus.newInMemory();
+
+ private Path execRoot;
+ private Root clientRoot;
+ private Root clientRoRoot;
+ private Root outRoot;
+
+ private PathFragment fooPath;
+ private PackageIdentifier fooPackage;
+ private PathFragment fooRelative;
+
+ private PathFragment barPath;
+ private PackageIdentifier barPackage;
+ private PathFragment barRelative;
+
+ private ArtifactFactory artifactFactory;
+
+ @Before
+ public void setUp() throws Exception {
+ execRoot = scratch.dir("/output/workspace");
+ clientRoot = Root.asSourceRoot(scratch.dir("/client/workspace"));
+ clientRoRoot = Root.asSourceRoot(scratch.dir("/client/RO/workspace"));
+ outRoot = Root.asDerivedRoot(execRoot, execRoot.getRelative("out-root/x/bin"));
+
+ fooPath = new PathFragment("foo");
+ fooPackage = PackageIdentifier.createInDefaultRepo(fooPath);
+ fooRelative = fooPath.getRelative("foosource.txt");
+
+ barPath = new PathFragment("foo/bar");
+ barPackage = PackageIdentifier.createInDefaultRepo(barPath);
+ barRelative = barPath.getRelative("barsource.txt");
+
+ artifactFactory = new ArtifactFactory(execRoot);
+ setupRoots();
+ }
+
+ private void setupRoots() {
+ Map<PackageIdentifier, Root> packageRootMap = new HashMap<>();
+ packageRootMap.put(fooPackage, clientRoot);
+ packageRootMap.put(barPackage, clientRoRoot);
+ artifactFactory.setPackageRoots(packageRootMap);
+ artifactFactory.setDerivedArtifactRoots(ImmutableList.of(outRoot));
+ }
+
+ @Test
+ public void testGetSourceArtifactYieldsSameArtifact() throws Exception {
+ assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot),
+ artifactFactory.getSourceArtifact(fooRelative, clientRoot));
+ }
+
+ @Test
+ public void testGetSourceArtifactUnnormalized() throws Exception {
+ assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot),
+ artifactFactory.getSourceArtifact(new PathFragment("foo/./foosource.txt"),
+ clientRoot));
+ }
+
+ @Test
+ public void testResolveArtifact_noDerived_simpleSource() throws Exception {
+ assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot),
+ artifactFactory.resolveSourceArtifact(fooRelative));
+ assertSame(artifactFactory.getSourceArtifact(barRelative, clientRoRoot),
+ artifactFactory.resolveSourceArtifact(barRelative));
+ }
+
+ @Test
+ public void testResolveArtifact_noDerived_derivedRoot() throws Exception {
+ assertNull(artifactFactory.resolveSourceArtifact(
+ outRoot.getPath().getRelative(fooRelative).relativeTo(execRoot)));
+ assertNull(artifactFactory.resolveSourceArtifact(
+ outRoot.getPath().getRelative(barRelative).relativeTo(execRoot)));
+ }
+
+ @Test
+ public void testResolveArtifact_noDerived_simpleSource_other() throws Exception {
+ Artifact actual = artifactFactory.resolveSourceArtifact(fooRelative);
+ assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot), actual);
+ actual = artifactFactory.resolveSourceArtifact(barRelative);
+ assertSame(artifactFactory.getSourceArtifact(barRelative, clientRoRoot), actual);
+ }
+
+ @Test
+ public void testClearResetsFactory() {
+ Artifact fooArtifact = artifactFactory.getSourceArtifact(fooRelative, clientRoot);
+ artifactFactory.clear();
+ setupRoots();
+ assertNotSame(fooArtifact, artifactFactory.getSourceArtifact(fooRelative, clientRoot));
+ }
+
+ @Test
+ public void testFindDerivedRoot() throws Exception {
+ assertSame(outRoot,
+ artifactFactory.findDerivedRoot(outRoot.getPath().getRelative(fooRelative)));
+ assertSame(outRoot,
+ artifactFactory.findDerivedRoot(outRoot.getPath().getRelative(barRelative)));
+ }
+
+ @Test
+ public void testSetGeneratingActionIdempotenceNewActionGraph() throws Exception {
+ Artifact a = artifactFactory.getDerivedArtifact(fooRelative, outRoot, NULL_ARTIFACT_OWNER);
+ Artifact b = artifactFactory.getDerivedArtifact(barRelative, outRoot, NULL_ARTIFACT_OWNER);
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Action originalAction = new ActionsTestUtil.NullAction(NULL_ACTION_OWNER, a);
+ actionGraph.registerAction(originalAction);
+
+ // Creating a second Action referring to the Artifact should create a conflict.
+ try {
+ Action action = new ActionsTestUtil.NullAction(NULL_ACTION_OWNER, a, b);
+ actionGraph.registerAction(action);
+ fail();
+ } catch (ActionConflictException e) {
+ assertSame(a, e.getArtifact());
+ assertSame(originalAction, actionGraph.getGeneratingAction(a));
+ }
+ }
+
+ @Test
+ public void testGetDerivedArtifact() throws Exception {
+ PathFragment toolPath = new PathFragment("_bin/tool");
+ Artifact artifact = artifactFactory.getDerivedArtifact(toolPath);
+ assertEquals(toolPath, artifact.getExecPath());
+ assertEquals(Root.asDerivedRoot(execRoot), artifact.getRoot());
+ assertEquals(execRoot.getRelative(toolPath), artifact.getPath());
+ assertNull(artifact.getOwner());
+ }
+
+ @Test
+ public void testGetDerivedArtifactFailsForAbsolutePath() throws Exception {
+ try {
+ artifactFactory.getDerivedArtifact(new PathFragment("/_bin/b"));
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected exception
+ }
+ }
+
+ private static class MockPackageRootResolver implements PackageRootResolver {
+ private Map<PathFragment, Root> packageRoots = Maps.newHashMap();
+
+ public void setPackageRoots(Map<PackageIdentifier, Root> packageRoots) {
+ for (Entry<PackageIdentifier, Root> packageRoot : packageRoots.entrySet()) {
+ this.packageRoots.put(packageRoot.getKey().getPackageFragment(), packageRoot.getValue());
+ }
+ }
+
+ @Override
+ public Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths) {
+ Map<PathFragment, Root> result = new HashMap<>();
+ for (PathFragment execPath : execPaths) {
+ for (PathFragment dir = execPath.getParentDirectory(); dir != null;
+ dir = dir.getParentDirectory()) {
+ if (packageRoots.get(dir) != null) {
+ result.put(execPath, packageRoots.get(dir));
+ }
+ }
+ if (result.get(execPath) == null) {
+ result.put(execPath, null);
+ }
+ }
+ return result;
+ }
+ }
+
+ @Test
+ public void testArtifactDeserializationWithoutReusedArtifacts() throws Exception {
+ PathFragment derivedPath = outRoot.getExecPath().getRelative("fruit/banana");
+ artifactFactory.clear();
+ artifactFactory.setDerivedArtifactRoots(ImmutableList.of(outRoot));
+ MockPackageRootResolver rootResolver = new MockPackageRootResolver();
+ rootResolver.setPackageRoots(
+ ImmutableMap.of(PackageIdentifier.createInDefaultRepo(""), clientRoot));
+ Artifact artifact1 = artifactFactory.deserializeArtifact(derivedPath, rootResolver);
+ Artifact artifact2 = artifactFactory.deserializeArtifact(derivedPath, rootResolver);
+ assertEquals(artifact1, artifact2);
+ assertNull(artifact1.getOwner());
+ assertNull(artifact2.getOwner());
+ assertEquals(derivedPath, artifact1.getExecPath());
+ assertEquals(derivedPath, artifact2.getExecPath());
+
+ // Source artifacts are always reused
+ PathFragment sourcePath = clientRoot.getExecPath().getRelative("fruit/mango");
+ artifact1 = artifactFactory.deserializeArtifact(sourcePath, rootResolver);
+ artifact2 = artifactFactory.deserializeArtifact(sourcePath, rootResolver);
+ assertSame(artifact1, artifact2);
+ assertEquals(sourcePath, artifact1.getExecPath());
+ }
+
+ @Test
+ public void testDeserializationWithInvalidPath() throws Exception {
+ artifactFactory.clear();
+ PathFragment randomPath = new PathFragment("maracuja/lemon/kiwi");
+ Artifact artifact = artifactFactory.deserializeArtifact(randomPath,
+ new MockPackageRootResolver());
+ assertNull(artifact);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java b/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java
new file mode 100644
index 0000000..d493e42
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java
@@ -0,0 +1,312 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.testutil.MoreAsserts.assertSameContents;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.actions.util.LabelArtifactOwner;
+import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
+import com.google.devtools.build.lib.rules.java.JavaSemantics;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.testutil.Scratch;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@RunWith(JUnit4.class)
+public class ArtifactTest {
+ private Scratch scratch;
+ private Path execDir;
+ private Root rootDir;
+
+ @Before
+ public void setUp() throws Exception {
+ scratch = new Scratch();
+ execDir = scratch.dir("/exec");
+ rootDir = Root.asDerivedRoot(scratch.dir("/exec/root"));
+ }
+
+ @Test
+ public void testConstruction_badRootDir() throws IOException {
+ Path f1 = scratch.file("/exec/dir/file.ext");
+ Path bogusDir = scratch.file("/exec/dir/bogus");
+ try {
+ new Artifact(f1, Root.asDerivedRoot(bogusDir), f1.relativeTo(execDir));
+ fail("Expected IllegalArgumentException constructing artifact with a bad root dir");
+ } catch (IllegalArgumentException expected) {}
+ }
+
+ @Test
+ public void testEquivalenceRelation() throws Exception {
+ PathFragment aPath = new PathFragment("src/a");
+ PathFragment bPath = new PathFragment("src/b");
+ assertEquals(new Artifact(aPath, rootDir),
+ new Artifact(aPath, rootDir));
+ assertEquals(new Artifact(bPath, rootDir),
+ new Artifact(bPath, rootDir));
+ assertFalse(new Artifact(aPath, rootDir).equals(
+ new Artifact(bPath, rootDir)));
+ }
+
+ @Test
+ public void testComparison() throws Exception {
+ PathFragment aPath = new PathFragment("src/a");
+ PathFragment bPath = new PathFragment("src/b");
+ Artifact aArtifact = new Artifact(aPath, rootDir);
+ Artifact bArtifact = new Artifact(bPath, rootDir);
+ assertEquals(-1, aArtifact.compareTo(bArtifact));
+ assertEquals(0, aArtifact.compareTo(aArtifact));
+ assertEquals(0, bArtifact.compareTo(bArtifact));
+ assertEquals(1, bArtifact.compareTo(aArtifact));
+ }
+
+ @Test
+ public void testRootPrefixedExecPath_normal() throws IOException {
+ Path f1 = scratch.file("/exec/root/dir/file.ext");
+ Artifact a1 = new Artifact(f1, rootDir, f1.relativeTo(execDir));
+ assertEquals("root:dir/file.ext", Artifact.asRootPrefixedExecPath(a1));
+ }
+
+ @Test
+ public void testRootPrefixedExecPath_noRoot() throws IOException {
+ Path f1 = scratch.file("/exec/dir/file.ext");
+ Artifact a1 = new Artifact(f1.relativeTo(execDir), Root.asDerivedRoot(execDir));
+ assertEquals(":dir/file.ext", Artifact.asRootPrefixedExecPath(a1));
+ }
+
+ @Test
+ public void testRootPrefixedExecPath_nullRootDir() throws IOException {
+ Path f1 = scratch.file("/exec/dir/file.ext");
+ try {
+ new Artifact(f1, null, f1.relativeTo(execDir));
+ fail("Expected IllegalArgumentException creating artifact with null root");
+ } catch (IllegalArgumentException expected) {}
+ }
+
+ @Test
+ public void testRootPrefixedExecPaths() throws IOException {
+ Path f1 = scratch.file("/exec/root/dir/file1.ext");
+ Path f2 = scratch.file("/exec/root/dir/dir/file2.ext");
+ Path f3 = scratch.file("/exec/root/dir/dir/dir/file3.ext");
+ Artifact a1 = new Artifact(f1, rootDir, f1.relativeTo(execDir));
+ Artifact a2 = new Artifact(f2, rootDir, f2.relativeTo(execDir));
+ Artifact a3 = new Artifact(f3, rootDir, f3.relativeTo(execDir));
+ List<String> strings = new ArrayList<>();
+ Artifact.addRootPrefixedExecPaths(Lists.newArrayList(a1, a2, a3), strings);
+ assertThat(strings).containsExactly(
+ "root:dir/file1.ext",
+ "root:dir/dir/file2.ext",
+ "root:dir/dir/dir/file3.ext").inOrder();
+ }
+
+ @Test
+ public void testGetFilename() throws Exception {
+ Root root = Root.asSourceRoot(scratch.dir("/foo"));
+ Artifact javaFile = new Artifact(scratch.file("/foo/Bar.java"), root);
+ Artifact generatedHeader = new Artifact(scratch.file("/foo/bar.proto.h"), root);
+ Artifact generatedCc = new Artifact(scratch.file("/foo/bar.proto.cc"), root);
+ Artifact aCPlusPlusFile = new Artifact(scratch.file("/foo/bar.cc"), root);
+ assertTrue(JavaSemantics.JAVA_SOURCE.matches(javaFile.getFilename()));
+ assertTrue(CppFileTypes.CPP_HEADER.matches(generatedHeader.getFilename()));
+ assertTrue(CppFileTypes.CPP_SOURCE.matches(generatedCc.getFilename()));
+ assertTrue(CppFileTypes.CPP_SOURCE.matches(aCPlusPlusFile.getFilename()));
+ }
+
+ @Test
+ public void testMangledPath() {
+ String path = "dir/sub_dir/name:end";
+ assertEquals("dir_Ssub_Udir_Sname_Cend", Actions.escapedPath(path));
+ }
+
+ private List<Artifact> getFooBarArtifacts(MutableActionGraph actionGraph, boolean collapsedList)
+ throws Exception {
+ Root root = Root.asSourceRoot(scratch.dir("/foo"));
+ Artifact aHeader1 = new Artifact(scratch.file("/foo/bar1.h"), root);
+ Artifact aHeader2 = new Artifact(scratch.file("/foo/bar2.h"), root);
+ Artifact aHeader3 = new Artifact(scratch.file("/foo/bar3.h"), root);
+ Artifact middleman = new Artifact(new PathFragment("middleman"),
+ Root.middlemanRoot(scratch.dir("/foo"), scratch.dir("/foo/out")));
+ actionGraph.registerAction(new MiddlemanAction(ActionsTestUtil.NULL_ACTION_OWNER,
+ ImmutableList.of(aHeader1, aHeader2, aHeader3), middleman, "desc",
+ MiddlemanType.AGGREGATING_MIDDLEMAN));
+ return collapsedList ? Lists.newArrayList(aHeader1, middleman) :
+ Lists.newArrayList(aHeader1, aHeader2, middleman);
+ }
+
+ @Test
+ public void testAddExecPaths() throws Exception {
+ List<String> paths = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Artifact.addExecPaths(getFooBarArtifacts(actionGraph, false), paths);
+ assertSameContents(ImmutableList.of("bar1.h", "bar2.h"), paths);
+ }
+
+ @Test
+ public void testAddExpandedExecPathStrings() throws Exception {
+ List<String> paths = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Artifact.addExpandedExecPathStrings(getFooBarArtifacts(actionGraph, true), paths,
+ ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+ assertSameContents(ImmutableList.of("bar1.h", "bar2.h", "bar3.h"), paths);
+ }
+
+ @Test
+ public void testAddExpandedExecPaths() throws Exception {
+ List<PathFragment> paths = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Artifact.addExpandedExecPaths(getFooBarArtifacts(actionGraph, true), paths,
+ ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+ assertSameContents(ImmutableList.of(
+ new PathFragment("bar1.h"), new PathFragment("bar2.h"), new PathFragment("bar3.h")),
+ paths);
+ }
+
+ @Test
+ public void testAddExpandedArtifacts() throws Exception {
+ List<Artifact> expanded = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ List<Artifact> original = getFooBarArtifacts(actionGraph, true);
+ Artifact.addExpandedArtifacts(original, expanded,
+ ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+
+ List<Artifact> manuallyExpanded = new ArrayList<>();
+ for (Artifact artifact : original) {
+ Action action = actionGraph.getGeneratingAction(artifact);
+ if (artifact.isMiddlemanArtifact()) {
+ Iterables.addAll(manuallyExpanded, action.getInputs());
+ } else {
+ manuallyExpanded.add(artifact);
+ }
+ }
+ assertSameContents(manuallyExpanded, expanded);
+ }
+
+ @Test
+ public void testAddExecPathsNewActionGraph() throws Exception {
+ List<String> paths = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Artifact.addExecPaths(getFooBarArtifacts(actionGraph, false), paths);
+ assertSameContents(ImmutableList.of("bar1.h", "bar2.h"), paths);
+ }
+
+ @Test
+ public void testAddExpandedExecPathStringsNewActionGraph() throws Exception {
+ List<String> paths = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Artifact.addExpandedExecPathStrings(getFooBarArtifacts(actionGraph, true), paths,
+ ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+ assertSameContents(ImmutableList.of("bar1.h", "bar2.h", "bar3.h"), paths);
+ }
+
+ @Test
+ public void testAddExpandedExecPathsNewActionGraph() throws Exception {
+ List<PathFragment> paths = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ Artifact.addExpandedExecPaths(getFooBarArtifacts(actionGraph, true), paths,
+ ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+ assertSameContents(ImmutableList.of(
+ new PathFragment("bar1.h"), new PathFragment("bar2.h"), new PathFragment("bar3.h")),
+ paths);
+ }
+
+ @Test
+ public void testAddExpandedArtifactsNewActionGraph() throws Exception {
+ List<Artifact> expanded = new ArrayList<>();
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ List<Artifact> original = getFooBarArtifacts(actionGraph, true);
+ Artifact.addExpandedArtifacts(original, expanded,
+ ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+
+ List<Artifact> manuallyExpanded = new ArrayList<>();
+ for (Artifact artifact : original) {
+ Action action = actionGraph.getGeneratingAction(artifact);
+ if (artifact.isMiddlemanArtifact()) {
+ Iterables.addAll(manuallyExpanded, action.getInputs());
+ } else {
+ manuallyExpanded.add(artifact);
+ }
+ }
+ assertSameContents(manuallyExpanded, expanded);
+ }
+
+ @Test
+ public void testRootRelativePathIsSameAsExecPath() throws Exception {
+ Root root = Root.asSourceRoot(scratch.dir("/foo"));
+ Artifact a = new Artifact(scratch.file("/foo/bar1.h"), root);
+ assertSame(a.getExecPath(), a.getRootRelativePath());
+ }
+
+ @Test
+ public void testToDetailString() throws Exception {
+ Artifact a = new Artifact(scratch.file("/a/b/c"), Root.asDerivedRoot(scratch.dir("/a/b")),
+ new PathFragment("b/c"));
+ assertEquals("[[/a]b]c", a.toDetailString());
+ }
+
+ @Test
+ public void testWeirdArtifact() throws Exception {
+ try {
+ new Artifact(scratch.file("/a/b/c"), Root.asDerivedRoot(scratch.dir("/a")),
+ new PathFragment("c"));
+ fail();
+ } catch (IllegalArgumentException e) {
+ assertEquals("c: illegal execPath doesn't end with b/c at /a/b/c with root /a[derived]",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSerializeToString() throws Exception {
+ assertEquals("b/c /3",
+ new Artifact(scratch.file("/a/b/c"),
+ Root.asDerivedRoot(scratch.dir("/a"))).serializeToString());
+ }
+
+ @Test
+ public void testSerializeToStringWithExecPath() throws Exception {
+ Path path = scratch.file("/aaa/bbb/ccc");
+ Root root = Root.asDerivedRoot(scratch.dir("/aaa/bbb"));
+ PathFragment execPath = new PathFragment("bbb/ccc");
+
+ assertEquals("bbb/ccc /3", new Artifact(path, root, execPath).serializeToString());
+ }
+
+ @Test
+ public void testSerializeToStringWithOwner() throws Exception {
+ assertEquals("b/c /3 //foo:bar",
+ new Artifact(scratch.file("/aa/b/c"), Root.asDerivedRoot(scratch.dir("/aa")),
+ new PathFragment("b/c"),
+ new LabelArtifactOwner(Label.parseAbsoluteUnchecked("//foo:bar"))).serializeToString());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElementTest.java b/src/test/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElementTest.java
new file mode 100644
index 0000000..28f185c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElementTest.java
@@ -0,0 +1,175 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.testing.GcFinalization;
+import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.testutil.TestUtils;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.lang.ref.WeakReference;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Tests for ConcurrentMultimapWithHeadElement.
+ */
+@RunWith(JUnit4.class)
+public class ConcurrentMultimapWithHeadElementTest {
+ @Test
+ public void testSmoke() throws Exception {
+ ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<String, String>();
+ assertEquals("val", multimap.putAndGet("key", "val"));
+ assertEquals("val", multimap.get("key"));
+ assertEquals("val", multimap.putAndGet("key", "val2"));
+ multimap.remove("key", "val2");
+ assertEquals("val", multimap.get("key"));
+ assertEquals("val", multimap.putAndGet("key", "val2"));
+ multimap.remove("key", "val");
+ assertEquals("val2", multimap.get("key"));
+ }
+
+ @Test
+ public void testDuplicate() throws Exception {
+ ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<String, String>();
+ assertEquals("val", multimap.putAndGet("key", "val"));
+ assertEquals("val", multimap.get("key"));
+ assertEquals("val", multimap.putAndGet("key", "val"));
+ multimap.remove("key", "val");
+ assertEquals(null, multimap.get("key"));
+ }
+
+ @Test
+ public void testDuplicateWithEqualsObject() throws Exception {
+ ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<>();
+ assertEquals(new String("val"), multimap.putAndGet("key", new String("val")));
+ assertEquals(new String("val"), multimap.get("key"));
+ assertEquals(new String("val"), multimap.putAndGet("key", new String("val")));
+ multimap.remove("key", new String("val"));
+ assertEquals(null, multimap.get("key"));
+ }
+
+ @Test
+ public void testFailedRemoval() throws Exception {
+ ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<String, String>();
+ assertEquals("val", multimap.putAndGet("key", "val"));
+ multimap.remove("key", "val2");
+ assertEquals("val", multimap.get("key"));
+ }
+
+ @Test
+ public void testNotEmpty() throws Exception {
+ ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<String, String>();
+ assertEquals("val", multimap.putAndGet("key", "val"));
+ multimap.remove("key", "val2");
+ assertEquals("val", multimap.get("key"));
+ }
+
+ @Test
+ public void testKeyRemoved() throws Exception {
+ String key = new String("key");
+ ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<String, String>();
+ assertEquals("val", multimap.putAndGet(key, "val"));
+ WeakReference<String> weakKey = new WeakReference<String>(key);
+ multimap.remove(key, "val");
+ key = null;
+ GcFinalization.awaitClear(weakKey);
+ }
+
+ @Test
+ public void testKeyRemovedAndAddedConcurrently() throws Exception {
+ final ConcurrentMultimapWithHeadElement<String, String> multimap =
+ new ConcurrentMultimapWithHeadElement<String, String>();
+ // Because we have two threads racing, run the test many times. Before fixed, there was a 90%
+ // chance of failure in 10,000 runs.
+ for (int i = 0; i < 10000; i++) {
+ assertEquals("val", multimap.putAndGet("key", "val"));
+ final CountDownLatch threadStart = new CountDownLatch(1);
+ TestThread testThread = new TestThread() {
+ @Override
+ public void runTest() throws Exception {
+ threadStart.countDown();
+ multimap.remove("key", "val");
+ }
+ };
+ testThread.start();
+ assertTrue(threadStart.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ assertNotNull(multimap.putAndGet("key", "val2")); // Removal may not have happened yet.
+ assertNotNull(multimap.get("key")); // If put failed, this will be null.
+ testThread.joinAndAssertState(2000);
+ multimap.clear();
+ }
+ }
+
+ private class StressTester extends AbstractQueueVisitor {
+ private final ConcurrentMultimapWithHeadElement<Boolean, Integer> multimap =
+ new ConcurrentMultimapWithHeadElement<Boolean, Integer>();
+ private final AtomicInteger actionCount = new AtomicInteger(0);
+
+ private StressTester() {
+ super(/*concurrent=*/true, 200, 200, 1, TimeUnit.SECONDS,
+ /*failFastOnException=*/true, /*failFastOnInterrupt=*/true, "action-graph-test");
+ }
+
+ private void addAndRemove(final Boolean key, final Integer add, final Integer remove) {
+ enqueue(new Runnable() {
+ @Override
+ public void run() {
+ assertNotNull(multimap.putAndGet(key, add));
+ multimap.remove(key, remove);
+ doRandom();
+ }
+ });
+ }
+
+ private Integer getRandomInt() {
+ return (int) Math.round(Math.random() * 3.0);
+ }
+
+ private void doRandom() {
+ if (actionCount.incrementAndGet() > 100000) {
+ return;
+ }
+ Boolean key = Math.random() < 0.5;
+ addAndRemove(key, getRandomInt(), getRandomInt());
+ }
+
+ private void work() throws InterruptedException {
+ work(/*failFastOnInterrupt=*/true);
+ }
+ }
+
+ @Test
+ public void testStressTest() throws Exception {
+ StressTester stressTester = new StressTester();
+ stressTester.doRandom();
+ stressTester.work();
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/CustomCommandLineTest.java b/src/test/java/com/google/devtools/build/lib/actions/CustomCommandLineTest.java
new file mode 100644
index 0000000..3a7db34
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/CustomCommandLineTest.java
@@ -0,0 +1,161 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomArgv;
+import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomMultiArgv;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.syntax.Label.SyntaxException;
+import com.google.devtools.build.lib.testutil.Scratch;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for CustomCommandLine.
+ */
+@RunWith(JUnit4.class)
+public class CustomCommandLineTest {
+
+ private Scratch scratch;
+ private Root rootDir;
+ private Artifact artifact1;
+ private Artifact artifact2;
+
+ @Before
+ public void setUp() throws Exception {
+ scratch = new Scratch();
+ rootDir = Root.asDerivedRoot(scratch.dir("/exec/root"));
+ artifact1 = new Artifact(scratch.file("/exec/root/dir/file1.txt"), rootDir);
+ artifact2 = new Artifact(scratch.file("/exec/root/dir/file2.txt"), rootDir);
+ }
+
+ @Test
+ public void testStringArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().add("--arg1").add("--arg2").build();
+ assertEquals(ImmutableList.of("--arg1", "--arg2"), cl.arguments());
+ }
+
+ @Test
+ public void testLabelArgs() throws SyntaxException {
+ CustomCommandLine cl = CustomCommandLine.builder().add(Label.parseAbsolute("//a:b")).build();
+ assertEquals(ImmutableList.of("//a:b"), cl.arguments());
+ }
+
+ @Test
+ public void testStringsArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().add("--arg",
+ ImmutableList.of("a", "b")).build();
+ assertEquals(ImmutableList.of("--arg", "a", "b"), cl.arguments());
+ }
+
+ @Test
+ public void testArtifactExecPathArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addExecPath("--path", artifact1).build();
+ assertEquals(ImmutableList.of("--path", "dir/file1.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testArtifactExecPathsArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addExecPaths("--path",
+ ImmutableList.of(artifact1, artifact2)).build();
+ assertEquals(ImmutableList.of("--path", "dir/file1.txt", "dir/file2.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testNestedSetArtifactExecPathsArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addExecPaths(
+ NestedSetBuilder.<Artifact>stableOrder().add(artifact1).add(artifact2).build()).build();
+ assertEquals(ImmutableList.of("dir/file1.txt", "dir/file2.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testArtifactJoinExecPathArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addJoinExecPaths("--path", ":",
+ ImmutableList.of(artifact1, artifact2)).build();
+ assertEquals(ImmutableList.of("--path", "dir/file1.txt:dir/file2.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testPathArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addPath(artifact1.getExecPath()).build();
+ assertEquals(ImmutableList.of("dir/file1.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testJoinPathArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addJoinPaths(":",
+ ImmutableList.of(artifact1.getExecPath(), artifact2.getExecPath())).build();
+ assertEquals(ImmutableList.of("dir/file1.txt:dir/file2.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testPathsArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().addPaths("%s:%s",
+ artifact1.getExecPath(), artifact1.getRootRelativePath()).build();
+ assertEquals(ImmutableList.of("dir/file1.txt:dir/file1.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testCustomArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().add(new CustomArgv() {
+ @Override
+ public String argv() {
+ return "--arg";
+ }
+ }).build();
+ assertEquals(ImmutableList.of("--arg"), cl.arguments());
+ }
+
+ @Test
+ public void testCustomMultiArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder().add(new CustomMultiArgv() {
+ @Override
+ public ImmutableList<String> argv() {
+ return ImmutableList.of("--arg1", "--arg2");
+ }
+ }).build();
+ assertEquals(ImmutableList.of("--arg1", "--arg2"), cl.arguments());
+ }
+
+ @Test
+ public void testCombinedArgs() {
+ CustomCommandLine cl = CustomCommandLine.builder()
+ .add("--arg")
+ .add("--args", ImmutableList.of("abc"))
+ .addExecPaths("--path1", ImmutableList.of(artifact1))
+ .addExecPath("--path2", artifact2)
+ .build();
+ assertEquals(ImmutableList.of("--arg", "--args", "abc", "--path1", "dir/file1.txt", "--path2",
+ "dir/file2.txt"), cl.arguments());
+ }
+
+ @Test
+ public void testAddNulls() {
+ CustomCommandLine cl = CustomCommandLine.builder()
+ .add("--args", null)
+ .addExecPaths(null, ImmutableList.of(artifact1))
+ .addExecPath(null, null)
+ .build();
+ assertEquals(ImmutableList.of(), cl.arguments());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java
new file mode 100644
index 0000000..2ed24a6
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java
@@ -0,0 +1,145 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Strings;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests for DigestUtils.
+ */
+@RunWith(JUnit4.class)
+public class DigestUtilsTest {
+
+ private static void assertMd5CalculationConcurrency(boolean expectConcurrent,
+ final boolean fastDigest, final int fileSize1, final int fileSize2) throws Exception {
+ final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads.
+ final CountDownLatch readyLatch = new CountDownLatch(1); // Used to block main thread.
+
+ FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) {
+ @Override
+ protected byte[] getMD5Digest(Path path) throws IOException {
+ try {
+ barrierLatch.countDown();
+ readyLatch.countDown();
+ // Either both threads will be inside getMD5Digest at the same time or they
+ // both will be blocked.
+ barrierLatch.await();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return super.getMD5Digest(path);
+ }
+
+ @Override
+ protected String getFastDigestFunctionType(Path path) {
+ return "MD5";
+ }
+
+ @Override
+ protected byte[] getFastDigest(Path path) throws IOException {
+ return fastDigest ? super.getMD5Digest(path) : null;
+ }
+ };
+
+ final Path myFile1 = myfs.getPath("/f1.dat");
+ final Path myFile2 = myfs.getPath("/f2.dat");
+ FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
+ FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));
+
+ TestThread thread1 = new TestThread () {
+ @Override public void runTest() throws Exception {
+ DigestUtils.getDigestOrFail(myFile1, fileSize1);
+ }
+ };
+
+ TestThread thread2 = new TestThread () {
+ @Override public void runTest() throws Exception {
+ DigestUtils.getDigestOrFail(myFile2, fileSize2);
+ }
+ };
+
+ thread1.start();
+ thread2.start();
+ if (!expectConcurrent) { // Synchronized case.
+ // Wait until at least one thread reached getMD5Digest().
+ assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ // Only 1 thread should be inside getMD5Digest().
+ assertEquals(1, barrierLatch.getCount());
+ barrierLatch.countDown(); // Release barrier latch, allowing both threads to proceed.
+ }
+ // Test successful execution within 5 seconds.
+ thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ }
+
+ /**
+ * Ensures that MD5 calculation is synchronized for files
+ * greater than 4096 bytes if MD5 is not available cheaply,
+ * so machines with rotating drives don't become unusable.
+ */
+ @Test
+ public void testMd5CalculationConcurrency() throws Exception {
+ assertMd5CalculationConcurrency(true, true, 4096, 4096);
+ assertMd5CalculationConcurrency(true, true, 4097, 4097);
+ assertMd5CalculationConcurrency(true, false, 4096, 4096);
+ assertMd5CalculationConcurrency(false, false, 4097, 4097);
+ assertMd5CalculationConcurrency(true, false, 1024, 4097);
+ assertMd5CalculationConcurrency(true, false, 1024, 1024);
+ }
+
+ @Test
+ public void testRecoverFromMalformedDigest() throws Exception {
+ final byte[] malformed = {0, 0, 0};
+ FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) {
+ @Override
+ protected String getFastDigestFunctionType(Path path) {
+ return "MD5";
+ }
+
+ @Override
+ protected byte[] getFastDigest(Path path) throws IOException {
+ // MD5 digests are supposed to be 16 bytes.
+ return malformed;
+ }
+ };
+ Path path = myFS.getPath("/file");
+ FileSystemUtils.writeContentAsLatin1(path, "a");
+ byte[] result = DigestUtils.getDigestOrFail(path, 1);
+ assertArrayEquals(path.getMD5Digest(), result);
+ assertNotSame(malformed, result);
+ assertEquals(16, result.length);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java b/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java
new file mode 100644
index 0000000..50d0f25
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java
@@ -0,0 +1,105 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.actions.util.DummyExecutor;
+import com.google.devtools.build.lib.analysis.actions.ExecutableSymlinkAction;
+import com.google.devtools.build.lib.exec.SingleBuildFileCache;
+import com.google.devtools.build.lib.testutil.Scratch;
+import com.google.devtools.build.lib.testutil.TestFileOutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class ExecutableSymlinkActionTest {
+ private Scratch scratch = new Scratch();
+ private Root inputRoot;
+ private Root outputRoot;
+ TestFileOutErr outErr;
+ private Executor executor;
+
+ @Before
+ public void setUp() throws Exception {
+ final Path inputDir = scratch.dir("/in");
+ inputRoot = Root.asDerivedRoot(inputDir);
+ outputRoot = Root.asDerivedRoot(scratch.dir("/out"));
+ outErr = new TestFileOutErr();
+ executor = new DummyExecutor(inputDir);
+ }
+
+ private ActionExecutionContext createContext() {
+ Path execRoot = executor.getExecRoot();
+ return new ActionExecutionContext(
+ executor,
+ new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()),
+ null, outErr, null);
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ Path inputFile = inputRoot.getPath().getChild("some-file");
+ Path outputFile = outputRoot.getPath().getChild("some-output");
+ FileSystemUtils.createEmptyFile(inputFile);
+ inputFile.setExecutable(/*executable=*/true);
+ Artifact input = new Artifact(inputFile, inputRoot);
+ Artifact output = new Artifact(outputFile, outputRoot);
+ ExecutableSymlinkAction action = new ExecutableSymlinkAction(NULL_ACTION_OWNER, input, output);
+ action.execute(createContext());
+ assertEquals(inputFile, outputFile.resolveSymbolicLinks());
+ }
+
+ @Test
+ public void testFailIfInputIsNotAFile() throws Exception {
+ Path dir = inputRoot.getPath().getChild("some-dir");
+ FileSystemUtils.createDirectoryAndParents(dir);
+ Artifact input = new Artifact(dir, inputRoot);
+ Artifact output = new Artifact(outputRoot.getPath().getChild("some-output"), outputRoot);
+ ExecutableSymlinkAction action = new ExecutableSymlinkAction(NULL_ACTION_OWNER, input, output);
+ try {
+ action.execute(createContext());
+ fail();
+ } catch (ActionExecutionException e) {
+ assertTrue(e.getMessage().contains("'some-dir' is not a file"));
+ }
+ }
+
+ @Test
+ public void testFailIfInputIsNotExecutable() throws Exception {
+ Path file = inputRoot.getPath().getChild("some-file");
+ FileSystemUtils.createEmptyFile(file);
+ file.setExecutable(/*executable=*/false);
+ Artifact input = new Artifact(file, inputRoot);
+ Artifact output = new Artifact(outputRoot.getPath().getChild("some-output"), outputRoot);
+ ExecutableSymlinkAction action = new ExecutableSymlinkAction(NULL_ACTION_OWNER, input, output);
+ try {
+ action.execute(createContext());
+ fail();
+ } catch (ActionExecutionException e) {
+ String want = "'some-file' is not executable";
+ String got = e.getMessage();
+ assertTrue(String.format("got %s, want %s", got, want), got.contains(want));
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/FailActionTest.java b/src/test/java/com/google/devtools/build/lib/actions/FailActionTest.java
new file mode 100644
index 0000000..7838fca
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/FailActionTest.java
@@ -0,0 +1,81 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER;
+import static com.google.devtools.build.lib.testutil.MoreAsserts.assertSameContents;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.testutil.Scratch;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Collection;
+import java.util.Collections;
+
+@RunWith(JUnit4.class)
+public class FailActionTest {
+
+ private Scratch scratch = new Scratch();
+
+ private String errorMessage;
+ private Artifact anOutput;
+ private Collection<Artifact> outputs;
+ private FailAction failAction;
+
+ protected MutableActionGraph actionGraph = new MapBasedActionGraph();
+
+ @Before
+ public void setUp() throws Exception {
+ errorMessage = "An error just happened.";
+ anOutput = new Artifact(scratch.file("/out/foo"),
+ Root.asDerivedRoot(scratch.dir("/"), scratch.dir("/out")));
+ outputs = ImmutableList.of(anOutput);
+ failAction = new FailAction(NULL_ACTION_OWNER, outputs, errorMessage);
+ actionGraph.registerAction(failAction);
+ assertSame(failAction, actionGraph.getGeneratingAction(anOutput));
+ }
+
+ @Test
+ public void testExecutingItYieldsExceptionWithErrorMessage() {
+ try {
+ failAction.execute(null);
+ fail();
+ } catch (ActionExecutionException e) {
+ assertEquals(errorMessage, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testInputsAreEmptySet() {
+ assertSameContents(Collections.emptySet(), failAction.getInputs());
+ }
+
+ @Test
+ public void testRetainsItsOutputs() {
+ assertSameContents(outputs, failAction.getOutputs());
+ }
+
+ @Test
+ public void testPrimaryOutput() {
+ assertSame(anOutput, failAction.getPrimaryOutput());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/LocalHostCapacityTest.java b/src/test/java/com/google/devtools/build/lib/actions/LocalHostCapacityTest.java
new file mode 100644
index 0000000..5e2e1a8
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/LocalHostCapacityTest.java
@@ -0,0 +1,434 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class LocalHostCapacityTest {
+
+ private FsApparatus scratch = FsApparatus.newNative();
+
+ @Test
+ public void testNonHyperthreadedMachine() throws Exception {
+ String cpuinfoContent = StringUtilities.joinLines(
+ "processor\t: 0",
+ "vendor_id\t: GenuineIntel",
+ "cpu family\t: 15",
+ "model\t\t: 4",
+ "model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
+ "stepping\t: 10",
+ "cpu MHz\t\t: 3400.000",
+ "cache size\t: 2048 KB",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 5",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca "
+ + "cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
+ + "syscall nx lm constant_tsc up pni monitor ds_cpl est cid cx16 "
+ + "xtpr lahf_lm",
+ "bogomips\t: 6803.83",
+ "clflush size\t: 64",
+ "cache_alignment\t: 128",
+ "address sizes\t: 36 bits physical, 48 bits virtual",
+ "power management:"
+ );
+ String cpuinfoFile =
+ scratch.file("test_cpuinfo_nonht", cpuinfoContent).getPathString();
+ String meminfoContent = StringUtilities.joinLines(
+ "MemTotal: 3091732 kB",
+ "MemFree: 2167344 kB",
+ "Buffers: 60644 kB",
+ "Cached: 509940 kB",
+ "SwapCached: 0 kB",
+ "Active: 636892 kB",
+ "Inactive: 212760 kB",
+ "HighTotal: 0 kB",
+ "HighFree: 0 kB",
+ "LowTotal: 3091732 kB",
+ "LowFree: 2167344 kB",
+ "SwapTotal: 9124880 kB",
+ "SwapFree: 9124880 kB",
+ "Dirty: 0 kB",
+ "Writeback: 0 kB",
+ "AnonPages: 279028 kB",
+ "Mapped: 54404 kB",
+ "Slab: 42820 kB",
+ "PageTables: 5184 kB",
+ "NFS_Unstable: 0 kB",
+ "Bounce: 0 kB",
+ "CommitLimit: 10670744 kB",
+ "Committed_AS: 665840 kB",
+ "VmallocTotal: 34359738367 kB",
+ "VmallocUsed: 300484 kB",
+ "VmallocChunk: 34359437307 kB",
+ "HugePages_Total: 0",
+ "HugePages_Free: 0",
+ "HugePages_Rsvd: 0",
+ "Hugepagesize: 2048 kB"
+ );
+ String stat1Content = StringUtilities.joinLines(
+ "cpu 29793342 260290 3479274 636259369 6683218 656426 714057 0",
+ "cpu0 29793342 260290 3479274 636259369 6683218 656426 714057 0",
+ "intr 2870488853 2486517107 3 0 0 2 0 5 0 0 0 0 0 3 0 74363716 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 52483586 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 973315 0 0 0 0 0 0 0 46 " +
+ "0 0 0 0 0 0 0 98792358 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 114339590 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43019122 0 0 0 0 0",
+ "ctxt 15053799843",
+ "btime 1199289688",
+ "processes 25799993",
+ "procs_running 1",
+ "procs_blocked 0"
+ );
+ String stat2Content = StringUtilities.joinLines(
+ "cpu 29794509 260290 3479474 636287862 6683283 656450 714087 0 0",
+ "cpu0 29794509 260290 3479474 636287862 6683283 656450 714087 0 0",
+ "intr 2870488853 2486517107 3 0 0 2 0 5 0 0 0 0 0 3 0 74363716 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 52483586 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 973315 0 0 0 0 0 0 0 46 " +
+ "0 0 0 0 0 0 0 98792358 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 114339590 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " +
+ "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43019122 0 0 0 0 0",
+ "ctxt 15053799843",
+ "btime 1199289688",
+ "processes 25799993",
+ "procs_running 1",
+ "procs_blocked 0"
+ );
+ String meminfoFile =
+ scratch.file("test_meminfo_nonht", meminfoContent).getPathString();
+ String stat1File =
+ scratch.file("proc_stat_1", stat1Content).getPathString();
+ String stat2File =
+ scratch.file("proc_stat_2", stat2Content).getPathString();
+ assertEquals(1, LocalHostCapacity.getLogicalCpuCount(cpuinfoContent));
+ assertEquals(1, LocalHostCapacity.getPhysicalCpuCount(cpuinfoContent, 1));
+ assertEquals(1, LocalHostCapacity.getCoresPerCpu(cpuinfoContent));
+ ResourceSet capacity =
+ LocalHostCapacity.getLocalHostCapacity(cpuinfoFile, meminfoFile);
+ assertEquals(1.0, capacity.getCpuUsage(), 0.01);
+ assertEquals(3091.732, capacity.getMemoryMb(), 0.1); // +/- 0.1MB
+ LocalHostCapacity.setLocalHostCapacity(capacity);
+ assertSame(capacity, LocalHostCapacity.getLocalHostCapacity());
+ Clock mockedClock = new Clock() {
+ private int callCount = 0;
+
+ @Override
+ public long currentTimeMillis() {
+ throw new AssertionError("unexpected method call");
+ }
+
+ @Override
+ public long nanoTime() {
+ callCount++;
+ if (callCount == 1) {
+ return 0;
+ } else if (callCount == 2) {
+ return 100 * 1000000;
+ } else if (callCount == 3) {
+ return 200 * 1000000;
+ } else {
+ throw new AssertionError("unexpected method call");
+ }
+ }
+ };
+ LocalHostCapacity.FreeResources freeStats =
+ LocalHostCapacity.getFreeResources(mockedClock, meminfoFile, stat1File, null);
+ assertNotNull(freeStats);
+ assertEquals(2356.756, freeStats.getFreeMb(), 0.001);
+ assertEquals(0.0, freeStats.getAvgFreeCpu(), 0);
+ // The next call to the mock clock returns a timestamp as if 100 ms have passed.
+ assertTrue(freeStats.getReadingAge() > 50);
+ // Fake another 100 ms going by for the next call.
+ freeStats = LocalHostCapacity.getFreeResources(mockedClock, meminfoFile, stat2File, freeStats);
+ assertNotNull(freeStats);
+ assertEquals(2356.756, freeStats.getFreeMb(), 0.001);
+ assertTrue(freeStats.getInterval() > 100);
+ assertEquals(0.95, freeStats.getAvgFreeCpu(), 0.001);
+ }
+
+ @Test
+ public void testHyperthreadedMachine() throws Exception {
+ String cpuinfoContent = StringUtilities.joinLines(
+ "processor\t: 0",
+ "vendor_id\t: GenuineIntel",
+ "cpu family\t: 15",
+ "model\t\t: 4",
+ "model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
+ "stepping\t: 1",
+ "cpu MHz\t\t: 3400.245",
+ "cache size\t: 1024 KB",
+ "physical id\t: 0",
+ "siblings\t: 2",
+ "core id\t\t: 0",
+ "cpu cores\t: 1",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 5",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge "
+ + "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
+ + "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr",
+ "bogomips\t: 6806.31",
+ "clflush size\t: 64",
+ "cache_alignment\t: 128",
+ "address sizes\t: 36 bits physical, 48 bits virtual",
+ "power management:",
+ "",
+ "processor\t: 1",
+ "vendor_id\t: GenuineIntel",
+ "cpu family\t: 15",
+ "model\t\t: 4",
+ "model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
+ "stepping\t: 1",
+ "cpu MHz\t\t: 3400.245",
+ "cache size\t: 1024 KB",
+ "physical id\t: 0",
+ "siblings\t: 2",
+ "core id\t\t: 0",
+ "cpu cores\t: 1",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 5",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge "
+ + "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
+ + "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr",
+ "bogomips\t: 6800.76",
+ "clflush size\t: 64",
+ "cache_alignment\t: 128",
+ "address sizes\t: 36 bits physical, 48 bits virtual",
+ "power management:",
+ ""
+ );
+ String cpuinfoFile =
+ scratch.file("test_cpuinfo_ht", cpuinfoContent).getPathString();
+ String meminfoContent = StringUtilities.joinLines(
+ "MemTotal: 3092004 kB",
+ "MemFree: 26124 kB",
+ "Buffers: 3836 kB",
+ "Cached: 52400 kB",
+ "SwapCached: 68204 kB",
+ "Active: 2281464 kB",
+ "Inactive: 260908 kB",
+ "HighTotal: 0 kB",
+ "HighFree: 0 kB",
+ "LowTotal: 3092004 kB",
+ "LowFree: 26124 kB",
+ "SwapTotal: 9124880 kB",
+ "SwapFree: 8264920 kB",
+ "Dirty: 616 kB",
+ "Writeback: 0 kB",
+ "AnonPages: 2466336 kB",
+ "Mapped: 37576 kB",
+ "Slab: 483004 kB",
+ "PageTables: 11912 kB",
+ "NFS_Unstable: 0 kB",
+ "Bounce: 0 kB",
+ "CommitLimit: 10670880 kB",
+ "Committed_AS: 3627984 kB",
+ "VmallocTotal: 34359738367 kB",
+ "VmallocUsed: 300460 kB",
+ "VmallocChunk: 34359437307 kB",
+ "HugePages_Total: 0",
+ "HugePages_Free: 0",
+ "HugePages_Rsvd: 0",
+ "Hugepagesize: 2048 kB"
+ );
+ String meminfoFile =
+ scratch.file("test_meminfo_ht", meminfoContent).getPathString();
+ assertEquals(2, LocalHostCapacity.getLogicalCpuCount(cpuinfoContent));
+ assertEquals(1, LocalHostCapacity.getPhysicalCpuCount(cpuinfoContent, 2));
+ assertEquals(1, LocalHostCapacity.getCoresPerCpu(cpuinfoContent));
+ ResourceSet capacity =
+ LocalHostCapacity.getLocalHostCapacity(cpuinfoFile, meminfoFile);
+ assertEquals(1.2, capacity.getCpuUsage(), .0001);
+ assertEquals(3092.004, capacity.getMemoryMb(), 0.1); // +/- 0.1MB
+ }
+
+ @Test
+ public void testAMDMachine() throws Exception {
+ String cpuinfoContent = StringUtilities.joinLines(
+ "processor\t: 0",
+ "vendor_id\t: AuthenticAMD",
+ "cpu family\t: 15",
+ "model\t\t: 65",
+ "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
+ "stepping\t: 2",
+ "cpu MHz\t\t: 2200.000",
+ "cache size\t: 1024 KB",
+ "physical id\t: 0",
+ "siblings\t: 2",
+ "core id\t\t: 0",
+ "cpu cores\t: 2",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 1",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ + "cmp_legacy svm cr8_legacy",
+ "bogomips\t: 4425.84",
+ "TLB size\t: 1024 4K pages",
+ "clflush size\t: 64",
+ "cache_alignment\t: 64",
+ "address sizes\t: 40 bits physical, 48 bits virtual",
+ "power management: ts fid vid ttp tm stc",
+ "",
+ "processor\t: 1",
+ "vendor_id\t: AuthenticAMD",
+ "cpu family\t: 15",
+ "model\t\t: 65",
+ "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
+ "stepping\t: 2",
+ "cpu MHz\t\t: 2200.000",
+ "cache size\t: 1024 KB",
+ "physical id\t: 0",
+ "siblings\t: 2",
+ "core id\t\t: 1",
+ "cpu cores\t: 2",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 1",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ + "cmp_legacy svm cr8_legacy",
+ "bogomips\t: 4460.61",
+ "TLB size\t: 1024 4K pages",
+ "clflush size\t: 64",
+ "cache_alignment\t: 64",
+ "address sizes\t: 40 bits physical, 48 bits virtual",
+ "power management: ts fid vid ttp tm stc",
+ "",
+ "processor\t: 2",
+ "vendor_id\t: AuthenticAMD",
+ "cpu family\t: 15",
+ "model\t\t: 65",
+ "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
+ "stepping\t: 2",
+ "cpu MHz\t\t: 2200.000",
+ "cache size\t: 1024 KB",
+ "physical id\t: 1",
+ "siblings\t: 2",
+ "core id\t\t: 0",
+ "cpu cores\t: 2",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 1",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ + "cmp_legacy svm cr8_legacy",
+ "bogomips\t: 4420.45",
+ "TLB size\t: 1024 4K pages",
+ "clflush size\t: 64",
+ "cache_alignment\t: 64",
+ "address sizes\t: 40 bits physical, 48 bits virtual",
+ "power management: ts fid vid ttp tm stc",
+ "",
+ "processor\t: 3",
+ "vendor_id\t: AuthenticAMD",
+ "cpu family\t: 15",
+ "model\t\t: 65",
+ "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
+ "stepping\t: 2",
+ "cpu MHz\t\t: 2200.000",
+ "cache size\t: 1024 KB",
+ "physical id\t: 1",
+ "siblings\t: 2",
+ "core id\t\t: 1",
+ "cpu cores\t: 2",
+ "fpu\t\t: yes",
+ "fpu_exception\t: yes",
+ "cpuid level\t: 1",
+ "wp\t\t: yes",
+ "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ + "cmp_legacy svm cr8_legacy",
+ "bogomips\t: 4460.39",
+ "TLB size\t: 1024 4K pages",
+ "clflush size\t: 64",
+ "cache_alignment\t: 64",
+ "address sizes\t: 40 bits physical, 48 bits virtual",
+ "power management: ts fid vid ttp tm stc",
+ ""
+ );
+ String cpuinfoFile =
+ scratch.file("test_cpuinfo_amd", cpuinfoContent).getPathString();
+ String meminfoContent = StringUtilities.joinLines(
+ "MemTotal: 8223956 kB",
+ "MemFree: 3670396 kB",
+ "Buffers: 374068 kB",
+ "Cached: 3366980 kB",
+ "SwapCached: 0 kB",
+ "Active: 3275860 kB",
+ "Inactive: 737816 kB",
+ "HighTotal: 0 kB",
+ "HighFree: 0 kB",
+ "LowTotal: 8223956 kB",
+ "LowFree: 3670396 kB",
+ "SwapTotal: 6024332 kB",
+ "SwapFree: 6024332 kB",
+ "Dirty: 84 kB",
+ "Writeback: 0 kB",
+ "AnonPages: 272308 kB",
+ "Mapped: 62604 kB",
+ "Slab: 506140 kB",
+ "PageTables: 4608 kB",
+ "NFS_Unstable: 0 kB",
+ "Bounce: 0 kB",
+ "CommitLimit: 10136308 kB",
+ "Committed_AS: 600672 kB",
+ "VmallocTotal: 34359738367 kB",
+ "VmallocUsed: 299068 kB",
+ "VmallocChunk: 34359438843 kB",
+ "HugePages_Total: 0",
+ "HugePages_Free: 0",
+ "HugePages_Rsvd: 0",
+ "Hugepagesize: 2048 kB");
+ String meminfoFile =
+ scratch.file("test_meminfo_amd", meminfoContent).getPathString();
+ assertEquals(4, LocalHostCapacity.getLogicalCpuCount(cpuinfoContent));
+ assertEquals(2, LocalHostCapacity.getPhysicalCpuCount(cpuinfoContent, 4));
+ assertEquals(2, LocalHostCapacity.getCoresPerCpu(cpuinfoContent));
+ ResourceSet capacity =
+ LocalHostCapacity.getLocalHostCapacity(cpuinfoFile, meminfoFile);
+ assertEquals(capacity.getCpuUsage(), 4.0, 0.01);
+ assertEquals(8223.956, capacity.getMemoryMb(), 0.1); // +/- 0.1MB
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/MapBasedActionGraphTest.java b/src/test/java/com/google/devtools/build/lib/actions/MapBasedActionGraphTest.java
new file mode 100644
index 0000000..cde9aed3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/MapBasedActionGraphTest.java
@@ -0,0 +1,147 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil.UncheckedActionConflictException;
+import com.google.devtools.build.lib.actions.util.TestAction;
+import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Tests for {@link MapBasedActionGraph}.
+ */
+@RunWith(JUnit4.class)
+public class MapBasedActionGraphTest {
+ @Test
+ public void testSmoke() throws Exception {
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ FileSystem fileSystem = new InMemoryFileSystem(BlazeClock.instance());
+ Path path = fileSystem.getPath("/root/foo");
+ Artifact output = new Artifact(path, Root.asDerivedRoot(path));
+ Action action = new TestAction(TestAction.NO_EFFECT,
+ ImmutableSet.<Artifact>of(), ImmutableSet.of(output));
+ actionGraph.registerAction(action);
+ actionGraph.unregisterAction(action);
+ path = fileSystem.getPath("/root/bar");
+ output = new Artifact(path, Root.asDerivedRoot(path));
+ Action action2 = new TestAction(TestAction.NO_EFFECT,
+ ImmutableSet.<Artifact>of(), ImmutableSet.of(output));
+ actionGraph.registerAction(action);
+ actionGraph.registerAction(action2);
+ actionGraph.unregisterAction(action);
+ }
+
+ @Test
+ public void testNoActionConflictWhenUnregisteringSharedAction() throws Exception {
+ MutableActionGraph actionGraph = new MapBasedActionGraph();
+ FileSystem fileSystem = new InMemoryFileSystem(BlazeClock.instance());
+ Path path = fileSystem.getPath("/root/foo");
+ Artifact output = new Artifact(path, Root.asDerivedRoot(path));
+ Action action = new TestAction(TestAction.NO_EFFECT,
+ ImmutableSet.<Artifact>of(), ImmutableSet.of(output));
+ actionGraph.registerAction(action);
+ Action otherAction = new TestAction(TestAction.NO_EFFECT,
+ ImmutableSet.<Artifact>of(), ImmutableSet.of(output));
+ actionGraph.registerAction(otherAction);
+ actionGraph.unregisterAction(action);
+ }
+
+ private class ActionRegisterer extends AbstractQueueVisitor {
+ private final MutableActionGraph graph = new MapBasedActionGraph();
+ private final Artifact output;
+ // Just to occasionally add actions that were already present.
+ private final Set<Action> allActions = Sets.newConcurrentHashSet();
+ private final AtomicInteger actionCount = new AtomicInteger(0);
+
+ private ActionRegisterer() {
+ super(/*concurrent=*/true, 200, 200, 1, TimeUnit.SECONDS,
+ /*failFastOnException=*/true, /*failFastOnInterrupt=*/true, "action-graph-test");
+ FileSystem fileSystem = new InMemoryFileSystem(BlazeClock.instance());
+ Path path = fileSystem.getPath("/root/foo");
+ output = new Artifact(path, Root.asDerivedRoot(path));
+ allActions.add(new TestAction(TestAction.NO_EFFECT,
+ ImmutableSet.<Artifact>of(), ImmutableSet.of(output)));
+ }
+
+ private void registerAction(final Action action) {
+ enqueue(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ graph.registerAction(action);
+ } catch (ActionConflictException e) {
+ throw new UncheckedActionConflictException(e);
+ }
+ doRandom();
+ }
+ });
+ }
+
+ private void unregisterAction(final Action action) {
+ enqueue(new Runnable() {
+ @Override
+ public void run() {
+ graph.unregisterAction(action);
+ doRandom();
+ }
+ });
+ }
+
+ private void doRandom() {
+ if (actionCount.incrementAndGet() > 10000) {
+ return;
+ }
+ Action action = null;
+ if (Math.random() < 0.5) {
+ action = Iterables.getFirst(allActions, null);
+ } else {
+ action = new TestAction(TestAction.NO_EFFECT,
+ ImmutableSet.<Artifact>of(), ImmutableSet.of(output));
+ allActions.add(action);
+ }
+ if (Math.random() < 0.5) {
+ registerAction(action);
+ } else {
+ unregisterAction(action);
+ }
+ }
+
+ private void work() throws InterruptedException {
+ work(/*failFastOnInterrupt=*/true);
+ }
+ }
+
+ @Test
+ public void testSharedActionStressTest() throws Exception {
+ ActionRegisterer actionRegisterer = new ActionRegisterer();
+ actionRegisterer.doRandom();
+ actionRegisterer.work();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java
new file mode 100644
index 0000000..4955288
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java
@@ -0,0 +1,400 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+
+/**
+ *
+ * Tests for @{link ResourceManager}.
+ */
+@RunWith(JUnit4.class)
+public class ResourceManagerTest {
+
+ private final ActionMetadata resourceOwner = new ResourceOwnerStub();
+ private final ResourceManager rm = ResourceManager.instanceForTestingOnly();
+ private AtomicInteger counter;
+ CyclicBarrier sync;
+ CyclicBarrier sync2;
+
+ @Before
+ public void setUp() throws Exception {
+ rm.setRamUtilizationPercentage(100);
+ rm.setAvailableResources(new ResourceSet(1000, 1, 1));
+ rm.setEventBus(new EventBus());
+ counter = new AtomicInteger(0);
+ sync = new CyclicBarrier(2);
+ sync2 = new CyclicBarrier(2);
+ rm.resetResourceUsage();
+ }
+
+ private void acquire(double ram, double cpu, double io) throws InterruptedException {
+ rm.acquireResources(resourceOwner, new ResourceSet(ram, cpu, io));
+ }
+
+ private boolean acquireNonblocking(double ram, double cpu, double io) {
+ return rm.tryAcquire(resourceOwner, new ResourceSet(ram, cpu, io));
+ }
+
+ private void release(double ram, double cpu, double io) {
+ rm.releaseResources(resourceOwner, new ResourceSet(ram, cpu, io));
+ }
+
+ private void validate (int count) {
+ assertEquals(count, counter.incrementAndGet());
+ }
+
+ @Test
+ public void testIndependentLargeRequests() throws Exception {
+ // Available: 1000 RAM and 1 CPU.
+ assertFalse(rm.inUse());
+ acquire(10000, 0, 0); // Available: 0 RAM 1 CPU 1 IO.
+ acquire(0, 100, 0); // Available: 0 RAM 0 CPU 1 IO.
+ acquire(0, 0, 1); // Available: 0 RAM 0 CPU 0 IO.
+ assertTrue(rm.inUse());
+ release(9500, 0, 0); // Available: 500 RAM 0 CPU 0 IO.
+ acquire(400, 0, 0); // Available: 100 RAM 0 CPU 0 IO.
+ release(0, 99.5, 0.6); // Available: 100 RAM 0.5 CPU 0.4 IO.
+ acquire(100, 0.5, 0.4); // Available: 0 RAM 0 CPU 0 IO.
+ release(1000, 1, 1); // Available: 1000 RAM 1 CPU 1 IO.
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testOverallocation() throws Exception {
+ // Since ResourceManager.MIN_NECESSARY_RAM_RATIO = 1.0, overallocation is
+ // enabled only for the CPU resource.
+ assertFalse(rm.inUse());
+ acquire(900, 0.5, 0.1); // Available: 100 RAM 0.5 CPU 0.9 IO.
+ acquire(100, 0.6, 0.9); // Available: 0 RAM 0 CPU 0 IO.
+ release(100, 0.6, 0.9); // Available: 100 RAM 0.5 CPU 0.9 IO.
+ acquire(100, 0.1, 0.1); // Available: 0 RAM 0.4 CPU 0.8 IO.
+ acquire(0, 0.5, 0.8); // Available: 0 RAM 0 CPU 0.8 IO.
+ release(1020, 1.1, 1.05); // Available: 1000 RAM 1 CPU 1 IO.
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testNonblocking() throws Exception {
+ assertFalse(rm.inUse());
+ assertTrue(acquireNonblocking(900, 0.5, 0)); // Available: 100 RAM 0.5 CPU 1 IO.
+ assertTrue(acquireNonblocking(100, 0.5, 0.2)); // Available: 0 RAM 0 CPU 0.8 IO.
+ assertFalse(acquireNonblocking(.1, .01, 0.0));
+ assertFalse(acquireNonblocking(0, 0, 0.9));
+ assertTrue(acquireNonblocking(0, 0, 0.8)); // Available: 0 RAM 0 CPU 0 IO.
+ release(100, 0.5, 0.1); // Available: 100 RAM 0.5 CPU 0.1 IO.
+ assertTrue(acquireNonblocking(100, 0.1, 0.1)); // Available: 0 RAM 0.4 CPU 0 IO.
+ assertFalse(acquireNonblocking(5, .5, 0));
+ assertFalse(acquireNonblocking(0, .5, 0.1));
+ assertTrue(acquireNonblocking(0, 0.4, 0)); // Available: 0 RAM 0 CPU 0 IO.
+ release(1000, 1, 1); // Available: 1000 RAM 1 CPU 1 IO.
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testHasResources() throws Exception {
+ assertFalse(rm.inUse());
+ assertFalse(rm.threadHasResources());
+ acquire(1, .1, .1);
+ assertTrue(rm.threadHasResources());
+
+ // We have resources in this thread - make sure other threads
+ // are not affected.
+ TestThread thread1 = new TestThread () {
+ @Override public void runTest() throws Exception {
+ assertFalse(rm.threadHasResources());
+ acquire(1, 0, 0);
+ assertTrue(rm.threadHasResources());
+ release(1, 0, 0);
+ assertFalse(rm.threadHasResources());
+ acquire(0, 0.1, 0);
+ assertTrue(rm.threadHasResources());
+ release(0, 0.1, 0);
+ assertFalse(rm.threadHasResources());
+ acquire(0, 0, 0.1);
+ assertTrue(rm.threadHasResources());
+ release(0, 0, 0.1);
+ assertFalse(rm.threadHasResources());
+ }
+ };
+ thread1.start();
+ thread1.joinAndAssertState(10000);
+
+ release(1, .1, .1);
+ assertFalse(rm.threadHasResources());
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testConcurrentLargeRequests() throws Exception {
+ assertFalse(rm.inUse());
+ TestThread thread1 = new TestThread () {
+ @Override public void runTest() throws Exception {
+ acquire(2000, 2, 0);
+ sync.await();
+ validate(1);
+ sync.await();
+ // Wait till other thread will be locked.
+ while (rm.getWaitCount() == 0) {
+ Thread.yield();
+ }
+ release(2000, 2, 0);
+ assertEquals(0, rm.getWaitCount());
+ acquire(2000, 2, 0); // Will be blocked by the thread2.
+ validate(3);
+ release(2000, 2, 0);
+ }
+ };
+ TestThread thread2 = new TestThread () {
+ @Override public void runTest() throws Exception {
+ sync2.await();
+ assertFalse(rm.isAvailable(2000, 2, 0));
+ acquire(2000, 2, 0); // Will be blocked by the thread1.
+ validate(2);
+ sync2.await();
+ // Wait till other thread will be locked.
+ while (rm.getWaitCount() == 0) {
+ Thread.yield();
+ }
+ release(2000, 2, 0);
+ }
+ };
+
+ thread1.start();
+ thread2.start();
+ sync.await(1, TimeUnit.SECONDS);
+ assertTrue(rm.inUse());
+ assertEquals(0, rm.getWaitCount());
+ sync2.await(1, TimeUnit.SECONDS);
+ sync.await(1, TimeUnit.SECONDS);
+ sync2.await(1, TimeUnit.SECONDS);
+ thread1.joinAndAssertState(1000);
+ thread2.joinAndAssertState(1000);
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testOutOfOrderAllocation() throws Exception {
+ assertFalse(rm.inUse());
+ TestThread thread1 = new TestThread () {
+ @Override public void runTest() throws Exception {
+ sync.await();
+ acquire(900, 0.5, 0); // Will be blocked by the main thread.
+ validate(5);
+ release(900, 0.5, 0);
+ sync.await();
+ }
+ };
+ TestThread thread2 = new TestThread() {
+ @Override public void runTest() throws Exception {
+ // Wait till other thread will be locked
+ while (rm.getWaitCount() == 0) {
+ Thread.yield();
+ }
+ acquire(100, 0.1, 0);
+ validate(2);
+ release(100, 0.1, 0);
+ sync2.await();
+ acquire(200, 0.5, 0);
+ validate(4);
+ sync2.await();
+ release(200, 0.5, 0);
+ }
+ };
+ acquire(900, 0.9, 0);
+ validate(1);
+ thread1.start();
+ sync.await(1, TimeUnit.SECONDS);
+ thread2.start();
+ sync2.await(1, TimeUnit.SECONDS);
+ //Waiting till both threads are locked.
+ while (rm.getWaitCount() < 2) {
+ Thread.yield();
+ }
+ validate(3); // Thread1 is now first in the queue and Thread2 is second.
+ release(100, 0.4, 0); // This allows Thread2 to continue out of order.
+ sync2.await(1, TimeUnit.SECONDS);
+ release(750, 0.3, 0); // At this point thread1 will finally acquire resources.
+ sync.await(1, TimeUnit.SECONDS);
+ release(50, 0.2, 0);
+ thread1.join();
+ thread2.join();
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testSingleton() throws Exception {
+ ResourceManager.instance();
+ }
+
+ /**
+ * Checks that that resource manager
+ * can recover from LocalHostCapacity.getFreeResources() failure.
+ */
+ @Test
+ public void testAutoSenseFailure() throws Exception {
+ boolean isDisabled = LocalHostCapacity.isDisabled;
+ assertFalse(rm.inUse());
+ try {
+ rm.setAutoSensing(true);
+ // Resource manager autosense state should be enabled now if
+ // LocalHostCapacity class supports it.
+ assertEquals(rm.isAutoSensingEnabled(), !LocalHostCapacity.isDisabled);
+ rm.setAutoSensing(false);
+ assertFalse(rm.isAutoSensingEnabled());
+
+ // Emulate failure to parse /proc/* filesystem.
+ LocalHostCapacity.isDisabled = true;
+ rm.setAutoSensing(true);
+ assertFalse(rm.isAutoSensingEnabled());
+ rm.setAutoSensing(false);
+ assertFalse(rm.isAutoSensingEnabled());
+ } finally {
+ LocalHostCapacity.isDisabled = isDisabled;
+ rm.setAutoSensing(false);
+ }
+ assertFalse(rm.inUse());
+ }
+
+ @Test
+ public void testResourceSetConverter() throws Exception {
+ ResourceSet.ResourceSetConverter converter = new ResourceSet.ResourceSetConverter();
+
+ ResourceSet resources = converter.convert("1,0.5,2");
+ assertEquals(1.0, resources.getMemoryMb(), 0.01);
+ assertEquals(0.5, resources.getCpuUsage(), 0.01);
+ assertEquals(2.0, resources.getIoUsage(), 0.01);
+
+ try {
+ converter.convert("0,0,");
+ fail();
+ } catch (OptionsParsingException ope) {
+ // expected
+ }
+
+ try {
+ converter.convert("0,0,0,0");
+ fail();
+ } catch (OptionsParsingException ope) {
+ // expected
+ }
+
+ try {
+ converter.convert("-1,0,0");
+ fail();
+ } catch (OptionsParsingException ope) {
+ // expected
+ }
+ }
+
+ private static class ResourceOwnerStub implements ActionMetadata {
+
+ @Override
+ @Nullable
+ public String getProgressMessage() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public ActionOwner getOwner() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public String prettyPrint() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public String getMnemonic() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean inputsKnown() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean discoversInputs() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Iterable<Artifact> getInputs() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public int getInputCount() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public ImmutableSet<Artifact> getOutputs() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Artifact getPrimaryInput() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Artifact getPrimaryOutput() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Iterable<Artifact> getMandatoryInputs() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public String getKey() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ @Nullable
+ public String describeKey() {
+ throw new IllegalStateException();
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/RootTest.java b/src/test/java/com/google/devtools/build/lib/actions/RootTest.java
new file mode 100644
index 0000000..c8fc14b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/RootTest.java
@@ -0,0 +1,132 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.testutil.Scratch;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+
+/**
+ * Tests for {@link Root}.
+ */
+@RunWith(JUnit4.class)
+public class RootTest {
+ private Scratch scratch = new Scratch();
+
+ @Test
+ public void testAsSourceRoot() throws IOException {
+ Path sourceDir = scratch.dir("/source");
+ Root root = Root.asSourceRoot(sourceDir);
+ assertTrue(root.isSourceRoot());
+ assertEquals(PathFragment.EMPTY_FRAGMENT, root.getExecPath());
+ assertEquals(sourceDir, root.getPath());
+ assertEquals("/source[source]", root.toString());
+ }
+
+ @Test
+ public void testBadAsSourceRoot() {
+ try {
+ Root.asSourceRoot(null);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void testAsDerivedRoot() throws IOException {
+ Path execRoot = scratch.dir("/exec");
+ Path rootDir = scratch.dir("/exec/root");
+ Root root = Root.asDerivedRoot(execRoot, rootDir);
+ assertFalse(root.isSourceRoot());
+ assertEquals(new PathFragment("root"), root.getExecPath());
+ assertEquals(rootDir, root.getPath());
+ assertEquals("/exec/root[derived]", root.toString());
+ }
+
+ @Test
+ public void testBadAsDerivedRoot() throws IOException {
+ try {
+ Path execRoot = scratch.dir("/exec");
+ Path outsideDir = scratch.dir("/not_exec");
+ Root.asDerivedRoot(execRoot, outsideDir);
+ fail();
+ } catch (IllegalArgumentException expected) {
+ }
+ }
+
+ @Test
+ public void testBadAsDerivedRootSameForBoth() throws IOException {
+ try {
+ Path execRoot = scratch.dir("/exec");
+ Root.asDerivedRoot(execRoot, execRoot);
+ fail();
+ } catch (IllegalArgumentException expected) {
+ }
+ }
+
+ @Test
+ public void testBadAsDerivedRootNullDir() throws IOException {
+ try {
+ Path execRoot = scratch.dir("/exec");
+ Root.asDerivedRoot(execRoot, null);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void testBadAsDerivedRootNullExecRoot() throws IOException {
+ try {
+ Path execRoot = scratch.dir("/exec");
+ Root.asDerivedRoot(null, execRoot);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void testEquals() throws IOException {
+ Path execRoot = scratch.dir("/exec");
+ Path rootDir = scratch.dir("/exec/root");
+ Path otherRootDir = scratch.dir("/");
+ Path sourceDir = scratch.dir("/source");
+ Root rootA = Root.asDerivedRoot(execRoot, rootDir);
+ assertEqualsAndHashCode(true, rootA, Root.asDerivedRoot(execRoot, rootDir));
+ assertEqualsAndHashCode(false, rootA, Root.asSourceRoot(sourceDir));
+ assertEqualsAndHashCode(false, rootA, Root.asSourceRoot(rootDir));
+ assertEqualsAndHashCode(false, rootA, Root.asDerivedRoot(otherRootDir, rootDir));
+ }
+
+ public void assertEqualsAndHashCode(boolean expected, Object a, Object b) {
+ if (expected) {
+ assertTrue(a.equals(b));
+ assertTrue(a.hashCode() == b.hashCode());
+ } else {
+ assertFalse(a.equals(b));
+ assertFalse(a.hashCode() == b.hashCode());
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCacheTest.java b/src/test/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCacheTest.java
new file mode 100644
index 0000000..5fc1f4f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCacheTest.java
@@ -0,0 +1,190 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+
+/**
+ * Test for the CompactPersistentActionCache class.
+ */
+@RunWith(JUnit4.class)
+public class CompactPersistentActionCacheTest {
+
+ private static class ManualClock implements Clock {
+ private long currentTime = 0L;
+
+ ManualClock() { }
+
+ @Override public long currentTimeMillis() {
+ return currentTime;
+ }
+
+ @Override public long nanoTime() {
+ return 0;
+ }
+ }
+
+ private FsApparatus scratch = FsApparatus.newInMemory();
+ private Path dataRoot;
+ private Path mapFile;
+ private Path journalFile;
+ private ManualClock clock = new ManualClock();
+ private CompactPersistentActionCache cache;
+
+ @Before
+ public void setUp() throws Exception {
+ dataRoot = scratch.path("/cache/test.dat");
+ cache = new CompactPersistentActionCache(dataRoot, clock);
+ mapFile = CompactPersistentActionCache.cacheFile(dataRoot);
+ journalFile = CompactPersistentActionCache.journalFile(dataRoot);
+ }
+
+ @Test
+ public void testGetInvalidKey() {
+ assertNull(cache.get("key"));
+ }
+
+ @Test
+ public void testPutAndGet() {
+ String key = "key";
+ putKey(key);
+ ActionCache.Entry readentry = cache.get(key);
+ assertTrue(readentry != null);
+ assertEquals(cache.get(key).toString(), readentry.toString());
+ assertFalse(mapFile.exists());
+ }
+
+ @Test
+ public void testPutAndRemove() {
+ String key = "key";
+ putKey(key);
+ cache.remove(key);
+ assertNull(cache.get(key));
+ assertFalse(mapFile.exists());
+ }
+
+ @Test
+ public void testSave() throws IOException {
+ String key = "key";
+ putKey(key);
+ cache.save();
+ assertTrue(mapFile.exists());
+ assertFalse(journalFile.exists());
+
+ CompactPersistentActionCache newcache =
+ new CompactPersistentActionCache(dataRoot, clock);
+ ActionCache.Entry readentry = newcache.get(key);
+ assertTrue(readentry != null);
+ assertEquals(cache.get(key).toString(), readentry.toString());
+ }
+
+ @Test
+ public void testIncrementalSave() throws IOException {
+ for (int i = 0; i < 300; i++) {
+ putKey(Integer.toString(i));
+ }
+ assertFullSave();
+
+ // Add 2 entries to 300. Might as well just leave them in the journal.
+ putKey("abc");
+ putKey("123");
+ assertIncrementalSave(cache);
+
+ // Make sure we have all the entries, including those in the journal,
+ // after deserializing into a new cache.
+ CompactPersistentActionCache newcache =
+ new CompactPersistentActionCache(dataRoot, clock);
+ for (int i = 0; i < 100; i++) {
+ assertKeyEquals(cache, newcache, Integer.toString(i));
+ }
+ assertKeyEquals(cache, newcache, "abc");
+ assertKeyEquals(cache, newcache, "123");
+ putKey("xyz", newcache);
+ assertIncrementalSave(newcache);
+
+ // Make sure we can see previous journal values after a second incremental save.
+ CompactPersistentActionCache newerCache =
+ new CompactPersistentActionCache(dataRoot, clock);
+ for (int i = 0; i < 100; i++) {
+ assertKeyEquals(cache, newerCache, Integer.toString(i));
+ }
+ assertKeyEquals(cache, newerCache, "abc");
+ assertKeyEquals(cache, newerCache, "123");
+ assertNotNull(newerCache.get("xyz"));
+ assertNull(newerCache.get("not_a_key"));
+
+ // Add another 10 entries. This should not be incremental.
+ for (int i = 300; i < 310; i++) {
+ putKey(Integer.toString(i));
+ }
+ assertFullSave();
+ }
+
+ // Regression test to check that CompactActionCacheEntry.toString does not mutate the object.
+ // Mutations may result in IllegalStateException.
+ @Test
+ public void testEntryToStringIsIdempotent() throws Exception {
+ ActionCache.Entry entry = new ActionCache.Entry("actionKey");
+ entry.toString();
+ entry.addFile(new PathFragment("foo/bar"), Metadata.CONSTANT_METADATA);
+ entry.toString();
+ entry.getFileDigest();
+ entry.toString();
+ }
+
+ private static void assertKeyEquals(ActionCache cache1, ActionCache cache2, String key) {
+ Object entry = cache1.get(key);
+ assertNotNull(entry);
+ assertEquals(entry.toString(), cache2.get(key).toString());
+ }
+
+ private void assertFullSave() throws IOException {
+ cache.save();
+ assertTrue(mapFile.exists());
+ assertFalse(journalFile.exists());
+ }
+
+ private void assertIncrementalSave(ActionCache ac) throws IOException {
+ ac.save();
+ assertTrue(mapFile.exists());
+ assertTrue(journalFile.exists());
+ }
+
+ private void putKey(String key) {
+ putKey(key, cache);
+ }
+
+ private void putKey(String key, ActionCache ac) {
+ ActionCache.Entry entry = ac.createEntry(key);
+ entry.getFileDigest();
+ ac.put(key, entry);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/cache/MetadataTest.java b/src/test/java/com/google/devtools/build/lib/actions/cache/MetadataTest.java
new file mode 100644
index 0000000..fe37af2
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/cache/MetadataTest.java
@@ -0,0 +1,45 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+
+import com.google.common.io.BaseEncoding;
+import com.google.common.testing.EqualsTester;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class MetadataTest {
+
+ private static byte[] toBytes(String hex) {
+ return BaseEncoding.base16().upperCase().decode(hex);
+ }
+
+ @Test
+ public void testEqualsAndHashCode() throws Exception {
+ // Each "equality group" is checked for equality within itself (including hashCode equality)
+ // and inequality with members of other equality groups.
+ new EqualsTester()
+ .addEqualityGroup(new Metadata(toBytes("00112233445566778899AABBCCDDEEFF")),
+ new Metadata(toBytes("00112233445566778899AABBCCDDEEFF")))
+ .addEqualityGroup(new Metadata(1))
+ .addEqualityGroup(new Metadata(toBytes("FFFFFF00000000000000000000000000")))
+ .addEqualityGroup(new Metadata(2),
+ new Metadata(2))
+ .addEqualityGroup("a string")
+ .testEquals();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexerTest.java b/src/test/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexerTest.java
new file mode 100644
index 0000000..7fc0cb1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexerTest.java
@@ -0,0 +1,387 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.cache;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * Test for the PersistentStringIndexer class.
+ */
+@RunWith(JUnit4.class)
+public class PersistentStringIndexerTest {
+
+ private static class ManualClock implements Clock {
+ private long currentTime = 0L;
+
+ ManualClock() { }
+
+ @Override public long currentTimeMillis() {
+ throw new AssertionError("unexpected method call");
+ }
+
+ @Override public long nanoTime() {
+ return currentTime;
+ }
+
+ void advance(long time) {
+ currentTime += time;
+ }
+ }
+
+ private PersistentStringIndexer psi;
+ private Map<Integer, String> mappings = new ConcurrentHashMap<>();
+ private FsApparatus scratch = FsApparatus.newInMemory();
+ private ManualClock clock = new ManualClock();
+ private Path dataPath;
+ private Path journalPath;
+
+
+ @Before
+ public void setUp() throws Exception {
+ dataPath = scratch.path("/cache/test.dat");
+ journalPath = scratch.path("/cache/test.journal");
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ }
+
+ private void assertSize(int expected) {
+ assertEquals(expected, psi.size());
+ }
+
+ private void assertIndex(int expected, String s) {
+ int index = psi.getOrCreateIndex(s);
+ assertEquals(expected, index);
+ mappings.put(expected, s);
+ }
+
+ private void assertContent() {
+ for (int i = 0; i < psi.size(); i++) {
+ if(mappings.get(i) != null) {
+ assertEquals(mappings.get(i), psi.getStringForIndex(i));
+ }
+ }
+ }
+
+
+ private void setupTestContent() {
+ assertSize(0);
+ assertIndex(0, "abcdefghi"); // Create leafs
+ assertIndex(1, "abcdefjkl");
+ assertIndex(2, "abcdefmno");
+ assertIndex(3, "abcdefjklpr");
+ assertIndex(3, "abcdefjklpr");
+ assertIndex(4, "abcdstr");
+ assertIndex(5, "012345");
+ assertSize(6);
+ assertIndex(6, "abcdef"); // Validate inner nodes
+ assertIndex(7, "abcd");
+ assertIndex(8, "");
+ assertSize(9);
+ assertContent();
+ }
+
+ /**
+ * Writes lots of entries with labels "fooconcurrent[int]" at the same time.
+ * The set of labels written is deterministic, but the label:index mapping is
+ * not.
+ */
+ private void writeLotsOfEntriesConcurrently(final int numToWrite) throws InterruptedException {
+ final int NUM_THREADS = 10;
+ final CountDownLatch synchronizerLatch = new CountDownLatch(NUM_THREADS);
+
+ class IndexAdder extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ for (int i = 0; i < numToWrite; i++) {
+ synchronizerLatch.countDown();
+ synchronizerLatch.await();
+
+ String value = "fooconcurrent" + i;
+ mappings.put(psi.getOrCreateIndex(value), value);
+ }
+ }
+ }
+
+ Collection<TestThread> threads = new ArrayList<>();
+ for (int i = 0; i < NUM_THREADS; i++) {
+ TestThread thread = new IndexAdder();
+ thread.start();
+ threads.add(thread);
+ }
+
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+ }
+
+ @Test
+ public void testNormalOperation() throws Exception {
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+ setupTestContent();
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ clock.advance(4);
+ assertIndex(9, "xyzqwerty"); // This should flush journal to disk.
+ assertFalse(dataPath.exists());
+ assertTrue(journalPath.exists());
+
+ psi.save(); // Successful save will remove journal file.
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ // Now restore data from file and verify it.
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ assertFalse(journalPath.exists());
+ clock.advance(4);
+ assertSize(10);
+ assertContent();
+ assertFalse(journalPath.exists());
+ }
+
+ @Test
+ public void testJournalRecoveryWithoutMainDataFile() throws Exception {
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+ setupTestContent();
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ clock.advance(4);
+ assertIndex(9, "abc1234"); // This should flush journal to disk.
+ assertFalse(dataPath.exists());
+ assertTrue(journalPath.exists());
+
+ // Now restore data from file and verify it. All data should be restored from journal;
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+ clock.advance(4);
+ assertSize(10);
+ assertContent();
+ assertFalse(journalPath.exists());
+ }
+
+ @Test
+ public void testJournalRecovery() throws Exception {
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+ setupTestContent();
+ psi.save();
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+ long oldDataFileLen = dataPath.getFileSize();
+
+ clock.advance(4);
+ assertIndex(9, "another record"); // This should flush journal to disk.
+ assertSize(10);
+ assertTrue(dataPath.exists());
+ assertTrue(journalPath.exists());
+
+ // Now restore data from file and verify it. All data should be restored from journal;
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+ assertTrue(dataPath.getFileSize() > oldDataFileLen); // data file should have been updated
+ clock.advance(4);
+ assertSize(10);
+ assertContent();
+ assertFalse(journalPath.exists());
+ }
+
+ @Test
+ public void testConcurrentWritesJournalRecovery() throws Exception {
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+ setupTestContent();
+ psi.save();
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+ long oldDataFileLen = dataPath.getFileSize();
+
+ int size = psi.size();
+ int numToWrite = 50000;
+ writeLotsOfEntriesConcurrently(numToWrite);
+ assertFalse(journalPath.exists());
+ clock.advance(4);
+ assertIndex(size + numToWrite, "another record"); // This should flush journal to disk.
+ assertSize(size + numToWrite + 1);
+ assertTrue(dataPath.exists());
+ assertTrue(journalPath.exists());
+
+ // Now restore data from file and verify it. All data should be restored from journal;
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+ assertTrue(dataPath.getFileSize() > oldDataFileLen); // data file should have been updated
+ clock.advance(4);
+ assertSize(size + numToWrite + 1);
+ assertContent();
+ assertFalse(journalPath.exists());
+ }
+
+ @Test
+ public void testCorruptedJournal() throws Exception {
+ FileSystemUtils.createDirectoryAndParents(journalPath.getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(journalPath, "bogus content");
+ try {
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ fail();
+ } catch (IOException e) {
+ assertThat(e.getMessage()).contains("too short: Only 13 bytes");
+ }
+
+ journalPath.delete();
+ setupTestContent();
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ clock.advance(4);
+ assertIndex(9, "abc1234"); // This should flush journal to disk.
+ assertFalse(dataPath.exists());
+ assertTrue(journalPath.exists());
+
+ byte[] journalContent = FileSystemUtils.readContent(journalPath);
+
+ // Now restore data from file and verify it. All data should be restored from journal;
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ // Now put back truncated journal. We should get an error.
+ assertTrue(dataPath.delete());
+ FileSystemUtils.writeContent(journalPath,
+ Arrays.copyOf(journalContent, journalContent.length - 1));
+ try {
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ fail();
+ } catch (EOFException e) {
+ // Expected.
+ }
+
+ // Corrupt the journal with a negative size value.
+ byte[] journalCopy = Arrays.copyOf(journalContent, journalContent.length);
+ // Flip this bit to make the key size negative.
+ journalCopy[95] = -2;
+ FileSystemUtils.writeContent(journalPath, journalCopy);
+ try {
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ fail();
+ } catch (IOException e) {
+ // Expected.
+ assertThat(e.getMessage()).contains("corrupt key length");
+ }
+
+ // Now put back corrupted journal. We should get an error.
+ journalContent[journalContent.length - 13] = 100;
+ FileSystemUtils.writeContent(journalPath, journalContent);
+ try {
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ fail();
+ } catch (IOException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testDupeIndexCorruption() throws Exception {
+ setupTestContent();
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ assertIndex(9, "abc1234"); // This should flush journal to disk.
+ psi.save();
+ assertTrue(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ byte[] content = FileSystemUtils.readContent(dataPath);
+
+ // We remove the data file, and instead create a corrupt journal.
+ //
+ // The journal has a header followed by a sequence of (String, int) pairs, where each int is a
+ // unique value. The String is encoded by the length (as an int), and the int is simply encoded
+ // as an int. Note that the DataOutputStream class uses big endian by default, so the low-order
+ // bits are at the end.
+ //
+ // For the purpose of this test, we want to make the journal contain two entries with the same
+ // index (which is illegal). The PersistentStringIndexer assigns int values in the usual order,
+ // starting with zero, and it now contains 9 entries. We simply change the last entry to an
+ // index that is guaranteed to already exist. If it is the index 1, we change it to 2, otherwise
+ // we change it to 1 - in both cases, the code currently guarantees that the duplicate comes
+ // earlier in the stream.
+ assertTrue(dataPath.delete());
+ content[content.length - 1] = content[content.length - 1] == 1 ? (byte) 2 : (byte) 1;
+ FileSystemUtils.writeContent(journalPath, content);
+
+ try {
+ psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock);
+ fail();
+ } catch (IOException e) {
+ // Expected.
+ assertThat(e.getMessage()).contains("Corrupted filename index has duplicate entry");
+ }
+ }
+
+ @Test
+ public void testDeferredIOFailure() throws Exception {
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+ setupTestContent();
+ assertFalse(dataPath.exists());
+ assertFalse(journalPath.exists());
+
+ // Ensure that journal cannot be saved.
+ FileSystemUtils.createDirectoryAndParents(journalPath);
+
+ clock.advance(4);
+ assertIndex(9, "abc1234"); // This should flush journal to disk (and fail at that).
+ assertFalse(dataPath.exists());
+
+ // Subsequent updates should succeed even though journaling is disabled at this point.
+ clock.advance(4);
+ assertIndex(10, "another record");
+ try {
+ // Save should actually save main data file but then return us deferred IO failure
+ // from failed journal write.
+ psi.save();
+ fail();
+ } catch(IOException e) {
+ assertThat(e.getMessage()).contains(journalPath.getPathString() + " (Is a directory)");
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java
new file mode 100644
index 0000000..1db0249
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java
@@ -0,0 +1,42 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.util;
+
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+
+import java.io.PrintStream;
+
+/**
+ * Utilities for tests that use the action cache.
+ */
+public class ActionCacheTestHelper {
+ private ActionCacheTestHelper() {}
+
+ /** A cache which does not remember anything. Causes perpetual rebuilds! */
+ public static final ActionCache AMNESIAC_CACHE =
+ new ActionCache() {
+ @Override
+ public void put(String fingerprint, Entry entry) {}
+ @Override
+ public Entry get(String fingerprint) { return null; }
+ @Override
+ public void remove(String key) {}
+ @Override
+ public Entry createEntry(String key) { return new ActionCache.Entry(key); }
+ @Override
+ public long save() { return -1; }
+ @Override
+ public void dump(PrintStream out) { }
+ };
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
new file mode 100644
index 0000000..2f601d4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
@@ -0,0 +1,440 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.util;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.AbstractActionOwner;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.MutableActionGraph;
+import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.exec.SingleBuildFileCache;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A bunch of utilities that are useful for test concerning actions, artifacts,
+ * etc.
+ */
+public final class ActionsTestUtil {
+
+ private final ActionGraph actionGraph;
+
+ public ActionsTestUtil(ActionGraph actionGraph) {
+ this.actionGraph = actionGraph;
+ }
+
+ private static final Label NULL_LABEL = Label.parseAbsoluteUnchecked("//null/action:owner");
+
+ public static ActionExecutionContext createContext(Executor executor, FileOutErr fileOutErr,
+ Path execRoot, MetadataHandler metadataHandler, @Nullable ActionGraph actionGraph) {
+ return new ActionExecutionContext(
+ executor,
+ new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()),
+ metadataHandler, fileOutErr,
+ actionGraph == null
+ ? null
+ : ActionInputHelper.actionGraphMiddlemanExpander(actionGraph));
+ }
+
+ /**
+ * A dummy ActionOwner implementation for use in tests.
+ */
+ public static class NullActionOwner extends AbstractActionOwner {
+ @Override
+ public Label getLabel() {
+ return NULL_LABEL;
+ }
+
+ @Override
+ public final String getConfigurationName() {
+ return "dummy-configuration";
+ }
+
+ @Override
+ public String getConfigurationMnemonic() {
+ return "dummy-configuration-mnemonic";
+ }
+
+ @Override
+ public final String getConfigurationShortCacheKey() {
+ return "dummy-configuration";
+ }
+ }
+
+ public static final Artifact DUMMY_ARTIFACT = new Artifact(
+ new PathFragment("dummy"),
+ Root.asSourceRoot(new InMemoryFileSystem().getRootDirectory()));
+
+ public static final ActionOwner NULL_ACTION_OWNER = new NullActionOwner();
+
+ public static final ArtifactOwner NULL_ARTIFACT_OWNER =
+ new ArtifactOwner() {
+ @Override
+ public Label getLabel() {
+ return NULL_LABEL;
+ }
+ };
+
+ public static class UncheckedActionConflictException extends RuntimeException {
+ public UncheckedActionConflictException(ActionConflictException e) {
+ super(e);
+ }
+ }
+
+ /**
+ * A dummy Action class for use in tests.
+ */
+ public static class NullAction extends AbstractAction {
+
+ public NullAction() {
+ super(NULL_ACTION_OWNER, Artifact.NO_ARTIFACTS, ImmutableList.of(DUMMY_ARTIFACT));
+ }
+
+ public NullAction(ActionOwner owner, Artifact... outputs) {
+ super(owner, Artifact.NO_ARTIFACTS, ImmutableList.copyOf(outputs));
+ }
+
+ public NullAction(Artifact... outputs) {
+ super(NULL_ACTION_OWNER, Artifact.NO_ARTIFACTS, ImmutableList.copyOf(outputs));
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext) {
+ }
+
+ @Override protected String computeKey() { return "action"; }
+ @Override public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+ @Override
+ public String getMnemonic() {
+ return "Null";
+ }
+ }
+
+ /**
+ * For a bunch of actions, gets the basenames of the paths and accumulates
+ * them in a space separated string, like <code>foo.o bar.o baz.a</code>.
+ */
+ public static String baseNamesOf(Iterable<Artifact> artifacts) {
+ List<String> baseNames = baseArtifactNames(artifacts);
+ return Joiner.on(' ').join(baseNames);
+ }
+
+ /**
+ * For a bunch of actions, gets the basenames of the paths, sorts them in alphabetical
+ * order and accumulates them in a space separated string, for example
+ * <code>bar.o baz.a foo.o</code>.
+ */
+ public static String sortedBaseNamesOf(Iterable<Artifact> artifacts) {
+ List<String> baseNames = baseArtifactNames(artifacts);
+ Collections.sort(baseNames);
+ return Joiner.on(' ').join(baseNames);
+ }
+
+ /**
+ * For a bunch of artifacts, gets the basenames and accumulates them in a
+ * List.
+ */
+ public static List<String> baseArtifactNames(Iterable<Artifact> artifacts) {
+ List<String> baseNames = new ArrayList<>();
+ for (Artifact artifact : artifacts) {
+ baseNames.add(artifact.getExecPath().getBaseName());
+ }
+ return baseNames;
+ }
+
+ /**
+ * For a bunch of artifacts, gets the exec paths and accumulates them in a
+ * List.
+ */
+ public static List<String> execPaths(Iterable<Artifact> artifacts) {
+ List<String> names = new ArrayList<>();
+ for (Artifact artifact : artifacts) {
+ names.add(artifact.getExecPathString());
+ }
+ return names;
+ }
+
+ /**
+ * For a bunch of artifacts, gets the pretty printed names and accumulates them in a List. Note
+ * that this returns the root-relative paths, not the exec paths.
+ */
+ public static List<String> prettyArtifactNames(Iterable<Artifact> artifacts) {
+ List<String> result = new ArrayList<>();
+ for (Artifact artifact : artifacts) {
+ result.add(artifact.prettyPrint());
+ }
+ return result;
+ }
+
+ public static List<String> prettyJarNames(Iterable<Artifact> jars) {
+ List<String> result = new ArrayList<>();
+ for (Artifact jar : jars) {
+ result.add(jar.prettyPrint());
+ }
+ return result;
+ }
+
+ /**
+ * Returns the closure of the predecessors of any of the given types, joining the basenames of the
+ * artifacts into a space-separated string like "libfoo.a libbar.a libbaz.a".
+ */
+ public String predecessorClosureOf(Artifact artifact, FileType... types) {
+ return predecessorClosureOf(Collections.singleton(artifact), types);
+ }
+
+ /**
+ * Returns the closure of the predecessors of any of the given types.
+ */
+ public Collection<String> predecessorClosureAsCollection(Artifact artifact, FileType... types) {
+ return predecessorClosureAsCollection(Collections.singleton(artifact), types);
+ }
+
+ /**
+ * Returns the closure of the predecessors of any of the given types, joining the basenames of the
+ * artifacts into a space-separated string like "libfoo.a libbar.a libbaz.a".
+ */
+ public String predecessorClosureOf(Iterable<Artifact> artifacts, FileType... types) {
+ Set<Artifact> visited = artifactClosureOf(artifacts);
+ return baseNamesOf(FileType.filter(visited, types));
+ }
+
+ /**
+ * Returns the closure of the predecessors of any of the given types.
+ */
+ public Collection<String> predecessorClosureAsCollection(Iterable<Artifact> artifacts,
+ FileType... types) {
+ return baseArtifactNames(FileType.filter(artifactClosureOf(artifacts), types));
+ }
+
+ public String predecessorClosureOfJars(Iterable<Artifact> artifacts, FileType... types) {
+ return baseNamesOf(FileType.filter(artifactClosureOf(artifacts), types));
+ }
+
+ public Collection<String> predecessorClosureJarsAsCollection(Iterable<Artifact> artifacts,
+ FileType... types) {
+ Set<Artifact> visited = artifactClosureOf(artifacts);
+ return baseArtifactNames(FileType.filter(visited, types));
+ }
+
+ /**
+ * Returns the closure over the input files of an action.
+ */
+ public Set<Artifact> inputClosureOf(Action action) {
+ return artifactClosureOf(action.getInputs());
+ }
+
+ /**
+ * Returns the closure over the input files of an artifact.
+ */
+ public Set<Artifact> artifactClosureOf(Artifact artifact) {
+ return artifactClosureOf(Collections.singleton(artifact));
+ }
+
+ /**
+ * Returns the closure over the input files of an artifact, filtered by the given matcher.
+ */
+ public Set<Artifact> filteredArtifactClosureOf(Artifact artifact, Predicate<Artifact> matcher) {
+ return ImmutableSet.copyOf(Iterables.filter(artifactClosureOf(artifact), matcher));
+ }
+
+ /**
+ * Returns the closure over the input files of a set of artifacts.
+ */
+ public Set<Artifact> artifactClosureOf(Iterable<Artifact> artifacts) {
+ Set<Artifact> visited = new LinkedHashSet<>();
+ List<Artifact> toVisit = Lists.newArrayList(artifacts);
+ while (!toVisit.isEmpty()) {
+ Artifact current = toVisit.remove(0);
+ if (!visited.add(current)) {
+ continue;
+ }
+ Action generatingAction = actionGraph.getGeneratingAction(current);
+ if (generatingAction != null) {
+ Iterables.addAll(toVisit, generatingAction.getInputs());
+ }
+ }
+ return visited;
+ }
+
+ /**
+ * Returns the closure over the input files of a set of artifacts, filtered by the given matcher.
+ */
+ public Set<Artifact> filteredArtifactClosureOf(Iterable<Artifact> artifacts,
+ Predicate<Artifact> matcher) {
+ return ImmutableSet.copyOf(Iterables.filter(artifactClosureOf(artifacts), matcher));
+ }
+
+ /**
+ * Returns a predicate to match {@link Artifact}s with the given root-relative path suffix.
+ */
+ public static Predicate<Artifact> getArtifactSuffixMatcher(final String suffix) {
+ return new Predicate<Artifact>() {
+ @Override
+ public boolean apply(Artifact input) {
+ return input.getRootRelativePath().getPathString().endsWith(suffix);
+ }
+ };
+ }
+
+ /**
+ * Finds all the actions that are instances of <code>actionClass</code>
+ * in the transitive closure of prerequisites.
+ */
+ public <A extends Action> List<A> findTransitivePrerequisitesOf(Artifact artifact,
+ Class<A> actionClass, Predicate<Artifact> allowedArtifacts) {
+ List<A> actions = new ArrayList<>();
+ Set<Artifact> visited = new LinkedHashSet<>();
+ List<Artifact> toVisit = new LinkedList<>();
+ toVisit.add(artifact);
+ while (!toVisit.isEmpty()) {
+ Artifact current = toVisit.remove(0);
+ if (!visited.add(current)) {
+ continue;
+ }
+ Action generatingAction = actionGraph.getGeneratingAction(current);
+ if (generatingAction != null) {
+ Iterables.addAll(toVisit, Iterables.filter(generatingAction.getInputs(), allowedArtifacts));
+ if (actionClass.isInstance(generatingAction)) {
+ actions.add(actionClass.cast(generatingAction));
+ }
+ }
+ }
+ return actions;
+ }
+
+ public <A extends Action> List<A> findTransitivePrerequisitesOf(
+ Artifact artifact, Class<A> actionClass) {
+ return findTransitivePrerequisitesOf(artifact, actionClass, Predicates.<Artifact>alwaysTrue());
+ }
+
+ /**
+ * Looks in the given artifacts Iterable for the first Artifact whose path ends with the given
+ * suffix and returns its generating Action.
+ */
+ public Action getActionForArtifactEndingWith(Iterable<Artifact> artifacts, String suffix) {
+ Artifact a = getFirstArtifactEndingWith(artifacts, suffix);
+ return a != null ? actionGraph.getGeneratingAction(a) : null;
+ }
+
+ /**
+ * Looks in the given artifacts Iterable for the first Artifact whose path ends with the given
+ * suffix and returns the Artifact.
+ */
+ public static Artifact getFirstArtifactEndingWith(
+ Iterable<Artifact> artifacts, String suffix) {
+ for (Artifact a : artifacts) {
+ if (a.getExecPath().getPathString().endsWith(suffix)) {
+ return a;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the first artifact which is an input to "action" and has the
+ * specified basename. An assertion error is raised if none is found.
+ */
+ public static Artifact getInput(Action action, String basename) {
+ for (Artifact artifact : action.getInputs()) {
+ if (artifact.getExecPath().getBaseName().equals(basename)) {
+ return artifact;
+ }
+ }
+ throw new AssertionError("No input with basename '" + basename + "' in action " + action);
+ }
+
+ /**
+ * Returns true if an artifact that is an input to "action" with the specific
+ * basename exists.
+ */
+ public static boolean hasInput(Action action, String basename) {
+ try {
+ getInput(action, basename);
+ return true;
+ } catch (AssertionError e) {
+ return false;
+ }
+ }
+
+ /**
+ * Assert that an artifact is the primary output of its generating action.
+ */
+ public void assertPrimaryInputAndOutputArtifacts(Artifact input, Artifact output) {
+ Action generatingAction = actionGraph.getGeneratingAction(output);
+ assertThat(generatingAction).isNotNull();
+ assertThat(generatingAction.getPrimaryOutput()).isEqualTo(output);
+ assertThat(generatingAction.getPrimaryInput()).isEqualTo(input);
+ }
+
+ /**
+ * Returns the first artifact which is an output of "action" and has the
+ * specified basename. An assertion error is raised if none is found.
+ */
+ public static Artifact getOutput(Action action, String basename) {
+ for (Artifact artifact : action.getOutputs()) {
+ if (artifact.getExecPath().getBaseName().equals(basename)) {
+ return artifact;
+ }
+ }
+ throw new AssertionError("No output with basename '" + basename + "' in action " + action);
+ }
+
+ public static void registerActionWith(Action action, MutableActionGraph actionGraph) {
+ try {
+ actionGraph.registerAction(action);
+ } catch (ActionConflictException e) {
+ throw new UncheckedActionConflictException(e);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/DummyExecutor.java b/src/test/java/com/google/devtools/build/lib/actions/util/DummyExecutor.java
new file mode 100644
index 0000000..409227d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/DummyExecutor.java
@@ -0,0 +1,86 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.util;
+
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.OptionsClassProvider;
+
+/**
+ * A dummy implementation of Executor.
+ */
+public final class DummyExecutor implements Executor {
+ private final Path inputDir;
+
+ /**
+ * @param inputDir
+ */
+ public DummyExecutor(Path inputDir) {
+ this.inputDir = inputDir;
+ }
+
+ @Override
+ public Path getExecRoot() {
+ return inputDir;
+ }
+
+ @Override
+ public Clock getClock() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public EventBus getEventBus() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean getVerboseFailures() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T extends ActionContext> T getContext(Class<? extends T> type) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SpawnActionContext getSpawnActionContext(String mnemonic) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public OptionsClassProvider getOptions() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean reportsSubcommands() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reportSubcommand(String reason, String message) {
+ throw new UnsupportedOperationException();
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/LabelArtifactOwner.java b/src/test/java/com/google/devtools/build/lib/actions/util/LabelArtifactOwner.java
new file mode 100644
index 0000000..8e200e73
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/LabelArtifactOwner.java
@@ -0,0 +1,49 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.syntax.Label;
+
+import java.util.Objects;
+
+/** ArtifactOwner wrapper for Labels, for use in tests. */
+@VisibleForTesting
+public class LabelArtifactOwner implements ArtifactOwner {
+ private final Label label;
+
+ @VisibleForTesting
+ public LabelArtifactOwner(Label label) {
+ this.label = label;
+ }
+
+ @Override
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public int hashCode() {
+ return label == null ? super.hashCode() : label.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (!(that instanceof LabelArtifactOwner)) {
+ return false;
+ }
+ return Objects.equals(this.label, ((LabelArtifactOwner) that).label);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/TestAction.java b/src/test/java/com/google/devtools/build/lib/actions/util/TestAction.java
new file mode 100644
index 0000000..0861384
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/TestAction.java
@@ -0,0 +1,176 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions.util;
+
+import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+
+/**
+ * A dummy action for testing. Its execution runs the specified
+ * Runnable or Callable, which is defined by the test case,
+ * and touches all the output files.
+ */
+public class TestAction extends AbstractAction {
+
+ public static final Runnable NO_EFFECT = new Runnable() { @Override public void run() {} };
+
+ private static final ResourceSet RESOURCES =
+ new ResourceSet(/*memoryMb=*/1.0, /*cpu=*/0.1, /*io=*/0.0);
+
+ private final Callable<Void> effect;
+
+ /** Use this constructor if the effect can't throw exceptions. */
+ public TestAction(Runnable effect,
+ Collection<Artifact> inputs,
+ Collection<Artifact> outputs) {
+ super(NULL_ACTION_OWNER, inputs, outputs);
+ this.effect = Executors.callable(effect, null);
+ }
+
+ /**
+ * Use this constructor if the effect can throw exceptions.
+ * Any checked exception thrown will be repackaged as an
+ * ActionExecutionException.
+ */
+ public TestAction(Callable<Void> effect,
+ Collection<Artifact> inputs,
+ Collection<Artifact> outputs) {
+ super(NULL_ACTION_OWNER, inputs, outputs);
+ this.effect = effect;
+ }
+
+ @Override
+ public Collection<Artifact> getMandatoryInputs() {
+ List<Artifact> mandatoryInputs = new ArrayList<>();
+ for (Artifact input : getInputs()) {
+ if (!input.getExecPath().getBaseName().endsWith(".optional")) {
+ mandatoryInputs.add(input);
+ }
+ }
+ return mandatoryInputs;
+ }
+
+ @Override
+ public boolean discoversInputs() {
+ for (Artifact input : getInputs()) {
+ if (!input.getExecPath().getBaseName().endsWith(".optional")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void discoverInputs(ActionExecutionContext actionExecutionContext) {
+ Preconditions.checkState(discoversInputs(), this);
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ for (Artifact artifact : getInputs()) {
+ // Do not check *.optional artifacts - artifacts with such extension are
+ // used by tests to specify artifacts that may or may not be missing.
+ // This is used, e.g., to test Blaze behavior when action has missing
+ // input artifacts but still is successfully executed.
+ if (!artifact.getPath().exists() &&
+ !artifact.getExecPath().getBaseName().endsWith(".optional")) {
+ throw new IllegalStateException("action's input file does not exist: "
+ + artifact.getPath());
+ }
+ }
+
+ try {
+ effect.call();
+ } catch (RuntimeException | Error e) {
+ throw e;
+ } catch (Exception e) {
+ throw new ActionExecutionException("TestAction failed due to exception",
+ e, this, false);
+ }
+
+ try {
+ for (Artifact artifact: getOutputs()) {
+ FileSystemUtils.touchFile(artifact.getPath());
+ }
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ protected String computeKey() {
+ List<String> outputsList = new ArrayList<>();
+ for (Artifact output : getOutputs()) {
+ outputsList.add(output.getPath().getPathString());
+ }
+ // This could use a functional iterable and avoid creating a list
+ return "test " + StringUtilities.combineKeys(outputsList);
+ }
+
+ @Override
+ public String getMnemonic() { return "Test"; }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return RESOURCES;
+ }
+
+
+ /** No-op action that has exactly one output, and can be a middleman action. */
+ public static class DummyAction extends TestAction {
+ private static final Runnable NOOP = new Runnable() {
+ @Override
+ public void run() {}
+ };
+
+ private final MiddlemanType type;
+
+ public DummyAction(Collection<Artifact> inputs, Artifact output, MiddlemanType type) {
+ super(NOOP, inputs, ImmutableList.of(output));
+ this.type = type;
+ }
+
+ public DummyAction(Collection<Artifact> inputs, Artifact output) {
+ this(inputs, output, MiddlemanType.NORMAL);
+ }
+
+ @Override
+ public MiddlemanType getActionType() {
+ return type;
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/CollectionUtilsTest.java b/src/test/java/com/google/devtools/build/lib/collect/CollectionUtilsTest.java
new file mode 100644
index 0000000..2505501
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/CollectionUtilsTest.java
@@ -0,0 +1,175 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Tests for {@link CollectionUtils}.
+ */
+
+@RunWith(JUnit4.class)
+public class CollectionUtilsTest {
+
+ @Test
+ public void testDuplicatedElementsOf() {
+ assertDups(ImmutableList.<Integer>of(), ImmutableSet.<Integer>of());
+ assertDups(ImmutableList.of(0), ImmutableSet.<Integer>of());
+ assertDups(ImmutableList.of(0, 0, 0), ImmutableSet.of(0));
+ assertDups(ImmutableList.of(1, 2, 3, 1, 2, 3), ImmutableSet.of(1, 2, 3));
+ assertDups(ImmutableList.of(1, 2, 3, 1, 2, 3, 4), ImmutableSet.of(1, 2, 3));
+ assertDups(ImmutableList.of(1, 2, 3, 4), ImmutableSet.<Integer>of());
+ }
+
+ private static void assertDups(List<Integer> collection, Set<Integer> dups) {
+ assertEquals(dups, CollectionUtils.duplicatedElementsOf(collection));
+ }
+
+ @Test
+ public void testIsImmutable() throws Exception {
+ assertTrue(CollectionUtils.isImmutable(ImmutableList.of(1, 2, 3)));
+ assertTrue(CollectionUtils.isImmutable(ImmutableSet.of(1, 2, 3)));
+
+ NestedSet<Integer> ns = NestedSetBuilder.<Integer>compileOrder()
+ .add(1).add(2).add(3).build();
+ assertTrue(CollectionUtils.isImmutable(ns));
+
+ NestedSet<Integer> ns2 = NestedSetBuilder.<Integer>linkOrder().add(1).add(2).add(3).build();
+ assertTrue(CollectionUtils.isImmutable(ns2));
+
+ IterablesChain<Integer> chain = IterablesChain.<Integer>builder().addElement(1).build();
+
+ assertTrue(CollectionUtils.isImmutable(chain));
+
+ assertFalse(CollectionUtils.isImmutable(Lists.newArrayList()));
+ assertFalse(CollectionUtils.isImmutable(Lists.newLinkedList()));
+ assertFalse(CollectionUtils.isImmutable(Sets.newHashSet()));
+ assertFalse(CollectionUtils.isImmutable(Sets.newLinkedHashSet()));
+
+ // The result of Iterables.concat() actually is immutable, but we have no way of checking if
+ // a given Iterable comes from concat().
+ assertFalse(CollectionUtils.isImmutable(Iterables.concat(ns, ns2)));
+
+ // We can override the check by using the ImmutableIterable wrapper.
+ assertTrue(CollectionUtils.isImmutable(
+ ImmutableIterable.from(Iterables.concat(ns, ns2))));
+ }
+
+ @Test
+ public void testCheckImmutable() throws Exception {
+ CollectionUtils.checkImmutable(ImmutableList.of(1, 2, 3));
+ CollectionUtils.checkImmutable(ImmutableSet.of(1, 2, 3));
+
+ try {
+ CollectionUtils.checkImmutable(Lists.newArrayList(1, 2, 3));
+ } catch (IllegalStateException e) {
+ return;
+ }
+ fail();
+ }
+
+ @Test
+ public void testMakeImmutable() throws Exception {
+ Iterable<Integer> immutableList = ImmutableList.of(1, 2, 3);
+ assertSame(immutableList, CollectionUtils.makeImmutable(immutableList));
+
+ Iterable<Integer> mutableList = Lists.newArrayList(1, 2, 3);
+ Iterable<Integer> converted = CollectionUtils.makeImmutable(mutableList);
+ assertNotSame(mutableList, converted);
+ assertEquals(mutableList, ImmutableList.copyOf(converted));
+ }
+
+ private static enum Small { ALPHA, BRAVO }
+ private static enum Large {
+ L0, L1, L2, L3, L4, L5, L6, L7, L8, L9,
+ L10, L11, L12, L13, L14, L15, L16, L17, L18, L19,
+ L20, L21, L22, L23, L24, L25, L26, L27, L28, L29,
+ L30, L31,
+ }
+
+ private static enum TooLarge {
+ T0, T1, T2, T3, T4, T5, T6, T7, T8, T9,
+ T10, T11, T12, T13, T14, T15, T16, T17, T18, T19,
+ T20, T21, T22, T23, T24, T25, T26, T27, T28, T29,
+ T30, T31, T32,
+ }
+
+ private static enum Medium {
+ ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT,
+ }
+
+ private <T extends Enum<T>> void assertAllDifferent(Class<T> clazz) throws Exception {
+ Set<EnumSet<T>> allSets = new HashSet<>();
+
+ int maxBits = 1 << clazz.getEnumConstants().length;
+ for (int i = 0; i < maxBits; i++) {
+ EnumSet<T> set = CollectionUtils.fromBits(i, clazz);
+ int back = CollectionUtils.toBits(set);
+ assertEquals(back, i); // Assert that a roundtrip is idempotent
+ allSets.add(set);
+ }
+
+ assertEquals(maxBits, allSets.size()); // Assert that every decoded value is different
+ }
+
+ @Test
+ public void testEnumBitfields() throws Exception {
+ assertEquals(0, CollectionUtils.<Small>toBits());
+ assertEquals(EnumSet.noneOf(Small.class), CollectionUtils.fromBits(0, Small.class));
+ assertEquals(3, CollectionUtils.toBits(Small.ALPHA, Small.BRAVO));
+ assertEquals(10, CollectionUtils.toBits(Medium.TWO, Medium.FOUR));
+ assertEquals(EnumSet.of(Medium.SEVEN, Medium.EIGHT),
+ CollectionUtils.fromBits(192, Medium.class));
+
+ assertAllDifferent(Small.class);
+ assertAllDifferent(Medium.class);
+ assertAllDifferent(Large.class);
+
+ try {
+ CollectionUtils.toBits(TooLarge.T32);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // good
+ }
+
+ try {
+ CollectionUtils.fromBits(0, TooLarge.class);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // good
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimapTest.java b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimapTest.java
new file mode 100644
index 0000000..1249f6d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimapTest.java
@@ -0,0 +1,352 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.testing.google.UnmodifiableCollectionTests;
+import com.google.common.testing.EqualsTester;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A test for {@link ImmutableSortedKeyListMultimap}. Started out as a copy of
+ * ImmutableListMultimapTest.
+ */
+@RunWith(JUnit4.class)
+public class ImmutableSortedKeyListMultimapTest {
+
+ @Test
+ public void builderPutAllIterable() {
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll("foo", Arrays.asList(1, 2, 3));
+ builder.putAll("bar", Arrays.asList(4, 5));
+ builder.putAll("foo", Arrays.asList(6, 7));
+ Multimap<String, Integer> multimap = builder.build();
+ assertEquals(Arrays.asList(1, 2, 3, 6, 7), multimap.get("foo"));
+ assertEquals(Arrays.asList(4, 5), multimap.get("bar"));
+ assertEquals(7, multimap.size());
+ }
+
+ @Test
+ public void builderPutAllVarargs() {
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll("foo", 1, 2, 3);
+ builder.putAll("bar", 4, 5);
+ builder.putAll("foo", 6, 7);
+ Multimap<String, Integer> multimap = builder.build();
+ assertEquals(Arrays.asList(1, 2, 3, 6, 7), multimap.get("foo"));
+ assertEquals(Arrays.asList(4, 5), multimap.get("bar"));
+ assertEquals(7, multimap.size());
+ }
+
+ @Test
+ public void builderPutAllMultimap() {
+ Multimap<String, Integer> toPut = LinkedListMultimap.create();
+ toPut.put("foo", 1);
+ toPut.put("bar", 4);
+ toPut.put("foo", 2);
+ toPut.put("foo", 3);
+ Multimap<String, Integer> moreToPut = LinkedListMultimap.create();
+ moreToPut.put("foo", 6);
+ moreToPut.put("bar", 5);
+ moreToPut.put("foo", 7);
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll(toPut);
+ builder.putAll(moreToPut);
+ Multimap<String, Integer> multimap = builder.build();
+ assertEquals(Arrays.asList(1, 2, 3, 6, 7), multimap.get("foo"));
+ assertEquals(Arrays.asList(4, 5), multimap.get("bar"));
+ assertEquals(7, multimap.size());
+ }
+
+ @Test
+ public void builderPutAllWithDuplicates() {
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll("foo", 1, 2, 3);
+ builder.putAll("bar", 4, 5);
+ builder.putAll("foo", 1, 6, 7);
+ ImmutableSortedKeyListMultimap<String, Integer> multimap = builder.build();
+ assertEquals(Arrays.asList(1, 2, 3, 1, 6, 7), multimap.get("foo"));
+ assertEquals(Arrays.asList(4, 5), multimap.get("bar"));
+ assertEquals(8, multimap.size());
+ }
+
+ @Test
+ public void builderPutWithDuplicates() {
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll("foo", 1, 2, 3);
+ builder.putAll("bar", 4, 5);
+ builder.put("foo", 1);
+ ImmutableSortedKeyListMultimap<String, Integer> multimap = builder.build();
+ assertEquals(Arrays.asList(1, 2, 3, 1), multimap.get("foo"));
+ assertEquals(Arrays.asList(4, 5), multimap.get("bar"));
+ assertEquals(6, multimap.size());
+ }
+
+ @Test
+ public void builderPutAllMultimapWithDuplicates() {
+ Multimap<String, Integer> toPut = LinkedListMultimap.create();
+ toPut.put("foo", 1);
+ toPut.put("bar", 4);
+ toPut.put("foo", 2);
+ toPut.put("foo", 1);
+ toPut.put("bar", 5);
+ Multimap<String, Integer> moreToPut = LinkedListMultimap.create();
+ moreToPut.put("foo", 6);
+ moreToPut.put("bar", 4);
+ moreToPut.put("foo", 7);
+ moreToPut.put("foo", 2);
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll(toPut);
+ builder.putAll(moreToPut);
+ Multimap<String, Integer> multimap = builder.build();
+ assertEquals(Arrays.asList(1, 2, 1, 6, 7, 2), multimap.get("foo"));
+ assertEquals(Arrays.asList(4, 5, 4), multimap.get("bar"));
+ assertEquals(9, multimap.size());
+ }
+
+ @Test
+ public void builderPutNullKey() {
+ Multimap<String, Integer> toPut = LinkedListMultimap.create();
+ toPut.put("foo", null);
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ try {
+ builder.put(null, 1);
+ fail();
+ } catch (NullPointerException expected) {}
+ try {
+ builder.putAll(null, Arrays.asList(1, 2, 3));
+ fail();
+ } catch (NullPointerException expected) {}
+ try {
+ builder.putAll(null, 1, 2, 3);
+ fail();
+ } catch (NullPointerException expected) {}
+ try {
+ builder.putAll(toPut);
+ fail();
+ } catch (NullPointerException expected) {}
+ }
+
+ @Test
+ public void builderPutNullValue() {
+ Multimap<String, Integer> toPut = LinkedListMultimap.create();
+ toPut.put(null, 1);
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ try {
+ builder.put("foo", null);
+ fail();
+ } catch (NullPointerException expected) {}
+ try {
+ builder.putAll("foo", Arrays.asList(1, null, 3));
+ fail();
+ } catch (NullPointerException expected) {}
+ try {
+ builder.putAll("foo", 1, null, 3);
+ fail();
+ } catch (NullPointerException expected) {}
+ try {
+ builder.putAll(toPut);
+ fail();
+ } catch (NullPointerException expected) {}
+ }
+
+ @Test
+ public void copyOf() {
+ ArrayListMultimap<String, Integer> input = ArrayListMultimap.create();
+ input.put("foo", 1);
+ input.put("bar", 2);
+ input.put("foo", 3);
+ Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.copyOf(input);
+ assertEquals(multimap, input);
+ assertEquals(input, multimap);
+ }
+
+ @Test
+ public void copyOfWithDuplicates() {
+ ArrayListMultimap<String, Integer> input = ArrayListMultimap.create();
+ input.put("foo", 1);
+ input.put("bar", 2);
+ input.put("foo", 3);
+ input.put("foo", 1);
+ Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.copyOf(input);
+ assertEquals(multimap, input);
+ assertEquals(input, multimap);
+ }
+
+ @Test
+ public void copyOfEmpty() {
+ ArrayListMultimap<String, Integer> input = ArrayListMultimap.create();
+ Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.copyOf(input);
+ assertEquals(multimap, input);
+ assertEquals(input, multimap);
+ }
+
+ @Test
+ public void copyOfImmutableListMultimap() {
+ Multimap<String, Integer> multimap = createMultimap();
+ assertSame(multimap, ImmutableSortedKeyListMultimap.copyOf(multimap));
+ }
+
+ @Test
+ public void copyOfNullKey() {
+ ArrayListMultimap<String, Integer> input = ArrayListMultimap.create();
+ input.put(null, 1);
+ try {
+ ImmutableSortedKeyListMultimap.copyOf(input);
+ fail();
+ } catch (NullPointerException expected) {}
+ }
+
+ @Test
+ public void copyOfNullValue() {
+ ArrayListMultimap<String, Integer> input = ArrayListMultimap.create();
+ input.putAll("foo", Arrays.asList(1, null, 3));
+ try {
+ ImmutableSortedKeyListMultimap.copyOf(input);
+ fail();
+ } catch (NullPointerException expected) {}
+ }
+
+ @Test
+ public void emptyMultimapReads() {
+ Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.of();
+ assertFalse(multimap.containsKey("foo"));
+ assertFalse(multimap.containsValue(1));
+ assertFalse(multimap.containsEntry("foo", 1));
+ assertTrue(multimap.entries().isEmpty());
+ assertTrue(multimap.equals(ArrayListMultimap.create()));
+ assertEquals(Collections.emptyList(), multimap.get("foo"));
+ assertEquals(0, multimap.hashCode());
+ assertTrue(multimap.isEmpty());
+ assertEquals(HashMultiset.create(), multimap.keys());
+ assertEquals(Collections.emptySet(), multimap.keySet());
+ assertEquals(0, multimap.size());
+ assertTrue(multimap.values().isEmpty());
+ assertEquals("{}", multimap.toString());
+ }
+
+ @Test
+ public void emptyMultimapWrites() {
+ Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.of();
+ UnmodifiableCollectionTests.assertMultimapIsUnmodifiable(
+ multimap, "foo", 1);
+ }
+
+ private Multimap<String, Integer> createMultimap() {
+ return ImmutableSortedKeyListMultimap.<String, Integer>builder()
+ .put("foo", 1).put("bar", 2).put("foo", 3).build();
+ }
+
+ @Test
+ public void multimapReads() {
+ Multimap<String, Integer> multimap = createMultimap();
+ assertTrue(multimap.containsKey("foo"));
+ assertFalse(multimap.containsKey("cat"));
+ assertTrue(multimap.containsValue(1));
+ assertFalse(multimap.containsValue(5));
+ assertTrue(multimap.containsEntry("foo", 1));
+ assertFalse(multimap.containsEntry("cat", 1));
+ assertFalse(multimap.containsEntry("foo", 5));
+ assertFalse(multimap.entries().isEmpty());
+ assertEquals(3, multimap.size());
+ assertFalse(multimap.isEmpty());
+ assertEquals("{bar=[2], foo=[1, 3]}", multimap.toString());
+ }
+
+ @Test
+ public void multimapWrites() {
+ Multimap<String, Integer> multimap = createMultimap();
+ UnmodifiableCollectionTests.assertMultimapIsUnmodifiable(
+ multimap, "bar", 2);
+ }
+
+ @Test
+ public void multimapEquals() {
+ Multimap<String, Integer> multimap = createMultimap();
+ Multimap<String, Integer> arrayListMultimap
+ = ArrayListMultimap.create();
+ arrayListMultimap.putAll("foo", Arrays.asList(1, 3));
+ arrayListMultimap.put("bar", 2);
+
+ new EqualsTester()
+ .addEqualityGroup(multimap, createMultimap(), arrayListMultimap,
+ ImmutableSortedKeyListMultimap.<String, Integer>builder()
+ .put("bar", 2).put("foo", 1).put("foo", 3).build())
+ .addEqualityGroup(ImmutableSortedKeyListMultimap.<String, Integer>builder()
+ .put("bar", 2).put("foo", 3).put("foo", 1).build())
+ .addEqualityGroup(ImmutableSortedKeyListMultimap.<String, Integer>builder()
+ .put("foo", 2).put("foo", 3).put("foo", 1).build())
+ .addEqualityGroup(ImmutableSortedKeyListMultimap.<String, Integer>builder()
+ .put("bar", 2).put("foo", 3).build())
+ .testEquals();
+ }
+
+ @Test
+ public void asMap() {
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll("foo", Arrays.asList(1, 2, 3));
+ builder.putAll("bar", Arrays.asList(4, 5));
+ Map<String, Collection<Integer>> map = builder.build().asMap();
+ assertEquals(Arrays.asList(1, 2, 3), map.get("foo"));
+ assertEquals(Arrays.asList(4, 5), map.get("bar"));
+ assertEquals(2, map.size());
+ assertTrue(map.containsKey("foo"));
+ assertTrue(map.containsKey("bar"));
+ assertFalse(map.containsKey("notfoo"));
+ }
+
+ @Test
+ public void asMapEntries() {
+ ImmutableSortedKeyListMultimap.Builder<String, Integer> builder
+ = ImmutableSortedKeyListMultimap.builder();
+ builder.putAll("foo", Arrays.asList(1, 2, 3));
+ builder.putAll("bar", Arrays.asList(4, 5));
+ Set<Map.Entry<String, Collection<Integer>>> set = builder.build().asMap().entrySet();
+ Set<Map.Entry<String, Collection<Integer>>> other =
+ ImmutableSet.<Map.Entry<String, Collection<Integer>>>builder()
+ .add(new SimpleImmutableEntry<String, Collection<Integer>>("foo", Arrays.asList(1, 2, 3)))
+ .add(new SimpleImmutableEntry<String, Collection<Integer>>("bar", Arrays.asList(4, 5)))
+ .build();
+ assertEquals(other, set);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMapTest.java b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMapTest.java
new file mode 100644
index 0000000..c712695
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMapTest.java
@@ -0,0 +1,288 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Maps;
+import com.google.common.testing.NullPointerTester;
+import com.google.devtools.build.lib.collect.ImmutableSortedKeyMap.Builder;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * A test for {@link ImmutableSortedKeyListMultimap}. Started out as a blatant copy of
+ * ImmutableListMapTest.
+ */
+@RunWith(JUnit4.class)
+public class ImmutableSortedKeyMapTest {
+
+ @Test
+ public void emptyBuilder() {
+ ImmutableSortedKeyMap<String, Integer> map
+ = ImmutableSortedKeyMap.<String, Integer>builder().build();
+ assertEquals(Collections.<String, Integer>emptyMap(), map);
+ }
+
+ @Test
+ public void singletonBuilder() {
+ ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder()
+ .put("one", 1)
+ .build();
+ assertMapEquals(map, "one", 1);
+ }
+
+ @Test
+ public void builder() {
+ ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder()
+ .put("one", 1)
+ .put("two", 2)
+ .put("three", 3)
+ .put("four", 4)
+ .put("five", 5)
+ .build();
+ assertMapEquals(map,
+ "five", 5, "four", 4, "one", 1, "three", 3, "two", 2);
+ }
+
+ @Test
+ public void builderPutAllWithEmptyMap() {
+ ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder()
+ .putAll(Collections.<String, Integer>emptyMap())
+ .build();
+ assertEquals(Collections.<String, Integer>emptyMap(), map);
+ }
+
+ @Test
+ public void builderPutAll() {
+ Map<String, Integer> toPut = new LinkedHashMap<>();
+ toPut.put("one", 1);
+ toPut.put("two", 2);
+ toPut.put("three", 3);
+ Map<String, Integer> moreToPut = new LinkedHashMap<>();
+ moreToPut.put("four", 4);
+ moreToPut.put("five", 5);
+
+ ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder()
+ .putAll(toPut)
+ .putAll(moreToPut)
+ .build();
+ assertMapEquals(map,
+ "five", 5, "four", 4, "one", 1, "three", 3, "two", 2);
+ }
+
+ @Test
+ public void builderReuse() {
+ ImmutableSortedKeyMap.Builder<String, Integer> builder =
+ ImmutableSortedKeyMap.<String, Integer>builder();
+ ImmutableSortedKeyMap<String, Integer> mapOne = builder
+ .put("one", 1)
+ .put("two", 2)
+ .build();
+ ImmutableSortedKeyMap<String, Integer> mapTwo = builder
+ .put("three", 3)
+ .put("four", 4)
+ .build();
+
+ assertMapEquals(mapOne, "one", 1, "two", 2);
+ assertMapEquals(mapTwo, "four", 4, "one", 1, "three", 3, "two", 2);
+ }
+
+ @Test
+ public void builderPutNullKey() {
+ Builder<String, Integer> builder = new Builder<>();
+ try {
+ builder.put(null, 1);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void builderPutNullValue() {
+ Builder<String, Integer> builder = new Builder<>();
+ try {
+ builder.put("one", null);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void builderPutNullKeyViaPutAll() {
+ Builder<String, Integer> builder = new Builder<>();
+ try {
+ builder.putAll(Collections.<String, Integer>singletonMap(null, 1));
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void builderPutNullValueViaPutAll() {
+ Builder<String, Integer> builder = new Builder<>();
+ try {
+ builder.putAll(Collections.<String, Integer>singletonMap("one", null));
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void of() {
+ assertMapEquals(
+ ImmutableSortedKeyMap.of("one", 1),
+ "one", 1);
+ assertMapEquals(
+ ImmutableSortedKeyMap.of("one", 1, "two", 2),
+ "one", 1, "two", 2);
+ }
+
+ @Test
+ public void ofNullKey() {
+ try {
+ ImmutableSortedKeyMap.of((String) null, 1);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+
+ try {
+ ImmutableSortedKeyMap.of("one", 1, null, 2);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void ofNullValue() {
+ try {
+ ImmutableSortedKeyMap.of("one", null);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+
+ try {
+ ImmutableSortedKeyMap.of("one", 1, "two", null);
+ fail();
+ } catch (NullPointerException expected) {
+ }
+ }
+
+ @Test
+ public void copyOfEmptyMap() {
+ ImmutableSortedKeyMap<String, Integer> copy
+ = ImmutableSortedKeyMap.copyOf(Collections.<String, Integer>emptyMap());
+ assertEquals(Collections.<String, Integer>emptyMap(), copy);
+ assertSame(copy, ImmutableSortedKeyMap.copyOf(copy));
+ }
+
+ @Test
+ public void copyOfSingletonMap() {
+ ImmutableSortedKeyMap<String, Integer> copy
+ = ImmutableSortedKeyMap.copyOf(Collections.singletonMap("one", 1));
+ assertMapEquals(copy, "one", 1);
+ assertSame(copy, ImmutableSortedKeyMap.copyOf(copy));
+ }
+
+ @Test
+ public void copyOf() {
+ Map<String, Integer> original = new LinkedHashMap<>();
+ original.put("one", 1);
+ original.put("two", 2);
+ original.put("three", 3);
+
+ ImmutableSortedKeyMap<String, Integer> copy = ImmutableSortedKeyMap.copyOf(original);
+ assertMapEquals(copy, "one", 1, "three", 3, "two", 2);
+ assertSame(copy, ImmutableSortedKeyMap.copyOf(copy));
+ }
+
+ @Test
+ public void nullGet() {
+ ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.of("one", 1);
+ assertNull(map.get(null));
+ }
+
+ @Test
+ public void nullPointers() {
+ NullPointerTester tester = new NullPointerTester();
+ tester.testAllPublicStaticMethods(ImmutableSortedKeyMap.class);
+ tester.testAllPublicInstanceMethods(
+ new ImmutableSortedKeyMap.Builder<String, Object>());
+ tester.testAllPublicInstanceMethods(ImmutableSortedKeyMap.<String, Integer>of());
+ tester.testAllPublicInstanceMethods(ImmutableSortedKeyMap.of("one", 1));
+ tester.testAllPublicInstanceMethods(
+ ImmutableSortedKeyMap.of("one", 1, "two", 2));
+ }
+
+ private static <K, V> void assertMapEquals(Map<K, V> map,
+ Object... alternatingKeysAndValues) {
+ assertEquals(map.size(), alternatingKeysAndValues.length / 2);
+ int i = 0;
+ for (Entry<K, V> entry : map.entrySet()) {
+ assertEquals(alternatingKeysAndValues[i++], entry.getKey());
+ assertEquals(alternatingKeysAndValues[i++], entry.getValue());
+ }
+ }
+
+ private static class IntHolder implements Serializable {
+ public int value;
+
+ public IntHolder(int value) {
+ this.value = value;
+ }
+
+ @Override public boolean equals(Object o) {
+ return (o instanceof IntHolder) && ((IntHolder) o).value == value;
+ }
+
+ @Override public int hashCode() {
+ return value;
+ }
+
+ private static final long serialVersionUID = 5;
+ }
+
+ @Test
+ public void mutableValues() {
+ IntHolder holderA = new IntHolder(1);
+ IntHolder holderB = new IntHolder(2);
+ Map<String, IntHolder> map = ImmutableSortedKeyMap.of("a", holderA, "b", holderB);
+ holderA.value = 3;
+ assertTrue(map.entrySet().contains(
+ Maps.immutableEntry("a", new IntHolder(3))));
+ Map<String, Integer> intMap = ImmutableSortedKeyMap.of("a", 3, "b", 2);
+ assertEquals(intMap.hashCode(), map.entrySet().hashCode());
+ assertEquals(intMap.hashCode(), map.hashCode());
+ }
+
+ @Test
+ public void toStringTest() {
+ Map<String, Integer> map = ImmutableSortedKeyMap.of("a", 1, "b", 2);
+ assertEquals("{a=1, b=2}", map.toString());
+ map = ImmutableSortedKeyMap.of();
+ assertEquals("{}", map.toString());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/IterablesChainTest.java b/src/test/java/com/google/devtools/build/lib/collect/IterablesChainTest.java
new file mode 100644
index 0000000..734d801
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/IterablesChainTest.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+
+/**
+ * A test for {@link IterablesChain}.
+ */
+@RunWith(JUnit4.class)
+public class IterablesChainTest {
+
+ @Test
+ public void addElement() {
+ IterablesChain.Builder<String> builder = IterablesChain.builder();
+ builder.addElement("a");
+ builder.addElement("b");
+ assertEquals(Arrays.asList("a", "b"), ImmutableList.copyOf(builder.build()));
+ }
+
+ @Test
+ public void add() {
+ IterablesChain.Builder<String> builder = IterablesChain.builder();
+ builder.add(ImmutableList.of("a", "b"));
+ assertEquals(Arrays.asList("a", "b"), ImmutableList.copyOf(builder.build()));
+ }
+
+ @Test
+ public void isEmpty() {
+ IterablesChain.Builder<String> builder = IterablesChain.builder();
+ assertTrue(builder.isEmpty());
+ builder.addElement("a");
+ assertFalse(builder.isEmpty());
+ builder = IterablesChain.builder();
+ assertTrue(builder.isEmpty());
+ builder.add(ImmutableList.of("a"));
+ assertFalse(builder.isEmpty());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpanderTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpanderTest.java
new file mode 100644
index 0000000..926ce2d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpanderTest.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+
+/**
+ * Tests for {@link CompileOrderExpander}.
+ */
+@RunWith(JUnit4.class)
+public class CompileOrderExpanderTest extends ExpanderTestBase {
+
+ @Override
+ protected Order expanderOrder() {
+ return Order.COMPILE_ORDER;
+ }
+
+ @Override
+ protected List<String> nestedResult() {
+ return ImmutableList.of("c", "a", "e", "b", "d");
+ }
+
+ @Override
+ protected List<String> nestedDuplicatesResult() {
+ return ImmutableList.of("c", "a", "e", "b", "d");
+ }
+
+ @Override
+ protected List<String> chainResult() {
+ return ImmutableList.of("c", "b", "a");
+ }
+
+ @Override
+ protected List<String> diamondResult() {
+ return ImmutableList.of("d", "b", "c", "a");
+ }
+
+ @Override
+ protected List<String> extendedDiamondResult() {
+ return ImmutableList.of("d", "e", "b", "c", "a");
+ }
+
+ @Override
+ protected List<String> extendedDiamondRightArmResult() {
+ return ImmutableList.of("d", "e", "b", "c2", "c", "a");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java
new file mode 100644
index 0000000..25448c6
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java
@@ -0,0 +1,330 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Base class for tests of {@link NestedSetExpander} implementations.
+ *
+ * <p>This class provides test cases for representative nested set structures; the expected
+ * results must be provided by overriding the corresponding methods.
+ */
+public abstract class ExpanderTestBase extends TestCase {
+
+ /**
+ * Returns the type of the expander under test.
+ */
+ protected abstract Order expanderOrder();
+
+ @Test
+ public void simple() {
+ NestedSet<String> s = prepareBuilder("c", "a", "b").build();
+
+ assertTrue(Arrays.equals(simpleResult().toArray(), s.directMembers()));
+ assertSetContents(simpleResult(), s);
+ }
+
+ @Test
+ public void simpleNoDuplicates() {
+ NestedSet<String> s = prepareBuilder("c", "a", "a", "a", "b").build();
+
+ assertTrue(Arrays.equals(simpleResult().toArray(), s.directMembers()));
+ assertSetContents(simpleResult(), s);
+ }
+
+ @Test
+ public void nesting() {
+ NestedSet<String> subset = prepareBuilder("c", "a", "e").build();
+ NestedSet<String> s = prepareBuilder("b", "d").addTransitive(subset).build();
+
+ assertSetContents(nestedResult(), s);
+ }
+
+ @Test
+ public void builderReuse() {
+ NestedSetBuilder<String> builder = prepareBuilder();
+ assertSetContents(Collections.<String>emptyList(), builder.build());
+
+ builder.add("b");
+ assertSetContents(ImmutableList.of("b"), builder.build());
+
+ builder.addAll(ImmutableList.of("d"));
+ Collection<String> expected = ImmutableList.copyOf(prepareBuilder("b", "d").build());
+ assertSetContents(expected, builder.build());
+
+ NestedSet<String> child = prepareBuilder("c", "a", "e").build();
+ builder.addTransitive(child);
+ assertSetContents(nestedResult(), builder.build());
+ }
+
+ @Test
+ public void builderChaining() {
+ NestedSet<String> s = prepareBuilder().add("b").addAll(ImmutableList.of("d"))
+ .addTransitive(prepareBuilder("c", "a", "e").build()).build();
+ assertSetContents(nestedResult(), s);
+ }
+
+ @Test
+ public void addAllOrdering() {
+ NestedSet<String> s1 = prepareBuilder().add("a").add("c").add("b").build();
+ NestedSet<String> s2 = prepareBuilder().addAll(ImmutableList.of("a", "c", "b")).build();
+
+ assertTrue(Arrays.equals(s1.directMembers(), s2.directMembers()));
+ assertCollectionsEqual(s1.toCollection(), s2.toCollection());
+ assertCollectionsEqual(s1.toList(), s2.toList());
+ assertCollectionsEqual(Lists.newArrayList(s1), Lists.newArrayList(s2));
+ }
+
+ @Test
+ public void mixedAddAllOrdering() {
+ NestedSet<String> s1 = prepareBuilder().add("a").add("b").add("c").add("d").build();
+ NestedSet<String> s2 = prepareBuilder().add("a").addAll(ImmutableList.of("b", "c")).add("d")
+ .build();
+
+ assertTrue(Arrays.equals(s1.directMembers(), s2.directMembers()));
+ assertCollectionsEqual(s1.toCollection(), s2.toCollection());
+ assertCollectionsEqual(s1.toList(), s2.toList());
+ assertCollectionsEqual(Lists.newArrayList(s1), Lists.newArrayList(s2));
+ }
+
+ @Test
+ public void transitiveDepsHandledSeparately() {
+ NestedSet<String> subset = prepareBuilder("c", "a", "e").build();
+ NestedSetBuilder<String> b = prepareBuilder();
+ // The fact that we add the transitive subset between the add("b") and add("d") calls should
+ // not change the result.
+ b.add("b");
+ b.addTransitive(subset);
+ b.add("d");
+ NestedSet<String> s = b.build();
+
+ assertSetContents(nestedResult(), s);
+ }
+
+ @Test
+ public void nestingNoDuplicates() {
+ NestedSet<String> subset = prepareBuilder("c", "a", "e").build();
+ NestedSet<String> s = prepareBuilder("b", "d", "e").addTransitive(subset).build();
+
+ assertSetContents(nestedDuplicatesResult(), s);
+ }
+
+ @Test
+ public void chain() {
+ NestedSet<String> c = prepareBuilder("c").build();
+ NestedSet<String> b = prepareBuilder("b").addTransitive(c).build();
+ NestedSet<String> a = prepareBuilder("a").addTransitive(b).build();
+
+ assertTrue(Arrays.equals(new String[]{"a"}, a.directMembers()));
+ assertSetContents(chainResult(), a);
+ }
+
+ @Test
+ public void diamond() {
+ NestedSet<String> d = prepareBuilder("d").build();
+ NestedSet<String> c = prepareBuilder("c").addTransitive(d).build();
+ NestedSet<String> b = prepareBuilder("b").addTransitive(d).build();
+ NestedSet<String> a = prepareBuilder("a").addTransitive(b).addTransitive(c).build();
+
+ assertTrue(Arrays.equals(new String[]{"a"}, a.directMembers()));
+ assertSetContents(diamondResult(), a);
+ }
+
+ @Test
+ public void extendedDiamond() {
+ NestedSet<String> d = prepareBuilder("d").build();
+ NestedSet<String> e = prepareBuilder("e").build();
+ NestedSet<String> b = prepareBuilder("b").addTransitive(d).addTransitive(e).build();
+ NestedSet<String> c = prepareBuilder("c").addTransitive(e).addTransitive(d).build();
+ NestedSet<String> a = prepareBuilder("a").addTransitive(b).addTransitive(c).build();
+ assertSetContents(extendedDiamondResult(), a);
+ }
+
+ @Test
+ public void extendedDiamondRightArm() {
+ NestedSet<String> d = prepareBuilder("d").build();
+ NestedSet<String> e = prepareBuilder("e").build();
+ NestedSet<String> b = prepareBuilder("b").addTransitive(d).addTransitive(e).build();
+ NestedSet<String> c2 = prepareBuilder("c2").addTransitive(e).addTransitive(d).build();
+ NestedSet<String> c = prepareBuilder("c").addTransitive(c2).build();
+ NestedSet<String> a = prepareBuilder("a").addTransitive(b).addTransitive(c).build();
+ assertSetContents(extendedDiamondRightArmResult(), a);
+ }
+
+ @Test
+ public void orderConflict() {
+ NestedSet<String> child1 = prepareBuilder("a", "b").build();
+ NestedSet<String> child2 = prepareBuilder("b", "a").build();
+ NestedSet<String> parent = prepareBuilder().addTransitive(child1).addTransitive(child2).build();
+ assertSetContents(orderConflictResult(), parent);
+ }
+
+ @Test
+ public void orderConflictNested() {
+ NestedSet<String> a = prepareBuilder("a").build();
+ NestedSet<String> b = prepareBuilder("b").build();
+ NestedSet<String> child1 = prepareBuilder().addTransitive(a).addTransitive(b).build();
+ NestedSet<String> child2 = prepareBuilder().addTransitive(b).addTransitive(a).build();
+ NestedSet<String> parent = prepareBuilder().addTransitive(child1).addTransitive(child2).build();
+ assertSetContents(orderConflictResult(), parent);
+ }
+
+ @Test
+ public void getOrderingEmpty() {
+ NestedSet<String> s = prepareBuilder().build();
+ assertTrue(s.isEmpty());
+ assertEquals(expanderOrder(), s.getOrder());
+ }
+
+ @Test
+ public void getOrdering() {
+ NestedSet<String> s = prepareBuilder("a", "b").build();
+ assertTrue(!s.isEmpty());
+ assertEquals(expanderOrder(), s.getOrder());
+ }
+
+ /**
+ * In case we have inner NestedSets with different order (allowed by the builder). We should
+ * maintain the order of the top-level NestedSet.
+ */
+ @Test
+ public void regressionOnOneTransitiveDep() {
+ NestedSet<String> subsub = NestedSetBuilder.<String>stableOrder().add("c").add("a").add("e")
+ .build();
+ NestedSet<String> sub = NestedSetBuilder.<String>stableOrder().add("b").add("d")
+ .addTransitive(subsub).build();
+ NestedSet<String> top = prepareBuilder().addTransitive(sub).build();
+ assertSetContents(nestedResult(), top);
+ }
+
+ @Test
+ public void nestingValidation() {
+ for (Order ordering : Order.values()) {
+ NestedSet<String> a = prepareBuilder("a", "b").build();
+ NestedSetBuilder<String> b = new NestedSetBuilder<>(ordering);
+ try {
+ b.addTransitive(a);
+ if (ordering != expanderOrder() && ordering != Order.STABLE_ORDER) {
+ fail(); // An exception was expected.
+ }
+ } catch (IllegalStateException e) {
+ if (ordering == expanderOrder() || ordering == Order.STABLE_ORDER) {
+ fail(); // No exception was expected.
+ }
+ }
+ }
+ }
+
+ private NestedSetBuilder<String> prepareBuilder(String... directMembers) {
+ NestedSetBuilder<String> builder = new NestedSetBuilder<>(expanderOrder());
+ builder.addAll(Lists.newArrayList(directMembers));
+ return builder;
+ }
+
+ protected final void assertSetContents(Collection<String> expected, NestedSet<String> set) {
+ assertEquals(expected, Lists.newArrayList(set));
+ assertEquals(expected, Lists.newArrayList(set.toCollection()));
+ assertEquals(expected, Lists.newArrayList(set.toList()));
+ assertEquals(expected, Lists.newArrayList(set.toSet()));
+ }
+
+ protected final void assertCollectionsEqual(
+ Collection<String> expected, Collection<String> actual) {
+ assertEquals(Lists.newArrayList(expected), Lists.newArrayList(actual));
+ }
+
+ /**
+ * Returns the enumeration of the nested set {"c", "a", "b"} in the
+ * implementation's enumeration order.
+ *
+ * @see #testSimple()
+ * @see #testSimpleNoDuplicates()
+ */
+ protected List<String> simpleResult() {
+ return ImmutableList.of("c", "a", "b");
+ }
+
+ /**
+ * Returns the enumeration of the nested set {"b", "d", {"c", "a", "e"}} in
+ * the implementation's enumeration order.
+ *
+ * @see #testNesting()
+ */
+ protected abstract List<String> nestedResult();
+
+ /**
+ * Returns the enumeration of the nested set {"b", "d", "e", {"c", "a", "e"}} in
+ * the implementation's enumeration order.
+ *
+ * @see #testNestingNoDuplicates()
+ */
+ protected abstract List<String> nestedDuplicatesResult();
+
+ /**
+ * Returns the enumeration of nested set {"a", {"b", {"c"}}} in the
+ * implementation's enumeration order.
+ *
+ * @see #testChain()
+ */
+ protected abstract List<String> chainResult();
+
+ /**
+ * Returns the enumeration of the nested set {"a", {"b", D}, {"c", D}}, where
+ * D is {"d"}, in the implementation's enumeration order.
+ *
+ * @see #testDiamond()
+ */
+ protected abstract List<String> diamondResult();
+
+ /**
+ * Returns the enumeration of the nested set {"a", {"b", E, D}, {"c", D, E}}, where
+ * D is {"d"} and E is {"e"}, in the implementation's enumeration order.
+ *
+ * @see #testExtendedDiamond()
+ */
+ protected abstract List<String> extendedDiamondResult();
+
+ /**
+ * Returns the enumeration of the nested set {"a", {"b", E, D}, {"c", C2}}, where
+ * D is {"d"}, E is {"e"} and C2 is {"c2", D, E}, in the implementation's enumeration order.
+ *
+ * @see #testExtendedDiamondRightArm()
+ */
+ protected abstract List<String> extendedDiamondRightArmResult();
+
+ /**
+ * Returns the enumeration of the nested set {{"a", "b"}, {"b", "a"}}.
+ *
+ * @see #testOrderConflict()
+ * @see #testOrderConflictNested()
+ */
+ protected List<String> orderConflictResult() {
+ return ImmutableList.of("a", "b");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpanderTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpanderTest.java
new file mode 100644
index 0000000..5d69882
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpanderTest.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+
+/**
+ * Tests for {@link LinkOrderExpander}.
+ */
+@RunWith(JUnit4.class)
+public class LinkOrderExpanderTest extends ExpanderTestBase {
+
+ @Override
+ protected Order expanderOrder() {
+ return Order.LINK_ORDER;
+ }
+
+ @Override
+ protected List<String> nestedResult() {
+ return ImmutableList.of("b", "d", "c", "a", "e");
+ }
+
+ @Override
+ protected List<String> nestedDuplicatesResult() {
+ return ImmutableList.of("b", "d", "c", "a", "e");
+ }
+
+ @Override
+ protected List<String> chainResult() {
+ return ImmutableList.of("a", "b", "c");
+ }
+
+ @Override
+ protected List<String> diamondResult() {
+ return ImmutableList.of("a", "b", "c", "d");
+ }
+
+ @Override
+ protected List<String> orderConflictResult() {
+ // Rightmost branch determines the order.
+ return ImmutableList.of("b", "a");
+ }
+
+ @Override
+ protected List<String> extendedDiamondResult() {
+ return ImmutableList.of("a", "b", "c", "e", "d");
+ }
+
+ @Override
+ protected List<String> extendedDiamondRightArmResult() {
+ return ImmutableList.of("a", "b", "c", "c2", "e", "d");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpanderTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpanderTest.java
new file mode 100644
index 0000000..80faf7a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpanderTest.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+
+/**
+ * Tests for {@link NaiveLinkOrderExpander}.
+ */
+@RunWith(JUnit4.class)
+public class NaiveLinkOrderExpanderTest extends ExpanderTestBase {
+
+ @Override
+ protected Order expanderOrder() {
+ return Order.NAIVE_LINK_ORDER;
+ }
+
+ @Override
+ protected List<String> nestedResult() {
+ return ImmutableList.of("b", "d", "c", "a", "e");
+ }
+
+ @Override
+ protected List<String> nestedDuplicatesResult() {
+ return ImmutableList.of("b", "d", "e", "c", "a");
+ }
+
+ @Override
+ protected List<String> chainResult() {
+ return ImmutableList.of("a", "b", "c");
+ }
+
+ @Override
+ protected List<String> diamondResult() {
+ // This case illustrates why this implementation is called "naive".
+ return ImmutableList.of("a", "b", "d", "c");
+ }
+
+ @Override
+ protected List<String> orderConflictResult() {
+ // Leftmost branch determines the order.
+ return ImmutableList.of("a", "b");
+ }
+
+ @Override
+ protected List<String> extendedDiamondResult() {
+ return ImmutableList.of("a", "b", "d", "e", "c");
+ }
+
+ @Override
+ protected List<String> extendedDiamondRightArmResult() {
+ return ImmutableList.of("a", "b", "d", "e", "c", "c2");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetImplTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetImplTest.java
new file mode 100644
index 0000000..e5cfae3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetImplTest.java
@@ -0,0 +1,245 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.testing.EqualsTester;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+
+/**
+ * Tests for {@link com.google.devtools.build.lib.collect.nestedset.NestedSet}.
+ */
+@RunWith(JUnit4.class)
+public class NestedSetImplTest extends TestCase {
+ @SafeVarargs
+ private static NestedSetBuilder<String> nestedSetBuilder(String... directMembers) {
+ NestedSetBuilder<String> builder = NestedSetBuilder.stableOrder();
+ builder.addAll(Lists.newArrayList(directMembers));
+ return builder;
+ }
+
+ @Test
+ public void simple() {
+ NestedSet<String> set = nestedSetBuilder("a").build();
+
+ assertTrue(Arrays.equals(new String[]{"a"}, set.directMembers()));
+ assertEquals(0, set.transitiveSets().length);
+ assertEquals(false, set.isEmpty());
+ }
+
+ @Test
+ public void flatToString() {
+ assertEquals("{}", nestedSetBuilder().build().toString());
+ assertEquals("{a}", nestedSetBuilder("a").build().toString());
+ assertEquals("{a, b}", nestedSetBuilder("a", "b").build().toString());
+ }
+
+ @Test
+ public void nestedToString() {
+ NestedSet<String> b = nestedSetBuilder("b").build();
+ NestedSet<String> c = nestedSetBuilder("c").build();
+
+ assertEquals("{a, {b}}",
+ nestedSetBuilder("a").addTransitive(b).build().toString());
+ assertEquals("{a, {b}, {c}}",
+ nestedSetBuilder("a").addTransitive(b).addTransitive(c).build().toString());
+
+ assertEquals("{b}", nestedSetBuilder().addTransitive(b).build().toString());
+ }
+
+ @Test
+ public void isEmpty() {
+ NestedSet<String> triviallyEmpty = nestedSetBuilder().build();
+ assertTrue(triviallyEmpty.isEmpty());
+
+ NestedSet<String> emptyLevel1 = nestedSetBuilder().addTransitive(triviallyEmpty).build();
+ assertTrue(emptyLevel1.isEmpty());
+
+ NestedSet<String> emptyLevel2 = nestedSetBuilder().addTransitive(emptyLevel1).build();
+ assertTrue(emptyLevel2.isEmpty());
+
+ NestedSet<String> triviallyNonEmpty = nestedSetBuilder("mango").build();
+ assertFalse(triviallyNonEmpty.isEmpty());
+
+ NestedSet<String> nonEmptyLevel1 = nestedSetBuilder().addTransitive(triviallyNonEmpty).build();
+ assertFalse(nonEmptyLevel1.isEmpty());
+
+ NestedSet<String> nonEmptyLevel2 = nestedSetBuilder().addTransitive(nonEmptyLevel1).build();
+ assertFalse(nonEmptyLevel2.isEmpty());
+ }
+
+ @Test
+ public void canIncludeAnyOrderInStableOrderAndViceVersa() {
+ NestedSetBuilder.stableOrder()
+ .addTransitive(NestedSetBuilder.compileOrder()
+ .addTransitive(NestedSetBuilder.stableOrder().build()).build())
+ .addTransitive(NestedSetBuilder.linkOrder()
+ .addTransitive(NestedSetBuilder.stableOrder().build()).build())
+ .addTransitive(NestedSetBuilder.naiveLinkOrder()
+ .addTransitive(NestedSetBuilder.stableOrder().build()).build()).build();
+ try {
+ NestedSetBuilder.compileOrder().addTransitive(NestedSetBuilder.linkOrder().build()).build();
+ fail("Shouldn't be able to include a non-stable order inside a different non-stable order!");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ /**
+ * A handy wrapper that allows us to use EqualsTester to test shallowEquals and shallowHashCode.
+ */
+ private static class SetWrapper<E> {
+ NestedSet<E> set;
+
+ SetWrapper(NestedSet<E> wrapped) {
+ set = wrapped;
+ }
+
+ @Override
+ public int hashCode() {
+ return set.shallowHashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof SetWrapper)) {
+ return false;
+ }
+ try {
+ @SuppressWarnings("unchecked")
+ SetWrapper<E> other = (SetWrapper<E>) o;
+ return set.shallowEquals(other.set);
+ } catch (ClassCastException e) {
+ return false;
+ }
+ }
+ }
+
+ @SafeVarargs
+ private static <E> SetWrapper<E> flat(E... directMembers) {
+ NestedSetBuilder<E> builder = NestedSetBuilder.stableOrder();
+ builder.addAll(Lists.newArrayList(directMembers));
+ return new SetWrapper<E>(builder.build());
+ }
+
+ // Same as flat(), but allows duplicate elements.
+ @SafeVarargs
+ private static <E> SetWrapper<E> flatWithDuplicates(E... directMembers) {
+ return new SetWrapper<E>(
+ NestedSetBuilder.wrap(Order.STABLE_ORDER, ImmutableList.copyOf(directMembers)));
+ }
+
+ @SafeVarargs
+ private static <E> SetWrapper<E> nest(SetWrapper<E>... nested) {
+ NestedSetBuilder<E> builder = NestedSetBuilder.stableOrder();
+ for (SetWrapper<E> wrap : nested) {
+ builder.addTransitive(wrap.set);
+ }
+ return new SetWrapper<E>(builder.build());
+ }
+
+ @SafeVarargs
+ // Restricted to <Integer> to avoid ambiguity with the other nest() function.
+ private static SetWrapper<Integer> nest(Integer elem, SetWrapper<Integer>... nested) {
+ NestedSetBuilder<Integer> builder = NestedSetBuilder.stableOrder();
+ builder.add(elem);
+ for (SetWrapper<Integer> wrap : nested) {
+ builder.addTransitive(wrap.set);
+ }
+ return new SetWrapper<Integer>(builder.build());
+ }
+
+ @Test
+ public void shallowEquality() {
+ // Used below to check that inner nested sets can be compared by reference equality.
+ SetWrapper<Integer> myRef = nest(nest(flat(7, 8)), flat(9));
+
+ // Each "equality group" contains elements that are equal to one another
+ // (according to equals() and hashCode()), yet distinct from all elements
+ // of all other equality groups.
+ new EqualsTester()
+ .addEqualityGroup(flat(),
+ flat(),
+ nest(flat())) // Empty set elision.
+ .addEqualityGroup(NestedSetBuilder.<Integer>linkOrder().build())
+ .addEqualityGroup(flat(3),
+ flat(3),
+ flat(3, 3)) // Element de-duplication.
+ .addEqualityGroup(flatWithDuplicates(3, 3))
+ .addEqualityGroup(flat(4),
+ nest(flat(4))) // Automatic elision of one-element nested sets.
+ .addEqualityGroup(NestedSetBuilder.<Integer>linkOrder().add(4).build())
+ .addEqualityGroup(nestedSetBuilder("4").build()) // Like flat("4").
+ .addEqualityGroup(flat(3, 4),
+ flat(3, 4))
+ // Shallow equality means that {{3},{5}} != {{3},{5}}.
+ .addEqualityGroup(nest(flat(3), flat(5)))
+ .addEqualityGroup(nest(flat(3), flat(5)))
+ .addEqualityGroup(nest(myRef),
+ nest(myRef),
+ nest(myRef, myRef)) // Set de-duplication.
+ .addEqualityGroup(nest(3, myRef))
+ .addEqualityGroup(nest(4, myRef))
+ .testEquals();
+
+ // Some things that are not tested by the above:
+ // - ordering among direct members
+ // - ordering among transitive sets
+ }
+
+ /** Checks that the builder always return a nested set with the correct order. */
+ @Test
+ public void correctOrder() {
+ for (Order order : Order.values()) {
+ for (int numDirects = 0; numDirects < 3; numDirects++) {
+ for (int numTransitives = 0; numTransitives < 3; numTransitives++) {
+ assertEquals(order, createNestedSet(order, numDirects, numTransitives, order).getOrder());
+ // We allow mixing orders if one of them is stable. This tests that the top level order is
+ // the correct one.
+ assertEquals(order,
+ createNestedSet(order, numDirects, numTransitives, Order.STABLE_ORDER).getOrder());
+ }
+ }
+ }
+ }
+
+ private NestedSet<Integer> createNestedSet(Order order, int numDirects, int numTransitives,
+ Order transitiveOrder) {
+ NestedSetBuilder<Integer> builder = new NestedSetBuilder<>(order);
+
+ for (int direct = 0; direct < numDirects; direct++) {
+ builder.add(direct);
+ }
+ for (int transitive = 0; transitive < numTransitives; transitive++) {
+ builder.addTransitive(new NestedSetBuilder<Integer>(transitiveOrder).add(transitive).build());
+ }
+ return builder.build();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifierTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifierTest.java
new file mode 100644
index 0000000..9764f4d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifierTest.java
@@ -0,0 +1,134 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.collect.nestedset;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Tests for {@link RecordingUniqueifier}.
+ */
+@RunWith(JUnit4.class)
+public class RecordingUniqueifierTest extends TestCase {
+
+ private static final Random RANDOM = new Random();
+
+ private static final int VERY_SMALL = 3; // one byte
+ private static final int SMALL = 11; // two bytes
+ private static final int MEDIUM = 18; // three bytes -- unmemoed
+ // For this one, the "* 8" is a bytes to bits (1 memo is 1 bit)
+ private static final int LARGE = (RecordingUniqueifier.LENGTH_THRESHOLD * 8) + 3;
+
+ private static final int[] SIZES = new int[] {VERY_SMALL, SMALL, MEDIUM, LARGE};
+
+ private void doTest(int uniqueInputs, int deterministicHeadSize) throws Exception {
+ Preconditions.checkArgument(deterministicHeadSize <= uniqueInputs,
+ "deterministicHeadSize must be smaller than uniqueInputs");
+
+ // Setup
+
+ List<Integer> inputList = new ArrayList<>(uniqueInputs);
+ Collection<Integer> inputsDeduped = new LinkedHashSet<>(uniqueInputs);
+
+ for (int i = 0; i < deterministicHeadSize; i++) { // deterministic head
+ inputList.add(i);
+ inputsDeduped.add(i);
+ }
+
+ while (inputsDeduped.size() < uniqueInputs) { // random selectees
+ Integer i = RANDOM.nextInt(uniqueInputs);
+ inputList.add(i);
+ inputsDeduped.add(i);
+ }
+
+ // Unmemoed run
+
+ List<Integer> firstList = new ArrayList<>(uniqueInputs);
+ RecordingUniqueifier recordingUniqueifier = new RecordingUniqueifier();
+ for (Integer i : inputList) {
+ if (recordingUniqueifier.isUnique(i)) {
+ firstList.add(i);
+ }
+ }
+
+ // Potentially memo'ed run
+
+ List<Integer> secondList = new ArrayList<>(uniqueInputs);
+ Object memo = recordingUniqueifier.getMemo();
+ Uniqueifier uniqueifier = RecordingUniqueifier.createReplayUniqueifier(memo);
+ for (Integer i : inputList) {
+ if (uniqueifier.isUnique(i)) {
+ secondList.add(i);
+ }
+ }
+
+ // Evaluate results
+
+ inputsDeduped = ImmutableList.copyOf(inputsDeduped);
+ assertEquals("Unmemo'ed run has unexpected contents", inputsDeduped, firstList);
+ assertEquals("Memo'ed run has unexpected contents", inputsDeduped, secondList);
+ }
+
+ private void doTestWithLucidException(int uniqueInputs, int deterministicHeadSize)
+ throws Exception {
+ try {
+ doTest(uniqueInputs, deterministicHeadSize);
+ } catch (Exception e) {
+ throw new Exception("Failure in size: " + uniqueInputs, e);
+ }
+ }
+
+ @Test
+ public void noInputs() throws Exception {
+ doTestWithLucidException(0, 0);
+ }
+
+ @Test
+ public void allUnique() throws Exception {
+ for (int size : SIZES) {
+ doTestWithLucidException(size, size);
+ }
+ }
+
+ @Test
+ public void fuzzedWithDeterministic2() throws Exception {
+ // The way that it is used, we know that the first two additions are not equal.
+ // Optimizations were made for this case in small memos.
+ for (int size : SIZES) {
+ doTestWithLucidException(size, 2);
+ }
+ }
+
+ @Test
+ public void fuzzedWithDeterministic2_otherSizes() throws Exception {
+ for (int i = 0; i < 100; i++) {
+ int size = RANDOM.nextInt(10000) + 2;
+ doTestWithLucidException(size, 2);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitorTest.java b/src/test/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitorTest.java
new file mode 100644
index 0000000..8a6485c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitorTest.java
@@ -0,0 +1,493 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.testutil.TestUtils;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Tests for AbstractQueueVisitor.
+ */
+@RunWith(JUnit4.class)
+public class AbstractQueueVisitorTest {
+
+ private static final RuntimeException THROWABLE = new RuntimeException();
+
+ @Test
+ public void simpleCounter() throws Exception {
+ CountingQueueVisitor counter = new CountingQueueVisitor();
+ counter.enqueue();
+ counter.work(false);
+ assertSame(10, counter.getCount());
+ }
+
+ @Test
+ public void callerOwnedPool() throws Exception {
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 5, 0, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>());
+ assertSame(0, executor.getActiveCount());
+
+ CountingQueueVisitor counter = new CountingQueueVisitor(executor);
+ counter.enqueue();
+ counter.work(false);
+ assertSame(10, counter.getCount());
+
+ executor.shutdown();
+ assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void doubleCounter() throws Exception {
+ CountingQueueVisitor counter = new CountingQueueVisitor();
+ counter.enqueue();
+ counter.enqueue();
+ counter.work(false);
+ assertSame(10, counter.getCount());
+ }
+
+ @Test
+ public void exceptionFromWorkerThread() {
+ final RuntimeException myException = new IllegalStateException();
+ ConcreteQueueVisitor visitor = new ConcreteQueueVisitor();
+ visitor.enqueue(new Runnable() {
+ @Override
+ public void run() {
+ throw myException;
+ }
+ });
+
+ try {
+ // The exception from the worker thread should be
+ // re-thrown from the main thread.
+ visitor.work(false);
+ fail();
+ } catch (Exception e) {
+ assertSame(myException, e);
+ }
+ }
+
+ // Regression test for "AbstractQueueVisitor loses track of jobs if thread allocation fails".
+ @Test
+ public void threadPoolThrowsSometimes() throws Exception {
+ // In certain cases (for example, if the address space is almost entirely consumed by a huge
+ // JVM heap), thread allocation can fail with an OutOfMemoryError. If the queue visitor
+ // does not handle this gracefully, we lose track of tasks and hang the visitor indefinitely.
+
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>()) {
+ private final AtomicLong count = new AtomicLong();
+
+ @Override
+ public void execute(Runnable command) {
+ long count = this.count.incrementAndGet();
+ if (count == 6) {
+ throw new Error("Could not create thread (fakeout)");
+ }
+ super.execute(command);
+ }
+ };
+
+ CountingQueueVisitor counter = new CountingQueueVisitor(executor);
+ counter.enqueue();
+ try {
+ counter.work(false);
+ fail();
+ } catch (Error expected) {
+ assertEquals("Could not create thread (fakeout)", expected.getMessage());
+ }
+ assertSame(5, counter.getCount());
+
+ executor.shutdown();
+ assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+ }
+
+ // Regression test to make sure that AbstractQueueVisitor doesn't swallow unchecked exceptions if
+ // it is interrupted concurrently with the unchecked exception being thrown.
+ @Test
+ public void interruptAndThrownIsInterruptedAndThrown() throws Exception {
+ final ConcreteQueueVisitor visitor = new ConcreteQueueVisitor();
+ // Use a latch to make sure the thread gets a chance to start.
+ final CountDownLatch threadStarted = new CountDownLatch(1);
+ visitor.enqueue(new Runnable() {
+ @Override
+ public void run() {
+ threadStarted.countDown();
+ assertTrue(Uninterruptibles.awaitUninterruptibly(
+ visitor.getInterruptionLatchForTestingOnly(), 2, TimeUnit.SECONDS));
+ throw THROWABLE;
+ }
+ });
+ assertTrue(threadStarted.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ // Interrupt will not be processed until work starts.
+ Thread.currentThread().interrupt();
+ try {
+ visitor.work(/*interruptWorkers=*/true);
+ fail();
+ } catch (Exception e) {
+ assertEquals(THROWABLE, e);
+ assertTrue(Thread.interrupted());
+ }
+ }
+
+ @Test
+ public void interruptionWithoutInterruptingWorkers() throws Exception {
+ final Thread mainThread = Thread.currentThread();
+ final CountDownLatch latch1 = new CountDownLatch(1);
+ final CountDownLatch latch2 = new CountDownLatch(1);
+ final boolean[] workerThreadCompleted = { false };
+ final ConcreteQueueVisitor visitor = new ConcreteQueueVisitor();
+
+ visitor.enqueue(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ latch1.countDown();
+ latch2.await();
+ workerThreadCompleted[0] = true;
+ } catch (InterruptedException e) {
+ // Do not set workerThreadCompleted to true
+ }
+ }
+ });
+
+ TestThread interrupterThread = new TestThread() {
+ @Override
+ public void runTest() throws Exception {
+ latch1.await();
+ mainThread.interrupt();
+ assertTrue(visitor.awaitInterruptionForTestingOnly(TestUtils.WAIT_TIMEOUT_MILLISECONDS,
+ TimeUnit.MILLISECONDS));
+ latch2.countDown();
+ }
+ };
+
+ interrupterThread.start();
+
+ try {
+ visitor.work(false);
+ fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+
+ interrupterThread.joinAndAssertState(400);
+ assertTrue(workerThreadCompleted[0]);
+ }
+
+ @Test
+ public void interruptionWithInterruptingWorkers() throws Exception {
+ assertInterruptWorkers(null);
+
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>());
+ assertInterruptWorkers(executor);
+ executor.shutdown();
+ executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ private void assertInterruptWorkers(ThreadPoolExecutor executor) throws Exception {
+ final CountDownLatch latch1 = new CountDownLatch(1);
+ final CountDownLatch latch2 = new CountDownLatch(1);
+ final boolean[] workerThreadInterrupted = { false };
+ ConcreteQueueVisitor visitor = (executor == null)
+ ? new ConcreteQueueVisitor()
+ : new ConcreteQueueVisitor(executor, true);
+
+ visitor.enqueue(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ latch1.countDown();
+ latch2.await();
+ } catch (InterruptedException e) {
+ workerThreadInterrupted[0] = true;
+ }
+ }
+ });
+
+ latch1.await();
+ Thread.currentThread().interrupt();
+
+ try {
+ visitor.work(true);
+ fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+
+ assertTrue(workerThreadInterrupted[0]);
+ }
+
+ @Test
+ public void failFast() throws Exception {
+ // In failFast mode, we only run actions queued before the exception.
+ assertFailFast(null, true, false, false, "a", "b");
+
+ // In !failFast mode, we complete all queued actions.
+ assertFailFast(null, false, false, false, "a", "b", "1", "2");
+
+ // Now check fail-fast on interrupt:
+ assertFailFast(null, false, true, true, "a", "b");
+ assertFailFast(null, false, false, true, "a", "b", "1", "2");
+ }
+
+ @Test
+ public void failFastNoShutdown() throws Exception {
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 5, 0, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>());
+ // In failFast mode, we only run actions queued before the exception.
+ assertFailFast(executor, true, false, false, "a", "b");
+
+ // In !failFast mode, we complete all queued actions.
+ assertFailFast(executor, false, false, false, "a", "b", "1", "2");
+
+ // Now check fail-fast on interrupt:
+ assertFailFast(executor, false, true, true, "a", "b");
+ assertFailFast(executor, false, false, true, "a", "b", "1", "2");
+
+ executor.shutdown();
+ assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ }
+
+ private void assertFailFast(ThreadPoolExecutor executor,
+ boolean failFastOnException, boolean failFastOnInterrupt,
+ boolean interrupt, String... expectedVisited) throws Exception {
+ assertTrue(executor == null || !executor.isShutdown());
+ AbstractQueueVisitor visitor = (executor == null)
+ ? new ConcreteQueueVisitor(failFastOnException, failFastOnInterrupt)
+ : new ConcreteQueueVisitor(executor, failFastOnException, failFastOnInterrupt);
+
+ List<String> visitedList = Collections.synchronizedList(Lists.<String>newArrayList());
+
+ // Runnable "ra" will await the uncaught exception from
+ // "throwingRunnable", then add "a" to the list and
+ // enqueue "r1". Runnable "r1" should be
+ // executed iff !failFast.
+
+ CountDownLatch latchA = new CountDownLatch(1);
+ CountDownLatch latchB = new CountDownLatch(1);
+
+ Runnable r1 = awaitAddAndEnqueueRunnable(interrupt, visitor, null, visitedList, "1", null);
+ Runnable r2 = awaitAddAndEnqueueRunnable(interrupt, visitor, null, visitedList, "2", null);
+ Runnable ra = awaitAddAndEnqueueRunnable(interrupt, visitor, latchA, visitedList, "a", r1);
+ Runnable rb = awaitAddAndEnqueueRunnable(interrupt, visitor, latchB, visitedList, "b", r2);
+
+ visitor.enqueue(ra);
+ visitor.enqueue(rb);
+ latchA.await();
+ latchB.await();
+ visitor.enqueue(interrupt ? interruptingRunnable(Thread.currentThread()) : throwingRunnable());
+
+ try {
+ visitor.work(false);
+ fail();
+ } catch (Exception e) {
+ if (interrupt) {
+ assertTrue(e instanceof InterruptedException);
+ } else {
+ assertSame(THROWABLE, e);
+ }
+ }
+ assertTrue(
+ "got: " + visitedList + "\nwant: " + Arrays.toString(expectedVisited),
+ Sets.newHashSet(visitedList).equals(Sets.newHashSet(expectedVisited)));
+
+ if (executor != null) {
+ assertFalse(executor.isShutdown());
+ assertEquals(0, visitor.getTaskCount());
+ }
+ }
+
+ @Test
+ public void jobIsInterruptedWhenOtherFails() throws Exception {
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>());
+
+ final QueueVisitorWithCriticalError visitor = new QueueVisitorWithCriticalError(executor);
+ final CountDownLatch latch1 = new CountDownLatch(1);
+ final AtomicBoolean wasInterrupted = new AtomicBoolean(false);
+
+ Runnable r1 = new Runnable() {
+
+ @Override
+ public void run() {
+ latch1.countDown();
+ try {
+ // Interruption is expected during a sleep. There is no sense in fail or assert call
+ // because exception is going to be swallowed inside AbstractQueueVisitior.
+ // We are using wasInterrupted flag to assert in the end of test.
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ wasInterrupted.set(true);
+ }
+ }
+ };
+
+ visitor.enqueue(r1);
+ latch1.await();
+ visitor.enqueue(throwingRunnable());
+
+ try {
+ visitor.work(true);
+ fail();
+ } catch (Exception e) {
+ assertSame(THROWABLE, e);
+ }
+
+ assertTrue(wasInterrupted.get());
+ assertTrue(executor.isShutdown());
+ }
+
+ private Runnable throwingRunnable() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ throw THROWABLE;
+ }
+ };
+ }
+
+ private Runnable interruptingRunnable(final Thread thread) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ thread.interrupt();
+ }
+ };
+ }
+
+ private static Runnable awaitAddAndEnqueueRunnable(final boolean interrupt,
+ final AbstractQueueVisitor visitor,
+ final CountDownLatch started,
+ final List<String> list,
+ final String toAdd,
+ final Runnable toEnqueue) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ if (started != null) {
+ started.countDown();
+ }
+
+ try {
+ assertTrue(interrupt
+ ? visitor.awaitInterruptionForTestingOnly(1, TimeUnit.MINUTES)
+ : visitor.getExceptionLatchForTestingOnly().await(1, TimeUnit.MINUTES));
+ } catch (InterruptedException e) {
+ // Unexpected.
+ throw new RuntimeException(e);
+ }
+ list.add(toAdd);
+ if (toEnqueue != null) {
+ visitor.enqueue(toEnqueue);
+ }
+ }
+ };
+ }
+
+ private static class CountingQueueVisitor extends AbstractQueueVisitor {
+
+ private final static String THREAD_NAME = "BlazeTest CountingQueueVisitor";
+
+ private int theInt = 0;
+ private final Object lock = new Object();
+
+ public CountingQueueVisitor() {
+ super(5, 5, 3L, TimeUnit.SECONDS, THREAD_NAME);
+ }
+
+ public CountingQueueVisitor(ThreadPoolExecutor executor) {
+ super(executor, false, true, true);
+ }
+
+ public void enqueue() {
+ super.enqueue(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (lock) {
+ if (theInt < 10) {
+ theInt++;
+ enqueue();
+ }
+ }
+ }
+ });
+ }
+
+ public int getCount() {
+ return theInt;
+ }
+ }
+
+ private static class ConcreteQueueVisitor extends AbstractQueueVisitor {
+
+ private final static String THREAD_NAME = "BlazeTest ConcreteQueueVisitor";
+
+ public ConcreteQueueVisitor() {
+ super(5, 5, 3L, TimeUnit.SECONDS, THREAD_NAME);
+ }
+
+ public ConcreteQueueVisitor(boolean failFast) {
+ super(true, 5, 5, 3L, TimeUnit.SECONDS, failFast, THREAD_NAME);
+ }
+
+ public ConcreteQueueVisitor(boolean failFast, boolean failFastOnInterrupt) {
+ super(true, 5, 5, 3L, TimeUnit.SECONDS, failFast, failFastOnInterrupt, THREAD_NAME);
+ }
+
+ public ConcreteQueueVisitor(ThreadPoolExecutor executor, boolean failFast,
+ boolean failFastOnInterrupt) {
+ super(executor, /*shutdownOnCompletion=*/false, failFast, failFastOnInterrupt);
+ }
+
+ public ConcreteQueueVisitor(ThreadPoolExecutor executor, boolean failFast) {
+ super(executor, /*shutdownOnCompletion=*/false, failFast, true);
+ }
+ }
+
+ private static class QueueVisitorWithCriticalError extends AbstractQueueVisitor {
+
+ public QueueVisitorWithCriticalError(ThreadPoolExecutor executor) {
+ super(executor, false);
+ }
+
+ @Override
+ protected boolean isCriticalError(Throwable e) {
+ return true;
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/concurrent/MoreFuturesTest.java b/src/test/java/com/google/devtools/build/lib/concurrent/MoreFuturesTest.java
new file mode 100644
index 0000000..60f29ac
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/concurrent/MoreFuturesTest.java
@@ -0,0 +1,151 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.devtools.build.lib.testutil.TestUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests for MoreFutures
+ */
+@RunWith(JUnit4.class)
+public class MoreFuturesTest {
+
+ private ExecutorService executorService;
+
+ @Before
+ public void setUp() throws Exception {
+ executorService = Executors.newFixedThreadPool(5);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ MoreExecutors.shutdownAndAwaitTermination(executorService, TestUtils.WAIT_TIMEOUT_SECONDS,
+ TimeUnit.SECONDS);
+
+ }
+
+ /** Test the normal path where everything is successful. */
+ @Test
+ public void allAsListOrCancelAllHappy() throws ExecutionException, InterruptedException {
+ final List<DelayedFuture> futureList = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ DelayedFuture future = new DelayedFuture(i);
+ executorService.execute(future);
+ futureList.add(future);
+ }
+ ListenableFuture<List<Object>> list = MoreFutures.allAsListOrCancelAll(futureList);
+ List<Object> result = list.get();
+ assertEquals(futureList.size(), result.size());
+ for (DelayedFuture delayedFuture : futureList) {
+ assertFalse(delayedFuture.wasCanceled);
+ assertFalse(delayedFuture.wasInterrupted);
+ assertNotNull(delayedFuture.get());
+ assertTrue(result.contains(delayedFuture.get()));
+ }
+ }
+
+ /** Test that if any of the futures in the list fails, we cancel all the futures immediately. */
+ @Test
+ public void allAsListOrCancelAllCancellation() throws InterruptedException {
+ final List<DelayedFuture> futureList = new ArrayList<>();
+ for (int i = 1; i < 6; i++) {
+ DelayedFuture future = new DelayedFuture(i * 1000);
+ executorService.execute(future);
+ futureList.add(future);
+ }
+ DelayedFuture toFail = new DelayedFuture(1000);
+ futureList.add(toFail);
+ toFail.makeItFail();
+ ListenableFuture<List<Object>> list = MoreFutures.allAsListOrCancelAll(futureList);
+
+ try {
+ list.get();
+ fail("This should fail");
+ } catch (InterruptedException | ExecutionException ignored) {
+ }
+ Thread.sleep(100);
+ for (DelayedFuture delayedFuture : futureList) {
+ assertTrue(delayedFuture.wasCanceled || delayedFuture == toFail);
+ assertFalse(delayedFuture.wasInterrupted);
+ }
+ }
+
+ /**
+ * A future that (if added to an executor) waits {@code delay} milliseconds before setting a
+ * response.
+ */
+ private static class DelayedFuture extends AbstractFuture<Object> implements Runnable {
+
+ private final int delay;
+ private final CountDownLatch latch = new CountDownLatch(1);
+ private boolean wasCanceled;
+ private boolean wasInterrupted;
+
+ public DelayedFuture(int delay) {
+ this.delay = delay;
+ }
+
+ @Override
+ public void run() {
+ try {
+ wasCanceled = latch.await(delay, TimeUnit.MILLISECONDS);
+ // Not canceled and not done (makeItFail sets the value, so in that case is done).
+ if (!wasCanceled && !isDone()) {
+ set(new Object());
+ }
+ } catch (InterruptedException e) {
+ wasInterrupted = true;
+ }
+ }
+
+ public void makeItFail() {
+ setException(new RuntimeException("I like to fail!!"));
+ latch.countDown();
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return super.cancel(mayInterruptIfRunning);
+ }
+
+ @Override
+ protected void interruptTask() {
+ latch.countDown();
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/concurrent/ThreadSafetyTest.java b/src/test/java/com/google/devtools/build/lib/concurrent/ThreadSafetyTest.java
new file mode 100644
index 0000000..8532bae
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/concurrent/ThreadSafetyTest.java
@@ -0,0 +1,313 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.concurrent;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This file just contains some examples of the use of
+ * annotations for different categories of thread safety:
+ * ThreadSafe
+ * ThreadCompatible
+ * ThreadHostile
+ * Immutable ThreadSafe
+ * Immutable ThreadHostile
+ *
+ * It doesn't really test much -- just that this code
+ * using those annotations compiles and runs.
+ *
+ * The main class here is annotated as being both ConditionallyThreadSafe
+ * and ConditionallyThreadCompatible, and accordingly we document here the
+ * conditions under which it is thread-safe and thread-compatible:
+ * - it is thread-safe if you only use the testThreadSafety() method,
+ * the ThreadSafeCounter class, and/or ImmutableThreadSafeCounter class;
+ * - it is thread-compatible if you use only those and/or the
+ * ThreadCompatibleCounter and/or ImmutableThreadCompatibleCounter class;
+ * - it is thread-hostile otherwise.
+ */
+@ConditionallyThreadSafe @ConditionallyThreadCompatible
+@RunWith(JUnit4.class)
+public class ThreadSafetyTest {
+
+ @ThreadSafe
+ public static final class ThreadSafeCounter {
+
+ // A ThreadSafe class can have public mutable fields,
+ // provided they are atomic or volatile.
+
+ public volatile boolean myBool;
+ public AtomicInteger myInt;
+
+ // A ThreadSafe class can have private mutable fields,
+ // provided that access to them is synchronized.
+ private int value;
+ public ThreadSafeCounter(int value) {
+ synchronized (this) { // is this needed?
+ this.value = value;
+ }
+ }
+ public synchronized int getValue() {
+ return value;
+ }
+ public synchronized void increment() {
+ value++;
+ }
+
+ // A ThreadSafe class can have private mutable members
+ // provided that the methods of the class synchronize access
+ // to them.
+ // These members could be static...
+ private static int numFoos = 0;
+ public static synchronized void foo() {
+ numFoos++;
+ }
+ public static synchronized int getNumFoos() {
+ return numFoos;
+ }
+ // ... or non-static.
+ private int numBars = 0;
+ public synchronized void bar() {
+ numBars++;
+ }
+ public synchronized int getNumBars() {
+ return numBars;
+ }
+ }
+
+ @ThreadCompatible
+ public static final class ThreadCompatibleCounter {
+
+ // A ThreadCompatible class can have public mutable fields.
+ public int value;
+ public ThreadCompatibleCounter(int value) {
+ this.value = value;
+ }
+ public int getValue() {
+ return value;
+ }
+ public void increment() {
+ value++;
+ }
+
+ // A ThreadCompatible class can have mutable static members
+ // provided that the methods of the class synchronize access
+ // to them.
+ private static int numFoos = 0;
+ public static synchronized void foo() {
+ numFoos++;
+ }
+ public static synchronized int getNumFoos() {
+ return numFoos;
+ }
+ }
+
+ @ThreadHostile
+ public static final class ThreadHostileCounter {
+
+ // A ThreadHostile class can have public mutable fields.
+ public int value;
+ public ThreadHostileCounter(int value) {
+ this.value = value;
+ }
+ public int getValue() {
+ return value;
+ }
+ public void increment() {
+ value++;
+ }
+
+ // A ThreadHostile class can perform unsynchronized access
+ // to mutable static data.
+ private static int numFoos = 0;
+ public static void foo() {
+ numFoos++;
+ }
+ public static int getNumFoos() {
+ return numFoos;
+ }
+ }
+
+ @Immutable @ThreadSafe
+ public static final class ImmutableThreadSafeCounter {
+
+ // An Immutable ThreadSafe class can have public fields,
+ // provided they are final and immutable.
+ public final int value;
+ public ImmutableThreadSafeCounter(int value) {
+ this.value = value;
+ }
+ public int getValue() {
+ return value;
+ }
+ public ImmutableThreadSafeCounter increment() {
+ return new ImmutableThreadSafeCounter(value + 1);
+ }
+
+ // An Immutable ThreadSafe class can have immutable static members.
+ public static final int NUM_STATIC_CACHE_ENTRIES = 3;
+ private static final ImmutableThreadSafeCounter[] staticCache =
+ new ImmutableThreadSafeCounter[] {
+ new ImmutableThreadSafeCounter(0),
+ new ImmutableThreadSafeCounter(1),
+ new ImmutableThreadSafeCounter(2)
+ };
+ public static ImmutableThreadSafeCounter makeUsingStaticCache(int value) {
+ if (value < NUM_STATIC_CACHE_ENTRIES) {
+ return staticCache[value];
+ } else {
+ return new ImmutableThreadSafeCounter(value);
+ }
+ }
+
+ // An Immutable ThreadSafe class can have private mutable members
+ // provided that the methods of the class synchronize access
+ // to them.
+ // These members could be static...
+ private static int cachedValue = 0;
+ private static ImmutableThreadSafeCounter cachedCounter =
+ new ImmutableThreadSafeCounter(0);
+ public static synchronized ImmutableThreadSafeCounter
+ makeUsingDynamicCache(int value) {
+ if (value != cachedValue) {
+ cachedValue = value;
+ cachedCounter = new ImmutableThreadSafeCounter(value);
+ }
+ return cachedCounter;
+ }
+ // ... or non-static.
+ private ImmutableThreadSafeCounter incrementCache = null;
+ public synchronized ImmutableThreadSafeCounter incrementUsingCache() {
+ if (incrementCache == null) {
+ incrementCache = new ImmutableThreadSafeCounter(value + 1);
+ }
+ return incrementCache;
+ }
+ // Methods of an Immutable class need not be deterministic.
+ private static Random random = new Random();
+ public int choose() {
+ return random.nextInt(value);
+ }
+ }
+
+ @Immutable @ThreadHostile
+ public static final class ImmutableThreadHostileCounter {
+
+ // An Immutable ThreadHostile class can have public fields,
+ // provided they are final and immutable.
+ public final int value;
+ public ImmutableThreadHostileCounter(int value) {
+ this.value = value;
+ }
+ public int getValue() {
+ return value;
+ }
+ public ImmutableThreadHostileCounter increment() {
+ return new ImmutableThreadHostileCounter(value + 1);
+ }
+
+ // An Immutable ThreadHostile class can have private mutable members,
+ // and doesn't need to synchronize access to them.
+ // These members could be static...
+ private static int cachedValue = 0;
+ private static ImmutableThreadHostileCounter cachedCounter =
+ new ImmutableThreadHostileCounter(0);
+ public static ImmutableThreadHostileCounter
+ makeUsingDynamicCache(int value) {
+ if (value != cachedValue) {
+ cachedValue = value;
+ cachedCounter = new ImmutableThreadHostileCounter(value);
+ }
+ return cachedCounter;
+ }
+ // ... or non-static.
+ private ImmutableThreadHostileCounter incrementCache = null;
+ public ImmutableThreadHostileCounter incrementUsingCache() {
+ if (incrementCache == null) {
+ incrementCache = new ImmutableThreadHostileCounter(value + 1);
+ }
+ return incrementCache;
+ }
+ }
+
+ @Test
+ public void threadSafety() throws InterruptedException {
+ final ThreadSafeCounter threadSafeCounterArray[] =
+ new ThreadSafeCounter[] {
+ new ThreadSafeCounter(1),
+ new ThreadSafeCounter(2),
+ new ThreadSafeCounter(3)
+ };
+ final ThreadCompatibleCounter threadCompatibleCounterArray[] =
+ new ThreadCompatibleCounter[] {
+ new ThreadCompatibleCounter(1),
+ new ThreadCompatibleCounter(2),
+ new ThreadCompatibleCounter(3)
+ };
+ final ThreadHostileCounter threadHostileCounter =
+ new ThreadHostileCounter(1);
+
+ class MyThread implements Runnable {
+
+ ThreadCompatibleCounter threadCompatibleCounter =
+ new ThreadCompatibleCounter(1);
+
+ @Override
+ public void run() {
+
+ // ThreadSafe objects can be accessed with without synchronization
+ for (ThreadSafeCounter counter : threadSafeCounterArray) {
+ counter.increment();
+ }
+
+ // ThreadCompatible objects can be accessed with without
+ // synchronization if they are thread-local
+ threadCompatibleCounter.increment();
+
+ // Access to ThreadCompatible objects must be synchronized
+ // if they could be concurrently accessed by other threads
+ for (ThreadCompatibleCounter counter : threadCompatibleCounterArray) {
+ synchronized (counter) {
+ counter.increment();
+ }
+ }
+
+ // Access to ThreadHostile objects must be synchronized.
+ synchronized (this.getClass()) {
+ threadHostileCounter.increment();
+ }
+
+ }
+ }
+
+ Thread thread1 = new Thread(new MyThread());
+ Thread thread2 = new Thread(new MyThread());
+ thread1.start();
+ thread2.start();
+ thread1.join();
+ thread2.join();
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/AbstractEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/AbstractEventHandlerTest.java
new file mode 100644
index 0000000..7033f17
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/AbstractEventHandlerTest.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Set;
+
+/**
+ * Tests {@link AbstractEventHandler}.
+ */
+@RunWith(JUnit4.class)
+public class AbstractEventHandlerTest {
+
+ private static AbstractEventHandler create(Set<EventKind> mask) {
+ return new AbstractEventHandler(mask) {
+ @Override
+ public void handle(Event event) {}
+ };
+ }
+
+ @Test
+ public void retainsEventMask() {
+ assertEquals(EventKind.ALL_EVENTS,
+ create(EventKind.ALL_EVENTS).getEventMask());
+ assertEquals(EventKind.ERRORS_AND_WARNINGS,
+ create(EventKind.ERRORS_AND_WARNINGS).getEventMask());
+ assertEquals(EventKind.ERRORS,
+ create(EventKind.ERRORS).getEventMask());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/EventCollectorTest.java b/src/test/java/com/google/devtools/build/lib/events/EventCollectorTest.java
new file mode 100644
index 0000000..332afac
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/EventCollectorTest.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+
+import com.google.devtools.build.lib.events.Event;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * Tests the {@link EventCollector} class.
+ */
+@RunWith(JUnit4.class)
+public class EventCollectorTest extends EventTestTemplate {
+
+ @Test
+ public void usesPassedInCollection() {
+ Collection<Event> events = new ArrayList<>();
+ EventCollector collector =
+ new EventCollector(EventKind.ERRORS_AND_WARNINGS, events);
+ collector.handle(event);
+ Event onlyEvent = events.iterator().next();
+ assertEquals(event.getMessage(), onlyEvent.getMessage());
+ assertSame(location, onlyEvent.getLocation());
+ assertEquals(event.getKind(), onlyEvent.getKind());
+ assertEquals(event.getLocation().getStartOffset(),
+ onlyEvent.getLocation().getStartOffset());
+ assertEquals(collector.count(), 1);
+ assertEquals(events.size(), 1);
+ }
+
+ @Test
+ public void collectsEvents() {
+ EventCollector collector =
+ new EventCollector(EventKind.ERRORS_AND_WARNINGS);
+ collector.handle(event);
+ Iterator<Event> collectedEventIt = collector.iterator();
+ Event onlyEvent = collectedEventIt.next();
+ assertEquals(event.getMessage(), onlyEvent.getMessage());
+ assertSame(location, onlyEvent.getLocation());
+ assertEquals(event.getKind(), onlyEvent.getKind());
+ assertEquals(event.getLocation().getStartOffset(),
+ onlyEvent.getLocation().getStartOffset());
+ assertFalse(collectedEventIt.hasNext());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/EventSensorTest.java b/src/test/java/com/google/devtools/build/lib/events/EventSensorTest.java
new file mode 100644
index 0000000..3c97b37
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/EventSensorTest.java
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test for {@link EventSensor}.
+ */
+@RunWith(JUnit4.class)
+public class EventSensorTest extends EventTestTemplate {
+
+ @Test
+ public void sensorStartsOutWithFalse() {
+ assertFalse(new EventSensor(EventKind.ALL_EVENTS).wasTriggered());
+ assertFalse(new EventSensor(EventKind.ERRORS).wasTriggered());
+ assertFalse(new EventSensor(EventKind.ERRORS_AND_WARNINGS).wasTriggered());
+ }
+
+ @Test
+ public void sensorNoticesEventsInItsMask() {
+ EventSensor sensor = new EventSensor(EventKind.ERRORS);
+ Reporter reporter = new Reporter(sensor);
+ reporter.handle(Event.error(location, "An ERROR event."));
+ assertTrue(sensor.wasTriggered());
+ }
+
+ @Test
+ public void sensorNoticesEventsInItsMask2() {
+ EventSensor sensor = new EventSensor(EventKind.ALL_EVENTS);
+ Reporter reporter = new Reporter(sensor);
+ reporter.handle(Event.error(location, "An ERROR event."));
+ reporter.handle(Event.warn(location, "A warning event."));
+ assertTrue(sensor.wasTriggered());
+ }
+
+ @Test
+ public void sensorIgnoresEventsNotInItsMask() {
+ EventSensor sensor = new EventSensor(EventKind.ERRORS_AND_WARNINGS);
+ Reporter reporter = new Reporter(sensor);
+ reporter.handle(Event.info(location, "An INFO event."));
+ assertFalse(sensor.wasTriggered());
+ }
+
+ @Test
+ public void sensorCanCount() {
+ EventSensor sensor = new EventSensor(EventKind.ERRORS_AND_WARNINGS);
+ Reporter reporter = new Reporter(sensor);
+ reporter.handle(Event.error(location, "An ERROR event."));
+ reporter.handle(Event.error(location, "Another ERROR event."));
+ reporter.handle(Event.warn(location, "A warning event."));
+ reporter.handle(Event.info(location, "An info event.")); // not in mask
+ assertEquals(3, sensor.getTriggerCount());
+ assertTrue(sensor.wasTriggered());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/EventTest.java b/src/test/java/com/google/devtools/build/lib/events/EventTest.java
new file mode 100644
index 0000000..50fe88a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/EventTest.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A super simple little test for the {@link Event} class.
+ */
+@RunWith(JUnit4.class)
+public class EventTest extends EventTestTemplate {
+
+ @Test
+ public void eventRetainsEventKind() {
+ assertEquals(EventKind.WARNING, event.getKind());
+ }
+
+ @Test
+ public void eventRetainsMessage() {
+ assertEquals("This is not an error message.", event.getMessage());
+ }
+
+ @Test
+ public void eventRetainsLocation() {
+ assertEquals(21, event.getLocation().getStartOffset());
+ assertEquals(31, event.getLocation().getEndOffset());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/EventTestTemplate.java b/src/test/java/com/google/devtools/build/lib/events/EventTestTemplate.java
new file mode 100644
index 0000000..612cdf0
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/EventTestTemplate.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import com.google.devtools.build.lib.events.Location.LineAndColumn;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Before;
+
+public abstract class EventTestTemplate {
+
+ protected Event event;
+ protected Path path;
+ protected Location location;
+ protected Location locationNoPath;
+ protected Location locationNoLineInfo;
+
+ private FsApparatus scratch = FsApparatus.newInMemory();
+
+ @Before
+ public void setUp() throws Exception {
+ String message = "This is not an error message.";
+ path = scratch.path("/my/sample/path.txt");
+
+ location = Location.fromPathAndStartColumn(path, 21, 31, new LineAndColumn(3, 4));
+
+ event = new Event(EventKind.WARNING, location, message);
+
+ locationNoPath = Location.fromPathAndStartColumn(null, 21, 31, new LineAndColumn(3, 4));
+
+ locationNoLineInfo = Location.fromFileAndOffsets(path, 21, 31);
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/LocationTest.java b/src/test/java/com/google/devtools/build/lib/events/LocationTest.java
new file mode 100644
index 0000000..a585b0c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/LocationTest.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class LocationTest extends EventTestTemplate {
+
+ @Test
+ public void fromFile() throws Exception {
+ Location location = Location.fromFile(path);
+ assertEquals(path.asFragment(), location.getPath());
+ assertEquals(0, location.getStartOffset());
+ assertEquals(0, location.getEndOffset());
+ assertNull(location.getStartLineAndColumn());
+ assertNull(location.getEndLineAndColumn());
+ assertEquals(path + ":1", location.print());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/PrintingEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/PrintingEventHandlerTest.java
new file mode 100644
index 0000000..4cdfcb4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/PrintingEventHandlerTest.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.util.io.RecordingOutErr;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests the {@link PrintingEventHandler}.
+ */
+@RunWith(JUnit4.class)
+public class PrintingEventHandlerTest extends EventTestTemplate {
+
+ @Test
+ public void collectsEvents() {
+ RecordingOutErr recordingOutErr = new RecordingOutErr();
+ PrintingEventHandler handler = new PrintingEventHandler(EventKind.ERRORS_AND_WARNINGS);
+ handler.setOutErr(recordingOutErr);
+ handler.handle(event);
+ MoreAsserts.assertEqualsUnifyingLineEnds("WARNING: /my/sample/path.txt:3:4: "
+ + "This is not an error message.\n",
+ recordingOutErr.errAsLatin1());
+ assertEquals("", recordingOutErr.outAsLatin1());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/ReporterStreamTest.java b/src/test/java/com/google/devtools/build/lib/events/ReporterStreamTest.java
new file mode 100644
index 0000000..092d940
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/ReporterStreamTest.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.PrintWriter;
+
+@RunWith(JUnit4.class)
+public class ReporterStreamTest {
+
+ private Reporter reporter;
+ private StringBuilder out;
+ private EventHandler outAppender;
+
+ @Before
+ public void setUp() throws Exception {
+ reporter = new Reporter();
+ out = new StringBuilder();
+ outAppender = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ out.append("[" + event.getKind() + ": " + event.getMessage() + "]\n");
+ }
+ };
+ }
+
+ @Test
+ public void reporterStream() throws Exception {
+ assertEquals("", out.toString());
+ reporter.addHandler(outAppender);
+ PrintWriter infoWriter = new PrintWriter(new ReporterStream(reporter, EventKind.INFO), true);
+ PrintWriter warnWriter = new PrintWriter(new ReporterStream(reporter, EventKind.WARNING), true);
+ try {
+ infoWriter.println("some info");
+ warnWriter.println("a warning");
+ } finally {
+ infoWriter.close();
+ warnWriter.close();
+ }
+ reporter.getOutErr().printOutLn("some output");
+ reporter.getOutErr().printErrLn("an error");
+ MoreAsserts.assertEqualsUnifyingLineEnds(
+ "[INFO: some info\n]\n"
+ + "[WARNING: a warning\n]\n"
+ + "[STDOUT: some output\n]\n"
+ + "[STDERR: an error\n]\n",
+ out.toString());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/ReporterTest.java b/src/test/java/com/google/devtools/build/lib/events/ReporterTest.java
new file mode 100644
index 0000000..f51451a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/ReporterTest.java
@@ -0,0 +1,100 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests the {@link Reporter} class.
+ */
+@RunWith(JUnit4.class)
+public class ReporterTest extends EventTestTemplate {
+
+ private Reporter reporter;
+ private StringBuilder out;
+ private AbstractEventHandler outAppender;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ reporter = new Reporter();
+ out = new StringBuilder();
+ outAppender = new AbstractEventHandler(EventKind.ERRORS) {
+ @Override
+ public void handle(Event event) {
+ out.append(event.getMessage());
+ }
+ };
+ }
+
+ @Test
+ public void reporterShowOutput() {
+ reporter.setOutputFilter(OutputFilter.RegexOutputFilter.forRegex("naughty"));
+ EventCollector collector = new EventCollector(EventKind.ALL_EVENTS);
+ reporter.addHandler(collector);
+ Event interesting = new Event(EventKind.WARNING, null, "show-me", "naughty");
+
+ reporter.handle(interesting);
+ reporter.handle(new Event(EventKind.WARNING, null, "ignore-me", "good"));
+
+ assertEquals(ImmutableList.copyOf(collector.iterator()), ImmutableList.of(interesting));
+ }
+
+ @Test
+ public void reporterCollectsEvents() {
+ ImmutableList<Event> want = ImmutableList.of(Event.warn("xyz"), Event.error("err"));
+ EventCollector collector = new EventCollector(EventKind.ALL_EVENTS);
+ reporter.addHandler(collector);
+ for (Event e : want) {
+ reporter.handle(e);
+ }
+ ImmutableList<Event> got = ImmutableList.copyOf(collector.iterator());
+ assertEquals(got, want);
+ }
+
+ @Test
+ public void reporterCopyConstructorCopiesHandlersList() {
+ reporter.addHandler(outAppender);
+ reporter.addHandler(outAppender);
+ Reporter copiedReporter = new Reporter(reporter);
+ copiedReporter.addHandler(outAppender); // Should have 3 handlers now.
+ reporter.addHandler(outAppender);
+ reporter.addHandler(outAppender); // Should have 4 handlers now.
+ copiedReporter.handle(Event.error(location, "."));
+ assertEquals("...", out.toString()); // The copied reporter has 3 handlers.
+ out = new StringBuilder();
+ reporter.handle(Event.error(location, "."));
+ assertEquals("....", out.toString()); // The old reporter has 4 handlers.
+ }
+
+ @Test
+ public void removeHandlerUndoesAddHandler() {
+ assertEquals("", out.toString());
+ reporter.addHandler(outAppender);
+ reporter.handle(Event.error(location, "Event gets registered."));
+ assertEquals("Event gets registered.", out.toString());
+ out = new StringBuilder();
+ reporter.removeHandler(outAppender);
+ reporter.handle(Event.error(location, "Event gets ignored."));
+ assertEquals("", out.toString());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/SimpleReportersTest.java b/src/test/java/com/google/devtools/build/lib/events/SimpleReportersTest.java
new file mode 100644
index 0000000..deed44f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/SimpleReportersTest.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test for {@link Reporter}.
+ */
+@RunWith(JUnit4.class)
+public class SimpleReportersTest extends EventTestTemplate {
+
+ private int handlerCount = 0;
+
+ @Test
+ public void addsHandlers() {
+ EventHandler handler = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ handlerCount++;
+ }
+
+ };
+
+ Reporter reporter = new Reporter(handler);
+ reporter.handle(new Event(EventKind.INFO, location, "Add to handlerCount."));
+ reporter.handle(new Event(EventKind.INFO, location, "Add to handlerCount."));
+ reporter.handle(new Event(EventKind.INFO, location, "Add to handlerCount."));
+ assertEquals(3, handlerCount);
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/StoredErrorEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/StoredErrorEventHandlerTest.java
new file mode 100644
index 0000000..d47cf86
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/StoredErrorEventHandlerTest.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+
+/**
+ * Tests the {@link StoredEventHandler} class.
+ */
+@RunWith(JUnit4.class)
+public class StoredErrorEventHandlerTest {
+
+ @Test
+ public void hasErrors() {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ assertFalse(eventHandler.hasErrors());
+ eventHandler.handle(Event.warn("warning"));
+ assertFalse(eventHandler.hasErrors());
+ eventHandler.handle(Event.info("info"));
+ assertFalse(eventHandler.hasErrors());
+ eventHandler.handle(Event.error("error"));
+ assertTrue(eventHandler.hasErrors());
+ }
+
+ @Test
+ public void replayOnWithoutEvents() {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ StoredEventHandler sink = new StoredEventHandler();
+
+ eventHandler.replayOn(sink);
+ assertTrue(sink.isEmpty());
+ }
+
+ @Test
+ public void replayOn() {
+ StoredEventHandler eventHandler = new StoredEventHandler();
+ StoredEventHandler sink = new StoredEventHandler();
+
+ List<Event> events = ImmutableList.of(
+ Event.warn("a"),
+ Event.error("b"),
+ Event.info("c"),
+ Event.warn("d"));
+ for (Event e : events) {
+ eventHandler.handle(e);
+ }
+
+ eventHandler.replayOn(sink);
+ assertEquals(events.size(), sink.getEvents().size());
+ for (int i = 0; i < events.size(); i++) {
+ assertEquals(events.get(i), sink.getEvents().get(i));
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandlerTest.java
new file mode 100644
index 0000000..ce6b1a9
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandlerTest.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests the {@link StoredEventHandler} class.
+ */
+@RunWith(JUnit4.class)
+public class WarningsAsErrorsEventHandlerTest {
+
+ @Test
+ public void hasErrors() {
+ ErrorSensingEventHandler delegate =
+ new ErrorSensingEventHandler(NullEventHandler.INSTANCE);
+ WarningsAsErrorsEventHandler eventHandler =
+ new WarningsAsErrorsEventHandler(delegate);
+
+ eventHandler.handle(Event.info("info"));
+ assertFalse(delegate.hasErrors());
+
+ eventHandler.handle(Event.warn("warning"));
+ assertTrue(delegate.hasErrors());
+
+ delegate.resetErrors();
+
+ eventHandler.handle(Event.error("error"));
+ assertTrue(delegate.hasErrors());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java b/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java
new file mode 100644
index 0000000..1513735
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java
@@ -0,0 +1,149 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events.util;
+
+import static com.google.common.truth.Truth.assertWithMessage;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventCollector;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.PrintingEventHandler;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.testutil.JunitTestUtils;
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * An apparatus for reporting / collecting events.
+ */
+public class EventCollectionApparatus {
+
+ /**
+ * The fail fast handler, which fails the test fail whenever we encounter
+ * an error event.
+ */
+ private static final EventHandler FAIL_FAST_HANDLER = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ assertWithMessage(event.toString()).that(EventKind.ERRORS_AND_WARNINGS)
+ .doesNotContain(event.getKind());
+ }
+ };
+ private Set<EventKind> customMask;
+
+ /*
+ * Determine which events the {@link #collector()} created by this apparatus
+ * will collect. Default: {@link EventKind#ERRORS_AND_WARNINGS}.
+ *
+ */
+ public EventCollectionApparatus(Set<EventKind> mask) {
+ this.customMask = mask;
+
+ eventCollector = new EventCollector(customMask);
+ reporter = new Reporter(eventCollector);
+ printingEventHandler = new PrintingEventHandler(EventKind.ERRORS_AND_WARNINGS_AND_OUTPUT);
+ reporter.addHandler(printingEventHandler);
+
+ this.setFailFast(true);
+ }
+
+ public EventCollectionApparatus() {
+ this(EventKind.ERRORS_AND_WARNINGS);
+ }
+
+ /* ---- Settings for the apparatus (configuration for creating state) ---- */
+
+ /* ---------- State that the apparatus initializes / operates on --------- */
+ private EventCollector eventCollector;
+ private Reporter reporter;
+ private PrintingEventHandler printingEventHandler;
+
+ /**
+ * Determine whether the {#link reporter()} created by this apparatus will
+ * fail fast, that is, throw an exception whenever we encounter an event of
+ * matching {@link EventKind#ERRORS_AND_WARNINGS}.
+ * Default: {@code true}.
+ */
+ public void setFailFast(boolean failFast) {
+ if (failFast) {
+ reporter.addHandler(FAIL_FAST_HANDLER);
+ } else {
+ reporter.removeHandler(FAIL_FAST_HANDLER);
+ }
+ }
+
+ /**
+ * Initializes the apparatus (if it's not been initialized yet) and returns
+ * the reporter created with the settings specified by this apparatus.
+ */
+ public Reporter reporter() {
+ return reporter;
+ }
+
+ /**
+ * Initializes the apparatus (if it's not been initialized yet) and returns
+ * the collector created with the settings specified by this apparatus.
+ */
+ public EventCollector collector() {
+ return eventCollector;
+ }
+
+ /**
+ * Redirects all output to the specified OutErr stream pair.
+ * Returns the previous OutErr.
+ */
+ public OutErr setOutErr(OutErr outErr) {
+ return printingEventHandler.setOutErr(outErr);
+ }
+
+ /**
+ * Utility method: Asserts that the {@link #collector()} has not collected
+ * any events.
+ *
+ * @throws IllegalStateException If the apparatus has not yet been
+ * initialized by calling {@link #reporter()} or {@link #collector()}.
+ */
+ public void assertNoEvents() {
+ JunitTestUtils.assertNoEvents(eventCollector);
+ }
+
+ /**
+ * Utility method: Assert that the {@link #collector()} has received an
+ * event with the {@code expectedMessage}.
+ */
+ public Event assertContainsEvent(String expectedMessage) {
+ return JunitTestUtils.assertContainsEvent(eventCollector,
+ expectedMessage);
+ }
+
+ public List<Event> assertContainsEventWithFrequency(String expectedMessage,
+ int expectedFrequency) {
+ return JunitTestUtils.assertContainsEventWithFrequency(eventCollector, expectedMessage,
+ expectedFrequency);
+ }
+
+ /**
+ * Utility method: Assert that the {@link #collector()} has received an
+ * event with the {@code expectedMessage} in quotes.
+ */
+
+ public Event assertContainsEventWithWordsInQuotes(String... words) {
+ return JunitTestUtils.assertContainsEventWithWordsInQuotes(
+ eventCollector, words);
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/events/util/LocationTestingUtil.java b/src/test/java/com/google/devtools/build/lib/events/util/LocationTestingUtil.java
new file mode 100644
index 0000000..66aaf30
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/events/util/LocationTestingUtil.java
@@ -0,0 +1,35 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.events.util;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.events.Location;
+
+/**
+ * Static utility methods for testing Locations.
+ */
+public class LocationTestingUtil {
+
+ private LocationTestingUtil() {
+ }
+
+ public static void assertEqualLocations(Location expected, Location actual) {
+ assertThat(actual.getStartOffset()).isEqualTo(expected.getStartOffset());
+ assertThat(actual.getStartLineAndColumn()).isEqualTo(expected.getStartLineAndColumn());
+ assertThat(actual.getEndOffset()).isEqualTo(expected.getEndOffset());
+ assertThat(actual.getEndLineAndColumn()).isEqualTo(expected.getEndLineAndColumn());
+ assertThat(actual.getPath()).isEqualTo(expected.getPath());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestSuiteBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestSuiteBuilder.java
new file mode 100644
index 0000000..d2f36a6
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestSuiteBuilder.java
@@ -0,0 +1,123 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.base.Predicate;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A base class for constructing test suites by searching the classpath for
+ * tests, possibly restricted to a predicate.
+ */
+public class BlazeTestSuiteBuilder {
+
+ /**
+ * @return a TestSuiteBuilder configured for Blaze.
+ */
+ protected TestSuiteBuilder getBuilder() {
+ return new TestSuiteBuilder()
+ .addPackageRecursive("com.google.devtools.build.lib");
+ }
+
+ /** A predicate that succeeds only for LARGE tests. */
+ public static final Predicate<Class<?>> TEST_IS_LARGE =
+ hasSize(Suite.LARGE_TESTS);
+
+ /** A predicate that succeeds only for MEDIUM tests. */
+ public static final Predicate<Class<?>> TEST_IS_MEDIUM =
+ hasSize(Suite.MEDIUM_TESTS);
+
+ /** A predicate that succeeds only for SMALL tests. */
+ public static final Predicate<Class<?>> TEST_IS_SMALL =
+ hasSize(Suite.SMALL_TESTS);
+
+ /** A predicate that succeeds only for non-flaky tests. */
+ public static final Predicate<Class<?>> TEST_IS_FLAKY = new Predicate<Class<?>>() {
+ @Override
+ public boolean apply(Class<?> testClass) {
+ return Suite.isFlaky(testClass);
+ }
+ };
+
+ private static Predicate<Class<?>> hasSize(final Suite size) {
+ return new Predicate<Class<?>>() {
+ @Override
+ public boolean apply(Class<?> testClass) {
+ return Suite.getSize(testClass) == size;
+ }
+ };
+ }
+
+ protected static Predicate<Class<?>> inSuite(final String suiteName) {
+ return new Predicate<Class<?>>() {
+ @Override
+ public boolean apply(Class<?> testClass) {
+ return Suite.getSuiteName(testClass).equalsIgnoreCase(suiteName);
+ }
+ };
+ }
+
+ /**
+ * Given a TestCase subclass, returns its designated suite annotation, if
+ * any, or the empty string otherwise.
+ */
+ public static String getSuite(Class<?> clazz) {
+ TestSpec spec = clazz.getAnnotation(TestSpec.class);
+ return spec == null ? "" : spec.suite();
+ }
+
+ /**
+ * Returns a predicate over TestCases that is true iff the TestCase has a
+ * TestSpec annotation whose suite="..." value (a comma-separated list of
+ * tags) matches all of the query operators specified in the system property
+ * {@code blaze.suite}. The latter is also a comma-separated list, but of
+ * query operators, each of which is either the name of a tag which must be
+ * present (e.g. "foo"), or the !-prefixed name of a tag that must be absent
+ * (e.g. "!foo").
+ */
+ public static Predicate<Class<?>> matchesSuiteQuery() {
+ final String suiteProperty = System.getProperty("blaze.suite");
+ if (suiteProperty == null) {
+ throw new IllegalArgumentException("blaze.suite property not found");
+ }
+ final Set<String> queryTokens = splitCommas(suiteProperty);
+ return new Predicate<Class<?>>() {
+ @Override
+ public boolean apply(Class<?> testClass) {
+ // Return true iff every queryToken is satisfied by suiteTags.
+ Set<String> suiteTags = splitCommas(getSuite(testClass));
+ for (String queryToken : queryTokens) {
+ if (queryToken.startsWith("!")) { // forbidden tag
+ if (suiteTags.contains(queryToken.substring(1))) {
+ return false;
+ }
+ } else { // mandatory tag
+ if (!suiteTags.contains(queryToken)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ };
+ }
+
+ private static Set<String> splitCommas(String s) {
+ return new HashSet<>(Arrays.asList(s.split(",")));
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestUtils.java b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestUtils.java
new file mode 100644
index 0000000..8588a08
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestUtils.java
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * Some static utility functions for testing Blaze code. In contrast to {@link TestUtils}, these
+ * functions are Blaze-specific.
+ */
+public class BlazeTestUtils {
+ private BlazeTestUtils() {}
+
+ /**
+ * Populates the _embedded_binaries/ directory, containing all binaries/libraries, by symlinking
+ * directories#getEmbeddedBinariesRoot() to the test's runfiles tree.
+ */
+ public static BinTools getIntegrationBinTools(BlazeDirectories directories) throws IOException {
+ Path embeddedDir = directories.getEmbeddedBinariesRoot();
+ FileSystemUtils.createDirectoryAndParents(embeddedDir);
+
+ Path runfiles = directories.getFileSystem().getPath(BlazeTestUtils.runfilesDir());
+ // Copy over everything in embedded_scripts.
+ Path embeddedScripts = runfiles.getRelative(TestConstants.EMBEDDED_SCRIPTS_PATH);
+ Collection<Path> files = new ArrayList<Path>();
+ if (embeddedScripts.exists()) {
+ files.addAll(embeddedScripts.getDirectoryEntries());
+ } else {
+ System.err.println("test does not have " + embeddedScripts);
+ }
+
+ for (Path fromFile : files) {
+ try {
+ embeddedDir.getChild(fromFile.getBaseName()).createSymbolicLink(fromFile);
+ } catch (IOException e) {
+ System.err.println("Could not symlink: " + e.getMessage());
+ }
+ }
+
+ return BinTools.forIntegrationTesting(
+ directories, embeddedDir.toString(), TestConstants.EMBEDDED_TOOLS);
+ }
+
+ /**
+ * Writes a FilesetRule to a String array.
+ *
+ * @param name the name of the rule.
+ * @param out the output directory.
+ * @param entries The FilesetEntry entries.
+ * @return the String array of the rule. One String for each line.
+ */
+ public static String[] createFilesetRule(String name, String out, String... entries) {
+ return new String[] {
+ String.format("Fileset(name = '%s', out = '%s',", name, out),
+ " entries = [" + Joiner.on(", ").join(entries) + "])"
+ };
+ }
+
+ public static File undeclaredOutputDir() {
+ String dir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR");
+ if (dir != null) {
+ return new File(dir);
+ }
+
+ return TestUtils.tmpDirFile();
+ }
+
+ public static String runfilesDir() {
+ String runfilesDirStr = TestUtils.getUserValue("TEST_SRCDIR");
+ Preconditions.checkState(runfilesDirStr != null && runfilesDirStr.length() > 0,
+ "TEST_SRCDIR unset or empty");
+ return new File(runfilesDirStr).getAbsolutePath();
+ }
+
+ /** Creates an empty file, along with all its parent directories. */
+ public static void makeEmptyFile(Path path) throws IOException {
+ FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
+ FileSystemUtils.createEmptyFile(path);
+ }
+
+ /**
+ * Changes the mtime of the file "path", which must exist. No guarantee is
+ * made about the new mtime except that it is different from the previous one.
+ *
+ * @throws IOException if the mtime could not be read or set.
+ */
+ public static void changeModtime(Path path)
+ throws IOException {
+ long prevMtime = path.getLastModifiedTime();
+ long newMtime = prevMtime;
+ do {
+ newMtime += 1000;
+ path.setLastModifiedTime(newMtime);
+ } while (path.getLastModifiedTime() == prevMtime);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleBuilder.java
new file mode 100644
index 0000000..9f770c5
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleBuilder.java
@@ -0,0 +1,202 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.packages.RuleClass;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility for quickly creating BUILD file rules for use in tests.
+ *
+ * <p>The use case for this class is writing BUILD files where simple
+ * readability for the sake of rules' relationship to the test framework
+ * is more important than detailed semantics and layout.
+ *
+ * <p>The behavior provided by this class is not meant to be exhaustive,
+ * but should handle a majority of simple cases.
+ *
+ * <p>Example:
+ *
+ * <pre>
+ * String text = new BuildRuleBuilder("java_library", "MyRule")
+ .setSources("First.java", "Second.java", "Third.java")
+ .setDeps(":library", "//java/com/google/common/collect")
+ .setResources("schema/myschema.xsd")
+ .build();
+ * </pre>
+ *
+ */
+public class BuildRuleBuilder {
+ protected final RuleClass ruleClass;
+ protected final String ruleName;
+ private Map<String, List<String>> multiValueAttributes;
+ private Map<String, Object> singleValueAttributes;
+ protected Map<String, RuleClass> ruleClassMap;
+
+ /**
+ * Create a new instance.
+ *
+ * @param ruleClass the rule class of the new rule
+ * @param ruleName the name of the new rule.
+ */
+ public BuildRuleBuilder(String ruleClass, String ruleName) {
+ this(ruleClass, ruleName, getDefaultRuleClassMap());
+ }
+
+ protected static Map<String, RuleClass> getDefaultRuleClassMap() {
+ return TestRuleClassProvider.getRuleClassProvider().getRuleClassMap();
+ }
+
+ public BuildRuleBuilder(String ruleClass, String ruleName, Map<String, RuleClass> ruleClassMap) {
+ this.ruleClass = ruleClassMap.get(ruleClass);
+ this.ruleName = ruleName;
+ this.multiValueAttributes = new HashMap<>();
+ this.singleValueAttributes = new HashMap<>();
+ this.ruleClassMap = ruleClassMap;
+ }
+
+ /**
+ * Sets the value of a single valued attribute
+ */
+ public BuildRuleBuilder setSingleValueAttribute(String attrName, Object value) {
+ Preconditions.checkState(!singleValueAttributes.containsKey(attrName),
+ "attribute '" + attrName + "' already set");
+ singleValueAttributes.put(attrName, value);
+ return this;
+ }
+
+ /**
+ * Sets the value of a list type attribute
+ */
+ public BuildRuleBuilder setMultiValueAttribute(String attrName, String... value) {
+ Preconditions.checkState(!multiValueAttributes.containsKey(attrName),
+ "attribute '" + attrName + "' already set");
+ multiValueAttributes.put(attrName, Lists.newArrayList(value));
+ return this;
+ }
+
+ /**
+ * Set the srcs attribute.
+ */
+ public BuildRuleBuilder setSources(String... sources) {
+ return setMultiValueAttribute("srcs", sources);
+ }
+
+ /**
+ * Set the deps attribute.
+ */
+ public BuildRuleBuilder setDeps(String... deps) {
+ return setMultiValueAttribute("deps", deps);
+ }
+
+ /**
+ * Set the resources attribute.
+ */
+ public BuildRuleBuilder setResources(String... resources) {
+ return setMultiValueAttribute("resources", resources);
+ }
+
+ /**
+ * Set the data attribute.
+ */
+ public BuildRuleBuilder setData(String... data) {
+ return setMultiValueAttribute("data", data);
+ }
+
+ /**
+ * Generate the rule
+ *
+ * @return a string representation of the rule.
+ */
+ public String build() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(ruleClass.getName()).append("(");
+ printNormal(sb, "name", ruleName);
+ for (Map.Entry<String, List<String>> entry : multiValueAttributes.entrySet()) {
+ printArray(sb, entry.getKey(), entry.getValue());
+ }
+ for (Map.Entry<String, Object> entry : singleValueAttributes.entrySet()) {
+ printNormal(sb, entry.getKey(), entry.getValue());
+ }
+ sb.append(")\n");
+ return sb.toString();
+ }
+
+ private void printArray(StringBuilder sb, String attr, List<String> values) {
+ if (values == null || values.isEmpty()) {
+ return;
+ }
+ sb.append(" ").append(attr).append(" = ");
+ printList(sb, values);
+ sb.append(",");
+ sb.append("\n");
+ }
+
+ private void printNormal(StringBuilder sb, String attr, Object value) {
+ if (value == null) {
+ return;
+ }
+ sb.append(" ").append(attr).append(" = ");
+ if (value instanceof Integer) {
+ sb.append(value);
+ } else {
+ sb.append("'").append(value).append("'");
+ }
+ sb.append(",");
+ sb.append("\n");
+ }
+
+ /**
+ * Turns iterable of {a b c} into string "['a', 'b', 'c']", appends to
+ * supplied StringBuilder.
+ */
+ private void printList(StringBuilder sb, List<String> elements) {
+ sb.append("[");
+ Joiner.on(",").appendTo(sb,
+ Iterables.transform(elements, new Function<String, String>() {
+ @Override
+ public String apply(String from) {
+ return "'" + from + "'";
+ }
+ }));
+ sb.append("]");
+ }
+
+ /**
+ * Returns the transitive closure of file names need to be generated in order
+ * for this rule to build.
+ */
+ public Collection<String> getFilesToGenerate() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the transitive closure of BuildRuleBuilders need to be generated in order
+ * for this rule to build.
+ */
+ public Collection<BuildRuleBuilder> getRulesToGenerate() {
+ return ImmutableList.of();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleWithDefaultsBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleWithDefaultsBuilder.java
new file mode 100644
index 0000000..4af7ad1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleWithDefaultsBuilder.java
@@ -0,0 +1,231 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Attribute.AllowedValueSet;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A helper class to generate valid rules with filled attributes if necessary.
+ */
+public class BuildRuleWithDefaultsBuilder extends BuildRuleBuilder {
+
+ private Set<String> generateFiles;
+ private Map<String, BuildRuleBuilder> generateRules;
+
+ public BuildRuleWithDefaultsBuilder(String ruleClass, String ruleName) {
+ super(ruleClass, ruleName);
+ this.generateFiles = new HashSet<>();
+ this.generateRules = new HashMap<>();
+ }
+
+ private BuildRuleWithDefaultsBuilder(String ruleClass, String ruleName,
+ Map<String, RuleClass> ruleClassMap, Set<String> generateFiles,
+ Map<String, BuildRuleBuilder> generateRules) {
+ super(ruleClass, ruleName, ruleClassMap);
+ this.generateFiles = generateFiles;
+ this.generateRules = generateRules;
+ }
+
+ /**
+ * Creates a dummy file with the given extension in the given package and returns a valid Blaze
+ * label referring to the file. Note, the created label depends on the package of the rule.
+ */
+ private String getDummyFileLabel(String rulePkg, String filePkg, String extension,
+ Type<?> attrType) {
+ boolean isInput = (attrType == Type.LABEL || attrType == Type.LABEL_LIST);
+ String fileName = (isInput ? "dummy_input" : "dummy_output") + extension;
+ generateFiles.add(filePkg + "/" + fileName);
+ if (rulePkg.equals(filePkg)) {
+ return ":" + fileName;
+ } else {
+ return filePkg + ":" + fileName;
+ }
+ }
+
+ private String getDummyRuleLabel(String rulePkg, RuleClass referencedRuleClass) {
+ String referencedRuleName = ruleName + "_ref_" + referencedRuleClass.getName()
+ .replace("$", "").replace(":", "");
+ // The new generated rule should have the same generatedFiles and generatedRules
+ // in order to avoid duplications
+ BuildRuleWithDefaultsBuilder builder = new BuildRuleWithDefaultsBuilder(
+ referencedRuleClass.getName(), referencedRuleName, ruleClassMap, generateFiles,
+ generateRules);
+ builder.popuplateAttributes(rulePkg, true);
+ generateRules.put(referencedRuleClass.getName(), builder);
+ return referencedRuleName;
+ }
+
+ public BuildRuleWithDefaultsBuilder popuplateLabelAttribute(String pkg, Attribute attribute) {
+ return popuplateLabelAttribute(pkg, pkg, attribute);
+ }
+
+ /**
+ * Populates the label type attribute with generated values. Populates with a file if possible, or
+ * generates an appropriate rule. Note, that the rules are always generated in the same package.
+ */
+ public BuildRuleWithDefaultsBuilder popuplateLabelAttribute(String rulePkg, String filePkg,
+ Attribute attribute) {
+ Type<?> attrType = attribute.getType();
+ String label = null;
+ if (attribute.getAllowedFileTypesPredicate() != FileTypeSet.NO_FILE) {
+ // Try to populate with files first
+ String extension = null;
+ if (attribute.getAllowedFileTypesPredicate() == FileTypeSet.ANY_FILE) {
+ extension = ".txt";
+ } else {
+ FileTypeSet fileTypes = attribute.getAllowedFileTypesPredicate();
+ // This argument should always hold, if not that means a Blaze design/implementation error
+ Preconditions.checkArgument(fileTypes.getExtensions().size() > 0);
+ extension = fileTypes.getExtensions().get(0);
+ }
+ label = getDummyFileLabel(rulePkg, filePkg, extension, attrType);
+ } else {
+ Predicate<RuleClass> allowedRuleClasses = attribute.getAllowedRuleClassesPredicate();
+ if (allowedRuleClasses != Predicates.<RuleClass>alwaysFalse()) {
+ // See if there is an applicable rule among the already enqueued rules
+ BuildRuleBuilder referencedRuleBuilder = getFirstApplicableRule(allowedRuleClasses);
+ if (referencedRuleBuilder != null) {
+ label = ":" + referencedRuleBuilder.ruleName;
+ } else {
+ RuleClass referencedRuleClass = getFirstApplicableRuleClass(allowedRuleClasses);
+ if (referencedRuleClass != null) {
+ // Generate a rule with the appropriate ruleClass and a label for it in
+ // the original rule
+ label = ":" + getDummyRuleLabel(rulePkg, referencedRuleClass);
+ }
+ }
+ }
+ }
+ if (label != null) {
+ if (attrType == Type.LABEL_LIST || attrType == Type.OUTPUT_LIST) {
+ setMultiValueAttribute(attribute.getName(), label);
+ } else {
+ setSingleValueAttribute(attribute.getName(), label);
+ }
+ }
+ return this;
+ }
+
+ private BuildRuleBuilder getFirstApplicableRule(Predicate<RuleClass> allowedRuleClasses) {
+ // There is no direct way to get the set of allowedRuleClasses from the Attribute
+ // The Attribute API probably should not be modified for sole testing purposes
+ for (Map.Entry<String, BuildRuleBuilder> entry : generateRules.entrySet()) {
+ if (allowedRuleClasses.apply(ruleClassMap.get(entry.getKey()))) {
+ return entry.getValue();
+ }
+ }
+ return null;
+ }
+
+ private RuleClass getFirstApplicableRuleClass(Predicate<RuleClass> allowedRuleClasses) {
+ // See comments in getFirstApplicableRule(Predicate<RuleClass> allowedRuleClasses)
+ for (RuleClass ruleClass : ruleClassMap.values()) {
+ if (allowedRuleClasses.apply(ruleClass)) {
+ return ruleClass;
+ }
+ }
+ return null;
+ }
+
+ public BuildRuleWithDefaultsBuilder popuplateStringListAttribute(Attribute attribute) {
+ setMultiValueAttribute(attribute.getName(), "x");
+ return this;
+ }
+
+ public BuildRuleWithDefaultsBuilder popuplateStringAttribute(Attribute attribute) {
+ setSingleValueAttribute(attribute.getName(), "x");
+ return this;
+ }
+
+ public BuildRuleWithDefaultsBuilder popuplateBooleanAttribute(Attribute attribute) {
+ setSingleValueAttribute(attribute.getName(), "false");
+ return this;
+ }
+
+ public BuildRuleWithDefaultsBuilder popuplateIntegerAttribute(Attribute attribute) {
+ setSingleValueAttribute(attribute.getName(), 1);
+ return this;
+ }
+
+ public BuildRuleWithDefaultsBuilder popuplateAttributes(String rulePkg, boolean heuristics) {
+ for (Attribute attribute : ruleClass.getAttributes()) {
+ if (attribute.isMandatory()) {
+ if (attribute.getType() == Type.LABEL_LIST || attribute.getType() == Type.OUTPUT_LIST) {
+ if (attribute.isNonEmpty()) {
+ popuplateLabelAttribute(rulePkg, attribute);
+ } else {
+ // TODO(bazel-team): actually here an empty list would be fine, but BuildRuleBuilder
+ // doesn't support that, and it makes little sense anyway
+ popuplateLabelAttribute(rulePkg, attribute);
+ }
+ } else if (attribute.getType() == Type.LABEL || attribute.getType() == Type.OUTPUT) {
+ popuplateLabelAttribute(rulePkg, attribute);
+ } else {
+ // Non label type attributes
+ if (attribute.getAllowedValues() instanceof AllowedValueSet) {
+ Collection<Object> allowedValues =
+ ((AllowedValueSet) attribute.getAllowedValues()).getAllowedValues();
+ setSingleValueAttribute(attribute.getName(), allowedValues.iterator().next());
+ } else if (attribute.getType() == Type.STRING) {
+ popuplateStringAttribute(attribute);
+ } else if (attribute.getType() == Type.BOOLEAN) {
+ popuplateBooleanAttribute(attribute);
+ } else if (attribute.getType() == Type.INTEGER) {
+ popuplateIntegerAttribute(attribute);
+ } else if (attribute.getType() == Type.STRING_LIST) {
+ popuplateStringListAttribute(attribute);
+ }
+ }
+ // TODO(bazel-team): populate for other data types
+ } else if (heuristics) {
+ populateAttributesHeuristics(rulePkg, attribute);
+ }
+ }
+ return this;
+ }
+
+ // Heuristics which might help to generate valid rules.
+ // This is a bit hackish, but it helps some generated ruleclasses to pass analysis phase.
+ private void populateAttributesHeuristics(String rulePkg, Attribute attribute) {
+ if (attribute.getName().equals("srcs") && attribute.getType() == Type.LABEL_LIST) {
+ // If there is a srcs attribute it might be better to populate it even if it's not mandatory
+ popuplateLabelAttribute(rulePkg, attribute);
+ } else if (attribute.getName().equals("main_class") && attribute.getType() == Type.STRING) {
+ popuplateStringAttribute(attribute);
+ }
+ }
+
+ @Override
+ public Collection<String> getFilesToGenerate() {
+ return generateFiles;
+ }
+
+ @Override
+ public Collection<BuildRuleBuilder> getRulesToGenerate() {
+ return generateRules.values();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ChattyAssertsTestCase.java b/src/test/java/com/google/devtools/build/lib/testutil/ChattyAssertsTestCase.java
new file mode 100644
index 0000000..f17d81e
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/ChattyAssertsTestCase.java
@@ -0,0 +1,237 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.ExitCode;
+
+import junit.framework.TestCase;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Most of this stuff is copied from junit's {@link junit.framework.Assert}
+ * class, and then customized to make the error messages a bit more informative.
+ */
+public abstract class ChattyAssertsTestCase extends TestCase {
+ private long currentTestStartTime = -1;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ currentTestStartTime = BlazeClock.instance().currentTimeMillis();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ JunitTestUtils.nullifyInstanceFields(this);
+ assertFalse("tearDown without setUp!", currentTestStartTime == -1);
+
+ super.tearDown();
+ }
+
+ /**
+ * Asserts that two objects are equal. If they are not
+ * an AssertionFailedError is thrown with the given message.
+ */
+ public static void assertEquals(String message, Object expected,
+ Object actual) {
+ if (Objects.equals(expected, actual)) {
+ return;
+ }
+ chattyFailNotEquals(message, expected, actual);
+ }
+
+ /**
+ * Asserts that two objects are equal. If they are not
+ * an AssertionFailedError is thrown.
+ */
+ public static void assertEquals(Object expected, Object actual) {
+ assertEquals(null, expected, actual);
+ }
+
+ /**
+ * Asserts that two Strings are equal.
+ */
+ public static void assertEquals(String message, String expected, String actual) {
+ assertWithMessage(message).that(actual).isEqualTo(expected);
+ }
+
+ /**
+ * Asserts that two Strings are equal.
+ */
+ public static void assertEquals(String expected, String actual) {
+ assertEquals(null, expected, actual);
+ }
+
+ /**
+ * Asserts that two Strings are equal considering the line separator to be \n
+ * independently of the operating system.
+ */
+ public static void assertEqualsUnifyingLineEnds(String expected, String actual) {
+ MoreAsserts.assertEqualsUnifyingLineEnds(expected, actual);
+ }
+
+ private static void chattyFailNotEquals(String message, Object expected,
+ Object actual) {
+ fail(MoreAsserts.chattyFormat(message, expected, actual));
+ }
+
+ /**
+ * Asserts that {@code e}'s exception message contains each of {@code strings}
+ * <b>surrounded by single quotation marks</b>.
+ */
+ public static void assertMessageContainsWordsWithQuotes(Exception e,
+ String... strings) {
+ assertContainsWordsWithQuotes(e.getMessage(), strings);
+ }
+
+ /**
+ * Asserts that {@code message} contains each of {@code strings}
+ * <b>surrounded by single quotation marks</b>.
+ */
+ public static void assertContainsWordsWithQuotes(String message,
+ String... strings) {
+ MoreAsserts.assertContainsWordsWithQuotes(message, strings);
+ }
+
+ public static void assertNonZeroExitCode(int exitCode, String stdout, String stderr) {
+ MoreAsserts.assertNonZeroExitCode(exitCode, stdout, stderr);
+ }
+
+ public static void assertZeroExitCode(int exitCode, String stdout, String stderr) {
+ assertExitCode(0, exitCode, stdout, stderr);
+ }
+
+ public static void assertExitCode(ExitCode expectedExitCode,
+ int exitCode, String stdout, String stderr) {
+ int expectedExitCodeValue = expectedExitCode.getNumericExitCode();
+ if (exitCode != expectedExitCodeValue) {
+ fail(String.format("expected exit code '%s' <%d> but exit code was <%d> and stdout was <%s> "
+ + "and stderr was <%s>",
+ expectedExitCode.name(), expectedExitCodeValue, exitCode, stdout, stderr));
+ }
+ }
+
+ public static void assertExitCode(int expectedExitCode,
+ int exitCode, String stdout, String stderr) {
+ MoreAsserts.assertExitCode(expectedExitCode, exitCode, stdout, stderr);
+ }
+
+ public static void assertStdoutContainsString(String expected, String stdout, String stderr) {
+ MoreAsserts.assertStdoutContainsString(expected, stdout, stderr);
+ }
+
+ public static void assertStderrContainsString(String expected, String stdout, String stderr) {
+ MoreAsserts.assertStderrContainsString(expected, stdout, stderr);
+ }
+
+ public static void assertStdoutContainsRegex(String expectedRegex,
+ String stdout, String stderr) {
+ MoreAsserts.assertStdoutContainsRegex(expectedRegex, stdout, stderr);
+ }
+
+ public static void assertStderrContainsRegex(String expectedRegex,
+ String stdout, String stderr) {
+ MoreAsserts.assertStderrContainsRegex(expectedRegex, stdout, stderr);
+ }
+
+
+
+ /********************************************************************
+ * *
+ * Other testing utilities (unrelated to "chattiness") *
+ * *
+ ********************************************************************/
+
+ /**
+ * Returns the elements from the given collection in a set.
+ */
+ protected static <T> Set<T> asSet(Iterable<T> collection) {
+ return Sets.newHashSet(collection);
+ }
+
+ /**
+ * Returns the arguments given as varargs as a set.
+ */
+ @SuppressWarnings({"unchecked", "varargs"})
+ protected static <T> Set<T> asSet(T... elements) {
+ return Sets.newHashSet(elements);
+ }
+
+ /**
+ * Returns the arguments given as varargs as a set of sorted Strings.
+ */
+ protected static Set<String> asStringSet(Iterable<?> collection) {
+ return MoreAsserts.asStringSet(collection);
+ }
+
+ /**
+ * An equivalence relation for Collection, based on mapping to Set.
+ *
+ * Oft-forgotten fact: for all x in Set, y in List, !x.equals(y) even if
+ * their elements are the same.
+ */
+ protected static <T> void
+ assertSameContents(Iterable<? extends T> expected, Iterable<? extends T> actual) {
+ MoreAsserts.assertSameContents(expected, actual);
+ }
+
+ /**
+ * Asserts the presence or absence of values in the collection.
+ */
+ protected <T> void assertPresence(Iterable<T> actual, Iterable<Presence<T>> expectedPresences) {
+ for (Presence<T> expected : expectedPresences) {
+ if (expected.presence) {
+ assertThat(actual).contains(expected.value);
+ } else {
+ assertThat(actual).doesNotContain(expected.value);
+ }
+ }
+ }
+
+ /** Creates a presence information with expected value. */
+ protected static <T> Presence<T> present(T expected) {
+ return new Presence<>(expected, true);
+ }
+
+ /** Creates an absence information with expected value. */
+ protected static <T> Presence<T> absent(T expected) {
+ return new Presence<>(expected, false);
+ }
+
+ /**
+ * Combines value with the boolean presence flag.
+ *
+ * @param <T> value type
+ */
+ protected final static class Presence <T> {
+ /** wrapped value */
+ public final T value;
+ /** boolean presence flag */
+ public final boolean presence;
+
+ /** Creates a tuple of value and a boolean presence flag. */
+ Presence(T value, boolean presence) {
+ this.value = value;
+ this.presence = presence;
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/Classpath.java b/src/test/java/com/google/devtools/build/lib/testutil/Classpath.java
new file mode 100644
index 0000000..94711ac
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/Classpath.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.base.Preconditions;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Enumeration;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+/**
+ * A helper class to find all classes on the current classpath. This is used to automatically create
+ * JUnit 3 and 4 test suites.
+ */
+final class Classpath {
+ private static final String CLASS_EXTENSION = ".class";
+
+ /**
+ * Finds all classes that live in or below the given package.
+ */
+ static Set<Class<?>> findClasses(String packageName) {
+ Set<Class<?>> result = new LinkedHashSet<>();
+ String pathPrefix = (packageName + '.').replace('.', '/');
+ for (String entryName : getClassPath()) {
+ File classPathEntry = new File(entryName);
+ if (classPathEntry.exists()) {
+ try {
+ Set<String> classNames;
+ if (classPathEntry.isDirectory()) {
+ classNames = findClassesInDirectory(classPathEntry, pathPrefix);
+ } else {
+ classNames = findClassesInJar(classPathEntry, pathPrefix);
+ }
+ for (String className : classNames) {
+ Class<?> clazz = Class.forName(className);
+ result.add(clazz);
+ }
+ } catch (IOException e) {
+ throw new AssertionError("Can't read classpath entry "
+ + entryName + ": " + e.getMessage());
+ } catch (ClassNotFoundException e) {
+ throw new AssertionError("Class not found even though it is on the classpath "
+ + entryName + ": " + e.getMessage());
+ }
+ }
+ }
+ return result;
+ }
+
+ private static Set<String> findClassesInDirectory(File classPathEntry, String pathPrefix) {
+ Set<String> result = new TreeSet<>();
+ File directory = new File(classPathEntry, pathPrefix);
+ innerFindClassesInDirectory(result, directory, pathPrefix);
+ return result;
+ }
+
+ /**
+ * Finds all classes and sub packages in the given directory that are below the given package and
+ * add them to the respective sets.
+ *
+ * @param directory Directory to inspect
+ * @param pathPrefix Prefix for the path to the classes that are requested
+ * (ex: {@code com/google/foo/bar})
+ */
+ private static void innerFindClassesInDirectory(Set<String> classNames, File directory,
+ String pathPrefix) {
+ Preconditions.checkArgument(pathPrefix.endsWith("/"));
+ if (directory.exists()) {
+ for (File f : directory.listFiles()) {
+ String name = f.getName();
+ if (name.endsWith(CLASS_EXTENSION)) {
+ String clzName = getClassName(pathPrefix + name);
+ classNames.add(clzName);
+ } else if (f.isDirectory()) {
+ findClassesInDirectory(f, pathPrefix + name + "/");
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns a set of all classes in the jar that start with the given prefix.
+ */
+ private static Set<String> findClassesInJar(File jarFile, String pathPrefix) throws IOException {
+ Set<String> classNames = new TreeSet<>();
+ try (ZipFile zipFile = new ZipFile(jarFile)) {
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ String entryName = entries.nextElement().getName();
+ if (entryName.startsWith(pathPrefix) && entryName.endsWith(CLASS_EXTENSION)) {
+ classNames.add(getClassName(entryName));
+ }
+ }
+ }
+ return classNames;
+ }
+
+ /**
+ * Given the absolute path of a class file, return the class name.
+ */
+ private static String getClassName(String className) {
+ int classNameEnd = className.length() - CLASS_EXTENSION.length();
+ return className.substring(0, classNameEnd).replace('/', '.');
+ }
+
+ /**
+ * Gets the class path from the System Property "java.class.path" and splits
+ * it up into the individual elements.
+ */
+ private static String[] getClassPath() {
+ String classPath = System.getProperty("java.class.path");
+ String separator = System.getProperty("path.separator", ":");
+ return classPath.split(Pattern.quote(separator));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ClasspathSuite.java b/src/test/java/com/google/devtools/build/lib/testutil/ClasspathSuite.java
new file mode 100644
index 0000000..ee880fc
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/ClasspathSuite.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import org.junit.runners.Suite;
+import org.junit.runners.model.RunnerBuilder;
+
+import java.util.Set;
+
+/**
+ * A suite implementation that finds all JUnit 3 and 4 classes on the current classpath in or below
+ * the package of the annotated class, except classes that are annotated with {@code ClasspathSuite}
+ * or {@link CustomSuite}.
+ *
+ * <p>If you need to specify a custom test class filter or a different package prefix, then use
+ * {@link CustomSuite} instead.
+ */
+public final class ClasspathSuite extends Suite {
+
+ /**
+ * Only called reflectively. Do not use programmatically.
+ */
+ public ClasspathSuite(Class<?> klass, RunnerBuilder builder) throws Throwable {
+ super(builder, klass, getClasses(klass));
+ }
+
+ private static Class<?>[] getClasses(Class<?> klass) {
+ Set<Class<?>> result = new TestSuiteBuilder().addPackageRecursive(klass.getPackage().getName())
+ .create();
+ return result.toArray(new Class<?>[result.size()]);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/CustomSuite.java b/src/test/java/com/google/devtools/build/lib/testutil/CustomSuite.java
new file mode 100644
index 0000000..6e3b6c5
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/CustomSuite.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import org.junit.runners.Suite;
+import org.junit.runners.model.RunnerBuilder;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Set;
+
+/**
+ * A JUnit4 suite implementation that delegates the class finding to a {@code suite()} method on the
+ * annotated class. To be used in combination with {@link TestSuiteBuilder}.
+ */
+public final class CustomSuite extends Suite {
+
+ /**
+ * Only called reflectively. Do not use programmatically.
+ */
+ public CustomSuite(Class<?> klass, RunnerBuilder builder) throws Throwable {
+ super(builder, klass, getClasses(klass));
+ }
+
+ private static Class<?>[] getClasses(Class<?> klass) {
+ Set<Class<?>> result = evalSuite(klass);
+ return result.toArray(new Class<?>[result.size()]);
+ }
+
+ @SuppressWarnings("unchecked") // unchecked cast to a generic type
+ private static Set<Class<?>> evalSuite(Class<?> klass) {
+ try {
+ Method m = klass.getMethod("suite");
+ if (!Modifier.isStatic(m.getModifiers())) {
+ throw new IllegalStateException("suite() must be static");
+ }
+ return (Set<Class<?>>) m.invoke(null);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/DebuggingEventHandler.java b/src/test/java/com/google/devtools/build/lib/testutil/DebuggingEventHandler.java
new file mode 100644
index 0000000..59fe5fb
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/DebuggingEventHandler.java
@@ -0,0 +1,41 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+
+import java.io.PrintStream;
+
+/**
+ * Prints all errors and warnings to {@link System#out}.
+ */
+public class DebuggingEventHandler implements EventHandler {
+
+ private PrintStream out;
+
+ public DebuggingEventHandler() {
+ this.out = System.out;
+ }
+
+ @Override
+ public void handle(Event e) {
+ if (e.getLocation() != null) {
+ out.println(e.getKind() + " " + e.getLocation() + ": " + e.getMessage());
+ } else {
+ out.println(e.getKind() + " " + e.getMessage());
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/FoundationTestCase.java b/src/test/java/com/google/devtools/build/lib/testutil/FoundationTestCase.java
new file mode 100644
index 0000000..5c04612
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/FoundationTestCase.java
@@ -0,0 +1,264 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventCollector;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This is a specialization of {@link ChattyAssertsTestCase} that's useful for
+ * implementing tests of the "foundation" library.
+ */
+public abstract class FoundationTestCase extends ChattyAssertsTestCase {
+
+ protected Path rootDirectory;
+
+ protected Path outputBase;
+
+ protected Path actionOutputBase;
+
+ // May be overridden by subclasses:
+ protected Reporter reporter;
+ protected EventCollector eventCollector;
+
+ private Scratch scratch;
+
+
+ // Individual tests can opt-out of this handler if they expect an error, by
+ // calling reporter.removeHandler(failFastHandler).
+ protected static final EventHandler failFastHandler = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ if (EventKind.ERRORS.contains(event.getKind())) {
+ fail(event.toString());
+ }
+ }
+ };
+
+ protected static final EventHandler printHandler = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ System.out.println(event);
+ }
+ };
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ scratch = new Scratch(createFileSystem());
+ outputBase = scratchDir("/usr/local/google/_blaze_jrluser/FAKEMD5/");
+ rootDirectory = scratchDir("/" + TestConstants.TEST_WORKSPACE_DIRECTORY);
+ copySkylarkFilesIfExist();
+ actionOutputBase = scratchDir("/usr/local/google/_blaze_jrluser/FAKEMD5/action_out/");
+ eventCollector = new EventCollector(EventKind.ERRORS_AND_WARNINGS);
+ reporter = new Reporter(eventCollector);
+ reporter.addHandler(failFastHandler);
+ }
+
+ /*
+ * Creates the file system; override to inject FS behavior.
+ */
+ protected FileSystem createFileSystem() {
+ return new InMemoryFileSystem(BlazeClock.instance());
+ }
+
+
+ private void copySkylarkFilesIfExist() throws IOException {
+ scratchFile(rootDirectory.getRelative("devtools/blaze/rules/BUILD").getPathString());
+ scratchFile(rootDirectory.getRelative("rules/BUILD").getPathString());
+ copySkylarkFilesIfExist("devtools/blaze/rules/staging", "devtools/blaze/rules");
+ copySkylarkFilesIfExist("devtools/blaze/bazel/base_workspace/tools/build_rules", "rules");
+ }
+
+ private void copySkylarkFilesIfExist(String from, String to) throws IOException {
+ File rulesDir = new File(from);
+ if (rulesDir.exists() && rulesDir.isDirectory()) {
+ for (String fileName : rulesDir.list()) {
+ File file = new File(from + "/" + fileName);
+ if (file.isFile() && fileName.endsWith(".bzl")) {
+ String context = loadFile(file);
+ Path path = rootDirectory.getRelative(to + "/" + fileName);
+ if (path.exists()) {
+ overwriteScratchFile(path.getPathString(), context);
+ } else {
+ scratchFile(path.getPathString(), context);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ Thread.interrupted(); // Clear any interrupt pending against this thread,
+ // so that we don't cause later tests to fail.
+
+ super.tearDown();
+ }
+
+ /**
+ * A scratch filesystem that is completely in-memory. Since this file system
+ * is "cached" in a private (but *not* static) field in the test class,
+ * each testFoo method in junit sees a fresh filesystem.
+ */
+ protected FileSystem scratchFS() {
+ return scratch.getFileSystem();
+ }
+
+ /**
+ * Create a scratch file in the scratch filesystem, with the given pathName,
+ * consisting of a set of lines. The method returns a Path instance for the
+ * scratch file.
+ */
+ protected Path scratchFile(String pathName, String... lines)
+ throws IOException {
+ return scratch.file(pathName, lines);
+ }
+
+ /**
+ * Like {@code scratchFile}, but the file is first deleted if it already
+ * exists.
+ */
+ protected Path overwriteScratchFile(String pathName, String... lines) throws IOException {
+ return scratch.overwriteFile(pathName, lines);
+ }
+
+ /**
+ * Deletes the specified scratch file, using the same specification as {@link Path#delete}.
+ */
+ protected boolean deleteScratchFile(String pathName) throws IOException {
+ return scratch.deleteFile(pathName);
+ }
+
+ /**
+ * Create a scratch file in the given filesystem, with the given pathName,
+ * consisting of a set of lines. The method returns a Path instance for the
+ * scratch file.
+ */
+ protected Path scratchFile(FileSystem fs, String pathName, String... lines)
+ throws IOException {
+ return scratch.file(fs, pathName, lines);
+ }
+
+ /**
+ * Create a scratch file in the given filesystem, with the given pathName,
+ * consisting of a set of lines. The method returns a Path instance for the
+ * scratch file.
+ */
+ protected Path scratchFile(FileSystem fs, String pathName, byte[] content)
+ throws IOException {
+ return scratch.file(fs, pathName, content);
+ }
+
+ /**
+ * Create a directory in the scratch filesystem, with the given path name.
+ */
+ public Path scratchDir(String pathName) throws IOException {
+ return scratch.dir(pathName);
+ }
+
+ /**
+ * If "expectedSuffix" is not a suffix of "actual", fails with an informative
+ * assertion.
+ */
+ protected void assertEndsWith(String expectedSuffix, String actual) {
+ if (!actual.endsWith(expectedSuffix)) {
+ fail("\"" + actual + "\" does not end with "
+ + "\"" + expectedSuffix + "\"");
+ }
+ }
+
+ /**
+ * If "expectedPrefix" is not a prefix of "actual", fails with an informative
+ * assertion.
+ */
+ protected void assertStartsWith(String expectedPrefix, String actual) {
+ if (!actual.startsWith(expectedPrefix)) {
+ fail("\"" + actual + "\" does not start with "
+ + "\"" + expectedPrefix + "\"");
+ }
+ }
+
+ // Mix-in assertions:
+
+ protected void assertNoEvents() {
+ JunitTestUtils.assertNoEvents(eventCollector);
+ }
+
+ protected Event assertContainsEvent(String expectedMessage) {
+ return JunitTestUtils.assertContainsEvent(eventCollector,
+ expectedMessage);
+ }
+
+ protected Event assertContainsEvent(String expectedMessage, Set<EventKind> kinds) {
+ return JunitTestUtils.assertContainsEvent(eventCollector,
+ expectedMessage,
+ kinds);
+ }
+
+ protected void assertContainsEventWithFrequency(String expectedMessage,
+ int expectedFrequency) {
+ JunitTestUtils.assertContainsEventWithFrequency(eventCollector, expectedMessage,
+ expectedFrequency);
+ }
+
+ protected void assertDoesNotContainEvent(String expectedMessage) {
+ JunitTestUtils.assertDoesNotContainEvent(eventCollector,
+ expectedMessage);
+ }
+
+ protected Event assertContainsEventWithWordsInQuotes(String... words) {
+ return JunitTestUtils.assertContainsEventWithWordsInQuotes(
+ eventCollector, words);
+ }
+
+ protected void assertContainsEventsInOrder(String... expectedMessages) {
+ JunitTestUtils.assertContainsEventsInOrder(eventCollector, expectedMessages);
+ }
+
+ @SuppressWarnings({"unchecked", "varargs"})
+ protected static <T> void assertContainsSublist(List<T> arguments,
+ T... expectedSublist) {
+ JunitTestUtils.assertContainsSublist(arguments, expectedSublist);
+ }
+
+ @SuppressWarnings({"unchecked", "varargs"})
+ protected static <T> void assertDoesNotContainSublist(List<T> arguments,
+ T... expectedSublist) {
+ JunitTestUtils.assertDoesNotContainSublist(arguments, expectedSublist);
+ }
+
+ protected static <T> void assertContainsSubset(Iterable<T> arguments,
+ Iterable<T> expectedSubset) {
+ JunitTestUtils.assertContainsSubset(arguments, expectedSubset);
+ }
+
+ protected String loadFile(File file) throws IOException {
+ return Files.toString(file, Charset.defaultCharset());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/JunitTestUtils.java b/src/test/java/com/google/devtools/build/lib/testutil/JunitTestUtils.java
new file mode 100644
index 0000000..efe1599
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/JunitTestUtils.java
@@ -0,0 +1,310 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventCollector;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.util.Pair;
+
+import junit.framework.TestCase;
+
+import java.lang.reflect.AccessibleObject;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This class contains a utility method {@link #nullifyInstanceFields(Object)}
+ * for setting all fields in an instance to {@code null}. This is needed for
+ * junit {@code TestCase} instances that keep expensive objects in fields.
+ * Basically junit holds onto the instances
+ * even after the test methods have run, and it creates one such instance
+ * per {@code testFoo} method.
+ */
+public class JunitTestUtils {
+
+ public static void nullifyInstanceFields(Object instance)
+ throws IllegalAccessException {
+ /**
+ * We're cleaning up this test case instance by assigning null pointers
+ * to all fields to reduce the memory overhead of test case instances
+ * staying around after the test methods have been executed. This is a
+ * bug in junit.
+ */
+ List<Field> instanceFields = new ArrayList<>();
+ for (Class<?> clazz = instance.getClass();
+ !clazz.equals(TestCase.class) && !clazz.equals(Object.class);
+ clazz = clazz.getSuperclass()) {
+ for (Field field : clazz.getDeclaredFields()) {
+ if (Modifier.isStatic(field.getModifiers())) {
+ continue;
+ }
+ if (field.getType().isPrimitive()) {
+ continue;
+ }
+ if (Modifier.isFinal(field.getModifiers())) {
+ String msg = "Please make field \"" + field + "\" non-final, or, if " +
+ "it's very simple and truly immutable and not too " +
+ "big, make it static.";
+ throw new AssertionError(msg);
+ }
+ instanceFields.add(field);
+ }
+ }
+ // Run setAccessible for efficiency
+ AccessibleObject.setAccessible(instanceFields.toArray(new Field[0]), true);
+ for (Field field : instanceFields) {
+ field.set(instance, null);
+ }
+ }
+
+ /********************************************************************
+ * *
+ * "Mix-in methods" *
+ * *
+ ********************************************************************/
+
+ // Java doesn't support mix-ins, but we need them in our tests so that we can
+ // inherit a bunch of useful methods, e.g. assertions over an EventCollector.
+ // We do this by hand, by delegating from instance methods in each TestCase
+ // to the static methods below.
+
+ /**
+ * If the specified EventCollector contains any events, an informative
+ * assertion fails in the context of the specified TestCase.
+ */
+ public static void assertNoEvents(Iterable<Event> eventCollector) {
+ String eventsString = eventsToString(eventCollector);
+ assertThat(eventsString).isEmpty();
+ }
+
+ /**
+ * If the specified EventCollector contains an unexpected number of events, an informative
+ * assertion fails in the context of the specified TestCase.
+ */
+ public static void assertEventCount(int expectedCount, EventCollector eventCollector) {
+ assertWithMessage(eventsToString(eventCollector))
+ .that(eventCollector.count()).isEqualTo(expectedCount);
+ }
+
+ /**
+ * If the specified EventCollector does not contain an event which has
+ * 'expectedEvent' as a substring, an informative assertion fails. Otherwise
+ * the matching event is returned.
+ */
+ public static Event assertContainsEvent(Iterable<Event> eventCollector,
+ String expectedEvent) {
+ return assertContainsEvent(eventCollector, expectedEvent, EventKind.ALL_EVENTS);
+ }
+
+ /**
+ * If the specified EventCollector does not contain an event of a kind of 'kinds' which has
+ * 'expectedEvent' as a substring, an informative assertion fails. Otherwise
+ * the matching event is returned.
+ */
+ public static Event assertContainsEvent(Iterable<Event> eventCollector,
+ String expectedEvent,
+ Set<EventKind> kinds) {
+ for (Event event : eventCollector) {
+ if (event.getMessage().contains(expectedEvent) && kinds.contains(event.getKind())) {
+ return event;
+ }
+ }
+ String eventsString = eventsToString(eventCollector);
+ assertWithMessage("Event '" + expectedEvent + "' not found"
+ + (eventsString.length() == 0 ? "" : ("; found these though:" + eventsString)))
+ .that(false).isTrue();
+ return null; // unreachable
+ }
+
+ /**
+ * If the specified EventCollector contains an event which has
+ * 'expectedEvent' as a substring, an informative assertion fails.
+ */
+ public static void assertDoesNotContainEvent(Iterable<Event> eventCollector,
+ String expectedEvent) {
+ for (Event event : eventCollector) {
+ assertWithMessage("Unexpected string '" + expectedEvent + "' matched following event:\n"
+ + event.getMessage()).that(event.getMessage()).doesNotContain(expectedEvent);
+ }
+ }
+
+ /**
+ * If the specified EventCollector does not contain an event which has
+ * each of {@code words} surrounded by single quotes as a substring, an
+ * informative assertion fails. Otherwise the matching event is returned.
+ */
+ public static Event assertContainsEventWithWordsInQuotes(
+ Iterable<Event> eventCollector,
+ String... words) {
+ for (Event event : eventCollector) {
+ boolean found = true;
+ for (String word : words) {
+ if (!event.getMessage().contains("'" + word + "'")) {
+ found = false;
+ break;
+ }
+ }
+ if (found) {
+ return event;
+ }
+ }
+ String eventsString = eventsToString(eventCollector);
+ assertWithMessage("Event containing words " + Arrays.toString(words) + " in "
+ + "single quotes not found"
+ + (eventsString.length() == 0 ? "" : ("; found these though:" + eventsString)))
+ .that(false).isTrue();
+ return null; // unreachable
+ }
+
+ /**
+ * Returns a string consisting of each event in the specified collector,
+ * preceded by a newline.
+ */
+ private static String eventsToString(Iterable<Event> eventCollector) {
+ StringBuilder buf = new StringBuilder();
+ eventLoop: for (Event event : eventCollector) {
+ for (String ignoredPrefix : TestConstants.IGNORED_MESSAGE_PREFIXES) {
+ if (event.getMessage().startsWith(ignoredPrefix)) {
+ continue eventLoop;
+ }
+ }
+ buf.append('\n').append(event);
+ }
+ return buf.toString();
+ }
+
+ /**
+ * If "expectedSublist" is not a sublist of "arguments", an informative
+ * assertion is failed in the context of the specified TestCase.
+ *
+ * Argument order mnemonic: assert(X)ContainsSublist(Y).
+ */
+ @SuppressWarnings({"unchecked", "varargs"})
+ public static <T> void assertContainsSublist(List<T> arguments, T... expectedSublist) {
+ List<T> sublist = Arrays.asList(expectedSublist);
+ try {
+ assertThat(Collections.indexOfSubList(arguments, sublist)).isNotEqualTo(-1);
+ } catch (AssertionError e) {
+ throw new AssertionError("Did not find " + sublist + " as a sublist of " + arguments, e);
+ }
+ }
+
+ /**
+ * If "expectedSublist" is a sublist of "arguments", an informative
+ * assertion is failed in the context of the specified TestCase.
+ *
+ * Argument order mnemonic: assert(X)DoesNotContainSublist(Y).
+ */
+ @SuppressWarnings({"unchecked", "varargs"})
+ public static <T> void assertDoesNotContainSublist(List<T> arguments, T... expectedSublist) {
+ List<T> sublist = Arrays.asList(expectedSublist);
+ try {
+ assertThat(Collections.indexOfSubList(arguments, sublist)).isEqualTo(-1);
+ } catch (AssertionError e) {
+ throw new AssertionError("Found " + sublist + " as a sublist of " + arguments, e);
+ }
+ }
+
+ /**
+ * If "arguments" does not contain "expectedSubset" as a subset, an
+ * informative assertion is failed in the context of the specified TestCase.
+ *
+ * Argument order mnemonic: assert(X)ContainsSubset(Y).
+ */
+ public static <T> void assertContainsSubset(Iterable<T> arguments,
+ Iterable<T> expectedSubset) {
+ Set<T> argumentsSet = arguments instanceof Set<?>
+ ? (Set<T>) arguments
+ : Sets.newHashSet(arguments);
+
+ for (T x : expectedSubset) {
+ assertWithMessage("assertContainsSubset failed: did not find element " + x
+ + "\nExpected subset = " + expectedSubset + "\nArguments = " + arguments)
+ .that(argumentsSet).contains(x);
+ }
+ }
+
+ /**
+ * Check to see if each element of expectedMessages is the beginning of a message
+ * in eventCollector, in order, as in {@link #containsSublistWithGapsAndEqualityChecker}.
+ * If not, an informative assertion is failed
+ */
+ protected static void assertContainsEventsInOrder(Iterable<Event> eventCollector,
+ String... expectedMessages) {
+ String failure = containsSublistWithGapsAndEqualityChecker(
+ ImmutableList.copyOf(eventCollector),
+ new Function<Pair<Event, String>, Boolean> () {
+ @Override
+ public Boolean apply(Pair<Event, String> pair) {
+ return pair.first.getMessage().contains(pair.second);
+ }
+ }, expectedMessages);
+
+ String eventsString = eventsToString(eventCollector);
+ assertWithMessage("Event '" + failure + "' not found in proper order"
+ + (eventsString.length() == 0 ? "" : ("; found these though:" + eventsString)))
+ .that(failure).isNull();
+ }
+
+ /**
+ * Check to see if each element of expectedSublist is in arguments, according to
+ * the equalityChecker, in the same order as in expectedSublist (although with
+ * other interspersed elements in arguments allowed).
+ * @param equalityChecker function that takes a Pair<S, T> element and returns true
+ * if the elements of the pair are equal by its lights.
+ * @return first element not in arguments in order, or null if success.
+ */
+ @SuppressWarnings({"unchecked"})
+ protected static <S, T> T containsSublistWithGapsAndEqualityChecker(List<S> arguments,
+ Function<Pair<S, T>, Boolean> equalityChecker, T... expectedSublist) {
+ Iterator<S> iter = arguments.iterator();
+ outerLoop:
+ for (T expected : expectedSublist) {
+ while (iter.hasNext()) {
+ S actual = iter.next();
+ if (equalityChecker.apply(Pair.of(actual, expected))) {
+ continue outerLoop;
+ }
+ }
+ return expected;
+ }
+ return null;
+ }
+
+ public static List<Event> assertContainsEventWithFrequency(Iterable<Event> events,
+ String expectedMessage, int expectedFrequency) {
+ ImmutableList.Builder<Event> builder = ImmutableList.builder();
+ for (Event event : events) {
+ if (event.getMessage().contains(expectedMessage)) {
+ builder.add(event);
+ }
+ }
+ List<Event> foundEvents = builder.build();
+ assertWithMessage(foundEvents.toString()).that(foundEvents).hasSize(expectedFrequency);
+ return foundEvents;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java
new file mode 100644
index 0000000..d4f6058
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.devtools.build.lib.util.Clock;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A fake clock for testing.
+ */
+public final class ManualClock implements Clock {
+ private long currentTimeMillis = 0L;
+
+ @Override
+ public long currentTimeMillis() {
+ return currentTimeMillis;
+ }
+
+ @Override
+ public long nanoTime() {
+ return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis);
+ }
+
+ public void advanceMillis(long time) {
+ currentTimeMillis += time;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/MoreAsserts.java b/src/test/java/com/google/devtools/build/lib/testutil/MoreAsserts.java
new file mode 100644
index 0000000..9224b8a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/MoreAsserts.java
@@ -0,0 +1,319 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static com.google.common.truth.Truth.assert_;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import java.lang.ref.Reference;
+import java.lang.reflect.Field;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * A helper class for tests providing a simple interface for asserts.
+ */
+public class MoreAsserts {
+
+ public static void assertContainsRegex(String regex, String actual) {
+ assertThat(actual).containsMatch(regex);
+ }
+
+ public static void assertContainsRegex(String msg, String regex, String actual) {
+ assertWithMessage(msg).that(actual).containsMatch(regex);
+ }
+
+ public static void assertNotContainsRegex(String regex, String actual) {
+ assertThat(actual).doesNotContainMatch(regex);
+ }
+
+ public static void assertNotContainsRegex(String msg, String regex, String actual) {
+ assertWithMessage(msg).that(actual).doesNotContainMatch(regex);
+ }
+
+ public static void assertMatchesRegex(String regex, String actual) {
+ assertThat(actual).matches(regex);
+ }
+
+ public static void assertMatchesRegex(String msg, String regex, String actual) {
+ assertWithMessage(msg).that(actual).matches(regex);
+ }
+
+ public static void assertNotMatchesRegex(String regex, String actual) {
+ assertThat(actual).doesNotMatch(regex);
+ }
+
+ public static <T> void assertEquals(T expected, T actual, Comparator<T> comp) {
+ assertThat(comp.compare(expected, actual)).isEqualTo(0);
+ }
+
+ public static <T> void assertContentsAnyOrder(
+ Iterable<? extends T> expected, Iterable<? extends T> actual,
+ Comparator<? super T> comp) {
+ assertThat(actual).hasSize(Iterables.size(expected));
+ int i = 0;
+ for (T e : expected) {
+ for (T a : actual) {
+ if (comp.compare(e, a) == 0) {
+ i++;
+ }
+ }
+ }
+ assertThat(actual).hasSize(i);
+ }
+
+ public static void assertGreaterThanOrEqual(long target, long actual) {
+ assertThat(actual).isAtLeast(target);
+ }
+
+ public static void assertGreaterThanOrEqual(String msg, long target, long actual) {
+ assertWithMessage(msg).that(actual).isAtLeast(target);
+ }
+
+ public static void assertGreaterThan(long target, long actual) {
+ assertThat(actual).isGreaterThan(target);
+ }
+
+ public static void assertGreaterThan(String msg, long target, long actual) {
+ assertWithMessage(msg).that(actual).isGreaterThan(target);
+ }
+
+ public static void assertLessThanOrEqual(long target, long actual) {
+ assertThat(actual).isAtMost(target);
+ }
+
+ public static void assertLessThanOrEqual(String msg, long target, long actual) {
+ assertWithMessage(msg).that(actual).isAtMost(target);
+ }
+
+ public static void assertLessThan(long target, long actual) {
+ assertThat(actual).isLessThan(target);
+ }
+
+ public static void assertLessThan(String msg, long target, long actual) {
+ assertWithMessage(msg).that(actual).isLessThan(target);
+ }
+
+ public static void assertEndsWith(String ending, String actual) {
+ assertThat(actual).endsWith(ending);
+ }
+
+ public static void assertStartsWith(String prefix, String actual) {
+ assertThat(actual).startsWith(prefix);
+ }
+
+ /**
+ * Scans if an instance of given class is strongly reachable from a given
+ * object.
+ * <p>Runs breadth-first search in object reachability graph to check if
+ * an instance of <code>clz</code> can be reached.
+ * <strong>Note:</strong> This method can take a long time if analyzed
+ * data structure spans across large part of heap and may need a lot of
+ * memory.
+ *
+ * @param start object to start the search from
+ * @param clazz class to look for
+ */
+ public static void assertInstanceOfNotReachable(
+ Object start, final Class<?> clazz) {
+ Predicate<Object> p = new Predicate<Object>() {
+ @Override
+ public boolean apply(Object obj) {
+ return clazz.isAssignableFrom(obj.getClass());
+ }
+ };
+ if (isRetained(p, start)) {
+ assert_().fail("Found an instance of " + clazz.getCanonicalName() +
+ " reachable from " + start.toString());
+ }
+ }
+
+ private static final Field NON_STRONG_REF;
+
+ static {
+ try {
+ NON_STRONG_REF = Reference.class.getDeclaredField("referent");
+ } catch (SecurityException e) {
+ throw new RuntimeException(e);
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static final Predicate<Field> ALL_STRONG_REFS = new Predicate<Field>() {
+ @Override
+ public boolean apply(Field field) {
+ return NON_STRONG_REF.equals(field);
+ }
+ };
+
+ private static boolean isRetained(Predicate<Object> predicate, Object start) {
+ Map<Object, Object> visited = Maps.newIdentityHashMap();
+ visited.put(start, start);
+ Queue<Object> toScan = Lists.newLinkedList();
+ toScan.add(start);
+
+ while (!toScan.isEmpty()) {
+ Object current = toScan.poll();
+ if (current.getClass().isArray()) {
+ if (current.getClass().getComponentType().isPrimitive()) {
+ continue;
+ }
+
+ for (Object ref : (Object[]) current) {
+ if (ref != null) {
+ if (predicate.apply(ref)) {
+ return true;
+ }
+ if (visited.put(ref, ref) == null) {
+ toScan.add(ref);
+ }
+ }
+ }
+ } else {
+ // iterate *all* fields (getFields() returns only accessible ones)
+ for (Class<?> clazz = current.getClass(); clazz != null;
+ clazz = clazz.getSuperclass()) {
+ for (Field f : clazz.getDeclaredFields()) {
+ if (f.getType().isPrimitive() || ALL_STRONG_REFS.apply(f)) {
+ continue;
+ }
+
+ f.setAccessible(true);
+ try {
+ Object ref = f.get(current);
+ if (ref != null) {
+ if (predicate.apply(ref)) {
+ return true;
+ }
+ if (visited.put(ref, ref) == null) {
+ toScan.add(ref);
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ throw new IllegalStateException("Error when scanning the heap", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException("Error when scanning the heap", e);
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private static String getClassDescription(Object object) {
+ return object == null
+ ? "null"
+ : ("instance of " + object.getClass().getName());
+ }
+
+ public static String chattyFormat(String message, Object expected, Object actual) {
+ String expectedClass = getClassDescription(expected);
+ String actualClass = getClassDescription(actual);
+
+ return Joiner.on('\n').join((message != null) ? ("\n" + message) : "",
+ " expected " + expectedClass + ": <" + expected + ">",
+ " but was " + actualClass + ": <" + actual + ">");
+ }
+
+ public static void assertEqualsUnifyingLineEnds(String expected, String actual) {
+ if (actual != null) {
+ actual = actual.replaceAll(System.getProperty("line.separator"), "\n");
+ }
+ assertThat(actual).isEqualTo(expected);
+ }
+
+ public static void assertContainsWordsWithQuotes(String message,
+ String... strings) {
+ for (String string : strings) {
+ assertTrue(message + " should contain '" + string + "' (with quotes)",
+ message.contains("'" + string + "'"));
+ }
+ }
+
+ public static void assertNonZeroExitCode(int exitCode, String stdout, String stderr) {
+ if (exitCode == 0) {
+ fail("expected non-zero exit code but exit code was 0 and stdout was <"
+ + stdout + "> and stderr was <" + stderr + ">");
+ }
+ }
+
+ public static void assertExitCode(int expectedExitCode,
+ int exitCode, String stdout, String stderr) {
+ if (exitCode != expectedExitCode) {
+ fail(String.format("expected exit code <%d> but exit code was <%d> and stdout was <%s> "
+ + "and stderr was <%s>", expectedExitCode, exitCode, stdout, stderr));
+ }
+ }
+
+ public static void assertStdoutContainsString(String expected, String stdout, String stderr) {
+ if (!stdout.contains(expected)) {
+ fail("expected stdout to contain string <" + expected + "> but stdout was <"
+ + stdout + "> and stderr was <" + stderr + ">");
+ }
+ }
+
+ public static void assertStderrContainsString(String expected, String stdout, String stderr) {
+ if (!stderr.contains(expected)) {
+ fail("expected stderr to contain string <" + expected + "> but stdout was <"
+ + stdout + "> and stderr was <" + stderr + ">");
+ }
+ }
+
+ public static void assertStdoutContainsRegex(String expectedRegex,
+ String stdout, String stderr) {
+ if (!Pattern.compile(expectedRegex).matcher(stdout).find()) {
+ fail("expected stdout to contain regex <" + expectedRegex + "> but stdout was <"
+ + stdout + "> and stderr was <" + stderr + ">");
+ }
+ }
+
+ public static void assertStderrContainsRegex(String expectedRegex,
+ String stdout, String stderr) {
+ if (!Pattern.compile(expectedRegex).matcher(stderr).find()) {
+ fail("expected stderr to contain regex <" + expectedRegex + "> but stdout was <"
+ + stdout + "> and stderr was <" + stderr + ">");
+ }
+ }
+
+ public static Set<String> asStringSet(Iterable<?> collection) {
+ Set<String> set = Sets.newTreeSet();
+ for (Object o : collection) {
+ set.add("\"" + String.valueOf(o) + "\"");
+ }
+ return set;
+ }
+
+ public static <T> void
+ assertSameContents(Iterable<? extends T> expected, Iterable<? extends T> actual) {
+ if (!Sets.newHashSet(expected).equals(Sets.newHashSet(actual))) {
+ fail("got string set: " + asStringSet(actual).toString()
+ + "\nwant: " + asStringSet(expected).toString());
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/Scratch.java b/src/test/java/com/google/devtools/build/lib/testutil/Scratch.java
new file mode 100644
index 0000000..229d2a7
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/Scratch.java
@@ -0,0 +1,150 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import java.io.IOException;
+
+/**
+ * Allow tests to easily manage scratch files in a FileSystem.
+ */
+public final class Scratch {
+
+ private final FileSystem fileSystem;
+
+ /**
+ * Create a new ScratchFileSystem using the {@link InMemoryFileSystem}
+ */
+ public Scratch() {
+ this(new InMemoryFileSystem(BlazeClock.instance()));
+ }
+
+ /**
+ * Create a new ScratchFileSystem using the supplied FileSystem.
+ */
+ public Scratch(FileSystem fileSystem) {
+ this.fileSystem = fileSystem;
+ }
+
+ /**
+ * Returns the FileSystem in use.
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ /**
+ * Create a directory in the scratch filesystem, with the given path name.
+ */
+ public Path dir(String pathName) throws IOException {
+ Path dir = getFileSystem().getPath(pathName);
+ if (!dir.exists()) {
+ FileSystemUtils.createDirectoryAndParents(dir);
+ }
+ if (!dir.isDirectory()) {
+ throw new IOException("Exists, but is not a directory: " + pathName);
+ }
+ return dir;
+ }
+
+ /**
+ * Create a scratch file in the scratch filesystem, with the given pathName,
+ * consisting of a set of lines. The method returns a Path instance for the
+ * scratch file.
+ */
+ public Path file(String pathName, String... lines)
+ throws IOException {
+ Path newFile = file(getFileSystem(), pathName, lines);
+ newFile.setLastModifiedTime(-1L);
+ return newFile;
+ }
+
+ /**
+ * Like {@code scratchFile}, but the file is first deleted if it already
+ * exists.
+ */
+ public Path overwriteFile(String pathName, String... lines) throws IOException {
+ Path oldFile = getFileSystem().getPath(pathName);
+ long newMTime = oldFile.exists() ? oldFile.getLastModifiedTime() + 1 : -1;
+ oldFile.delete();
+ Path newFile = file(getFileSystem(), pathName, lines);
+ newFile.setLastModifiedTime(newMTime);
+ return newFile;
+ }
+
+ /**
+ * Deletes the specified scratch file, using the same specification as {@link Path#delete}.
+ */
+ public boolean deleteFile(String pathName) throws IOException {
+ Path file = getFileSystem().getPath(pathName);
+ return file.delete();
+ }
+
+ /**
+ * Create a scratch file in the given filesystem, with the given pathName,
+ * consisting of a set of lines. The method returns a Path instance for the
+ * scratch file.
+ */
+ public Path file(FileSystem fs, String pathName, String... lines)
+ throws IOException {
+ Path file = newScratchFile(fs, pathName);
+ FileSystemUtils.writeContentAsLatin1(file, linesAsString(lines));
+ return file;
+ }
+
+ /**
+ * Create a scratch file in the given filesystem, with the given pathName,
+ * consisting of a set of lines. The method returns a Path instance for the
+ * scratch file.
+ */
+ public Path file(FileSystem fs, String pathName, byte[] content)
+ throws IOException {
+ Path file = newScratchFile(fs, pathName);
+ FileSystemUtils.writeContent(file, content);
+ return file;
+ }
+
+ /** Creates a new scratch file, ensuring parents exist. */
+ private Path newScratchFile(FileSystem fs, String pathName) throws IOException {
+ Path file = fs.getPath(pathName);
+ Path parentDir = file.getParentDirectory();
+ if (!parentDir.exists()) {
+ FileSystemUtils.createDirectoryAndParents(parentDir);
+ }
+ if (file.exists()) {
+ throw new IOException("Could not create scratch file (file exists) "
+ + pathName);
+ }
+ return file;
+ }
+
+ /**
+ * Converts the lines into a String with linebreaks. Useful for creating
+ * in-memory input for a file, for example.
+ */
+ private static String linesAsString(String... lines) {
+ StringBuilder builder = new StringBuilder();
+ for (String line : lines) {
+ builder.append(line);
+ builder.append('\n');
+ }
+ return builder.toString();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/Suite.java b/src/test/java/com/google/devtools/build/lib/testutil/Suite.java
new file mode 100644
index 0000000..43590d4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/Suite.java
@@ -0,0 +1,86 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+/**
+ * Test annotations used to select which tests to run in a given situation.
+ */
+public enum Suite {
+
+ /**
+ * It's so blazingly fast and lightweight we run it whenever we make any
+ * build.lib change. This size is the default.
+ */
+ SMALL_TESTS,
+
+ /**
+ * It's a bit too slow to run all the time, but it still tests some
+ * unit of functionality. May run external commands such as gcc, for example.
+ */
+ MEDIUM_TESTS,
+
+ /**
+ * I don't even want to think about running this one after every edit,
+ * but I don't mind if the continuous build runs it, and I'm happy to have
+ * it before making a release.
+ */
+ LARGE_TESTS,
+
+ /**
+ * These tests take a long time. They should only ever be run manually and probably from their
+ * own Blaze test target.
+ */
+ ENORMOUS_TESTS;
+
+ /**
+ * Given a class, determine the test size.
+ */
+ public static Suite getSize(Class<?> clazz) {
+ return getAnnotationElementOrDefault(clazz, "size");
+ }
+
+ /**
+ * Given a class, determine the suite it belongs to.
+ */
+ public static String getSuiteName(Class<?> clazz) {
+ return getAnnotationElementOrDefault(clazz, "suite");
+ }
+
+ /**
+ * Given a class, determine if it is flaky.
+ */
+ public static boolean isFlaky(Class<?> clazz) {
+ return getAnnotationElementOrDefault(clazz, "flaky");
+ }
+
+ /**
+ * Returns the value of the given element in the {@link TestSpec} annotation of the given class,
+ * or the default value of that element if the class doesn't have a {@link TestSpec} annotation.
+ */
+ @SuppressWarnings("unchecked")
+ private static <T> T getAnnotationElementOrDefault(Class<?> clazz, String elementName) {
+ TestSpec spec = clazz.getAnnotation(TestSpec.class);
+ try {
+ Method method = TestSpec.class.getMethod(elementName);
+ return spec != null ? (T) method.invoke(spec) : (T) method.getDefaultValue();
+ } catch (NoSuchMethodException e) {
+ throw new IllegalStateException("no such element " + elementName, e);
+ } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+ throw new IllegalStateException("can't invoke accessor for element " + elementName, e);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java b/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java
new file mode 100644
index 0000000..d9552ad
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java
@@ -0,0 +1,53 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Various constants required by the tests.
+ */
+public class TestConstants {
+ private TestConstants() {
+ }
+
+ /**
+ * A list of all embedded binaries that go into the regular Bazel binary.
+ */
+ public static final ImmutableList<String> EMBEDDED_TOOLS = ImmutableList.of(
+ "build-runfiles",
+ "process-wrapper",
+ "build_interface_so");
+
+
+ /**
+ * Location in the bazel repo where embedded binaries come from.
+ */
+ public static final String EMBEDDED_SCRIPTS_PATH = "DOES-NOT-WORK-YET";
+
+ /**
+ * Directory where we can find bazel's Java tests, relative to a test's runfiles directory.
+ */
+ public static final String JAVATESTS_ROOT = "src/test/java/";
+
+ /**
+ * The directory in InMemoryFileSystem where workspaces created during unit tests reside.
+ */
+ public static final String TEST_WORKSPACE_DIRECTORY = "bazel";
+
+ public static final String TEST_RULE_CLASS_PROVIDER =
+ "com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider";
+ public static final ImmutableList<String> IGNORED_MESSAGE_PREFIXES = ImmutableList.<String>of();
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestFileOutErr.java b/src/test/java/com/google/devtools/build/lib/testutil/TestFileOutErr.java
new file mode 100644
index 0000000..6f0494f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestFileOutErr.java
@@ -0,0 +1,127 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.util.io.RecordingOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An implementation of the FileOutErr that doesn't use a file.
+ * This is useful for tests, as they often test the action directly
+ * and would otherwise have to create files on the vfs.
+ */
+public class TestFileOutErr extends FileOutErr {
+
+ RecordingOutErr recorder;
+
+ public TestFileOutErr(TestFileOutErr arg) {
+ this(arg.getOutputStream(), arg.getErrorStream());
+ }
+
+ public TestFileOutErr() {
+ this(new ByteArrayOutputStream(), new ByteArrayOutputStream());
+ }
+
+ public TestFileOutErr(ByteArrayOutputStream stream) {
+ super(null, null); // This is a pretty brutal overloading - We're just inheriting for the type.
+ recorder = new RecordingOutErr(stream, stream);
+ }
+
+ public TestFileOutErr(ByteArrayOutputStream stream1, ByteArrayOutputStream stream2) {
+ super(null, null); // This is a pretty brutal overloading - We're just inheriting for the type.
+ recorder = new RecordingOutErr(stream1, stream2);
+ }
+
+
+ @Override
+ public Path getOutputFile() {
+ return null;
+ }
+
+ @Override
+ public Path getErrorFile() {
+ return null;
+ }
+
+ @Override
+ public ByteArrayOutputStream getOutputStream() {
+ return recorder.getOutputStream();
+ }
+
+ @Override
+ public ByteArrayOutputStream getErrorStream() {
+ return recorder.getErrorStream();
+ }
+
+ @Override
+ public void printOut(String s) {
+ recorder.printOut(s);
+ }
+
+ @Override
+ public void printErr(String s) {
+ recorder.printErr(s);
+ }
+
+ @Override
+ public String toString() {
+ return recorder.toString();
+ }
+
+ @Override
+ public boolean hasRecordedOutput() {
+ return recorder.hasRecordedOutput();
+ }
+
+ @Override
+ public String outAsLatin1() {
+ return recorder.outAsLatin1();
+ }
+
+ @Override
+ public String errAsLatin1() {
+ return recorder.errAsLatin1();
+ }
+
+ @Override
+ public void dumpOutAsLatin1(OutputStream out) {
+ try {
+ out.write(recorder.getOutputStream().toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dumpErrAsLatin1(OutputStream out) {
+ try {
+ out.write(recorder.getErrorStream().toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getRecordedOutput() {
+ return recorder.outAsLatin1() + recorder.errAsLatin1();
+ }
+
+ public void reset() {
+ recorder.reset();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestRuleClassProvider.java b/src/test/java/com/google/devtools/build/lib/testutil/TestRuleClassProvider.java
new file mode 100644
index 0000000..752605c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestRuleClassProvider.java
@@ -0,0 +1,84 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.INTEGER;
+import static com.google.devtools.build.lib.packages.Type.LABEL_LIST;
+import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST;
+import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
+
+import com.google.devtools.build.lib.analysis.BaseRuleClasses;
+import com.google.devtools.build.lib.analysis.BlazeRule;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.util.FileTypeSet;
+
+import java.lang.reflect.Method;
+
+/**
+ * Helper class to provide a RuleClassProvider for tests.
+ */
+public class TestRuleClassProvider {
+ private static ConfiguredRuleClassProvider ruleProvider = null;
+
+ /**
+ * Adds all the rule classes supported internally within the build tool to the given builder.
+ */
+ public static void addStandardRules(ConfiguredRuleClassProvider.Builder builder) {
+ try {
+ Class<?> providerClass = Class.forName(TestConstants.TEST_RULE_CLASS_PROVIDER);
+ Method setupMethod = providerClass.getMethod("setup",
+ ConfiguredRuleClassProvider.Builder.class);
+ setupMethod.invoke(null, builder);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Return a rule class provider.
+ */
+ public static ConfiguredRuleClassProvider getRuleClassProvider() {
+ if (ruleProvider == null) {
+ ConfiguredRuleClassProvider.Builder builder =
+ new ConfiguredRuleClassProvider.Builder();
+ addStandardRules(builder);
+ builder.addRuleDefinition(TestingDummyRule.class);
+ ruleProvider = builder.build();
+ }
+ return ruleProvider;
+ }
+
+ @BlazeRule(name = "testing_dummy_rule",
+ ancestors = { BaseRuleClasses.RuleBase.class },
+ // Instantiated only in tests
+ factoryClass = UnknownRuleConfiguredTarget.class)
+ public static final class TestingDummyRule implements RuleDefinition {
+ @Override
+ public RuleClass build(Builder builder, RuleDefinitionEnvironment env) {
+ return builder
+ .setUndocumented()
+ .add(attr("srcs", LABEL_LIST).allowedFileTypes(FileTypeSet.ANY_FILE))
+ .add(attr("outs", OUTPUT_LIST))
+ .add(attr("dummystrings", STRING_LIST))
+ .add(attr("dummyinteger", INTEGER))
+ .build();
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestSpec.java b/src/test/java/com/google/devtools/build/lib/testutil/TestSpec.java
new file mode 100644
index 0000000..316169c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestSpec.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation class which we use to attach a little meta data to test
+ * classes. For now, we use this to attach a {@link Suite}.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface TestSpec {
+
+ /**
+ * The size of the specified test, in terms of its resource consumption and
+ * execution time.
+ */
+ Suite size() default Suite.SMALL_TESTS;
+
+ /**
+ * The name of the suite to which this test belongs. Useful for creating
+ * test suites organised by function.
+ */
+ String suite() default "";
+
+ /**
+ * If the test will pass consistently without outside changes.
+ * This should be fixed as soon as possible.
+ */
+ boolean flaky() default false;
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestSuiteBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/TestSuiteBuilder.java
new file mode 100644
index 0000000..af90c52
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestSuiteBuilder.java
@@ -0,0 +1,139 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import junit.framework.TestCase;
+
+import org.junit.runner.RunWith;
+
+import java.lang.reflect.Modifier;
+import java.util.Comparator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * A collector for test classes, for both JUnit 3 and 4. To be used in combination with {@link
+ * CustomSuite}.
+ */
+public final class TestSuiteBuilder {
+
+ private Set<Class<?>> testClasses = Sets.newTreeSet(new TestClassNameComparator());
+ private Predicate<Class<?>> matchClassPredicate = Predicates.alwaysTrue();
+
+ /**
+ * Adds the tests found (directly) in class {@code c} to the set of tests
+ * this builder will search.
+ */
+ public TestSuiteBuilder addTestClass(Class<?> c) {
+ testClasses.add(c);
+ return this;
+ }
+
+ /**
+ * Adds all the test classes (top-level or nested) found in package
+ * {@code pkgName} or its subpackages to the set of tests this builder will
+ * search.
+ */
+ public TestSuiteBuilder addPackageRecursive(String pkgName) {
+ for (Class<?> c : getClassesRecursive(pkgName)) {
+ addTestClass(c);
+ }
+ return this;
+ }
+
+ private Set<Class<?>> getClassesRecursive(String pkgName) {
+ Set<Class<?>> result = new LinkedHashSet<>();
+ for (Class<?> clazz : Classpath.findClasses(pkgName)) {
+ if (isTestClass(clazz)) {
+ result.add(clazz);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Specifies a predicate returns false for classes we want to exclude.
+ */
+ public TestSuiteBuilder matchClasses(Predicate<Class<?>> predicate) {
+ matchClassPredicate = predicate;
+ return this;
+ }
+
+ /**
+ * Creates and returns a TestSuite containing the tests from the given
+ * classes and/or packages which matched the given tags.
+ */
+ public Set<Class<?>> create() {
+ Set<Class<?>> result = new LinkedHashSet<>();
+ // We have some cases where the resulting test suite is empty, which some of our test
+ // infrastructure treats as an error.
+ result.add(TautologyTest.class);
+ for (Class<?> testClass : Iterables.filter(testClasses, matchClassPredicate)) {
+ result.add(testClass);
+ }
+ return result;
+ }
+
+ /**
+ * Determines if a given class is a test class.
+ *
+ * @param container class to test
+ * @return <code>true</code> if the test is a test class.
+ */
+ private static boolean isTestClass(Class<?> container) {
+ return (isJunit4Test(container) || isJunit3Test(container))
+ && !isSuite(container)
+ && Modifier.isPublic(container.getModifiers())
+ && !Modifier.isAbstract(container.getModifiers());
+ }
+
+ private static boolean isJunit4Test(Class<?> container) {
+ return container.getAnnotation(RunWith.class) != null;
+ }
+
+ private static boolean isJunit3Test(Class<?> container) {
+ return TestCase.class.isAssignableFrom(container);
+ }
+
+ /**
+ * Classes that have a {@code RunWith} annotation for {@link ClasspathSuite} or {@link
+ * CustomSuite} are automatically excluded to avoid picking up the suite class itself.
+ */
+ private static boolean isSuite(Class<?> container) {
+ RunWith runWith = container.getAnnotation(RunWith.class);
+ return (runWith != null)
+ && ((runWith.value() == ClasspathSuite.class) || (runWith.value() == CustomSuite.class));
+ }
+
+ private static class TestClassNameComparator implements Comparator<Class<?>> {
+ @Override
+ public int compare(Class<?> o1, Class<?> o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ }
+
+ /**
+ * A test that does nothing and always passes. We have some cases where an empty test suite is
+ * treated as an error, so we use this test to make sure that the test suite is always non-empty.
+ */
+ public static class TautologyTest extends TestCase {
+ public void testThatNothingHappens() {
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestThread.java b/src/test/java/com/google/devtools/build/lib/testutil/TestThread.java
new file mode 100644
index 0000000..e043025
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestThread.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import static com.google.common.truth.Truth.assertWithMessage;
+
+/**
+ * Test thread implementation that allows the use of assertions within
+ * spawned threads.
+ *
+ * Main test method must call {@link TestThread#joinAndAssertState(long)}
+ * for each spawned test thread.
+ */
+public abstract class TestThread extends Thread {
+ Throwable testException = null;
+ boolean isSucceeded = false;
+
+ /**
+ * Specific test thread implementation overrides this method.
+ */
+ abstract public void runTest() throws Exception;
+
+ @Override public final void run() {
+ try {
+ runTest();
+ isSucceeded = true;
+ } catch (Exception e) {
+ testException = e;
+ } catch (AssertionError e) {
+ testException = e;
+ }
+ }
+
+ /**
+ * Joins test thread (waiting specified number of ms) and validates that
+ * it has been completed successfully.
+ */
+ public void joinAndAssertState(long timeout) throws InterruptedException {
+ join(timeout);
+ Throwable exception = this.testException;
+ if (isAlive()) {
+ exception = new AssertionError (
+ "Test thread " + getName() + " is still alive");
+ exception.setStackTrace(getStackTrace());
+ }
+ if(exception != null) {
+ AssertionError error = new AssertionError("Test thread " + getName() + " failed to execute");
+ error.initCause(exception);
+ throw error;
+ }
+ assertWithMessage("Test thread " + getName() + " has not run successfully").that(isSucceeded)
+ .isTrue();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestUtils.java b/src/test/java/com/google/devtools/build/lib/testutil/TestUtils.java
new file mode 100644
index 0000000..2ee5972
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/TestUtils.java
@@ -0,0 +1,152 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * Some static utility functions for testing.
+ */
+public class TestUtils {
+ public static final ThreadPoolExecutor POOL =
+ (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+
+ public static final UUID ZERO_UUID = UUID.fromString("00000000-0000-0000-0000-000000000000");
+
+ /**
+ * Wait until the {@link System#currentTimeMillis} / 1000 advances.
+ *
+ * This method takes 0-1000ms to run, 500ms on average.
+ */
+ public static void advanceCurrentTimeSeconds() throws InterruptedException {
+ long currentTimeSeconds = System.currentTimeMillis() / 1000;
+ do {
+ Thread.sleep(50);
+ } while (currentTimeSeconds == System.currentTimeMillis() / 1000);
+ }
+
+ public static ThreadPoolExecutor getPool() {
+ return POOL;
+ }
+
+ public static String tmpDir() {
+ return tmpDirFile().getAbsolutePath();
+ }
+
+ static String getUserValue(String key) {
+ String value = System.getProperty(key);
+ if (value == null) {
+ value = System.getenv(key);
+ }
+ return value;
+ }
+
+ public static File tmpDirFile() {
+ File tmpDir;
+
+ // Flag value specified in environment?
+ String tmpDirStr = getUserValue("TEST_TMPDIR");
+
+ if (tmpDirStr != null && tmpDirStr.length() > 0) {
+ tmpDir = new File(tmpDirStr);
+ } else {
+ // Fallback default $TEMP/$USER/tmp/$TESTNAME
+ String baseTmpDir = System.getProperty("java.io.tmpdir");
+ tmpDir = new File(baseTmpDir).getAbsoluteFile();
+
+ // .. Add username
+ String username = System.getProperty("user.name");
+ username = username.replace('/', '_');
+ username = username.replace('\\', '_');
+ username = username.replace('\000', '_');
+ tmpDir = new File(tmpDir, username);
+ tmpDir = new File(tmpDir, "tmp");
+ }
+
+ // Ensure tmpDir exists
+ if (!tmpDir.isDirectory()) {
+ tmpDir.mkdirs();
+ }
+ return tmpDir;
+ }
+
+ public static File makeTempDir() throws IOException {
+ File dir = File.createTempFile(TestUtils.class.getName(), ".temp", tmpDirFile());
+ if (!dir.delete()) {
+ throw new IOException("Cannot remove a temporary file " + dir);
+ }
+ if (!dir.mkdir()) {
+ throw new IOException("Cannot create a temporary directory " + dir);
+ }
+ return dir;
+ }
+
+ public static int getRandomSeed() {
+ // Default value if not running under framework
+ int randomSeed = 301;
+
+ // Value specified in environment by framework?
+ String value = getUserValue("TEST_RANDOM_SEED");
+ if ((value != null) && (value.length() > 0)) {
+ try {
+ randomSeed = Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ // throw new AssertionError("TEST_RANDOM_SEED must be an integer");
+ throw new RuntimeException("TEST_RANDOM_SEED must be an integer");
+ }
+ }
+
+ return randomSeed;
+ }
+
+ public static byte[] serializeObject(Object obj) throws IOException {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ try (ObjectOutputStream objectStream = new ObjectOutputStream(outputStream)) {
+ objectStream.writeObject(obj);
+ }
+ return outputStream.toByteArray();
+ }
+
+ public static Object deserializeObject(byte[] buf) throws IOException, ClassNotFoundException {
+ try (ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(buf))) {
+ return inStream.readObject();
+ }
+ }
+
+ /**
+ * Timeouts for asserting that an arbitrary event occurs eventually.
+ *
+ * <p>In general, it's not appropriate to use a small constant timeout for an arbitrary
+ * computation since there is no guarantee that a snippet of code will execute within a given
+ * amount of time - you are at the mercy of the jvm, your machine, and your OS. In theory we
+ * could try to take all of these factors into account but instead we took the simpler and
+ * obviously correct approach of not having timeouts.
+ *
+ * <p>If a test that uses these timeout values is failing due to a "timeout" at the
+ * 'blaze test' level, it could be because of a legitimate deadlock that would have been caught
+ * if the timeout values below were small. So you can rule out such a deadlock by changing these
+ * values to small numbers (also note that the --test_timeout blaze flag may be useful).
+ */
+ public static final long WAIT_TIMEOUT_MILLISECONDS = Long.MAX_VALUE;
+ public static final long WAIT_TIMEOUT_SECONDS = WAIT_TIMEOUT_MILLISECONDS / 1000;
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/UnknownRuleConfiguredTarget.java b/src/test/java/com/google/devtools/build/lib/testutil/UnknownRuleConfiguredTarget.java
new file mode 100644
index 0000000..d3e5457
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutil/UnknownRuleConfiguredTarget.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.testutil;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.FailAction;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
+import com.google.devtools.build.lib.analysis.RuleContext;
+import com.google.devtools.build.lib.analysis.Runfiles;
+import com.google.devtools.build.lib.analysis.RunfilesProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
+
+/**
+ * A null implementation of ConfiguredTarget for rules we don't know how to build.
+ */
+public class UnknownRuleConfiguredTarget implements RuleConfiguredTargetFactory {
+
+ @Override
+ public ConfiguredTarget create(RuleContext context) {
+ // TODO(bazel-team): (2009) why isn't this an error? It would stop the build more promptly...
+ context.ruleWarning("cannot build " + context.getRule().getRuleClass() + " rules");
+
+ ImmutableList<Artifact> outputArtifacts = context.getOutputArtifacts();
+ NestedSet<Artifact> filesToBuild;
+ if (outputArtifacts.isEmpty()) {
+ // Gotta build *something*...
+ filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER,
+ context.createOutputArtifact());
+ } else {
+ filesToBuild = NestedSetBuilder.wrap(Order.STABLE_ORDER, outputArtifacts);
+ }
+
+ Rule rule = context.getRule();
+ context.registerAction(new FailAction(context.getActionOwner(),
+ filesToBuild, "cannot build " + rule.getRuleClass() + " rules such as " + rule.getLabel()));
+ return new RuleConfiguredTargetBuilder(context)
+ .setFilesToBuild(filesToBuild)
+ .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY))
+ .build();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutiltests/JunitTestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/testutiltests/JunitTestUtilsTest.java
new file mode 100644
index 0000000..4bfe0fa
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutiltests/JunitTestUtilsTest.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutiltests;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.testutil.JunitTestUtils.assertContainsSublist;
+import static com.google.devtools.build.lib.testutil.JunitTestUtils.assertDoesNotContainSublist;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Tests {@link com.google.devtools.build.lib.testutil.JunitTestUtils}.
+ */
+@RunWith(JUnit4.class)
+public class JunitTestUtilsTest {
+
+ @Test
+ public void testAssertContainsSublistSuccess() {
+ List<String> actual = Arrays.asList("a", "b", "c");
+
+ // All single-string combinations.
+ assertContainsSublist(actual, "a");
+ assertContainsSublist(actual, "b");
+ assertContainsSublist(actual, "c");
+
+ // All two-string combinations.
+ assertContainsSublist(actual, "a", "b");
+ assertContainsSublist(actual, "b", "c");
+
+ // The whole list.
+ assertContainsSublist(actual, "a", "b", "c");
+ }
+
+ @Test
+ public void testAssertContainsSublistFailure() {
+ List<String> actual = Arrays.asList("a", "b", "c");
+
+ try {
+ assertContainsSublist(actual, "d");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e.getMessage()).startsWith("Did not find [d] as a sublist of [a, b, c]");
+ }
+
+ try {
+ assertContainsSublist(actual, "a", "c");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e.getMessage()).startsWith("Did not find [a, c] as a sublist of [a, b, c]");
+ }
+
+ try {
+ assertContainsSublist(actual, "b", "c", "d");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e.getMessage()).startsWith("Did not find [b, c, d] as a sublist of [a, b, c]");
+ }
+ }
+
+ @Test
+ public void testAssertDoesNotContainSublistSuccess() {
+ List<String> actual = Arrays.asList("a", "b", "c");
+ assertDoesNotContainSublist(actual, "d");
+ assertDoesNotContainSublist(actual, "a", "c");
+ assertDoesNotContainSublist(actual, "b", "c", "d");
+ }
+
+ @Test
+ public void testAssertDoesNotContainSublistFailure() {
+ List<String> actual = Arrays.asList("a", "b", "c");
+
+ // All single-string combinations.
+ try {
+ assertDoesNotContainSublist(actual, "a");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e).hasMessage("Found [a] as a sublist of [a, b, c]");
+ }
+ try {
+ assertDoesNotContainSublist(actual, "b");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e).hasMessage("Found [b] as a sublist of [a, b, c]");
+ }
+ try {
+ assertDoesNotContainSublist(actual, "c");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e).hasMessage("Found [c] as a sublist of [a, b, c]");
+ }
+
+ // All two-string combinations.
+ try {
+ assertDoesNotContainSublist(actual, "a", "b");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e).hasMessage("Found [a, b] as a sublist of [a, b, c]");
+ }
+ try {
+ assertDoesNotContainSublist(actual, "b", "c");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e).hasMessage("Found [b, c] as a sublist of [a, b, c]");
+ }
+
+ // The whole list.
+ try {
+ assertDoesNotContainSublist(actual, "a", "b", "c");
+ fail("no exception thrown");
+ } catch (AssertionError e) {
+ assertThat(e).hasMessage("Found [a, b, c] as a sublist of [a, b, c]");
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/testutiltests/TestSizeAnnotationTest.java b/src/test/java/com/google/devtools/build/lib/testutiltests/TestSizeAnnotationTest.java
new file mode 100644
index 0000000..5380cba
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/testutiltests/TestSizeAnnotationTest.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.testutiltests;
+
+import static com.google.devtools.build.lib.testutil.Suite.getSize;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.devtools.build.lib.testutil.Suite;
+import com.google.devtools.build.lib.testutil.TestSpec;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests {@link com.google.devtools.build.lib.testutil.Suite#getSize(Class)}.
+ */
+@RunWith(JUnit4.class)
+public class TestSizeAnnotationTest {
+
+ private static class HasNoTestSpecAnnotation {
+
+ }
+
+ @TestSpec(flaky = true)
+ private static class FlakyTestSpecAnnotation {
+
+ }
+
+ @TestSpec(suite = "foo")
+ private static class HasNoSizeAnnotationElement {
+
+ }
+
+ @TestSpec(size = Suite.SMALL_TESTS)
+ private static class IsAnnotatedWithSmallSize {
+
+ }
+
+ @TestSpec(size = Suite.MEDIUM_TESTS)
+ private static class IsAnnotatedWithMediumSize {
+
+ }
+
+ @TestSpec(size = Suite.LARGE_TESTS)
+ private static class IsAnnotatedWithLargeSize {
+
+ }
+
+ private static class SuperclassHasAnnotationButNoSizeElement
+ extends HasNoSizeAnnotationElement {
+
+ }
+
+ @TestSpec(size = Suite.LARGE_TESTS)
+ private static class HasSizeElementAndSuperclassHasAnnotationButNoSizeElement
+ extends HasNoSizeAnnotationElement {
+
+ }
+
+ private static class SuperclassHasAnnotationWithSizeElement
+ extends IsAnnotatedWithSmallSize {
+
+ }
+
+ @TestSpec(size = Suite.LARGE_TESTS)
+ private static class HasSizeElementAndSuperclassHasAnnotationWithSizeElement
+ extends IsAnnotatedWithSmallSize {
+
+ }
+
+ @Test
+ public void testHasNoTestSpecAnnotationIsSmall() {
+ assertEquals(Suite.SMALL_TESTS, getSize(HasNoTestSpecAnnotation.class));
+ }
+
+ @Test
+ public void testHasNoSizeAnnotationElementIsSmall() {
+ assertEquals(Suite.SMALL_TESTS, getSize(HasNoSizeAnnotationElement.class));
+ }
+
+ @Test
+ public void testIsAnnotatedWithSmallSizeIsSmall() {
+ assertEquals(Suite.SMALL_TESTS, getSize(IsAnnotatedWithSmallSize.class));
+ }
+
+ @Test
+ public void testIsAnnotatedWithMediumSizeIsMedium() {
+ assertEquals(Suite.MEDIUM_TESTS, getSize(IsAnnotatedWithMediumSize.class));
+ }
+
+ @Test
+ public void testIsAnnotatedWithLargeSizeIsLarge() {
+ assertEquals(Suite.LARGE_TESTS, getSize(IsAnnotatedWithLargeSize.class));
+ }
+
+ @Test
+ public void testSuperclassHasAnnotationButNoSizeElement() {
+ assertEquals(Suite.SMALL_TESTS, getSize(SuperclassHasAnnotationButNoSizeElement.class));
+ }
+
+ @Test
+ public void testHasSizeElementAndSuperclassHasAnnotationButNoSizeElement() {
+ assertEquals(Suite.LARGE_TESTS,
+ getSize(HasSizeElementAndSuperclassHasAnnotationButNoSizeElement.class));
+ }
+
+ @Test
+ public void testSuperclassHasAnnotationWithSizeElement() {
+ assertEquals(Suite.SMALL_TESTS, getSize(SuperclassHasAnnotationWithSizeElement.class));
+ }
+
+ @Test
+ public void testHasSizeElementAndSuperclassHasAnnotationWithSizeElement() {
+ assertEquals(Suite.LARGE_TESTS,
+ getSize(HasSizeElementAndSuperclassHasAnnotationWithSizeElement.class));
+ }
+
+ @Test
+ public void testIsNotFlaky() {
+ assertFalse(Suite.isFlaky(HasNoTestSpecAnnotation.class));
+ }
+
+ @Test
+ public void testIsFlaky() {
+ assertTrue(Suite.isFlaky(FlakyTestSpecAnnotation.class));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/unix/FilesystemUtilsTest.java b/src/test/java/com/google/devtools/build/lib/unix/FilesystemUtilsTest.java
new file mode 100644
index 0000000..48a67c1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/unix/FilesystemUtilsTest.java
@@ -0,0 +1,76 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.unix;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.hash.HashCode;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.UnixFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+import java.util.HashMap;
+
+/**
+ * This class tests the FilesystemUtils class.
+ */
+@RunWith(JUnit4.class)
+public class FilesystemUtilsTest {
+ private FileSystem testFS;
+ private Path workingDir;
+ private Path testFile;
+
+ @Before
+ public void setUp() throws Exception {
+ testFS = new UnixFileSystem();
+ workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath());
+ testFile = workingDir.getRelative("test");
+ FileSystemUtils.createEmptyFile(testFile);
+ }
+
+ /**
+ * This test validates that the md5sum() method returns hashes that match the official test
+ * vectors specified in RFC 1321, The MD5 Message-Digest Algorithm.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidateMd5Sum() throws Exception {
+ HashMap<String, String> testVectors = new HashMap<String, String>();
+ testVectors.put("", "d41d8cd98f00b204e9800998ecf8427e");
+ testVectors.put("a", "0cc175b9c0f1b6a831c399e269772661");
+ testVectors.put("abc", "900150983cd24fb0d6963f7d28e17f72");
+ testVectors.put("message digest", "f96b697d7cb7938d525a2f31aaf161d0");
+ testVectors.put("abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b");
+ testVectors.put("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
+ "d174ab98d277d9f5a5611c2c9f419d9f");
+ testVectors.put(
+ "12345678901234567890123456789012345678901234567890123456789012345678901234567890",
+ "57edf4a22be3c955ac49da2e2107b67a");
+
+ for (String testInput : testVectors.keySet()) {
+ FileSystemUtils.writeContentAsLatin1(testFile, testInput);
+ HashCode result = FilesystemUtils.md5sum(testFile.getPathString());
+ assertEquals(result.toString(), testVectors.get(testInput));
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStreamTest.java
new file mode 100644
index 0000000..41428cb
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStreamTest.java
@@ -0,0 +1,84 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+
+/**
+ * Tests for {@link AnsiStrippingOutputStream}.
+ */
+@RunWith(JUnit4.class)
+public class AnsiStrippingOutputStreamTest {
+ ByteArrayOutputStream output;
+ PrintStream input;
+
+ private static final String ESCAPE = "\u001b[";
+
+ @Before
+ public void setUp() throws Exception {
+ output = new ByteArrayOutputStream();
+ OutputStream inputStream = new AnsiStrippingOutputStream(output);
+ input = new PrintStream(inputStream);
+ }
+
+ private String getOutput(String... fragments) throws Exception {
+ for (String fragment: fragments) {
+ input.print(fragment);
+ }
+
+ return new String(output.toByteArray(), "ISO8859-1");
+ }
+
+ @Test
+ public void doesNotFailHorribly() throws Exception {
+ assertEquals("Love", getOutput("Love"));
+ }
+
+ @Test
+ public void canStripAnsiCode() throws Exception {
+ assertEquals("Love", getOutput(ESCAPE + "32mLove" + ESCAPE + "m"));
+ }
+
+ @Test
+ public void recognizesAnsiCodeWhenBrokenUp() throws Exception {
+ assertEquals("Love", getOutput("\u001b", "[", "mLove"));
+ }
+
+ @Test
+ public void handlesOnlyEscCorrectly() throws Exception {
+ assertEquals("\u001bLove", getOutput("\u001bLove"));
+ }
+
+ @Test
+ public void handlesEscInPlaceOfControlCharCorrectly() throws Exception {
+ assertEquals(ESCAPE + "31;42Love",
+ getOutput(ESCAPE + "31;42" + ESCAPE + "1mLove"));
+ }
+
+ @Test
+ public void handlesTwoEscapeSequencesCorrectly() throws Exception {
+ assertEquals("Love",
+ getOutput(ESCAPE + "32m" + ESCAPE + "1m" + "Love"));
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/CommandBuilderTest.java b/src/test/java/com/google/devtools/build/lib/util/CommandBuilderTest.java
new file mode 100644
index 0000000..566da25
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/CommandBuilderTest.java
@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+
+/**
+ * Tests for the {@link CommandBuilder} class.
+ */
+@RunWith(JUnit4.class)
+public class CommandBuilderTest {
+
+ private CommandBuilder linuxBuilder() {
+ return new CommandBuilder(OS.LINUX).useTempDir();
+ }
+
+ private CommandBuilder winBuilder() {
+ return new CommandBuilder(OS.WINDOWS).useTempDir();
+ }
+
+ private void assertArgv(CommandBuilder builder, String... expected) {
+ assertThat(Arrays.asList(builder.build().getCommandLineElements())).containsExactlyElementsIn(
+ Arrays.asList(expected)).inOrder();
+ }
+
+ private void assertWinCmdArgv(CommandBuilder builder, String expected) {
+ assertArgv(builder, "CMD.EXE", "/S", "/E:ON", "/V:ON", "/D", "/C", "\"" + expected + "\"");
+ }
+
+ private void assertFailure(CommandBuilder builder, String expected) {
+ try {
+ builder.build();
+ fail("Expected exception");
+ } catch (Exception e) {
+ assertEquals(expected, e.getMessage());
+ }
+ }
+
+ @Test
+ public void linuxBuilderTest() {
+ assertArgv(linuxBuilder().addArg("abc"), "abc");
+ assertArgv(linuxBuilder().addArg("abc def"), "abc def");
+ assertArgv(linuxBuilder().addArgs("abc", "def"), "abc", "def");
+ assertArgv(linuxBuilder().addArgs(ImmutableList.of("abc", "def")), "abc", "def");
+ assertArgv(linuxBuilder().addArg("abc").useShell(true), "/bin/sh", "-c", "abc");
+ assertArgv(linuxBuilder().addArg("abc def").useShell(true), "/bin/sh", "-c", "abc def");
+ assertArgv(linuxBuilder().addArgs("abc", "def").useShell(true), "/bin/sh", "-c", "abc def");
+ assertArgv(linuxBuilder().addArgs("/bin/sh", "-c", "abc").useShell(true),
+ "/bin/sh", "-c", "abc");
+ assertArgv(linuxBuilder().addArgs("/bin/sh", "-c"), "/bin/sh", "-c");
+ assertArgv(linuxBuilder().addArgs("/bin/bash", "-c"), "/bin/bash", "-c");
+ assertArgv(linuxBuilder().addArgs("/bin/sh", "-c").useShell(true), "/bin/sh", "-c");
+ assertArgv(linuxBuilder().addArgs("/bin/bash", "-c").useShell(true), "/bin/bash", "-c");
+ }
+
+ @Test
+ public void windowsBuilderTest() {
+ assertArgv(winBuilder().addArg("abc.exe"), "abc.exe");
+ assertArgv(winBuilder().addArg("abc.exe -o"), "abc.exe -o");
+ assertArgv(winBuilder().addArg("ABC.EXE"), "ABC.EXE");
+ assertWinCmdArgv(winBuilder().addArg("abc def.exe"), "abc def.exe");
+ assertArgv(winBuilder().addArgs("abc.exe", "def"), "abc.exe", "def");
+ assertArgv(winBuilder().addArgs(ImmutableList.of("abc.exe", "def")), "abc.exe", "def");
+ assertWinCmdArgv(winBuilder().addArgs("abc.exe", "def").useShell(true), "abc.exe def");
+ assertWinCmdArgv(winBuilder().addArg("abc"), "abc");
+ assertWinCmdArgv(winBuilder().addArgs("abc", "def"), "abc def");
+ assertWinCmdArgv(winBuilder().addArgs("/bin/sh", "-c", "abc", "def"), "abc def");
+ assertWinCmdArgv(winBuilder().addArgs("/bin/sh", "-c"), "");
+ assertWinCmdArgv(winBuilder().addArgs("/bin/bash", "-c"), "");
+ assertWinCmdArgv(winBuilder().addArgs("/bin/sh", "-c").useShell(true), "");
+ assertWinCmdArgv(winBuilder().addArgs("/bin/bash", "-c").useShell(true), "");
+ }
+
+ @Test
+ public void failureScenarios() {
+ assertFailure(linuxBuilder(), "At least one argument is expected");
+ assertFailure(new CommandBuilder(OS.UNKNOWN).useTempDir().addArg("a"),
+ "Unidentified operating system");
+ assertFailure(new CommandBuilder(OS.LINUX).addArg("a"),
+ "Working directory must be set");
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/CommandFailureUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/CommandFailureUtilsTest.java
new file mode 100644
index 0000000..f0c2c4a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/CommandFailureUtilsTest.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+@RunWith(JUnit4.class)
+public class CommandFailureUtilsTest {
+
+ @Test
+ public void describeCommandError() throws Exception {
+ String[] args = new String[40];
+ args[0] = "some_command";
+ for (int i = 1; i < args.length; i++) {
+ args[i] = "arg" + i;
+ }
+ args[7] = "with spaces"; // Test embedded spaces in argument.
+ args[9] = "*"; // Test shell meta characters.
+ Map<String, String> env = new HashMap<>();
+ env.put("PATH", "/usr/bin:/bin:/sbin");
+ env.put("FOO", "foo");
+ String cwd = "/my/working/directory";
+ String message = CommandFailureUtils.describeCommandError(false, Arrays.asList(args), env, cwd);
+ String verboseMessage = CommandFailureUtils.describeCommandError(true, Arrays.asList(args), env,
+ cwd);
+ assertEquals(
+ "error executing command some_command arg1 "
+ + "arg2 arg3 arg4 arg5 arg6 'with spaces' arg8 '*' arg10 "
+ + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 "
+ + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 "
+ + "arg27 arg28 arg29 arg30 arg31 "
+ + "... (remaining 8 argument(s) skipped)",
+ message);
+ assertEquals(
+ "error executing command \n"
+ + " (cd /my/working/directory && \\\n"
+ + " exec env - \\\n"
+ + " FOO=foo \\\n"
+ + " PATH=/usr/bin:/bin:/sbin \\\n"
+ + " some_command arg1 arg2 arg3 arg4 arg5 arg6 'with spaces' arg8 '*' arg10 "
+ + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 "
+ + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 "
+ + "arg27 arg28 arg29 arg30 arg31 arg32 arg33 arg34 "
+ + "arg35 arg36 arg37 arg38 arg39)",
+ verboseMessage);
+ }
+
+ @Test
+ public void describeCommandFailure() throws Exception {
+ String[] args = new String[3];
+ args[0] = "/bin/sh";
+ args[1] = "-c";
+ args[2] = "echo Some errors 1>&2; echo Some output; exit 42";
+ Map<String, String> env = new HashMap<>();
+ env.put("FOO", "foo");
+ env.put("PATH", "/usr/bin:/bin:/sbin");
+ String cwd = null;
+ String message = CommandFailureUtils.describeCommandFailure(false, Arrays.asList(args),
+ env, cwd);
+ String verboseMessage = CommandFailureUtils.describeCommandFailure(true, Arrays.asList(args),
+ env, cwd);
+ assertEquals(
+ "sh failed: error executing command "
+ + "/bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42'",
+ message);
+ assertEquals(
+ "sh failed: error executing command \n"
+ + " (exec env - \\\n"
+ + " FOO=foo \\\n"
+ + " PATH=/usr/bin:/bin:/sbin \\\n"
+ + " /bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42')",
+ verboseMessage);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/CommandUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/CommandUtilsTest.java
new file mode 100644
index 0000000..c7639a2
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/CommandUtilsTest.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+import java.util.Map;
+
+@RunWith(JUnit4.class)
+public class CommandUtilsTest {
+
+ @Test
+ public void longCommand() throws Exception {
+ String[] args = new String[40];
+ args[0] = "this_command_will_not_be_found";
+ for (int i = 1; i < args.length; i++) {
+ args[i] = "arg" + i;
+ }
+ Map<String, String> env = Maps.newTreeMap();
+ env.put("PATH", "/usr/bin:/bin:/sbin");
+ env.put("FOO", "foo");
+ File directory = new File("/tmp");
+ try {
+ new Command(args, env, directory).execute();
+ fail();
+ } catch (CommandException exception) {
+ String message = CommandUtils.describeCommandError(false, exception.getCommand());
+ String verboseMessage = CommandUtils.describeCommandError(true, exception.getCommand());
+ assertEquals(
+ "error executing command this_command_will_not_be_found arg1 "
+ + "arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 "
+ + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 "
+ + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 "
+ + "arg27 arg28 arg29 arg30 "
+ + "... (remaining 9 argument(s) skipped)",
+ message);
+ assertEquals(
+ "error executing command \n"
+ + " (cd /tmp && \\\n"
+ + " exec env - \\\n"
+ + " FOO=foo \\\n"
+ + " PATH=/usr/bin:/bin:/sbin \\\n"
+ + " this_command_will_not_be_found arg1 "
+ + "arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 "
+ + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 "
+ + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 "
+ + "arg27 arg28 arg29 arg30 arg31 arg32 arg33 arg34 "
+ + "arg35 arg36 arg37 arg38 arg39)",
+ verboseMessage);
+ }
+ }
+
+ @Test
+ public void failingCommand() throws Exception {
+ String[] args = new String[3];
+ args[0] = "/bin/sh";
+ args[1] = "-c";
+ args[2] = "echo Some errors 1>&2; echo Some output; exit 42";
+ Map<String, String> env = Maps.newTreeMap();
+ env.put("FOO", "foo");
+ env.put("PATH", "/usr/bin:/bin:/sbin");
+ try {
+ new Command(args, env, null).execute();
+ fail();
+ } catch (CommandException exception) {
+ String message = CommandUtils.describeCommandFailure(false, exception);
+ String verboseMessage = CommandUtils.describeCommandFailure(true, exception);
+ assertEquals(
+ "sh failed: error executing command " +
+ "/bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42': " +
+ "Process exited with status 42\n" +
+ "Some output\n" +
+ "Some errors\n",
+ message);
+ assertEquals(
+ "sh failed: error executing command \n" +
+ " (exec env - \\\n" +
+ " FOO=foo \\\n" +
+ " PATH=/usr/bin:/bin:/sbin \\\n" +
+ " /bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42'): " +
+ "Process exited with status 42\n" +
+ "Some output\n" +
+ "Some errors\n",
+ verboseMessage);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/DependencySetTest.java b/src/test/java/com/google/devtools/build/lib/util/DependencySetTest.java
new file mode 100644
index 0000000..40edf36
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/DependencySetTest.java
@@ -0,0 +1,230 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Collection;
+
+@RunWith(JUnit4.class)
+public class DependencySetTest {
+
+ private FsApparatus scratch = FsApparatus.newInMemory();
+
+ private DependencySet newDependencySet() {
+ return new DependencySet(scratch.fs().getRootDirectory());
+ }
+
+ @Test
+ public void dotDParser_simple() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ String filename = "hello.o";
+ Path dotd = scratch.file("/tmp/foo.d",
+ filename + ": \\",
+ " " + file1 + " \\",
+ " " + file2 + " ");
+ DependencySet depset = newDependencySet().read(dotd);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2),
+ depset.getDependencies());
+ assertEquals(depset.getOutputFileName(), filename);
+ }
+
+ @Test
+ public void dotDParser_simple_crlf() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ String filename = "hello.o";
+ Path dotd = scratch.file("/tmp/foo.d",
+ filename + ": \\\r",
+ " " + file1 + " \\\r",
+ " " + file2 + " ");
+ DependencySet depset = newDependencySet().read(dotd);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2),
+ depset.getDependencies());
+ assertEquals(depset.getOutputFileName(), filename);
+ }
+
+ @Test
+ public void dotDParser_simple_cr() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ String filename = "hello.o";
+ Path dotd = scratch.file("/tmp/foo.d",
+ filename + ": \\\r"
+ + " " + file1 + " \\\r"
+ + " " + file2 + " ");
+ DependencySet depset = newDependencySet().read(dotd);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2),
+ depset.getDependencies());
+ assertEquals(depset.getOutputFileName(), filename);
+ }
+
+ @Test
+ public void dotDParser_leading_crlf() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ String filename = "hello.o";
+ Path dotd = scratch.file("/tmp/foo.d",
+ "\r\n" + filename + ": \\\r\n"
+ + " " + file1 + " \\\r\n"
+ + " " + file2 + " ");
+ DependencySet depset = newDependencySet().read(dotd);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2),
+ depset.getDependencies());
+ assertEquals(depset.getOutputFileName(), filename);
+ }
+
+ @Test
+ public void dotDParser_oddFormatting() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h");
+ PathFragment file4 = new PathFragment("/usr/local/blah/blah/genhello/onemore.h");
+ String filename = "hello.o";
+ Path dotd = scratch.file("/tmp/foo.d",
+ filename + ": " + file1 + " \\",
+ " " + file2 + "\\",
+ " " + file3 + " " + file4);
+ DependencySet depset = newDependencySet().read(dotd);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2, file3, file4),
+ depset.getDependencies());
+ assertEquals(depset.getOutputFileName(), filename);
+ }
+
+ @Test
+ public void dotDParser_relativeFilenames() throws Exception {
+ PathFragment file1 = new PathFragment("hello.cc");
+ PathFragment file2 = new PathFragment("hello.h");
+ String filename = "hello.o";
+ Path dotd = scratch.file("/tmp/foo.d",
+ filename + ": \\",
+ " " + file1 + " \\",
+ " " + file2 + " ");
+ DependencySet depset = newDependencySet().read(dotd);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2),
+ depset.getDependencies());
+ assertEquals(depset.getOutputFileName(), filename);
+ }
+
+ @Test
+ public void dotDParser_emptyFile() throws Exception {
+ Path dotd = scratch.file("/tmp/empty.d");
+ DependencySet depset = newDependencySet().read(dotd);
+ Collection<PathFragment> headers = depset.getDependencies();
+ if (!headers.isEmpty()) {
+ fail("Not empty: " + headers.size() + " " + headers);
+ }
+ assertEquals(depset.getOutputFileName(), null);
+ }
+
+ @Test
+ public void dotDParser_multipleTargets() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ Path dotd = scratch.file("/tmp/foo.d",
+ "hello.o: \\",
+ " " + file1,
+ "hello2.o: \\",
+ " " + file2);
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2),
+ newDependencySet().read(dotd).getDependencies());
+ }
+
+ /*
+ * Regression test: if gcc fails to execute remotely, and we retry locally, then the behavior
+ * of gcc's DEPENDENCIES_OUTPUT option is to append, not overwrite, the .d file. As a result,
+ * during retry, a second stanza is written to the file.
+ *
+ * We handle this by merging all of the stanzas.
+ */
+ @Test
+ public void dotDParser_duplicateStanza() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h");
+ Path dotd = scratch.file("/tmp/foo.d",
+ "hello.o: \\",
+ " " + file1 + " \\",
+ " " + file2 + " ",
+ "hello.o: \\",
+ " " + file1 + " \\",
+ " " + file3 + " ");
+ MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2, file3),
+ newDependencySet().read(dotd).getDependencies());
+ }
+
+ @Test
+ public void writeSet() throws Exception {
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h");
+ String filename = "/usr/local/blah/blah/genhello/hello.o";
+
+ DependencySet depSet1 = newDependencySet();
+ depSet1.addDependency(file1);
+ depSet1.addDependency(file2);
+ depSet1.addDependency(file3);
+ depSet1.setOutputFileName(filename);
+
+ Path outfile = scratch.path(filename);
+ Path dotd = scratch.path("/usr/local/blah/blah/genhello/hello.d");
+ FileSystemUtils.createDirectoryAndParents(dotd.getParentDirectory());
+ depSet1.write(outfile, ".d");
+
+ String dotdContents = new String(FileSystemUtils.readContentAsLatin1(dotd));
+ String expected =
+ "usr/local/blah/blah/genhello/hello.o: \\\n" +
+ " /usr/local/blah/blah/genhello/hello.cc \\\n" +
+ " /usr/local/blah/blah/genhello/hello.h \\\n" +
+ " /usr/local/blah/blah/genhello/other.h\n";
+ assertEquals(expected, dotdContents);
+ assertEquals(filename, depSet1.getOutputFileName());
+ }
+
+ @Test
+ public void writeReadSet() throws Exception {
+ String filename = "/usr/local/blah/blah/genhello/hello.d";
+ PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc");
+ PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h");
+ PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h");
+ DependencySet depSet1 = newDependencySet();
+ depSet1.addDependency(file1);
+ depSet1.addDependency(file2);
+ depSet1.addDependency(file3);
+ depSet1.setOutputFileName(filename);
+
+ Path dotd = scratch.path(filename);
+ FileSystemUtils.createDirectoryAndParents(dotd.getParentDirectory());
+ depSet1.write(dotd, ".d");
+
+ DependencySet depSet2 = newDependencySet().read(dotd);
+ assertEquals(depSet1, depSet2);
+ // due to how pic.d files are written, absolute paths are changed into relatives
+ assertEquals(depSet1.getOutputFileName(), "/" + depSet2.getOutputFileName());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/DependencySetWindowsTest.java b/src/test/java/com/google/devtools/build/lib/util/DependencySetWindowsTest.java
new file mode 100644
index 0000000..cdda6d1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/DependencySetWindowsTest.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Set;
+
+@RunWith(JUnit4.class)
+public class DependencySetWindowsTest {
+
+ private FsApparatus scratch = FsApparatus.newInMemory();
+
+ private DependencySet newDependencySet() {
+ return new DependencySet(scratch.fs().getRootDirectory());
+ }
+
+ @Test
+ public void dotDParser_windowsPaths() throws Exception {
+ Path dotd = scratch.file("/tmp/foo.d",
+ "bazel-out/hello-lib/cpp/hello-lib.o: \\",
+ " cpp/hello-lib.cc cpp/hello-lib.h c:\\mingw\\include\\stdio.h \\",
+ " c:\\mingw\\include\\_mingw.h \\",
+ " c:\\mingw\\lib\\gcc\\mingw32\\4.8.1\\include\\stdarg.h");
+
+ Set<PathFragment> expected = Sets.newHashSet(
+ new PathFragment("cpp/hello-lib.cc"),
+ new PathFragment("cpp/hello-lib.h"),
+ new PathFragment("C:/mingw/include/stdio.h"),
+ new PathFragment("C:/mingw/include/_mingw.h"),
+ new PathFragment("C:/mingw/lib/gcc/mingw32/4.8.1/include/stdarg.h"));
+
+ MoreAsserts.assertSameContents(expected,
+ newDependencySet().read(dotd).getDependencies());
+ }
+
+ @Test
+ public void dotDParser_windowsPathsWithSpaces() throws Exception {
+ Path dotd = scratch.file("/tmp/foo.d",
+ "bazel-out/hello-lib/cpp/hello-lib.o: \\",
+ "C:\\Program\\ Files\\ (x86)\\LLVM\\stddef.h");
+ MoreAsserts.assertSameContents(
+ Sets.newHashSet(new PathFragment("C:/Program Files (x86)/LLVM/stddef.h")),
+ newDependencySet().read(dotd).getDependencies());
+ }
+
+ @Test
+ public void dotDParser_mixedWindowsPaths() throws Exception {
+ // This is (slightly simplified) actual output from clang. Yes, clang will happily mix
+ // forward slashes and backslashes in a single path, not to mention using backslashes as
+ // separators next to backslashes as escape characters.
+ Path dotd = scratch.file("/tmp/foo.d",
+ "bazel-out/hello-lib/cpp/hello-lib.o: \\",
+ "cpp/hello-lib.cc cpp/hello-lib.h /mingw/include\\stdio.h \\",
+ "/mingw/include\\_mingw.h \\",
+ "C:\\Program\\ Files\\ (x86)\\LLVM\\bin\\..\\lib\\clang\\3.5.0\\include\\stddef.h \\",
+ "C:\\Program\\ Files\\ (x86)\\LLVM\\bin\\..\\lib\\clang\\3.5.0\\include\\stdarg.h");
+
+ Set<PathFragment> expected = Sets.newHashSet(
+ new PathFragment("cpp/hello-lib.cc"),
+ new PathFragment("cpp/hello-lib.h"),
+ new PathFragment("/mingw/include/stdio.h"),
+ new PathFragment("/mingw/include/_mingw.h"),
+ new PathFragment("C:/Program Files (x86)/LLVM/lib/clang/3.5.0/include/stddef.h"),
+ new PathFragment("C:/Program Files (x86)/LLVM/lib/clang/3.5.0/include/stdarg.h"));
+
+ MoreAsserts.assertSameContents(expected,
+ newDependencySet().read(dotd).getDependencies());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/FileTypeTest.java b/src/test/java/com/google/devtools/build/lib/util/FileTypeTest.java
new file mode 100644
index 0000000..301875c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/FileTypeTest.java
@@ -0,0 +1,244 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.util.FileType.HasFilename;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+
+/**
+ * Test for {@link FileType} and {@link FileTypeSet}.
+ */
+@RunWith(JUnit4.class)
+public class FileTypeTest {
+ private static final FileType CFG = FileType.of(".cfg");
+ private static final FileType HTML = FileType.of(".html");
+ private static final FileType TEXT = FileType.of(".txt");
+ private static final FileType CPP_SOURCE = FileType.of(".cc", ".cpp", ".cxx", ".C");
+ private static final FileType JAVA_SOURCE = FileType.of(".java");
+ private static final FileType PYTHON_SOURCE = FileType.of(".py");
+
+ private static final class HasFilenameImpl implements HasFilename {
+ private final String path;
+
+ private HasFilenameImpl(String path) {
+ this.path = path;
+ }
+
+ @Override
+ public String getFilename() {
+ return path;
+ }
+
+ @Override
+ public String toString() {
+ return path;
+ }
+ }
+
+ @Test
+ public void simpleDotMatch() {
+ assertTrue(TEXT.matches("readme.txt"));
+ }
+
+ @Test
+ public void doubleDotMatches() {
+ assertTrue(TEXT.matches("read.me.txt"));
+ }
+
+ @Test
+ public void noExtensionMatches() {
+ assertTrue(FileType.NO_EXTENSION.matches("hello"));
+ assertTrue(FileType.NO_EXTENSION.matches("/path/to/hello"));
+ }
+
+ @Test
+ public void picksLastExtension() {
+ assertTrue(TEXT.matches("server.cfg.txt"));
+ }
+
+ @Test
+ public void onlyExtensionStillMatches() {
+ assertTrue(TEXT.matches(".txt"));
+ }
+
+ @Test
+ public void handlesPathObjects() {
+ Path readme = new InMemoryFileSystem().getPath("/readme.txt");
+ assertTrue(TEXT.matches(readme));
+ }
+
+ @Test
+ public void handlesPathFragmentObjects() {
+ PathFragment readme = new PathFragment("some/where/readme.txt");
+ assertTrue(TEXT.matches(readme));
+ }
+
+ @Test
+ public void fileTypeSetContains() {
+ FileTypeSet allowedTypes = FileTypeSet.of(TEXT, HTML);
+
+ assertTrue(allowedTypes.matches("readme.txt"));
+ assertTrue(!allowedTypes.matches("style.css"));
+ }
+
+ private List<HasFilename> getArtifacts() {
+ return Lists.<HasFilename>newArrayList(
+ new HasFilenameImpl("Foo.java"),
+ new HasFilenameImpl("bar.cc"),
+ new HasFilenameImpl("baz.py"));
+ }
+
+ private String filterAll(FileType... fileTypes) {
+ return Joiner.on(" ").join(FileType.filter(getArtifacts(), fileTypes));
+ }
+
+ @Test
+ public void justJava() {
+ assertEquals("Foo.java", filterAll(JAVA_SOURCE));
+ }
+
+ @Test
+ public void javaAndCpp() {
+ assertEquals("Foo.java bar.cc", filterAll(JAVA_SOURCE, CPP_SOURCE));
+ }
+
+ @Test
+ public void allThree() {
+ assertEquals("Foo.java bar.cc baz.py", filterAll(JAVA_SOURCE, CPP_SOURCE, PYTHON_SOURCE));
+ }
+
+ private HasFilename filename(final String name) {
+ return new HasFilename() {
+ @Override
+ public String getFilename() {
+ return name;
+ }
+ };
+ }
+
+ @Test
+ public void checkingSingleWithTypePredicate() throws Exception {
+ FileType.HasFilename item = filename("config.txt");
+
+ assertTrue(FileType.contains(item, TEXT));
+ assertFalse(FileType.contains(item, CFG));
+ }
+
+ @Test
+ public void checkingListWithTypePredicate() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("README.txt"));
+
+ assertTrue(FileType.contains(unfiltered, TEXT));
+ assertFalse(FileType.contains(unfiltered, CFG));
+ }
+
+ @Test
+ public void filteringWithTypePredicate() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("README.txt"),
+ filename("archive.zip"));
+
+ assertThat(FileType.filter(unfiltered, TEXT)).containsExactly(unfiltered.get(0),
+ unfiltered.get(2)).inOrder();
+ }
+
+ @Test
+ public void filteringWithMatcherPredicate() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("README.txt"),
+ filename("archive.zip"));
+
+ Predicate<String> textFileTypeMatcher = new Predicate<String>() {
+ @Override
+ public boolean apply(String input) {
+ return TEXT.matches(input);
+ }
+ };
+
+ assertThat(FileType.filter(unfiltered, textFileTypeMatcher)).containsExactly(unfiltered.get(0),
+ unfiltered.get(2)).inOrder();
+ }
+
+ @Test
+ public void filteringWithAlwaysFalse() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("binary"),
+ filename("archive.zip"));
+
+ assertThat(FileType.filter(unfiltered, FileTypeSet.NO_FILE)).isEmpty();
+ }
+
+ @Test
+ public void filteringWithAlwaysTrue() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("binary"),
+ filename("archive.zip"));
+
+ assertThat(FileType.filter(unfiltered, FileTypeSet.ANY_FILE)).containsExactly(unfiltered.get(0),
+ unfiltered.get(1), unfiltered.get(2), unfiltered.get(3)).inOrder();
+ }
+
+ @Test
+ public void exclusionWithTypePredicate() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("README.txt"),
+ filename("server.cfg"));
+
+ assertThat(FileType.except(unfiltered, TEXT)).containsExactly(unfiltered.get(1),
+ unfiltered.get(3)).inOrder();
+ }
+
+ @Test
+ public void listFiltering() throws Exception {
+ ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of(
+ filename("config.txt"),
+ filename("index.html"),
+ filename("README.txt"),
+ filename("server.cfg"));
+ FileTypeSet filter = FileTypeSet.of(HTML, CFG);
+
+ assertThat(FileType.filterList(unfiltered, filter)).containsExactly(unfiltered.get(1),
+ unfiltered.get(3)).inOrder();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/FingerprintTest.java b/src/test/java/com/google/devtools/build/lib/util/FingerprintTest.java
new file mode 100644
index 0000000..5158019
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/FingerprintTest.java
@@ -0,0 +1,137 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests for Fingerprint.
+ */
+@RunWith(JUnit4.class)
+public class FingerprintTest {
+
+ private static void assertFingerprintsDiffer(List<String> list1, List<String>list2) {
+ Fingerprint f1 = new Fingerprint();
+ Fingerprint f1Latin1 = new Fingerprint();
+ for (String s : list1) {
+ f1.addString(s);
+ f1Latin1.addStringLatin1(s);
+ }
+ Fingerprint f2 = new Fingerprint();
+ Fingerprint f2Latin1 = new Fingerprint();
+ for (String s : list2) {
+ f2.addString(s);
+ f2Latin1.addStringLatin1(s);
+ }
+ assertThat(f1.hexDigestAndReset()).isNotEqualTo(f2.hexDigestAndReset());
+ assertThat(f1Latin1.hexDigestAndReset()).isNotEqualTo(f2Latin1.hexDigestAndReset());
+ }
+
+ // You can validate the md5 of the simple string against
+ // echo -n 'Hello World!'| md5sum
+ @Test
+ public void bytesFingerprint() {
+ assertThat("ed076287532e86365e841e92bfc50d8c").isEqualTo(
+ new Fingerprint().addBytes("Hello World!".getBytes(UTF_8)).hexDigestAndReset());
+ assertThat("ed076287532e86365e841e92bfc50d8c").isEqualTo(Fingerprint.md5Digest("Hello World!"));
+ }
+
+ @Test
+ public void otherStringFingerprint() {
+ assertFingerprintsDiffer(ImmutableList.of("Hello World!"),
+ ImmutableList.of("Goodbye World."));
+ }
+
+ @Test
+ public void multipleUpdatesDiffer() throws Exception {
+ assertFingerprintsDiffer(ImmutableList.of("Hello ", "World!"),
+ ImmutableList.of("Hello World!"));
+ }
+
+ @Test
+ public void multipleUpdatesShiftedDiffer() throws Exception {
+ assertFingerprintsDiffer(ImmutableList.of("Hello ", "World!"),
+ ImmutableList.of("Hello", " World!"));
+ }
+
+ @Test
+ public void listFingerprintNotSameAsIndividualElements() throws Exception {
+ Fingerprint f1 = new Fingerprint();
+ f1.addString("Hello ");
+ f1.addString("World!");
+ Fingerprint f2 = new Fingerprint();
+ f2.addStrings(ImmutableList.of("Hello ", "World!"));
+ assertThat(f1.hexDigestAndReset()).isNotEqualTo(f2.hexDigestAndReset());
+ }
+
+ @Test
+ public void mapFingerprintNotSameAsIndividualElements() throws Exception {
+ Fingerprint f1 = new Fingerprint();
+ Map<String, String> map = new HashMap<>();
+ map.put("Hello ", "World!");
+ f1.addStringMap(map);
+ Fingerprint f2 = new Fingerprint();
+ f2.addStrings(ImmutableList.of("Hello ", "World!"));
+ assertThat(f1.hexDigestAndReset()).isNotEqualTo(f2.hexDigestAndReset());
+ }
+
+ @Test
+ public void toStringTest() throws Exception {
+ Fingerprint f1 = new Fingerprint();
+ f1.addString("Hello ");
+ f1.addString("World!");
+ String fp = f1.hexDigestAndReset();
+ Fingerprint f2 = new Fingerprint();
+ f2.addString("Hello ");
+ // make sure that you can call toString on the intermediate result
+ // and continue with the operation.
+ assertThat(fp).isNotEqualTo(f2.toString());
+ f2.addString("World!");
+ assertThat(fp).isEqualTo(f2.hexDigestAndReset());
+ }
+
+ @Test
+ public void addBoolean() throws Exception {
+ String f1 = new Fingerprint().addBoolean(true).hexDigestAndReset();
+ String f2 = new Fingerprint().addBoolean(false).hexDigestAndReset();
+ String f3 = new Fingerprint().addBoolean(true).hexDigestAndReset();
+
+ assertThat(f1).isEqualTo(f3);
+ assertThat(f1).isNotEqualTo(f2);
+ }
+
+ @Test
+ public void addPath() throws Exception {
+ PathFragment pf = new PathFragment("/etc/pwd");
+ assertThat("01cc3eeea3a2f58e447e824f9f62d3d1").isEqualTo(
+ new Fingerprint().addPath(pf).hexDigestAndReset());
+ Path p = new InMemoryFileSystem(BlazeClock.instance()).getPath(pf);
+ assertThat("01cc3eeea3a2f58e447e824f9f62d3d1").isEqualTo(
+ new Fingerprint().addPath(p).hexDigestAndReset());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/GroupedListTest.java b/src/test/java/com/google/devtools/build/lib/util/GroupedListTest.java
new file mode 100644
index 0000000..87cd8c9
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/GroupedListTest.java
@@ -0,0 +1,247 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assert_;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+@RunWith(JUnit4.class)
+public class GroupedListTest {
+ @Test
+ public void empty() {
+ createSizeN(0);
+ }
+
+ @Test
+ public void sizeOne() {
+ createSizeN(1);
+ }
+
+ @Test
+ public void sizeTwo() {
+ createSizeN(2);
+ }
+
+ @Test
+ public void sizeN() {
+ createSizeN(10);
+ }
+
+ private void createSizeN(int size) {
+ List<String> list = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ list.add("test" + i);
+ }
+ Object compressedList = createAndCompress(list);
+ assertTrue(Iterables.elementsEqual(iterable(compressedList), list));
+ assertElementsEqual(compressedList, list);
+ }
+
+ @Test
+ public void elementsNotEqualDifferentOrder() {
+ List<String> list = Lists.newArrayList("a", "b", "c");
+ Object compressedList = createAndCompress(list);
+ ArrayList<String> reversed = new ArrayList<>(list);
+ Collections.reverse(reversed);
+ assertFalse(elementsEqual(compressedList, reversed));
+ }
+
+ @Test
+ public void elementsNotEqualDifferentSizes() {
+ for (int size1 = 0; size1 < 10; size1++) {
+ List<String> firstList = new ArrayList<>();
+ for (int i = 0; i < size1; i++) {
+ firstList.add("test" + i);
+ }
+ Object array = createAndCompress(firstList);
+ for (int size2 = 0; size2 < 10; size2++) {
+ List<String> secondList = new ArrayList<>();
+ for (int i = 0; i < size2; i++) {
+ secondList.add("test" + i);
+ }
+ assertEquals(GroupedList.create(array) + ", " + secondList + ", " + size1 + ", " + size2,
+ size1 == size2, elementsEqual(array, secondList));
+ }
+ }
+ }
+
+ @Test
+ public void group() {
+ GroupedList<String> groupedList = new GroupedList<>();
+ assertTrue(groupedList.isEmpty());
+ GroupedListHelper<String> helper = new GroupedListHelper<>();
+ List<ImmutableList<String>> elements = ImmutableList.of(
+ ImmutableList.of("1"),
+ ImmutableList.of("2a", "2b"),
+ ImmutableList.of("3"),
+ ImmutableList.of("4"),
+ ImmutableList.of("5a", "5b", "5c"),
+ ImmutableList.of("6a", "6b", "6c")
+ );
+ List<String> allElts = new ArrayList<>();
+ for (List<String> group : elements) {
+ if (group.size() > 1) {
+ helper.startGroup();
+ }
+ for (String elt : group) {
+ helper.add(elt);
+ }
+ if (group.size() > 1) {
+ helper.endGroup();
+ }
+ allElts.addAll(group);
+ }
+ groupedList.append(helper);
+ assertEquals(allElts.size(), groupedList.size());
+ assertFalse(groupedList.isEmpty());
+ Object compressed = groupedList.compress();
+ assertElementsEqual(compressed, allElts);
+ assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder();
+ assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder();
+ }
+
+ @Test
+ public void singletonAndEmptyGroups() {
+ GroupedList<String> groupedList = new GroupedList<>();
+ assertTrue(groupedList.isEmpty());
+ GroupedListHelper<String> helper = new GroupedListHelper<>();
+ @SuppressWarnings("unchecked") // varargs
+ List<ImmutableList<String>> elements = Lists.newArrayList(
+ ImmutableList.of("1"),
+ ImmutableList.<String>of(),
+ ImmutableList.of("2a", "2b"),
+ ImmutableList.of("3")
+ );
+ List<String> allElts = new ArrayList<>();
+ for (List<String> group : elements) {
+ helper.startGroup(); // Start a group even if the group has only one element or is empty.
+ for (String elt : group) {
+ helper.add(elt);
+ }
+ helper.endGroup();
+ allElts.addAll(group);
+ }
+ groupedList.append(helper);
+ assertEquals(allElts.size(), groupedList.size());
+ assertFalse(groupedList.isEmpty());
+ Object compressed = groupedList.compress();
+ assertElementsEqual(compressed, allElts);
+ // Get rid of empty list -- it was not stored in groupedList.
+ elements.remove(1);
+ assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder();
+ assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder();
+ }
+
+ @Test
+ public void removeMakesEmpty() {
+ GroupedList<String> groupedList = new GroupedList<>();
+ assertTrue(groupedList.isEmpty());
+ GroupedListHelper<String> helper = new GroupedListHelper<>();
+ @SuppressWarnings("unchecked") // varargs
+ List<List<String>> elements = Lists.newArrayList(
+ (List<String>) ImmutableList.of("1"),
+ ImmutableList.<String>of(),
+ Lists.newArrayList("2a", "2b"),
+ ImmutableList.of("3"),
+ ImmutableList.of("removedGroup1", "removedGroup2"),
+ ImmutableList.of("4")
+ );
+ List<String> allElts = new ArrayList<>();
+ for (List<String> group : elements) {
+ helper.startGroup(); // Start a group even if the group has only one element or is empty.
+ for (String elt : group) {
+ helper.add(elt);
+ }
+ helper.endGroup();
+ allElts.addAll(group);
+ }
+ groupedList.append(helper);
+ Set<String> removed = ImmutableSet.of("2a", "3", "removedGroup1", "removedGroup2");
+ groupedList.remove(removed);
+ Object compressed = groupedList.compress();
+ allElts.removeAll(removed);
+ assertElementsEqual(compressed, allElts);
+ elements.get(2).remove("2a");
+ elements.remove(ImmutableList.of("3"));
+ elements.remove(ImmutableList.of());
+ elements.remove(ImmutableList.of("removedGroup1", "removedGroup2"));
+ assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder();
+ assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder();
+ }
+
+ @Test
+ public void removeGroupFromSmallList() {
+ GroupedList<String> groupedList = new GroupedList<>();
+ assertTrue(groupedList.isEmpty());
+ GroupedListHelper<String> helper = new GroupedListHelper<>();
+ List<List<String>> elements = new ArrayList<>();
+ List<String> group = Lists.newArrayList("1a", "1b", "1c", "1d");
+ elements.add(group);
+ List<String> allElts = new ArrayList<>();
+ helper.startGroup();
+ for (String item : elements.get(0)) {
+ helper.add(item);
+ }
+ allElts.addAll(group);
+ helper.endGroup();
+ groupedList.append(helper);
+ Set<String> removed = ImmutableSet.of("1b", "1c");
+ groupedList.remove(removed);
+ Object compressed = groupedList.compress();
+ allElts.removeAll(removed);
+ assertElementsEqual(compressed, allElts);
+ elements.get(0).removeAll(removed);
+ assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder();
+ assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder();
+ }
+
+ private static Object createAndCompress(Collection<String> list) {
+ GroupedList<String> result = new GroupedList<>();
+ result.append(GroupedListHelper.create(list));
+ return result.compress();
+ }
+
+ private static Iterable<String> iterable(Object compressed) {
+ return GroupedList.<String>create(compressed).toSet();
+ }
+
+ private static boolean elementsEqual(Object compressed, Iterable<String> expected) {
+ return Iterables.elementsEqual(GroupedList.<String>create(compressed).toSet(), expected);
+ }
+
+ private static void assertElementsEqual(Object compressed, Iterable<String> expected) {
+ assert_()
+ .that(GroupedList.<String>create(compressed).toSet())
+ .containsExactlyElementsIn(expected)
+ .inOrder();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/JavaClockTest.java b/src/test/java/com/google/devtools/build/lib/util/JavaClockTest.java
new file mode 100644
index 0000000..d5d39c0
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/JavaClockTest.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for the Clock instance based on the Java System class.
+ */
+@RunWith(JUnit4.class)
+public class JavaClockTest {
+
+ @Test
+ public void javaClockIsAdvancing() throws Exception {
+ Clock clock = new JavaClock();
+ long millis = clock.currentTimeMillis();
+ long nanos = clock.nanoTime();
+
+ Thread.sleep(10);
+
+ assertThat(clock.currentTimeMillis()).isNotEqualTo(millis);
+ assertThat(clock.nanoTime()).isNotEqualTo(nanos);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/OptionsUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/OptionsUtilsTest.java
new file mode 100644
index 0000000..9888061
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/OptionsUtilsTest.java
@@ -0,0 +1,157 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.util.OptionsUtils.PathFragmentListConverter;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test for {@link OptionsUtils}.
+ */
+@RunWith(JUnit4.class)
+public class OptionsUtilsTest {
+
+ public static class IntrospectionExample extends OptionsBase {
+ @Option(name = "alpha",
+ category = "one",
+ defaultValue = "alpha")
+ public String alpha;
+
+ @Option(name = "beta",
+ category = "one",
+ defaultValue = "beta")
+ public String beta;
+
+ @Option(name = "gamma",
+ category = "undocumented",
+ defaultValue = "gamma")
+ public String gamma;
+
+ @Option(name = "delta",
+ category = "undocumented",
+ defaultValue = "delta")
+ public String delta;
+
+ @Option(name = "echo",
+ category = "hidden",
+ defaultValue = "echo")
+ public String echo;
+ }
+
+ @Test
+ public void asStringOfExplicitOptions() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class);
+ parser.parse("--alpha=no", "--gamma=no", "--echo=no");
+ assertEquals("--alpha=no --gamma=no", OptionsUtils.asShellEscapedString(parser));
+ }
+
+ @Test
+ public void asStringOfExplicitOptionsCorrectSortingByPriority() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--alpha=no"));
+ parser.parse(OptionPriority.COMPUTED_DEFAULT, null, Arrays.asList("--beta=no"));
+ assertEquals("--beta=no --alpha=no", OptionsUtils.asShellEscapedString(parser));
+ }
+
+ public static class BooleanOpts extends OptionsBase {
+ @Option(name = "b_one",
+ category = "xyz",
+ defaultValue = "true")
+ public boolean bOne;
+
+ @Option(name = "b_two",
+ category = "123", // Not printed in usage messages!
+ defaultValue = "false")
+ public boolean bTwo;
+ }
+
+ @Test
+ public void asStringOfExplicitOptionsWithBooleans() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(BooleanOpts.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--b_one", "--nob_two"));
+ assertEquals("--b_one --nob_two", OptionsUtils.asShellEscapedString(parser));
+
+ parser = OptionsParser.newOptionsParser(BooleanOpts.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--b_one=true", "--b_two=0"));
+ assertTrue(parser.getOptions(BooleanOpts.class).bOne);
+ assertFalse(parser.getOptions(BooleanOpts.class).bTwo);
+ assertEquals("--b_one --nob_two", OptionsUtils.asShellEscapedString(parser));
+ }
+
+ @Test
+ public void asStringOfExplicitOptionsMultipleOptionsAreMultipleTimes() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--alpha=one"));
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--alpha=two"));
+ assertEquals("--alpha=one --alpha=two", OptionsUtils.asShellEscapedString(parser));
+ }
+
+ private static List<PathFragment> list(PathFragment... fragments) {
+ return Lists.newArrayList(fragments);
+ }
+
+ private PathFragment fragment(String string) {
+ return new PathFragment(string);
+ }
+
+ private List<PathFragment> convert(String input) throws Exception {
+ return new PathFragmentListConverter().convert(input);
+ }
+
+ @Test
+ public void emptyStringYieldsEmptyList() throws Exception {
+ assertEquals(list(), convert(""));
+ }
+
+ @Test
+ public void lonelyDotYieldsLonelyDot() throws Exception {
+ assertEquals(list(fragment(".")), convert("."));
+ }
+
+ @Test
+ public void converterSkipsEmptyStrings() throws Exception {
+ assertEquals(list(fragment("foo"), fragment("bar")), convert("foo::bar:"));
+ }
+
+ @Test
+ public void multiplePaths() throws Exception {
+ assertEquals(list(fragment("foo"), fragment("/bar/baz"), fragment("."),
+ fragment("/tmp/bang")), convert("foo:/bar/baz:.:/tmp/bang"));
+ }
+
+ @Test
+ public void valueisUnmodifiable() throws Exception {
+ try {
+ new PathFragmentListConverter().convert("value").add(new PathFragment("other"));
+ fail("could modify value");
+ } catch (UnsupportedOperationException expected) {}
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/PairTest.java b/src/test/java/com/google/devtools/build/lib/util/PairTest.java
new file mode 100644
index 0000000..f82e12f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/PairTest.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Objects;
+
+/**
+ * Tests for {@link Pair}.
+ */
+@RunWith(JUnit4.class)
+public class PairTest {
+
+ @Test
+ public void constructor() {
+ Object a = new Object();
+ Object b = new Object();
+ Pair<Object, Object> p = Pair.of(a, b);
+ assertSame(a, p.first);
+ assertSame(b, p.second);
+ assertEquals(Pair.of(a, b), p);
+ assertEquals(Objects.hash(a, b), p.hashCode());
+ }
+
+ @Test
+ public void nullable() {
+ Pair<Object, Object> p = Pair.of(null, null);
+ assertNull(p.first);
+ assertNull(p.second);
+ p.hashCode(); // Should not throw.
+ assertTrue(p.equals(p));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/PathFragmentFilterTest.java b/src/test/java/com/google/devtools/build/lib/util/PathFragmentFilterTest.java
new file mode 100644
index 0000000..e911324
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/PathFragmentFilterTest.java
@@ -0,0 +1,95 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test for {@link PathFragmentFilter}.
+ */
+@RunWith(JUnit4.class)
+public class PathFragmentFilterTest {
+ protected PathFragmentFilter filter = null;
+
+ protected void createFilter(String filterString) {
+ filter = new PathFragmentFilter.PathFragmentFilterConverter().convert(filterString);
+ }
+
+ protected void assertIncluded(String path) {
+ assertTrue(filter.isIncluded(new PathFragment(path)));
+ }
+
+ protected void assertExcluded(String path) {
+ assertFalse(filter.isIncluded(new PathFragment(path)));
+ }
+
+ @Test
+ public void emptyFilter() {
+ createFilter("");
+ assertIncluded("a/b/c");
+ assertIncluded("d");
+ }
+
+ @Test
+ public void inclusions() {
+ createFilter("a/b,c");
+ assertIncluded("a/b");
+ assertIncluded("a/b/c");
+ assertIncluded("c");
+ assertIncluded("c/d");
+ assertExcluded("a");
+ assertExcluded("a/c");
+ assertExcluded("d");
+ assertExcluded("e/f/g");
+ }
+
+ @Test
+ public void exclusions() {
+ createFilter("-a/b,-c");
+ assertExcluded("a/b");
+ assertExcluded("a/b/c");
+ assertExcluded("c");
+ assertExcluded("c/d");
+ assertIncluded("a");
+ assertIncluded("a/c");
+ assertIncluded("d");
+ assertIncluded("e/f/g");
+ }
+
+ @Test
+ public void inclusionsAndExclusions() {
+ createFilter("a,-c,,d,a/b/c,-a/b,a/b/d");
+ assertIncluded("a");
+ assertIncluded("a/c");
+ assertExcluded("a/b");
+ assertExcluded("a/b/c"); // Exclusions take precedence over inclusions. Order is not important.
+ assertExcluded("a/b/d"); // Exclusions take precedence over inclusions. Order is not important.
+ assertExcluded("c");
+ assertExcluded("c/d");
+ assertIncluded("d/e");
+ assertExcluded("e");
+ // When converted back to string, inclusion entries will be put first, followed by exclusion
+ // entries.
+ assertEquals("a,d,a/b/c,a/b/d,-c,-a/b", filter.toString());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/PersistentMapTest.java b/src/test/java/com/google/devtools/build/lib/util/PersistentMapTest.java
new file mode 100644
index 0000000..44aa538
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/PersistentMapTest.java
@@ -0,0 +1,225 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for the {@link PersistentMap}.
+ */
+@RunWith(JUnit4.class)
+public class PersistentMapTest {
+ public static class PersistentStringMap extends PersistentMap<String, String> {
+ boolean updateJournal = true;
+ boolean keepJournal = false;
+
+ public PersistentStringMap(Map<String, String> map, Path mapFile,
+ Path journalFile) throws IOException {
+ super(0x0, map, mapFile, journalFile);
+ load();
+ }
+
+ @Override
+ protected String readKey(DataInputStream in) throws IOException {
+ return in.readUTF();
+ }
+ @Override
+ protected String readValue(DataInputStream in) throws IOException {
+ return in.readUTF();
+ }
+ @Override
+ protected void writeKey(String key, DataOutputStream out)
+ throws IOException {
+ out.writeUTF(key);
+ }
+ @Override
+ protected void writeValue(String value, DataOutputStream out)
+ throws IOException {
+ out.writeUTF(value);
+ }
+ @Override
+ protected boolean updateJournal() {
+ return updateJournal;
+ }
+ @Override
+ protected boolean keepJournal() {
+ return keepJournal;
+ }
+ }
+
+ private FsApparatus scratch = FsApparatus.newInMemory();
+
+ private PersistentStringMap map;
+ private Path mapFile;
+ private Path journalFile;
+
+ @Before
+ public void setUp() throws Exception {
+ mapFile = scratch.fs().getPath("/tmp/map.txt");
+ journalFile = scratch.fs().getPath("/tmp/journal.txt");
+ createMap();
+ }
+
+ private void createMap() throws Exception {
+ Map<String, String> map = new HashMap<>();
+ this.map = new PersistentStringMap(map, mapFile, journalFile);
+ }
+
+ @Test
+ public void map() throws Exception {
+ createMap();
+ map.put("foo", "bar");
+ map.put("baz", "bang");
+ assertEquals("bar", map.get("foo"));
+ assertEquals("bang", map.get("baz"));
+ assertEquals(2, map.size());
+ long size = map.save();
+ assertEquals(mapFile.getFileSize(), size);
+ assertEquals("bar", map.get("foo"));
+ assertEquals("bang", map.get("baz"));
+ assertEquals(2, map.size());
+
+ createMap(); // create a new map
+ assertEquals("bar", map.get("foo"));
+ assertEquals("bang", map.get("baz"));
+ assertEquals(2, map.size());
+ }
+
+ @Test
+ public void remove() throws Exception {
+ createMap();
+ map.put("foo", "bar");
+ map.put("baz", "bang");
+ long size = map.save();
+ assertEquals(mapFile.getFileSize(), size);
+ assertFalse(journalFile.exists());
+ map.remove("foo");
+ assertEquals(1, map.size());
+ assertTrue(journalFile.exists());
+ createMap(); // create a new map
+ assertEquals(1, map.size());
+ }
+
+ @Test
+ public void clear() throws Exception {
+ createMap();
+ map.put("foo", "bar");
+ map.put("baz", "bang");
+ map.save();
+ assertTrue(mapFile.exists());
+ assertFalse(journalFile.exists());
+ map.clear();
+ assertEquals(0, map.size());
+ assertTrue(mapFile.exists());
+ assertFalse(journalFile.exists());
+ createMap(); // create a new map
+ assertEquals(0, map.size());
+ }
+
+ @Test
+ public void noUpdateJournal() throws Exception {
+ createMap();
+ map.put("foo", "bar");
+ map.put("baz", "bang");
+ map.save();
+ assertFalse(journalFile.exists());
+ // prevent updating the journal
+ map.updateJournal = false;
+ // remove an entry
+ map.remove("foo");
+ assertEquals(1, map.size());
+ // no journal file written
+ assertFalse(journalFile.exists());
+ createMap(); // create a new map
+ // both entries are still in the map on disk
+ assertEquals(2, map.size());
+ }
+
+ @Test
+ public void keepJournal() throws Exception {
+ createMap();
+ map.put("foo", "bar");
+ map.put("baz", "bang");
+ map.save();
+ assertFalse(journalFile.exists());
+
+ // Keep the journal through the save.
+ map.updateJournal = false;
+ map.keepJournal = true;
+
+ // remove an entry
+ map.remove("foo");
+ assertEquals(1, map.size());
+ // no journal file written
+ assertFalse(journalFile.exists());
+
+ long size = map.save();
+ assertEquals(1, map.size());
+ // The journal must be serialzed on save(), even if !updateJournal.
+ assertTrue(journalFile.exists());
+ assertEquals(journalFile.getFileSize() + mapFile.getFileSize(), size);
+
+ map.load();
+ assertEquals(1, map.size());
+ assertTrue(journalFile.exists());
+
+ createMap(); // create a new map
+ assertEquals(1, map.size());
+
+ map.keepJournal = false;
+ map.save();
+ assertEquals(1, map.size());
+ assertFalse(journalFile.exists());
+ }
+
+ @Test
+ public void multipleJournalUpdates() throws Exception {
+ createMap();
+ map.put("foo", "bar");
+ map.save();
+ assertFalse(journalFile.exists());
+ // add an entry
+ map.put("baz", "bang");
+ assertEquals(2, map.size());
+ // journal file written
+ assertTrue(journalFile.exists());
+ createMap(); // create a new map
+ // both entries are still in the map on disk
+ assertEquals(2, map.size());
+ // add another entry
+ map.put("baz2", "bang2");
+ assertEquals(3, map.size());
+ // journal file written
+ assertTrue(journalFile.exists());
+ createMap(); // create a new map
+ // all three entries are still in the map on disk
+ assertEquals(3, map.size());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/ProcMeminfoParserTest.java b/src/test/java/com/google/devtools/build/lib/util/ProcMeminfoParserTest.java
new file mode 100644
index 0000000..9d04f28
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/ProcMeminfoParserTest.java
@@ -0,0 +1,91 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.vfs.util.FsApparatus;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+
+/**
+ * Tests for ProcMeminfoParser.
+ */
+@RunWith(JUnit4.class)
+public class ProcMeminfoParserTest {
+
+ private FsApparatus scratch = FsApparatus.newNative();
+
+ @Test
+ public void memInfo() throws IOException {
+ String meminfoContent = StringUtilities.joinLines(
+ "MemTotal: 3091732 kB",
+ "MemFree: 2167344 kB",
+ "Buffers: 60644 kB",
+ "Cached: 509940 kB",
+ "SwapCached: 0 kB",
+ "Active: 636892 kB",
+ "Inactive: 212760 kB",
+ "HighTotal: 0 kB",
+ "HighFree: 0 kB",
+ "LowTotal: 3091732 kB",
+ "LowFree: 2167344 kB",
+ "SwapTotal: 9124880 kB",
+ "SwapFree: 9124880 kB",
+ "Dirty: 0 kB",
+ "Writeback: 0 kB",
+ "AnonPages: 279028 kB",
+ "Mapped: 54404 kB",
+ "Slab: 42820 kB",
+ "PageTables: 5184 kB",
+ "NFS_Unstable: 0 kB",
+ "Bounce: 0 kB",
+ "CommitLimit: 10670744 kB",
+ "Committed_AS: 665840 kB",
+ "VmallocTotal: 34359738367 kB",
+ "VmallocUsed: 300484 kB",
+ "VmallocChunk: 34359437307 kB",
+ "HugePages_Total: 0",
+ "HugePages_Free: 0",
+ "HugePages_Rsvd: 0",
+ "Hugepagesize: 2048 kB",
+ "Bogus: not_a_number",
+ "Bogus2: 1000000000000000000000000000000000000000000000000 kB"
+ );
+
+ String meminfoFile = scratch.file("test_meminfo", meminfoContent).getPathString();
+ ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile);
+
+ assertEquals(2356756, memInfo.getFreeRamKb());
+ assertEquals(509940, memInfo.getRamKb("Cached"));
+ assertEquals(3091732, memInfo.getTotalKb());
+ assertNotAvailable("Bogus", memInfo);
+ assertNotAvailable("Bogus2", memInfo);
+ }
+
+ private static void assertNotAvailable(String field, ProcMeminfoParser memInfo) {
+ try {
+ memInfo.getRamKb(field);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected.
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/RegexFilterTest.java b/src/test/java/com/google/devtools/build/lib/util/RegexFilterTest.java
new file mode 100644
index 0000000..bb502c7
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/RegexFilterTest.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test for {@link RegexFilter}.
+ */
+@RunWith(JUnit4.class)
+public class RegexFilterTest {
+ protected RegexFilter filter = null;
+
+ protected RegexFilter createFilter(String filterString) throws OptionsParsingException {
+ filter = new RegexFilter.RegexFilterConverter().convert(filterString);
+ return filter;
+ }
+
+ protected void assertIncluded(String value) {
+ assertTrue(filter.isIncluded(value));
+ }
+
+ protected void assertExcluded(String value) {
+ assertFalse(filter.isIncluded(value));
+ }
+
+ @Test
+ public void emptyFilter() throws Exception {
+ createFilter("");
+ assertIncluded("a/b/c");
+ assertIncluded("d");
+ }
+
+ @Test
+ public void inclusions() throws Exception {
+ createFilter("a/b,+^c,_test$");
+ assertEquals("(?:(?>a/b)|(?>^c)|(?>_test$))", filter.toString());
+ assertIncluded("a/b");
+ assertIncluded("a/b/c");
+ assertIncluded("c");
+ assertIncluded("c/d");
+ assertIncluded("e/a/b");
+ assertIncluded("f/1/2/3/_test");
+ assertExcluded("a");
+ assertExcluded("a/c");
+ assertExcluded("d");
+ assertExcluded("e/f/g");
+ assertExcluded("f/_test2");
+ }
+
+ @Test
+ public void exclusions() throws Exception {
+ createFilter("-a/b,-^c,-_test$");
+ assertEquals("-(?:(?>a/b)|(?>^c)|(?>_test$))", filter.toString());
+ assertExcluded("a/b");
+ assertExcluded("a/b/c");
+ assertExcluded("c");
+ assertExcluded("c/d");
+ assertExcluded("f/a/b/d");
+ assertExcluded("f/a_test");
+ assertIncluded("a");
+ assertIncluded("a/c");
+ assertIncluded("d");
+ assertIncluded("e/f/g");
+ assertIncluded("f/a_test_case");
+ }
+
+ @Test
+ public void inclusionsAndExclusions() throws Exception {
+ createFilter("a,-^c,,-,+,d,+a/b/c,-a/b,a/b/d");
+ assertEquals("(?:(?>a)|(?>d)|(?>a/b/c)|(?>a/b/d)),-(?:(?>^c)|(?>a/b))", filter.toString());
+ assertIncluded("a");
+ assertIncluded("a/c");
+ assertExcluded("a/b");
+ assertExcluded("a/b/c"); // Exclusions take precedence over inclusions. Order is not important.
+ assertExcluded("a/b/d"); // Exclusions take precedence over inclusions. Order is not important.
+ assertExcluded("a/c/a/b/d");
+ assertExcluded("c");
+ assertExcluded("c/d");
+ assertIncluded("d/e");
+ assertExcluded("e");
+ }
+
+ @Test
+ public void commas() throws Exception {
+ createFilter("a\\,b,c\\,d");
+ assertEquals("(?:(?>a\\,b)|(?>c\\,d))", filter.toString());
+ assertIncluded("a,b");
+ assertIncluded("c,d");
+ assertExcluded("a");
+ assertExcluded("b,c");
+ assertExcluded("d");
+ }
+
+ @Test
+ public void invalidExpression() throws Exception {
+ try {
+ createFilter("*a");
+ fail(); // OptionsParsingException should be thrown.
+ } catch (OptionsParsingException e) {
+ assertThat(e.getMessage())
+ .contains("Failed to build valid regular expression: Dangling meta character '*' "
+ + "near index");
+ }
+ }
+
+ @Test
+ public void equals() throws Exception {
+ new EqualsTester()
+ .addEqualityGroup(createFilter("a,b,c"), createFilter("a,b,c"))
+ .addEqualityGroup(createFilter("a,b,c,d"))
+ .addEqualityGroup(createFilter("a,b,-c"), createFilter("a,b,-c"))
+ .addEqualityGroup(createFilter("a,b,-c,-d"))
+ .addEqualityGroup(createFilter("-a,-b,-c"), createFilter("-a,-b,-c"))
+ .addEqualityGroup(createFilter("-a,-b,-c,-d"))
+ .addEqualityGroup(createFilter(""), createFilter(""))
+ .testEquals();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.java b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.java
new file mode 100644
index 0000000..e821ca1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+
+/**
+ * A test for {@link ResourceFileLoader}.
+ */
+@RunWith(JUnit4.class)
+public class ResourceFileLoaderTest {
+
+ @Test
+ public void loader() throws IOException {
+ String message = ResourceFileLoader.loadResource(
+ ResourceFileLoaderTest.class, "ResourceFileLoaderTest.message");
+ assertEquals("Hello, world.", message);
+ }
+
+ @Test
+ public void resourceNotFound() {
+ try {
+ ResourceFileLoader.loadResource(ResourceFileLoaderTest.class,
+ "does_not_exist.txt");
+ fail();
+ } catch (IOException e) {
+ assertEquals("does_not_exist.txt not found.", e.getMessage());
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.message b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.message
new file mode 100644
index 0000000..c872090
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.message
@@ -0,0 +1 @@
+Hello, world.
\ No newline at end of file
diff --git a/src/test/java/com/google/devtools/build/lib/util/ShellEscaperTest.java b/src/test/java/com/google/devtools/build/lib/util/ShellEscaperTest.java
new file mode 100644
index 0000000..ac5d413
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/ShellEscaperTest.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.devtools.build.lib.util.ShellEscaper.escapeString;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.Set;
+
+/**
+ * Tests for {@link ShellEscaper}.
+ *
+ * <p>Based on {@code com.google.io.base.shell.ShellUtilsTest}.
+ */
+@RunWith(JUnit4.class)
+public class ShellEscaperTest {
+
+ @Test
+ public void shellEscape() throws Exception {
+ assertEquals("''", escapeString(""));
+ assertEquals("foo", escapeString("foo"));
+ assertEquals("'foo bar'", escapeString("foo bar"));
+ assertEquals("''\\''foo'\\'''", escapeString("'foo'"));
+ assertEquals("'\\'\\''foo\\'\\'''", escapeString("\\'foo\\'"));
+ assertEquals("'${filename%.c}.o'", escapeString("${filename%.c}.o"));
+ assertEquals("'<html!>'", escapeString("<html!>"));
+ }
+
+ @Test
+ public void escapeAll() throws Exception {
+ Set<String> escaped = ImmutableSet.copyOf(
+ ShellEscaper.escapeAll(Arrays.asList("foo", "@bar", "baz'qux")));
+ assertEquals(ImmutableSet.of("foo", "@bar", "'baz'\\''qux'"), escaped);
+ }
+
+ @Test
+ public void escapeJoinAllIntoAppendable() throws Exception {
+ Appendable appendable = ShellEscaper.escapeJoinAll(
+ new StringBuilder("initial"), Arrays.asList("foo", "$BAR"));
+ assertEquals("initialfoo '$BAR'", appendable.toString());
+ }
+
+ @Test
+ public void escapeJoinAllIntoAppendableWithCustomJoiner() throws Exception {
+ Appendable appendable = ShellEscaper.escapeJoinAll(
+ new StringBuilder("initial"), Arrays.asList("foo", "$BAR"), Joiner.on('|'));
+ assertEquals("initialfoo|'$BAR'", appendable.toString());
+ }
+
+ @Test
+ public void escapeJoinAll() throws Exception {
+ String actual = ShellEscaper.escapeJoinAll(
+ Arrays.asList("foo", "@echo:-", "100", "$US", "a b", "\"qu'ot'es\"", "\"quot\"", "\\"));
+ assertEquals("foo @echo:- 100 '$US' 'a b' '\"qu'\\''ot'\\''es\"' '\"quot\"' '\\'", actual);
+ }
+
+ @Test
+ public void escapeJoinAllWithCustomJoiner() throws Exception {
+ String actual = ShellEscaper.escapeJoinAll(
+ Arrays.asList("foo", "@echo:-", "100", "$US", "a b", "\"qu'ot'es\"", "\"quot\"", "\\"),
+ Joiner.on('|'));
+ assertEquals("foo|@echo:-|100|'$US'|'a b'|'\"qu'\\''ot'\\''es\"'|'\"quot\"'|'\\'", actual);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/StringCanonicalizerTest.java b/src/test/java/com/google/devtools/build/lib/util/StringCanonicalizerTest.java
new file mode 100644
index 0000000..64acddc
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/StringCanonicalizerTest.java
@@ -0,0 +1,42 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertSame;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for String canonicalizer.
+ */
+@RunWith(JUnit4.class)
+public class StringCanonicalizerTest {
+
+ @Test
+ public void twoDifferentStringsAreDifferent() {
+ String stringA = StringCanonicalizer.intern("A");
+ String stringB = StringCanonicalizer.intern("B");
+ assertThat(stringA).isNotEqualTo(stringB);
+ }
+
+ @Test
+ public void twoSameStringsAreCanonicalized() {
+ String stringA1 = StringCanonicalizer.intern(new String("A"));
+ String stringA2 = StringCanonicalizer.intern(new String("A"));
+ assertSame(stringA1, stringA2);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/StringIndexerTest.java b/src/test/java/com/google/devtools/build/lib/util/StringIndexerTest.java
new file mode 100644
index 0000000..693e11a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/StringIndexerTest.java
@@ -0,0 +1,314 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Function;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.testutil.TestUtils;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+import java.util.SortedMap;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Test for the StringIndexer classes.
+ */
+public abstract class StringIndexerTest {
+
+ private static final int ATTEMPTS = 1000;
+ private SortedMap<Integer, String> mappings;
+
+ protected StringIndexer indexer;
+
+ @Before
+ public void setUp() throws Exception {
+ indexer = newIndexer();
+ mappings = Maps.newTreeMap();
+ }
+
+ protected abstract StringIndexer newIndexer();
+
+ protected void assertSize(int expected) {
+ assertEquals(expected, indexer.size());
+ }
+
+ protected void assertNoIndex(String s) {
+ int size = indexer.size();
+ assertEquals(-1, indexer.getIndex(s));
+ assertEquals(size, indexer.size());
+ }
+
+ protected void assertIndex(int expected, String s) {
+ // System.out.println("Adding " + s + ", expecting " + expected);
+ int index = indexer.getOrCreateIndex(s);
+ // System.out.println(csi);
+ assertEquals(expected, index);
+ mappings.put(expected, s);
+ }
+
+ protected void assertContent() {
+ for (int i = 0; i < indexer.size(); i++) {
+ assertNotNull(mappings.get(i));
+ assertEquals(mappings.get(i), indexer.getStringForIndex(i));
+ }
+ }
+
+ private void assertConcurrentUpdates(Function<Integer, String> keyGenerator) throws Exception {
+ final AtomicInteger safeIndex = new AtomicInteger(-1);
+ List<String> keys = Lists.newArrayListWithCapacity(ATTEMPTS);
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 5, TimeUnit.SECONDS,
+ new ArrayBlockingQueue<Runnable>(ATTEMPTS));
+ synchronized(indexer) {
+ for (int i = 0; i < ATTEMPTS; i++) {
+ final String key = keyGenerator.apply(i);
+ keys.add(key);
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ int index = indexer.getOrCreateIndex(key);
+ if (safeIndex.get() < index) { safeIndex.set(index); }
+ indexer.addString(key);
+ }
+ });
+ }
+ }
+ try {
+ while(!executor.getQueue().isEmpty()) {
+ // Validate that we can execute concurrent queries too.
+ if (safeIndex.get() >= 0) {
+ int index = safeIndex.get();
+ // Retrieve string using random existing index and validate reverse mapping.
+ String key = indexer.getStringForIndex(index);
+ assertNotNull(key);
+ assertEquals(index, indexer.getIndex(key));
+ }
+ }
+ } finally {
+ executor.shutdown();
+ executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ for (String key : keys) {
+ // Validate mapping between keys and indices.
+ assertEquals(key, indexer.getStringForIndex(indexer.getIndex(key)));
+ }
+ }
+
+ @Test
+ public void concurrentAddChildNode() throws Exception {
+ assertConcurrentUpdates(new Function<Integer, String>() {
+ @Override
+ public String apply(Integer from) { return Strings.repeat("a", from + 1); }
+ });
+ }
+
+ @Test
+ public void concurrentSplitNodeSuffix() throws Exception {
+ assertConcurrentUpdates(new Function<Integer, String>() {
+ @Override
+ public String apply(Integer from) { return Strings.repeat("b", ATTEMPTS - from); }
+ });
+ }
+
+ @Test
+ public void concurrentAddBranch() throws Exception {
+ assertConcurrentUpdates(new Function<Integer, String>() {
+ @Override
+ public String apply(Integer from) { return String.format("%08o", from); }
+ });
+ }
+
+ @RunWith(JUnit4.class)
+ public static class CompactStringIndexerTest extends StringIndexerTest {
+ @Override
+ protected StringIndexer newIndexer() {
+ return new CompactStringIndexer(1);
+ }
+
+ @Test
+ public void basicOperations() {
+ assertSize(0);
+ assertNoIndex("abcdef");
+ assertIndex(0, "abcdef"); // root node creation
+ assertIndex(0, "abcdef"); // root node match
+ assertSize(1);
+ assertIndex(2, "abddef"); // node branching, index 1 went to "ab" node.
+ assertSize(3);
+ assertIndex(1, "ab");
+ assertSize(3);
+ assertIndex(3, "abcdefghik"); // new leaf creation
+ assertSize(4);
+ assertIndex(4, "abcdefgh"); // node split
+ assertSize(5);
+ assertNoIndex("a");
+ assertNoIndex("abc");
+ assertNoIndex("abcdefg");
+ assertNoIndex("abcdefghil");
+ assertNoIndex("abcdefghikl");
+ assertContent();
+ indexer.clear();
+ assertSize(0);
+ assertNull(indexer.getStringForIndex(0));
+ assertNull(indexer.getStringForIndex(1000));
+ }
+
+ @Test
+ public void parentIndexUpdate() {
+ assertSize(0);
+ assertIndex(0, "abcdefghik"); // Create 3 nodes with single common parent "abcdefgh".
+ assertIndex(2, "abcdefghlm"); // Index 1 went to "abcdefgh".
+ assertIndex(3, "abcdefghxyz");
+ assertSize(4);
+ assertIndex(5, "abcdpqr"); // Split parent. Index 4 went to "abcd".
+ assertSize(6);
+ assertIndex(1, "abcdefgh"); // Check branch node indices.
+ assertIndex(4, "abcd");
+ assertSize(6);
+ assertContent();
+ }
+
+ @Test
+ public void emptyRootNode() {
+ assertSize(0);
+ assertIndex(0, "abc");
+ assertNoIndex("");
+ assertIndex(2, "def"); // root node key is now empty string and has index 1.
+ assertSize(3);
+ assertIndex(1, "");
+ assertSize(3);
+ assertContent();
+ }
+
+ protected void setupTestContent() {
+ assertSize(0);
+ assertIndex(0, "abcdefghi"); // Create leafs
+ assertIndex(2, "abcdefjkl");
+ assertIndex(3, "abcdefmno");
+ assertIndex(4, "abcdefjklpr");
+ assertIndex(6, "abcdstr");
+ assertIndex(8, "012345");
+ assertSize(9);
+ assertIndex(1, "abcdef"); // Validate inner nodes
+ assertIndex(5, "abcd");
+ assertIndex(7, "");
+ assertSize(9);
+ assertContent();
+ }
+
+ @Test
+ public void dumpContent() {
+ indexer = newIndexer();
+ indexer.addString("abc");
+ String content = indexer.toString();
+ assertThat(content).contains("size = 1");
+ assertThat(content).contains("contentSize = 5");
+ indexer = newIndexer();
+ setupTestContent();
+ content = indexer.toString();
+ assertThat(content).contains("size = 9");
+ assertThat(content).contains("contentSize = 60");
+ System.out.println(indexer);
+ }
+
+ @Test
+ public void addStringResult() {
+ assertSize(0);
+ assertTrue(indexer.addString("abcdef"));
+ assertTrue(indexer.addString("abcdgh"));
+ assertFalse(indexer.addString("abcd"));
+ assertTrue(indexer.addString("ab"));
+ }
+ }
+
+ @RunWith(JUnit4.class)
+ public static class CanonicalStringIndexerTest extends StringIndexerTest{
+ @Override
+ protected StringIndexer newIndexer() {
+ return new CanonicalStringIndexer(new MapMaker().<String, Integer>makeMap(),
+ new MapMaker().<Integer, String>makeMap());
+ }
+
+ @Test
+ public void basicOperations() {
+ assertSize(0);
+ assertNoIndex("abcdef");
+ assertIndex(0, "abcdef");
+ assertIndex(0, "abcdef");
+ assertSize(1);
+ assertIndex(1, "abddef");
+ assertSize(2);
+ assertIndex(2, "ab");
+ assertSize(3);
+ assertIndex(3, "abcdefghik");
+ assertSize(4);
+ assertIndex(4, "abcdefgh");
+ assertSize(5);
+ assertNoIndex("a");
+ assertNoIndex("abc");
+ assertNoIndex("abcdefg");
+ assertNoIndex("abcdefghil");
+ assertNoIndex("abcdefghikl");
+ assertContent();
+ indexer.clear();
+ assertSize(0);
+ assertNull(indexer.getStringForIndex(0));
+ assertNull(indexer.getStringForIndex(1000));
+ }
+
+ @Test
+ public void addStringResult() {
+ assertSize(0);
+ assertTrue(indexer.addString("abcdef"));
+ assertTrue(indexer.addString("abcdgh"));
+ assertTrue(indexer.addString("abcd"));
+ assertTrue(indexer.addString("ab"));
+ assertFalse(indexer.addString("ab"));
+ }
+
+ protected void setupTestContent() {
+ assertSize(0);
+ assertIndex(0, "abcdefghi");
+ assertIndex(1, "abcdefjkl");
+ assertIndex(2, "abcdefmno");
+ assertIndex(3, "abcdefjklpr");
+ assertIndex(4, "abcdstr");
+ assertIndex(5, "012345");
+ assertSize(6);
+ assertIndex(6, "abcdef");
+ assertIndex(7, "abcd");
+ assertIndex(8, "");
+ assertIndex(2, "abcdefmno");
+ assertSize(9);
+ assertContent();
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/StringTrieTest.java b/src/test/java/com/google/devtools/build/lib/util/StringTrieTest.java
new file mode 100644
index 0000000..dcb9205
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/StringTrieTest.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Unit tests for {@link StringTrie}.
+ */
+@RunWith(JUnit4.class)
+public class StringTrieTest {
+ @Test
+ public void empty() {
+ StringTrie<Integer> cut = new StringTrie<>();
+ assertNull(cut.get(""));
+ assertNull(cut.get("a"));
+ assertNull(cut.get("ab"));
+ }
+
+ @Test
+ public void simple() {
+ StringTrie<Integer> cut = new StringTrie<>();
+ cut.put("a", 1);
+ cut.put("b", 2);
+
+ assertNull(cut.get(""));
+ assertEquals(1, cut.get("a").intValue());
+ assertEquals(1, cut.get("ab").intValue());
+ assertEquals(1, cut.get("abc").intValue());
+
+ assertEquals(2, cut.get("b").intValue());
+ }
+
+ @Test
+ public void ancestors() {
+ StringTrie<Integer> cut = new StringTrie<>();
+ cut.put("abc", 3);
+ assertNull(cut.get(""));
+ assertNull(cut.get("a"));
+ assertNull(cut.get("ab"));
+ assertEquals(3, cut.get("abc").intValue());
+ assertEquals(3, cut.get("abcd").intValue());
+
+ cut.put("a", 1);
+ assertEquals(1, cut.get("a").intValue());
+ assertEquals(1, cut.get("ab").intValue());
+ assertEquals(3, cut.get("abc").intValue());
+
+ cut.put("", 0);
+ assertEquals(0, cut.get("").intValue());
+ assertEquals(0, cut.get("b").intValue());
+ assertEquals(1, cut.get("a").intValue());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/StringUtilTest.java b/src/test/java/com/google/devtools/build/lib/util/StringUtilTest.java
new file mode 100644
index 0000000..96d9a53
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/StringUtilTest.java
@@ -0,0 +1,110 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.devtools.build.lib.util.StringUtil.capitalize;
+import static com.google.devtools.build.lib.util.StringUtil.indent;
+import static com.google.devtools.build.lib.util.StringUtil.joinEnglishList;
+import static com.google.devtools.build.lib.util.StringUtil.splitAndInternString;
+import static com.google.devtools.build.lib.util.StringUtil.stripSuffix;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A test for {@link StringUtil}.
+ */
+@RunWith(JUnit4.class)
+public class StringUtilTest {
+
+ @Test
+ public void testJoinEnglishList() throws Exception {
+ assertEquals("nothing",
+ joinEnglishList(Collections.emptyList()));
+ assertEquals("one",
+ joinEnglishList(Arrays.asList("one")));
+ assertEquals("one or two",
+ joinEnglishList(Arrays.asList("one", "two")));
+ assertEquals("one and two",
+ joinEnglishList(Arrays.asList("one", "two"), "and"));
+ assertEquals("one, two or three",
+ joinEnglishList(Arrays.asList("one", "two", "three")));
+ assertEquals("one, two and three",
+ joinEnglishList(Arrays.asList("one", "two", "three"), "and"));
+ assertEquals("'one', 'two' and 'three'",
+ joinEnglishList(Arrays.asList("one", "two", "three"), "and", "'"));
+ }
+
+ @Test
+ public void splitAndIntern() throws Exception {
+ assertEquals(ImmutableList.of(), splitAndInternString(" "));
+ assertEquals(ImmutableList.of(), splitAndInternString(null));
+ List<String> list1 = splitAndInternString(" x y z z");
+ List<String> list2 = splitAndInternString("a z c z");
+
+ assertEquals(ImmutableList.of("x", "y", "z", "z"), list1);
+ assertEquals(ImmutableList.of("a", "z", "c", "z"), list2);
+ assertSame(list1.get(2), list1.get(3));
+ assertSame(list1.get(2), list2.get(1));
+ assertSame(list2.get(1), list2.get(3));
+ }
+
+ @Test
+ public void listItemsWithLimit() throws Exception {
+ assertEquals("begin/a, b, c/end", StringUtil.listItemsWithLimit(
+ new StringBuilder("begin/"), 3, ImmutableList.of("a", "b", "c")).append("/end").toString());
+
+ assertEquals("begin/a, b, c ...(omitting 2 more item(s))/end", StringUtil.listItemsWithLimit(
+ new StringBuilder("begin/"), 3, ImmutableList.of("a", "b", "c", "d", "e"))
+ .append("/end").toString());
+ }
+
+ @Test
+ public void testIndent() throws Exception {
+ assertEquals("", indent("", 0));
+ assertEquals("", indent("", 1));
+ assertEquals("a", indent("a", 1));
+ assertEquals("\n a", indent("\na", 2));
+ assertEquals("a\n b", indent("a\nb", 2));
+ assertEquals("a\n b\n c\n d", indent("a\nb\nc\nd", 1));
+ assertEquals("\n ", indent("\n", 1));
+ }
+
+ @Test
+ public void testStripSuffix() throws Exception {
+ assertEquals("", stripSuffix("", ""));
+ assertEquals(null, stripSuffix("", "a"));
+ assertEquals("a", stripSuffix("a", ""));
+ assertEquals("a", stripSuffix("aa", "a"));
+ assertEquals(null, stripSuffix("ab", "c"));
+ }
+
+ @Test
+ public void testCapitalize() throws Exception {
+ assertEquals("", capitalize(""));
+ assertEquals("Joe", capitalize("joe"));
+ assertEquals("Joe", capitalize("Joe"));
+ assertEquals("O", capitalize("o"));
+ assertEquals("O", capitalize("O"));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/StringUtilitiesTest.java b/src/test/java/com/google/devtools/build/lib/util/StringUtilitiesTest.java
new file mode 100644
index 0000000..c8ed641
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/StringUtilitiesTest.java
@@ -0,0 +1,195 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static com.google.devtools.build.lib.util.StringUtilities.combineKeys;
+import static com.google.devtools.build.lib.util.StringUtilities.joinLines;
+import static com.google.devtools.build.lib.util.StringUtilities.layoutTable;
+import static com.google.devtools.build.lib.util.StringUtilities.prettyPrintBytes;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Maps;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A test for {@link StringUtilities}.
+ */
+@RunWith(JUnit4.class)
+public class StringUtilitiesTest {
+
+ // Tests of StringUtilities.joinLines()
+
+ @Test
+ public void emptyLinesYieldsEmptyString() {
+ assertEquals("", joinLines());
+ }
+
+ @Test
+ public void twoLinesGetjoinedNicely() {
+ assertEquals("line 1\nline 2", joinLines("line 1", "line 2"));
+ }
+
+ @Test
+ public void aTrailingNewlineIsAvailableWhenYouNeedIt() {
+ assertEquals("two lines\nwith trailing newline\n",
+ joinLines("two lines", "with trailing newline", ""));
+ }
+
+ // Tests of StringUtilities.combineKeys()
+
+ /** Simple sanity test of format */
+ @Test
+ public void combineKeysFormat() {
+ assertEquals("<a><b!!c><!<d!>>", combineKeys("a", "b!c", "<d>"));
+ }
+
+ /**
+ * Test that combining different keys gives different results,
+ * i.e. that there are no collisions.
+ * We test all combinations of up to 3 keys from the test_keys
+ * array (defined below).
+ */
+ @Test
+ public void testCombineKeys() {
+ // This map is really just used as a set, but
+ // if the test fails, the values in the map may be
+ // useful for debugging.
+ Map<String,String[]> map = new HashMap<>();
+ for (int numKeys = 0; numKeys <= 3; numKeys++) {
+ testCombineKeys(map, numKeys, new String[numKeys]);
+ }
+ }
+
+ private void testCombineKeys(Map<String,String[]> map,
+ int n, String[] keys) {
+ if (n == 0) {
+ String[] keys_copy = keys.clone();
+ String combined_key = combineKeys(keys_copy);
+ String[] prev_keys = map.put(combined_key, keys_copy);
+ if (prev_keys != null) {
+ fail("combineKeys collision:\n" +
+ "key sequence 1: " + Arrays.deepToString(prev_keys) + "\n" +
+ "key sequence 2: " + Arrays.deepToString(keys_copy) + "\n" +
+ "combined key sequence 1: " + combineKeys(prev_keys) + "\n" +
+ "combined key sequence 2: " + combineKeys(keys_copy) + "\n");
+ }
+ } else {
+ for (String key : test_keys) {
+ keys[n - 1] = key;
+ testCombineKeys(map, n - 1, keys);
+ }
+ }
+ }
+
+ private static final String[] test_keys = {
+ // ordinary strings
+ "", "a", "word", "//depot/foo/bar",
+ // likely delimiter characters
+ " ", ",", "\\", "\"", "\'", "\0", "\u00ff",
+ // strings starting in special delimiter
+ " foo", ",foo", "\\foo", "\"foo", "\'foo", "\0foo", "\u00fffoo",
+ // strings ending in special delimiter
+ "bar ", "bar,", "bar\\", "bar\"", "bar\'", "bar\0", "bar\u00ff",
+ // white-box testing of the delimiters that combineKeys() uses
+ "<", ">", "!", "!<", "!>", "!!", "<!", ">!"
+ };
+
+ @Test
+ public void replaceAllLiteral() throws Exception {
+ assertEquals("ababab",
+ StringUtilities.replaceAllLiteral("bababa", "ba", "ab"));
+ assertEquals("",
+ StringUtilities.replaceAllLiteral("bababa", "ba", ""));
+ assertEquals("bababa",
+ StringUtilities.replaceAllLiteral("bababa", "", "ab"));
+ }
+
+ @Test
+ public void testLayoutTable() throws Exception {
+ Map<String, String> data = Maps.newTreeMap();
+ data.put("foo", "bar");
+ data.put("bang", "baz");
+ data.put("lengthy key", "lengthy value");
+
+ assertEquals(joinLines("bang: baz",
+ "foo: bar",
+ "lengthy key: lengthy value"), layoutTable(data));
+ }
+
+ @Test
+ public void testPrettyPrintBytes() {
+ String[] expected = {
+ "2B",
+ "23B",
+ "234B",
+ "2345B",
+ "23KB",
+ "234KB",
+ "2345KB",
+ "23MB",
+ "234MB",
+ "2345MB",
+ "23456MB",
+ "234GB",
+ "2345GB",
+ "23456GB",
+ };
+ double x = 2.3456;
+ for (int ii = 0; ii < expected.length; ++ii) {
+ assertEquals(expected[ii], prettyPrintBytes((long) x));
+ x = x * 10.0;
+ }
+ }
+
+ @Test
+ public void sanitizeControlChars() {
+ assertEquals("<?>", StringUtilities.sanitizeControlChars("\000"));
+ assertEquals("<?>", StringUtilities.sanitizeControlChars("\001"));
+ assertEquals("\\r", StringUtilities.sanitizeControlChars("\r"));
+ assertEquals(" abc123", StringUtilities.sanitizeControlChars(" abc123"));
+ }
+
+ @Test
+ public void containsSubarray() {
+ assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "ab".toCharArray()));
+ assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "de".toCharArray()));
+ assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "bc".toCharArray()));
+ assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "".toCharArray()));
+ }
+
+ @Test
+ public void notContainsSubarray() {
+ assertFalse(StringUtilities.containsSubarray("abc".toCharArray(), "abcd".toCharArray()));
+ assertFalse(StringUtilities.containsSubarray("abc".toCharArray(), "def".toCharArray()));
+ assertFalse(StringUtilities.containsSubarray("abcde".toCharArray(), "bd".toCharArray()));
+ }
+
+ @Test
+ public void toPythonStyleFunctionName() {
+ assertEquals("a", StringUtilities.toPythonStyleFunctionName("a"));
+ assertEquals("a_b", StringUtilities.toPythonStyleFunctionName("aB"));
+ assertEquals("a_b_c", StringUtilities.toPythonStyleFunctionName("aBC"));
+ assertEquals("a_bc_d", StringUtilities.toPythonStyleFunctionName("aBcD"));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/TargetUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/TargetUtilsTest.java
new file mode 100644
index 0000000..33b0fe3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/TargetUtilsTest.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.devtools.build.lib.packages.TargetUtils;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test for {@link TargetUtils}
+ */
+@RunWith(JUnit4.class)
+public class TargetUtilsTest {
+
+ @Test
+ public void getRuleLanguage() {
+ assertEquals("java", TargetUtils.getRuleLanguage("java_binary"));
+ assertEquals("foobar", TargetUtils.getRuleLanguage("foobar"));
+ assertEquals("", TargetUtils.getRuleLanguage(""));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinterTest.java b/src/test/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinterTest.java
new file mode 100644
index 0000000..d75208f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinterTest.java
@@ -0,0 +1,94 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * A test for {@link AnsiTerminalPrinter}.
+ */
+@RunWith(JUnit4.class)
+public class AnsiTerminalPrinterTest {
+ private ByteArrayOutputStream stream;
+ private AnsiTerminalPrinter printer;
+
+ @Before
+ public void setUp() throws Exception {
+ stream = new ByteArrayOutputStream(1000);
+ printer = new AnsiTerminalPrinter(stream, true);
+ }
+
+ private void setPlainPrinter() {
+ printer = new AnsiTerminalPrinter(stream, false);
+ }
+
+ private void assertString(String string) {
+ assertEquals(string, stream.toString());
+ }
+
+ private void assertRegex(String regex) {
+ MoreAsserts.assertStdoutContainsRegex(regex, stream.toString(), "");
+ }
+
+ @Test
+ public void testPlainPrinter() throws Exception {
+ setPlainPrinter();
+ printer.print("1" + Mode.INFO + "2" + Mode.ERROR + "3" + Mode.WARNING + "4"
+ + Mode.DEFAULT + "5");
+ assertString("12345");
+ }
+
+ @Test
+ public void testDefaultModeIsDefault() throws Exception {
+ printer.print("1" + Mode.DEFAULT + "2");
+ assertString("12");
+ }
+
+ @Test
+ public void testDuplicateMode() throws Exception {
+ printer.print("_A_" + Mode.INFO);
+ printer.print("_B_" + Mode.INFO + "_C_");
+ assertRegex("^_A_.+_B__C_$");
+ }
+
+ @Test
+ public void testModeCodes() throws Exception {
+ printer.print(Mode.INFO + "XXX" + Mode.ERROR + "XXX" + Mode.WARNING +"XXX" + Mode.DEFAULT
+ + "XXX" + Mode.INFO + "XXX" + Mode.ERROR + "XXX" + Mode.WARNING +"XXX" + Mode.DEFAULT);
+ String[] codes = stream.toString().split("XXX");
+ assertEquals(8, codes.length);
+ for (int i = 0; i < 4; i++) {
+ assertTrue(codes[i].length() > 0);
+ assertEquals(codes[i], codes[i+4]);
+ }
+ assertFalse(codes[0].equals(codes[1]));
+ assertFalse(codes[0].equals(codes[2]));
+ assertFalse(codes[0].equals(codes[3]));
+ assertFalse(codes[1].equals(codes[2]));
+ assertFalse(codes[1].equals(codes[3]));
+ assertFalse(codes[2].equals(codes[3]));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/DelegatingOutErrTest.java b/src/test/java/com/google/devtools/build/lib/util/io/DelegatingOutErrTest.java
new file mode 100644
index 0000000..ed644d1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/DelegatingOutErrTest.java
@@ -0,0 +1,72 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static com.google.devtools.build.lib.util.StringUtilities.joinLines;
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test for {@link DelegatingOutErr}.
+ */
+@RunWith(JUnit4.class)
+public class DelegatingOutErrTest {
+
+ @Test
+ public void testNewDelegateIsLikeDevNull() {
+ DelegatingOutErr delegate = new DelegatingOutErr();
+ delegate.printOut("Hello, world.\n");
+ delegate.printErr("Feel free to ignore me.\n");
+ }
+
+ @Test
+ public void testSubscribeAndUnsubscribeSink() {
+ DelegatingOutErr delegate = new DelegatingOutErr();
+ delegate.printOut("Nobody will listen to this.\n");
+ RecordingOutErr sink = new RecordingOutErr();
+ delegate.addSink(sink);
+ delegate.printOutLn("Hello, sink.");
+ delegate.removeSink(sink);
+ delegate.printOutLn("... and alone again ...");
+ delegate.addSink(sink);
+ delegate.printOutLn("How are things?");
+ assertEquals("Hello, sink.\nHow are things?\n", sink.outAsLatin1());
+ }
+
+ @Test
+ public void testSubscribeMultipleSinks() {
+ DelegatingOutErr delegate = new DelegatingOutErr();
+ RecordingOutErr left = new RecordingOutErr();
+ RecordingOutErr right = new RecordingOutErr();
+ delegate.addSink(left);
+ delegate.printOutLn("left only");
+ delegate.addSink(right);
+ delegate.printOutLn("both");
+ delegate.removeSink(left);
+ delegate.printOutLn("right only");
+ delegate.removeSink(right);
+ delegate.printOutLn("silence");
+ delegate.addSink(left);
+ delegate.addSink(right);
+ delegate.printOutLn("left and right");
+ assertEquals(joinLines("left only", "both", "left and right", ""),
+ left.outAsLatin1());
+ assertEquals(joinLines("both", "right only", "left and right", ""),
+ right.outAsLatin1());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStreamTest.java
new file mode 100644
index 0000000..b060beb
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStreamTest.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * Tests {@link LinePrefixingOutputStream}.
+ */
+@RunWith(JUnit4.class)
+public class LinePrefixingOutputStreamTest {
+
+ private byte[] bytes(String string) {
+ return string.getBytes(UTF_8);
+ }
+
+ private String string(byte[] bytes) {
+ return new String(bytes, UTF_8);
+ }
+
+ private ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private LinePrefixingOutputStream prefixOut =
+ new LinePrefixingOutputStream("Prefix: ", out);
+
+ @Test
+ public void testNoOutputUntilNewline() throws IOException {
+ prefixOut.write(bytes("We won't be seeing any output."));
+ assertEquals("", string(out.toByteArray()));
+ }
+
+ @Test
+ public void testOutputIfFlushed() throws IOException {
+ prefixOut.write(bytes("We'll flush after this line."));
+ prefixOut.flush();
+ assertEquals("Prefix: We'll flush after this line.\n",
+ string(out.toByteArray()));
+ }
+
+ @Test
+ public void testAutoflushUponNewline() throws IOException {
+ prefixOut.write(bytes("Hello, newline.\n"));
+ assertEquals("Prefix: Hello, newline.\n", string(out.toByteArray()));
+ }
+
+ @Test
+ public void testAutoflushUponEmbeddedNewLine() throws IOException {
+ prefixOut.write(bytes("Hello line1.\nHello line2.\nHello line3.\n"));
+ assertEquals(
+ "Prefix: Hello line1.\nPrefix: Hello line2.\nPrefix: Hello line3.\n",
+ string(out.toByteArray()));
+ }
+
+ @Test
+ public void testBufferMaxLengthFlush() throws IOException {
+ String junk = "lots of characters of non-newline junk. ";
+ while (junk.length() < LineFlushingOutputStream.BUFFER_LENGTH) {
+ junk = junk + junk;
+ }
+ junk = junk.substring(0, LineFlushingOutputStream.BUFFER_LENGTH);
+
+ // Also test bug where write on a full buffer blows up
+ prefixOut.write(bytes(junk + junk));
+ prefixOut.write(bytes(junk + junk));
+ prefixOut.write(bytes("x"));
+ assertEquals("Prefix: " + junk + "\n" + "Prefix: " + junk + "\n"
+ + "Prefix: " + junk + "\n" + "Prefix: " + junk + "\n",
+ string(out.toByteArray()));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/OutErrTest.java b/src/test/java/com/google/devtools/build/lib/util/io/OutErrTest.java
new file mode 100644
index 0000000..1419f42
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/OutErrTest.java
@@ -0,0 +1,79 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * Tests {@link OutErr}.
+ */
+@RunWith(JUnit4.class)
+public class OutErrTest {
+
+ private ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private ByteArrayOutputStream err = new ByteArrayOutputStream();
+ private OutErr outErr = OutErr.create(out, err);
+
+ @Test
+ public void testRetainsOutErr() {
+ assertSame(out, outErr.getOutputStream());
+ assertSame(err, outErr.getErrorStream());
+ }
+
+ @Test
+ public void testPrintsToOut() {
+ outErr.printOut("Hello, world.");
+ assertEquals("Hello, world.", new String(out.toByteArray()));
+ }
+
+ @Test
+ public void testPrintsToErr() {
+ outErr.printErr("Hello, moon.");
+ assertEquals("Hello, moon.", new String(err.toByteArray()));
+ }
+
+ @Test
+ public void testPrintsToOutWithANewline() {
+ outErr.printOutLn("With a newline.");
+ assertEquals("With a newline.\n", new String(out.toByteArray()));
+ }
+
+ @Test
+ public void testPrintsToErrWithANewline(){
+ outErr.printErrLn("With a newline.");
+ assertEquals("With a newline.\n", new String(err.toByteArray()));
+ }
+
+ @Test
+ public void testPrintsTwoLinesToOut() {
+ outErr.printOutLn("line 1");
+ outErr.printOutLn("line 2");
+ assertEquals("line 1\nline 2\n", new String(out.toByteArray()));
+ }
+
+ @Test
+ public void testPrintsTwoLinesToErr() {
+ outErr.printErrLn("line 1");
+ outErr.printErrLn("line 2");
+ assertEquals("line 1\nline 2\n", new String(err.toByteArray()));
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/RecordingOutErrTest.java b/src/test/java/com/google/devtools/build/lib/util/io/RecordingOutErrTest.java
new file mode 100644
index 0000000..b55eb4c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/RecordingOutErrTest.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.PrintWriter;
+
+/**
+ * A test for {@link RecordingOutErr}.
+ */
+@RunWith(JUnit4.class)
+public class RecordingOutErrTest {
+
+ protected RecordingOutErr getRecordingOutErr() {
+ return new RecordingOutErr();
+ }
+
+ @Test
+ public void testRecordingOutErrRecords() {
+ RecordingOutErr outErr = getRecordingOutErr();
+
+ outErr.printOut("Test");
+ outErr.printOutLn("out1");
+ PrintWriter writer = new PrintWriter(outErr.getOutputStream());
+ writer.println("Testout2");
+ writer.flush();
+
+ outErr.printErr("Test");
+ outErr.printErrLn("err1");
+ writer = new PrintWriter(outErr.getErrorStream());
+ writer.println("Testerr2");
+ writer.flush();
+
+ assertEquals(outErr.outAsLatin1(), "Testout1\nTestout2\n");
+ assertEquals(outErr.errAsLatin1(), "Testerr1\nTesterr2\n");
+
+ assertTrue(outErr.hasRecordedOutput());
+
+ outErr.reset();
+
+ assertEquals(outErr.outAsLatin1(), "");
+ assertEquals(outErr.errAsLatin1(), "");
+ assertFalse(outErr.hasRecordedOutput());
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/StreamDemultiplexerTest.java b/src/test/java/com/google/devtools/build/lib/util/io/StreamDemultiplexerTest.java
new file mode 100644
index 0000000..59277df5
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/StreamDemultiplexerTest.java
@@ -0,0 +1,150 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import com.google.devtools.build.lib.util.StringUtilities;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Random;
+
+/**
+ * Tests {@link StreamDemultiplexer}.
+ */
+@RunWith(JUnit4.class)
+public class StreamDemultiplexerTest {
+
+ private ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private ByteArrayOutputStream err = new ByteArrayOutputStream();
+ private ByteArrayOutputStream ctl = new ByteArrayOutputStream();
+
+ private byte[] lines(String... lines) {
+ try {
+ return StringUtilities.joinLines(lines).getBytes("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private String toAnsi(ByteArrayOutputStream stream) {
+ try {
+ return new String(stream.toByteArray(), "ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private byte[] inAnsi(String string) {
+ try {
+ return string.getBytes("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Test
+ public void testHelloWorldOnStandardOut() throws Exception {
+ byte[] multiplexed = lines("@1@", "Hello, world.");
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out);
+ try {
+ demux.write(multiplexed);
+ demux.flush();
+ } finally {
+ demux.close();
+ }
+ assertEquals("Hello, world.", out.toString("ISO-8859-1"));
+ }
+
+ @Test
+ public void testOutErrCtl() throws Exception {
+ byte[] multiplexed = lines("@1@", "out", "@2@", "err", "@3@", "ctl", "");
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out, err, ctl);
+ try {
+ demux.write(multiplexed);
+ demux.flush();
+ } finally {
+ demux.close();
+ }
+ assertEquals("out", toAnsi(out));
+ assertEquals("err", toAnsi(err));
+ assertEquals("ctl", toAnsi(ctl));
+ }
+
+ @Test
+ public void testWithoutLineBreaks() throws Exception {
+ byte[] multiplexed = lines("@1@", "just ", "@1@", "one ", "@1@", "line", "");
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out);
+ try {
+ demux.write(multiplexed);
+ demux.flush();
+ } finally {
+ demux.close();
+ }
+ assertEquals("just one line", out.toString("ISO-8859-1"));
+ }
+
+ @Test
+ public void testLineBreaks() throws Exception {
+ byte[] multiplexed = lines("@1", "two", "@1", "lines", "");
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out);
+ try {
+ demux.write(multiplexed);
+ demux.flush();
+ assertEquals("two\nlines\n", out.toString("ISO-8859-1"));
+ } finally {
+ demux.close();
+ }
+ }
+
+ @Test
+ public void testMultiplexAndBackWithHelloWorld() throws Exception {
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out);
+ StreamMultiplexer mux = new StreamMultiplexer(demux);
+ OutputStream out = mux.createStdout();
+ out.write(inAnsi("Hello, world."));
+ out.flush();
+ assertEquals("Hello, world.", toAnsi(this.out));
+ }
+
+ @Test
+ public void testMultiplexDemultiplexBinaryStress() throws Exception {
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out, err, ctl);
+ StreamMultiplexer mux = new StreamMultiplexer(demux);
+ OutputStream[] muxOuts = {mux.createStdout(), mux.createStderr(), mux.createControl()};
+ ByteArrayOutputStream[] expectedOuts =
+ {new ByteArrayOutputStream(), new ByteArrayOutputStream(), new ByteArrayOutputStream()};
+
+ Random random = new Random(0xdeadbeef);
+ for (int round = 0; round < 100; round++) {
+ byte[] buffer = new byte[random.nextInt(100)];
+ random.nextBytes(buffer);
+ int streamId = random.nextInt(3);
+ expectedOuts[streamId].write(buffer);
+ expectedOuts[streamId].flush();
+ muxOuts[streamId].write(buffer);
+ muxOuts[streamId].flush();
+ }
+ assertArrayEquals(expectedOuts[0].toByteArray(), out.toByteArray());
+ assertArrayEquals(expectedOuts[1].toByteArray(), err.toByteArray());
+ assertArrayEquals(expectedOuts[2].toByteArray(), ctl.toByteArray());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerParallelStressTest.java b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerParallelStressTest.java
new file mode 100644
index 0000000..db833a5
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerParallelStressTest.java
@@ -0,0 +1,128 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import com.google.common.io.ByteStreams;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Exercise {@link StreamMultiplexer} in a parallel setting and ensure there's
+ * no corruption.
+ */
+@RunWith(JUnit4.class)
+public class StreamMultiplexerParallelStressTest {
+
+ /**
+ * Characters that could likely cause corruption (they're used as control
+ * characters).
+ */
+ char[] toughCharsToTry = {'\n', '@', '1', '2', '\0', '0'};
+
+ /**
+ * We use a demultiplexer as a simple sanity checker only - that is, we don't
+ * care what the demultiplexer writes, but we are taking advantage of its
+ * built in error checking.
+ */
+ OutputStream devNull = ByteStreams.nullOutputStream();
+
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte)'1',
+ devNull, devNull, devNull);
+
+ /**
+ * The multiplexer under test.
+ */
+ StreamMultiplexer mux = new StreamMultiplexer(demux);
+
+ /**
+ * Streams is the out / err / control output streams of the multiplexer which
+ * we will write to in parallel.
+ */
+ OutputStream[] streams = {
+ mux.createStdout(), mux.createStderr(), mux.createControl()};
+
+ /**
+ * We will create a bunch of threads that write random data to the streams of
+ * the mux.
+ */
+ class RandomDataPump implements Callable<Object> {
+
+ private Random random;
+
+ public RandomDataPump(int threadId) {
+ random = new Random(threadId * 0xdeadbeefL);
+ }
+
+ @Override
+ public Object call() throws Exception {
+ Thread.yield();
+ OutputStream out = streams[random.nextInt(2)];
+ for (int i = 0; i < 10000; i++) {
+ switch (random.nextInt(5)) {
+ case 0:
+ out.write(random.nextInt());
+ break;
+ case 1:
+ int index = random.nextInt(toughCharsToTry.length);
+ out.write(toughCharsToTry[index]);
+ break;
+ case 2:
+ byte[] buffer = new byte[random.nextInt(312)];
+ random.nextBytes(buffer);
+ out.write(buffer);
+ break;
+ case 3:
+ out.flush();
+ break;
+ case 4:
+ out = streams[random.nextInt(3)];
+ break;
+ }
+ }
+ return null;
+ }
+ }
+
+ @Test
+ public void testSingleThreadedStress() throws Exception {
+ new RandomDataPump(1).call();
+ }
+
+ @Test
+ public void testMultiThreadedStress()
+ throws InterruptedException, ExecutionException {
+ ExecutorService service = Executors.newFixedThreadPool(50);
+
+ List<Future<?>> futures = new ArrayList<>();
+ for (int threadId = 0; threadId < 50; threadId++) {
+ futures.add(service.submit(new RandomDataPump(threadId)));
+ }
+ for (Future<?> future : futures) {
+ future.get();
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerTest.java b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerTest.java
new file mode 100644
index 0000000..aef6da3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerTest.java
@@ -0,0 +1,149 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.util.io;
+
+import static com.google.devtools.build.lib.util.StringUtilities.joinLines;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.io.ByteStreams;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+
+/**
+ * Test for {@link StreamMultiplexer}.
+ */
+@RunWith(JUnit4.class)
+public class StreamMultiplexerTest {
+
+ private ByteArrayOutputStream multiplexed;
+ private OutputStream out;
+ private OutputStream err;
+ private OutputStream ctl;
+
+ @Before
+ public void setUp() throws Exception {
+ multiplexed = new ByteArrayOutputStream();
+ StreamMultiplexer multiplexer = new StreamMultiplexer(multiplexed);
+ out = multiplexer.createStdout();
+ err = multiplexer.createStderr();
+ ctl = multiplexer.createControl();
+ }
+
+ @Test
+ public void testEmptyWire() throws IOException {
+ out.flush();
+ err.flush();
+ ctl.flush();
+ assertEquals(0, multiplexed.toByteArray().length);
+ }
+
+ private static byte[] getLatin(String string)
+ throws UnsupportedEncodingException {
+ return string.getBytes("ISO-8859-1");
+ }
+
+ private static String getLatin(byte[] bytes)
+ throws UnsupportedEncodingException {
+ return new String(bytes, "ISO-8859-1");
+ }
+
+ @Test
+ public void testHelloWorldOnStdOut() throws IOException {
+ out.write(getLatin("Hello, world."));
+ out.flush();
+ assertEquals(joinLines("@1@", "Hello, world.", ""),
+ getLatin(multiplexed.toByteArray()));
+ }
+
+ @Test
+ public void testInterleavedStdoutStderrControl() throws Exception {
+ out.write(getLatin("Hello, stdout."));
+ out.flush();
+ err.write(getLatin("Hello, stderr."));
+ err.flush();
+ ctl.write(getLatin("Hello, control."));
+ ctl.flush();
+ out.write(getLatin("... and back!"));
+ out.flush();
+ assertEquals(joinLines("@1@", "Hello, stdout.",
+ "@2@", "Hello, stderr.",
+ "@3@", "Hello, control.",
+ "@1@", "... and back!",
+ ""),
+ getLatin(multiplexed.toByteArray()));
+ }
+
+ @Test
+ public void testWillNotCommitToUnderlyingStreamUnlessFlushOrNewline()
+ throws Exception {
+ out.write(getLatin("There are no newline characters in here, so it won't" +
+ " get written just yet."));
+ assertArrayEquals(multiplexed.toByteArray(), new byte[0]);
+ }
+
+ @Test
+ public void testNewlineTriggersFlush() throws Exception {
+ out.write(getLatin("No newline just yet, so no flushing. "));
+ assertArrayEquals(multiplexed.toByteArray(), new byte[0]);
+ out.write(getLatin("OK, here we go:\nAnd more to come."));
+
+ String expected = joinLines("@1",
+ "No newline just yet, so no flushing. OK, here we go:", "");
+
+ assertEquals(expected, getLatin(multiplexed.toByteArray()));
+
+ out.write((byte) '\n');
+ expected += joinLines("@1", "And more to come.", "");
+
+ assertEquals(expected, getLatin(multiplexed.toByteArray()));
+ }
+
+ @Test
+ public void testFlush() throws Exception {
+ out.write(getLatin("Don't forget to flush!"));
+ assertArrayEquals(new byte[0], multiplexed.toByteArray());
+ out.flush(); // now the output will appear in multiplexed.
+ assertEquals(joinLines("@1@", "Don't forget to flush!", ""),
+ getLatin(multiplexed.toByteArray()));
+ }
+
+ @Test
+ public void testByteEncoding() throws IOException {
+ OutputStream devNull = ByteStreams.nullOutputStream();
+ StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', devNull);
+ StreamMultiplexer mux = new StreamMultiplexer(demux);
+ OutputStream out = mux.createStdout();
+
+ // When we cast 266 to a byte, we get 10. So basically, we ended up
+ // comparing 266 with 10 as an integer (because out.write takes an int),
+ // and then later cast it to 10. This way we'd end up with a control
+ // character \n in the middle of the payload which would then screw things
+ // up when the real control character arrived. The fixed version of the
+ // StreamMultiplexer avoids this problem by always casting to a byte before
+ // carrying out any comparisons.
+
+ out.write(266);
+ out.write(10);
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemConcurrencyTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemConcurrencyTest.java
new file mode 100644
index 0000000..c90de18
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemConcurrencyTest.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.vfs.util.FileSystems;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * (Slow) tests of FileSystem under concurrency.
+ *
+ * These tests are nondeterministic but provide good coverage nonetheless.
+ */
+@RunWith(JUnit4.class)
+public class FileSystemConcurrencyTest {
+
+ Path workingDir;
+
+ @Before
+ public void setUp() throws Exception {
+ FileSystem testFS = FileSystems.initDefaultAsNative();
+
+ // Resolve symbolic links in the temp dir:
+ workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath());
+ }
+
+ @Test
+ public void testConcurrentSymlinkModifications() throws Exception {
+ final Path xFile = workingDir.getRelative("file");
+ FileSystemUtils.createEmptyFile(xFile);
+
+ final Path xLinkToFile = workingDir.getRelative("link");
+
+ // "Boxed" for pass-by-reference.
+ final boolean[] run = { true };
+ final IOException[] exception = { null };
+ Thread createThread = new Thread() {
+ @Override
+ public void run() {
+ while (run[0]) {
+ if (!xLinkToFile.exists()) {
+ try {
+ xLinkToFile.createSymbolicLink(xFile);
+ } catch (IOException e) {
+ exception[0] = e;
+ return;
+ }
+ }
+ }
+ }
+ };
+ Thread deleteThread = new Thread() {
+ @Override
+ public void run() {
+ while (run[0]) {
+ if (xLinkToFile.exists(Symlinks.NOFOLLOW)) {
+ try {
+ xLinkToFile.delete();
+ } catch (IOException e) {
+ exception[0] = e;
+ return;
+ }
+ }
+ }
+ }
+ };
+ createThread.start();
+ deleteThread.start();
+ Thread.sleep(1000);
+ run[0] = false;
+ createThread.join(0);
+ deleteThread.join(0);
+
+ if (exception[0] != null) {
+ throw exception[0];
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemTest.java
new file mode 100644
index 0000000..b6e88d8
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemTest.java
@@ -0,0 +1,1356 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This class handles the generic tests that any filesystem must pass.
+ *
+ * <p>Each filesystem-test should inherit from this class, thereby obtaining
+ * all the tests.
+ */
+public abstract class FileSystemTest {
+
+ private long savedTime;
+ protected FileSystem testFS;
+ protected boolean supportsSymlinks;
+ protected Path workingDir;
+
+ // Some useful examples of various kinds of files (mnemonic: "x" = "eXample")
+ protected Path xNothing;
+ protected Path xFile;
+ protected Path xNonEmptyDirectory;
+ protected Path xNonEmptyDirectoryFoo;
+ protected Path xEmptyDirectory;
+
+ @Before
+ public void setUp() throws Exception {
+ testFS = getFreshFileSystem();
+ workingDir = testFS.getPath(getTestTmpDir());
+ cleanUpWorkingDirectory(workingDir);
+ supportsSymlinks = testFS.supportsSymbolicLinks();
+
+ // % ls -lR
+ // -rw-rw-r-- xFile
+ // drwxrwxr-x xNonEmptyDirectory
+ // -rw-rw-r-- xNonEmptyDirectory/foo
+ // drwxrwxr-x xEmptyDirectory
+
+ xNothing = absolutize("xNothing");
+ xFile = absolutize("xFile");
+ xNonEmptyDirectory = absolutize("xNonEmptyDirectory");
+ xNonEmptyDirectoryFoo = xNonEmptyDirectory.getChild("foo");
+ xEmptyDirectory = absolutize("xEmptyDirectory");
+
+ FileSystemUtils.createEmptyFile(xFile);
+ xNonEmptyDirectory.createDirectory();
+ FileSystemUtils.createEmptyFile(xNonEmptyDirectoryFoo);
+ xEmptyDirectory.createDirectory();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ destroyFileSystem(testFS);
+ }
+
+ /**
+ * Returns an instance of the file system to test.
+ */
+ protected abstract FileSystem getFreshFileSystem() throws IOException;
+
+ protected boolean isSymbolicLink(File file) {
+ return com.google.devtools.build.lib.unix.FilesystemUtils.isSymbolicLink(file);
+ }
+
+ protected void setWritable(File file) throws IOException {
+ com.google.devtools.build.lib.unix.FilesystemUtils.setWritable(file);
+ }
+
+ protected void setExecutable(File file) throws IOException {
+ com.google.devtools.build.lib.unix.FilesystemUtils.setExecutable(file);
+ }
+
+ private static final Pattern STAT_SUBDIR_ERROR = Pattern.compile("(.*) \\(Not a directory\\)");
+
+ // Test that file is not present, using statIfFound. Base implementation throws an exception, but
+ // subclasses may override statIfFound to return null, in which case their tests should override
+ // this method.
+ @SuppressWarnings("unused") // Subclasses may throw.
+ protected void expectNotFound(Path path) throws IOException {
+ try {
+ assertNull(path.statIfFound());
+ } catch (IOException e) {
+ // May be because of a non-directory path component. Parse exception to check this.
+ Matcher matcher = STAT_SUBDIR_ERROR.matcher(e.getMessage());
+ if (!matcher.matches() || !path.getPathString().startsWith(matcher.group(1))) {
+ // Throw if this doesn't match what an ENOTDIR error looks like.
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Removes all stuff from the test filesystem.
+ */
+ protected void destroyFileSystem(FileSystem fileSystem) throws IOException {
+ Preconditions.checkArgument(fileSystem.equals(workingDir.getFileSystem()));
+ cleanUpWorkingDirectory(workingDir);
+ }
+
+ /**
+ * Cleans up the working directory by removing everything.
+ */
+ protected void cleanUpWorkingDirectory(Path workingPath)
+ throws IOException {
+ if (workingPath.exists()) {
+ removeEntireDirectory(workingPath.getPathFile()); // uses java.io.File!
+ }
+ FileSystemUtils.createDirectoryAndParents(workingPath);
+ }
+
+ /**
+ * This function removes an entire directory and all of its contents.
+ * Much like rm -rf directoryToRemove
+ */
+ protected void removeEntireDirectory(File directoryToRemove)
+ throws IOException {
+ // make sure that we do not remove anything outside the test directory
+ Path testDirPath = testFS.getPath(getTestTmpDir());
+ if (!testFS.getPath(directoryToRemove.getAbsolutePath()).startsWith(testDirPath)) {
+ throw new IOException("trying to remove files outside of the testdata directory");
+ }
+ // Some tests set the directories read-only and/or non-executable, so
+ // override that:
+ setWritable(directoryToRemove);
+ setExecutable(directoryToRemove);
+
+ File[] files = directoryToRemove.listFiles();
+ if (files != null) {
+ for (File currentFile : files) {
+ boolean isSymbolicLink = isSymbolicLink(currentFile);
+ if (!isSymbolicLink && currentFile.isDirectory()) {
+ removeEntireDirectory(currentFile);
+ } else {
+ if (!isSymbolicLink) {
+ setWritable(currentFile);
+ }
+ if (!currentFile.delete()) {
+ throw new IOException("Failed to delete '" + currentFile + "'");
+ }
+ }
+ }
+ }
+ if (!directoryToRemove.delete()) {
+ throw new IOException("Failed to delete '" + directoryToRemove + "'");
+ }
+ }
+
+ /**
+ * Returns the directory to use as the FileSystem's working directory.
+ * Canonicalized to make tests hermetic against symbolic links in TEST_TMPDIR.
+ */
+ protected final String getTestTmpDir() throws IOException {
+ return new File(TestUtils.tmpDir()).getCanonicalPath() + "/testdir";
+ }
+
+ /**
+ * Indirection to create links so we can test FileSystems that do not support
+ * link creation. For example, JavaFileSystemTest overrides this method
+ * and creates the link with an alternate FileSystem.
+ */
+ protected void createSymbolicLink(Path link, Path target) throws IOException {
+ createSymbolicLink(link, target.asFragment());
+ }
+
+ /**
+ * Indirection to create links so we can test FileSystems that do not support
+ * link creation. For example, JavaFileSystemTest overrides this method
+ * and creates the link with an alternate FileSystem.
+ */
+ protected void createSymbolicLink(Path link, PathFragment target) throws IOException {
+ link.createSymbolicLink(target);
+ }
+
+ /**
+ * Indirection to setReadOnly(false) on FileSystems that do not
+ * support setReadOnly(false). For example, JavaFileSystemTest overrides this
+ * method and makes the Path writable with an alternate FileSystem.
+ */
+ protected void makeWritable(Path target) throws IOException {
+ target.setWritable(true);
+ }
+
+ /**
+ * Indirection to {@link Path#setExecutable(boolean)} on FileSystems that do
+ * not support setExecutable. For example, JavaFileSystemTest overrides this
+ * method and makes the Path executable with an alternate FileSystem.
+ */
+ protected void setExecutable(Path target, boolean mode) throws IOException {
+ target.setExecutable(mode);
+ }
+
+ // TODO(bazel-team): (2011) Put in a setLastModifiedTime into the various objects
+ // and clobber the current time of the object we're currently handling.
+ // Otherwise testing the thing might get a little hard, depending on the clock.
+ void storeReferenceTime(long timeToMark) {
+ savedTime = timeToMark;
+ }
+
+ boolean isLaterThanreferenceTime(long testTime) {
+ return (savedTime <= testTime);
+ }
+
+ Path getTestFile() throws IOException {
+ Path tempPath = absolutize("test-file");
+ FileSystemUtils.createEmptyFile(tempPath);
+ return tempPath;
+ }
+
+ protected Path absolutize(String relativePathName) {
+ return workingDir.getRelative(relativePathName);
+ }
+
+ // Here the tests begin.
+
+ @Test
+ public void testIsFileForNonexistingPath() {
+ Path nonExistingPath = testFS.getPath("/something/strange");
+ assertFalse(nonExistingPath.isFile());
+ }
+
+ @Test
+ public void testIsDirectoryForNonexistingPath() {
+ Path nonExistingPath = testFS.getPath("/something/strange");
+ assertFalse(nonExistingPath.isDirectory());
+ }
+
+ @Test
+ public void testIsLinkForNonexistingPath() {
+ Path nonExistingPath = testFS.getPath("/something/strange");
+ assertFalse(nonExistingPath.isSymbolicLink());
+ }
+
+ @Test
+ public void testExistsForNonexistingPath() throws Exception {
+ Path nonExistingPath = testFS.getPath("/something/strange");
+ assertFalse(nonExistingPath.exists());
+ expectNotFound(nonExistingPath);
+ }
+
+ @Test
+ public void testBadPermissionsThrowsExceptionOnStatIfFound() throws Exception {
+ Path inaccessible = absolutize("inaccessible");
+ inaccessible.createDirectory();
+ Path child = inaccessible.getChild("child");
+ FileSystemUtils.createEmptyFile(child);
+ inaccessible.setExecutable(false);
+ assertFalse(child.exists());
+ try {
+ child.statIfFound();
+ fail();
+ } catch (IOException expected) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testStatIfFoundReturnsNullForChildOfNonDir() throws Exception {
+ Path foo = absolutize("foo");
+ foo.createDirectory();
+ Path nonDir = foo.getRelative("bar");
+ FileSystemUtils.createEmptyFile(nonDir);
+ assertNull(nonDir.getRelative("file").statIfFound());
+ }
+
+ // The following tests check the handling of the current working directory.
+ @Test
+ public void testCreatePathRelativeToWorkingDirectory() {
+ Path relativeCreatedPath = absolutize("some-file");
+ Path expectedResult = workingDir.getRelative(new PathFragment("some-file"));
+
+ assertEquals(expectedResult, relativeCreatedPath);
+ }
+
+ // The following tests check the handling of the root directory
+ @Test
+ public void testRootIsDirectory() {
+ Path rootPath = testFS.getPath("/");
+ assertTrue(rootPath.isDirectory());
+ }
+
+ @Test
+ public void testRootHasNoParent() {
+ Path rootPath = testFS.getPath("/");
+ assertNull(rootPath.getParentDirectory());
+ }
+
+ // The following functions test the creation of files/links/directories.
+ @Test
+ public void testFileExists() throws Exception {
+ Path someFile = absolutize("some-file");
+ FileSystemUtils.createEmptyFile(someFile);
+ assertTrue(someFile.exists());
+ assertNotNull(someFile.statIfFound());
+ }
+
+ @Test
+ public void testFileIsFile() throws Exception {
+ Path someFile = absolutize("some-file");
+ FileSystemUtils.createEmptyFile(someFile);
+ assertTrue(someFile.isFile());
+ }
+
+ @Test
+ public void testFileIsNotDirectory() throws Exception {
+ Path someFile = absolutize("some-file");
+ FileSystemUtils.createEmptyFile(someFile);
+ assertFalse(someFile.isDirectory());
+ }
+
+ @Test
+ public void testFileIsNotSymbolicLink() throws Exception {
+ Path someFile = absolutize("some-file");
+ FileSystemUtils.createEmptyFile(someFile);
+ assertFalse(someFile.isSymbolicLink());
+ }
+
+ @Test
+ public void testDirectoryExists() throws Exception {
+ Path someDirectory = absolutize("some-dir");
+ someDirectory.createDirectory();
+ assertTrue(someDirectory.exists());
+ assertNotNull(someDirectory.statIfFound());
+ }
+
+ @Test
+ public void testDirectoryIsDirectory() throws Exception {
+ Path someDirectory = absolutize("some-dir");
+ someDirectory.createDirectory();
+ assertTrue(someDirectory.isDirectory());
+ }
+
+ @Test
+ public void testDirectoryIsNotFile() throws Exception {
+ Path someDirectory = absolutize("some-dir");
+ someDirectory.createDirectory();
+ assertFalse(someDirectory.isFile());
+ }
+
+ @Test
+ public void testDirectoryIsNotSymbolicLink() throws Exception {
+ Path someDirectory = absolutize("some-dir");
+ someDirectory.createDirectory();
+ assertFalse(someDirectory.isSymbolicLink());
+ }
+
+ @Test
+ public void testSymbolicFileLinkExists() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xFile);
+ assertTrue(someLink.exists());
+ assertNotNull(someLink.statIfFound());
+ }
+ }
+
+ @Test
+ public void testSymbolicFileLinkIsSymbolicLink() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xFile);
+ assertTrue(someLink.isSymbolicLink());
+ }
+ }
+
+ @Test
+ public void testSymbolicFileLinkIsFile() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xFile);
+ assertTrue(someLink.isFile());
+ }
+ }
+
+ @Test
+ public void testSymbolicFileLinkIsNotDirectory() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xFile);
+ assertFalse(someLink.isDirectory());
+ }
+ }
+
+ @Test
+ public void testSymbolicDirLinkExists() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xEmptyDirectory);
+ assertTrue(someLink.exists());
+ assertNotNull(someLink.statIfFound());
+ }
+ }
+
+ @Test
+ public void testSymbolicDirLinkIsSymbolicLink() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xEmptyDirectory);
+ assertTrue(someLink.isSymbolicLink());
+ }
+ }
+
+ @Test
+ public void testSymbolicDirLinkIsDirectory() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xEmptyDirectory);
+ assertTrue(someLink.isDirectory());
+ }
+ }
+
+ @Test
+ public void testSymbolicDirLinkIsNotFile() throws Exception {
+ if (supportsSymlinks) {
+ Path someLink = absolutize("some-link");
+ someLink.createSymbolicLink(xEmptyDirectory);
+ assertFalse(someLink.isFile());
+ }
+ }
+
+ @Test
+ public void testChildOfNonDirectory() throws Exception {
+ Path somePath = absolutize("file-name");
+ FileSystemUtils.createEmptyFile(somePath);
+ Path childOfNonDir = somePath.getChild("child");
+ assertFalse(childOfNonDir.exists());
+ expectNotFound(childOfNonDir);
+ }
+
+ @Test
+ public void testCreateDirectoryIsEmpty() throws Exception {
+ Path newPath = xEmptyDirectory.getChild("new-dir");
+ newPath.createDirectory();
+ assertEquals(newPath.getDirectoryEntries().size(), 0);
+ }
+
+ @Test
+ public void testCreateDirectoryIsOnlyChildInParent() throws Exception {
+ Path newPath = xEmptyDirectory.getChild("new-dir");
+ newPath.createDirectory();
+ assertEquals(1, newPath.getParentDirectory().getDirectoryEntries().size());
+ assertThat(newPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath);
+ }
+
+ @Test
+ public void testCreateDirectories() throws Exception {
+ Path newPath = absolutize("new-dir/sub/directory");
+ assertTrue(FileSystemUtils.createDirectoryAndParents(newPath));
+ }
+
+ @Test
+ public void testCreateDirectoriesIsDirectory() throws Exception {
+ Path newPath = absolutize("new-dir/sub/directory");
+ FileSystemUtils.createDirectoryAndParents(newPath);
+ assertTrue(newPath.isDirectory());
+ }
+
+ @Test
+ public void testCreateDirectoriesIsNotFile() throws Exception {
+ Path newPath = absolutize("new-dir/sub/directory");
+ FileSystemUtils.createDirectoryAndParents(newPath);
+ assertFalse(newPath.isFile());
+ }
+
+ @Test
+ public void testCreateDirectoriesIsNotSymbolicLink() throws Exception {
+ Path newPath = absolutize("new-dir/sub/directory");
+ FileSystemUtils.createDirectoryAndParents(newPath);
+ assertFalse(newPath.isSymbolicLink());
+ }
+
+ @Test
+ public void testCreateDirectoriesIsEmpty() throws Exception {
+ Path newPath = absolutize("new-dir/sub/directory");
+ FileSystemUtils.createDirectoryAndParents(newPath);
+ assertEquals(newPath.getDirectoryEntries().size(), 0);
+ }
+
+ @Test
+ public void testCreateDirectoriesIsOnlyChildInParent() throws Exception {
+ Path newPath = absolutize("new-dir/sub/directory");
+ FileSystemUtils.createDirectoryAndParents(newPath);
+ assertEquals(1, newPath.getParentDirectory().getDirectoryEntries().size());
+ assertThat(newPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath);
+ }
+
+ @Test
+ public void testCreateEmptyFileIsEmpty() throws Exception {
+ Path newPath = xEmptyDirectory.getChild("new-file");
+ FileSystemUtils.createEmptyFile(newPath);
+
+ assertEquals(newPath.getFileSize(), 0);
+ }
+
+ @Test
+ public void testCreateFileIsOnlyChildInParent() throws Exception {
+ Path newPath = xEmptyDirectory.getChild("new-file");
+ FileSystemUtils.createEmptyFile(newPath);
+ assertEquals(1, newPath.getParentDirectory().getDirectoryEntries().size());
+ assertThat(newPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath);
+ }
+
+ // The following functions test the behavior if errors occur during the
+ // creation of files/links/directories.
+ @Test
+ public void testCreateDirectoryWhereDirectoryAlreadyExists() throws Exception {
+ assertFalse(xEmptyDirectory.createDirectory());
+ }
+
+ @Test
+ public void testCreateDirectoryWhereFileAlreadyExists() {
+ try {
+ xFile.createDirectory();
+ fail();
+ } catch (IOException e) {
+ assertEquals(xFile + " (File exists)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateDirectoryWithoutExistingParent() throws Exception {
+ Path newPath = testFS.getPath("/deep/new-dir");
+ try {
+ newPath.createDirectory();
+ fail();
+ } catch (FileNotFoundException e) {
+ MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateDirectoryWithReadOnlyParent() throws Exception {
+ xEmptyDirectory.setWritable(false);
+ Path xChildOfReadonlyDir = xEmptyDirectory.getChild("x");
+ try {
+ xChildOfReadonlyDir.createDirectory();
+ fail();
+ } catch (IOException e) {
+ assertEquals(xChildOfReadonlyDir + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateFileWithoutExistingParent() throws Exception {
+ Path newPath = testFS.getPath("/non-existing-dir/new-file");
+ try {
+ FileSystemUtils.createEmptyFile(newPath);
+ fail();
+ } catch (FileNotFoundException e) {
+ MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateFileWithReadOnlyParent() throws Exception {
+ xEmptyDirectory.setWritable(false);
+ Path xChildOfReadonlyDir = xEmptyDirectory.getChild("x");
+ try {
+ FileSystemUtils.createEmptyFile(xChildOfReadonlyDir);
+ fail();
+ } catch (IOException e) {
+ assertEquals(xChildOfReadonlyDir + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateFileWithinFile() throws Exception {
+ Path newFilePath = absolutize("some-file");
+ FileSystemUtils.createEmptyFile(newFilePath);
+ Path wrongPath = absolutize("some-file/new-file");
+ try {
+ FileSystemUtils.createEmptyFile(wrongPath);
+ fail();
+ } catch (IOException e) {
+ MoreAsserts.assertEndsWith(" (Not a directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateDirectoryWithinFile() throws Exception {
+ Path newFilePath = absolutize("some-file");
+ FileSystemUtils.createEmptyFile(newFilePath);
+ Path wrongPath = absolutize("some-file/new-file");
+ try {
+ wrongPath.createDirectory();
+ fail();
+ } catch (IOException e) {
+ MoreAsserts.assertEndsWith(" (Not a directory)", e.getMessage());
+ }
+ }
+
+ // Test directory contents
+ @Test
+ public void testCreateMultipleChildren() throws Exception {
+ Path theDirectory = absolutize("foo/");
+ theDirectory.createDirectory();
+ Path newPath1 = absolutize("foo/new-file-1");
+ Path newPath2 = absolutize("foo/new-file-2");
+ Path newPath3 = absolutize("foo/new-file-3");
+
+ FileSystemUtils.createEmptyFile(newPath1);
+ FileSystemUtils.createEmptyFile(newPath2);
+ FileSystemUtils.createEmptyFile(newPath3);
+
+ assertThat(theDirectory.getDirectoryEntries()).containsExactly(newPath1, newPath2, newPath3);
+ }
+
+ @Test
+ public void testGetDirectoryEntriesThrowsExceptionWhenRunOnFile() throws Exception {
+ try {
+ xFile.getDirectoryEntries();
+ fail("No Exception thrown.");
+ } catch (IOException ex) {
+ if (ex instanceof FileNotFoundException) {
+ fail("The method should throw an object of class IOException.");
+ }
+ assertEquals(xFile + " (Not a directory)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetDirectoryEntriesThrowsExceptionForNonexistingPath() {
+ Path somePath = testFS.getPath("/non-existing-path");
+ try {
+ somePath.getDirectoryEntries();
+ fail("FileNotFoundException not thrown.");
+ } catch (Exception x) {
+ assertEquals(somePath + " (No such file or directory)", x.getMessage());
+ }
+ }
+
+ // Test the removal of items
+ @Test
+ public void testDeleteDirectory() throws Exception {
+ assertTrue(xEmptyDirectory.delete());
+ }
+
+ @Test
+ public void testDeleteDirectoryIsNotDirectory() throws Exception {
+ xEmptyDirectory.delete();
+ assertFalse(xEmptyDirectory.isDirectory());
+ }
+
+ @Test
+ public void testDeleteDirectoryParentSize() throws Exception {
+ int parentSize = workingDir.getDirectoryEntries().size();
+ xEmptyDirectory.delete();
+ assertEquals(workingDir.getDirectoryEntries().size(), parentSize - 1);
+ }
+
+ @Test
+ public void testDeleteFile() throws Exception {
+ assertTrue(xFile.delete());
+ }
+
+ @Test
+ public void testDeleteFileIsNotFile() throws Exception {
+ xFile.delete();
+ assertFalse(xEmptyDirectory.isFile());
+ }
+
+ @Test
+ public void testDeleteFileParentSize() throws Exception {
+ int parentSize = workingDir.getDirectoryEntries().size();
+ xFile.delete();
+ assertEquals(workingDir.getDirectoryEntries().size(), parentSize - 1);
+ }
+
+ @Test
+ public void testDeleteRemovesCorrectFile() throws Exception {
+ Path newPath1 = xEmptyDirectory.getChild("new-file-1");
+ Path newPath2 = xEmptyDirectory.getChild("new-file-2");
+ Path newPath3 = xEmptyDirectory.getChild("new-file-3");
+
+ FileSystemUtils.createEmptyFile(newPath1);
+ FileSystemUtils.createEmptyFile(newPath2);
+ FileSystemUtils.createEmptyFile(newPath3);
+
+ assertTrue(newPath2.delete());
+ assertThat(xEmptyDirectory.getDirectoryEntries()).containsExactly(newPath1, newPath3);
+ }
+
+ @Test
+ public void testDeleteNonExistingDir() throws Exception {
+ Path path = xEmptyDirectory.getRelative("non-existing-dir");
+ assertFalse(path.delete());
+ }
+
+ @Test
+ public void testDeleteNotADirectoryPath() throws Exception {
+ Path path = xFile.getChild("new-file");
+ assertFalse(path.delete());
+ }
+
+ // Here we test the situations where delete should throw exceptions.
+ @Test
+ public void testDeleteNonEmptyDirectoryThrowsException() throws Exception {
+ try {
+ xNonEmptyDirectory.delete();
+ fail();
+ } catch (IOException e) {
+ assertEquals(xNonEmptyDirectory + " (Directory not empty)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDeleteNonEmptyDirectoryNotDeletedDirectory() throws Exception {
+ try {
+ xNonEmptyDirectory.delete();
+ fail();
+ } catch (IOException e) {
+ // Expected
+ }
+
+ assertTrue(xNonEmptyDirectory.isDirectory());
+ }
+
+ @Test
+ public void testDeleteNonEmptyDirectoryNotDeletedFile() throws Exception {
+ try {
+ xNonEmptyDirectory.delete();
+ fail();
+ } catch (IOException e) {
+ // Expected
+ }
+
+ assertTrue(xNonEmptyDirectoryFoo.isFile());
+ }
+
+ @Test
+ public void testCannotRemoveRoot() {
+ Path rootDirectory = testFS.getRootDirectory();
+ try {
+ rootDirectory.delete();
+ fail();
+ } catch (IOException e) {
+ String msg = e.getMessage();
+ assertTrue(String.format("got %s want EBUSY or ENOTEMPTY", msg),
+ msg.endsWith(" (Directory not empty)")
+ || msg.endsWith(" (Device or resource busy)")
+ || msg.endsWith(" (Is a directory)")); // Happens on OS X.
+ }
+ }
+
+ // Test the date functions
+ @Test
+ public void testCreateFileChangesTimeOfDirectory() throws Exception {
+ storeReferenceTime(workingDir.getLastModifiedTime());
+ Path newPath = absolutize("new-file");
+ FileSystemUtils.createEmptyFile(newPath);
+ assertTrue(isLaterThanreferenceTime(workingDir.getLastModifiedTime()));
+ }
+
+ @Test
+ public void testRemoveFileChangesTimeOfDirectory() throws Exception {
+ Path newPath = absolutize("new-file");
+ FileSystemUtils.createEmptyFile(newPath);
+ storeReferenceTime(workingDir.getLastModifiedTime());
+ newPath.delete();
+ assertTrue(isLaterThanreferenceTime(workingDir.getLastModifiedTime()));
+ }
+
+ // This test is a little bit strange, as we cannot test the progression
+ // of the time directly. As the Java time and the OS time are slightly different.
+ // Therefore, we first create an unrelated file to get a notion
+ // of the current OS time and use that as a baseline.
+ @Test
+ public void testCreateFileTimestamp() throws Exception {
+ Path syncFile = absolutize("sync-file");
+ FileSystemUtils.createEmptyFile(syncFile);
+
+ Path newFile = absolutize("new-file");
+ storeReferenceTime(syncFile.getLastModifiedTime());
+ FileSystemUtils.createEmptyFile(newFile);
+ assertTrue(isLaterThanreferenceTime(newFile.getLastModifiedTime()));
+ }
+
+ @Test
+ public void testCreateDirectoryTimestamp() throws Exception {
+ Path syncFile = absolutize("sync-file");
+ FileSystemUtils.createEmptyFile(syncFile);
+
+ Path newPath = absolutize("new-dir");
+ storeReferenceTime(syncFile.getLastModifiedTime());
+ assertTrue(newPath.createDirectory());
+ assertTrue(isLaterThanreferenceTime(newPath.getLastModifiedTime()));
+ }
+
+ @Test
+ public void testWriteChangesModifiedTime() throws Exception {
+ storeReferenceTime(xFile.getLastModifiedTime());
+ FileSystemUtils.writeContentAsLatin1(xFile, "abc19");
+ assertTrue(isLaterThanreferenceTime(xFile.getLastModifiedTime()));
+ }
+
+ @Test
+ public void testGetLastModifiedTimeThrowsExceptionForNonexistingPath() throws Exception {
+ Path newPath = testFS.getPath("/non-existing-dir");
+ try {
+ newPath.getLastModifiedTime();
+ fail("FileNotFoundException not thrown!");
+ } catch (FileNotFoundException x) {
+ assertEquals(newPath + " (No such file or directory)", x.getMessage());
+ }
+ }
+
+ // Test file size
+ @Test
+ public void testFileSizeThrowsExceptionForNonexistingPath() throws Exception {
+ Path newPath = testFS.getPath("/non-existing-file");
+ try {
+ newPath.getFileSize();
+ fail("FileNotFoundException not thrown.");
+ } catch (FileNotFoundException e) {
+ assertEquals(newPath + " (No such file or directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testFileSizeAfterWrite() throws Exception {
+ String testData = "abc19";
+
+ FileSystemUtils.writeContentAsLatin1(xFile, testData);
+ assertEquals(testData.length(), xFile.getFileSize());
+ }
+
+ // Testing the input/output routines
+ @Test
+ public void testFileWriteAndReadAsLatin1() throws Exception {
+ String testData = "abc19";
+
+ FileSystemUtils.writeContentAsLatin1(xFile, testData);
+ String resultData = new String(FileSystemUtils.readContentAsLatin1(xFile));
+
+ assertEquals(testData,resultData);
+ }
+
+ @Test
+ public void testInputAndOutputStreamEOF() throws Exception {
+ OutputStream outStream = xFile.getOutputStream();
+ outStream.write(1);
+ outStream.close();
+
+ InputStream inStream = xFile.getInputStream();
+ inStream.read();
+ assertEquals(-1, inStream.read());
+ inStream.close();
+ }
+
+ @Test
+ public void testInputAndOutputStream() throws Exception {
+ OutputStream outStream = xFile.getOutputStream();
+ for (int i = 33; i < 126; i++) {
+ outStream.write(i);
+ }
+ outStream.close();
+
+ InputStream inStream = xFile.getInputStream();
+ for (int i = 33; i < 126; i++) {
+ int readValue = inStream.read();
+ assertEquals(i,readValue);
+ }
+ inStream.close();
+ }
+
+ @Test
+ public void testInputAndOutputStreamAppend() throws Exception {
+ OutputStream outStream = xFile.getOutputStream();
+ for (int i = 33; i < 126; i++) {
+ outStream.write(i);
+ }
+ outStream.close();
+
+ OutputStream appendOut = xFile.getOutputStream(true);
+ for (int i = 126; i < 155; i++) {
+ appendOut.write(i);
+ }
+ appendOut.close();
+
+ InputStream inStream = xFile.getInputStream();
+ for (int i = 33; i < 155; i++) {
+ int readValue = inStream.read();
+ assertEquals(i,readValue);
+ }
+ inStream.close();
+ }
+
+ @Test
+ public void testInputAndOutputStreamNoAppend() throws Exception {
+ OutputStream outStream = xFile.getOutputStream();
+ outStream.write(1);
+ outStream.close();
+
+ OutputStream noAppendOut = xFile.getOutputStream(false);
+ noAppendOut.close();
+
+ InputStream inStream = xFile.getInputStream();
+ assertEquals(-1, inStream.read());
+ inStream.close();
+ }
+
+ @Test
+ public void testGetOutputStreamCreatesFile() throws Exception {
+ Path newFile = absolutize("does_not_exist_yet.txt");
+
+ OutputStream out = newFile.getOutputStream();
+ out.write(42);
+ out.close();
+
+ assertTrue(newFile.isFile());
+ }
+
+ @Test
+ public void testInpuStreamThrowExceptionOnDirectory() throws Exception {
+ try {
+ xEmptyDirectory.getOutputStream();
+ fail("The Exception was not thrown!");
+ } catch (IOException ex) {
+ assertEquals(xEmptyDirectory + " (Is a directory)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testOutputStreamThrowExceptionOnDirectory() throws Exception {
+ try {
+ xEmptyDirectory.getInputStream();
+ fail("The Exception was not thrown!");
+ } catch (IOException ex) {
+ assertEquals(xEmptyDirectory + " (Is a directory)", ex.getMessage());
+ }
+ }
+
+ // Test renaming
+ @Test
+ public void testCanRenameToUnusedName() throws Exception {
+ xFile.renameTo(xNothing);
+ assertFalse(xFile.exists());
+ assertTrue(xNothing.isFile());
+ }
+
+ @Test
+ public void testCanRenameFileToExistingFile() throws Exception {
+ Path otherFile = absolutize("otherFile");
+ FileSystemUtils.createEmptyFile(otherFile);
+ xFile.renameTo(otherFile); // succeeds
+ assertFalse(xFile.exists());
+ assertTrue(otherFile.isFile());
+ }
+
+ @Test
+ public void testCanRenameDirToExistingEmptyDir() throws Exception {
+ xNonEmptyDirectory.renameTo(xEmptyDirectory); // succeeds
+ assertFalse(xNonEmptyDirectory.exists());
+ assertTrue(xEmptyDirectory.isDirectory());
+ assertFalse(xEmptyDirectory.getDirectoryEntries().isEmpty());
+ }
+
+ @Test
+ public void testCantRenameDirToExistingNonEmptyDir() throws Exception {
+ try {
+ xEmptyDirectory.renameTo(xNonEmptyDirectory);
+ fail();
+ } catch (IOException e) {
+ MoreAsserts.assertEndsWith(" (Directory not empty)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCantRenameDirToExistingNonEmptyDirNothingChanged() throws Exception {
+ try {
+ xEmptyDirectory.renameTo(xNonEmptyDirectory);
+ fail();
+ } catch (IOException e) {
+ // Expected
+ }
+
+ assertTrue(xNonEmptyDirectory.isDirectory());
+ assertTrue(xEmptyDirectory.isDirectory());
+ assertTrue(xEmptyDirectory.getDirectoryEntries().isEmpty());
+ assertFalse(xNonEmptyDirectory.getDirectoryEntries().isEmpty());
+ }
+
+ @Test
+ public void testCantRenameDirToExistingFile() {
+ try {
+ xEmptyDirectory.renameTo(xFile);
+ fail();
+ } catch (IOException e) {
+ assertEquals(xEmptyDirectory + " -> " + xFile + " (Not a directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCantRenameDirToExistingFileNothingChanged() {
+ try {
+ xEmptyDirectory.renameTo(xFile);
+ fail();
+ } catch (IOException e) {
+ // Expected
+ }
+
+ assertTrue(xEmptyDirectory.isDirectory());
+ assertTrue(xFile.isFile());
+ }
+
+ @Test
+ public void testCantRenameFileToExistingDir() {
+ try {
+ xFile.renameTo(xEmptyDirectory);
+ fail();
+ } catch (IOException e) {
+ assertEquals(xFile + " -> " + xEmptyDirectory + " (Is a directory)",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCantRenameFileToExistingDirNothingChanged() {
+ try {
+ xFile.renameTo(xEmptyDirectory);
+ fail();
+ } catch (IOException e) {
+ // Expected
+ }
+
+ assertTrue(xEmptyDirectory.isDirectory());
+ assertTrue(xFile.isFile());
+ }
+
+ @Test
+ public void testMoveOnNonExistingFileThrowsException() throws Exception {
+ Path nonExistingPath = absolutize("non-existing");
+ Path targetPath = absolutize("does-not-matter");
+ try {
+ nonExistingPath.renameTo(targetPath);
+ fail();
+ } catch (FileNotFoundException e) {
+ MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage());
+ }
+ }
+
+ // Test the Paths
+ @Test
+ public void testGetPathOnlyAcceptsAbsolutePath() {
+ try {
+ testFS.getPath("not-absolute");
+ fail("The expected Exception was not thrown.");
+ } catch (IllegalArgumentException ex) {
+ assertEquals("not-absolute (not an absolute path)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetPathOnlyAcceptsAbsolutePathFragment() {
+ try {
+ testFS.getPath(new PathFragment("not-absolute"));
+ fail("The expected Exception was not thrown.");
+ } catch (IllegalArgumentException ex) {
+ assertEquals("not-absolute (not an absolute path)", ex.getMessage());
+ }
+ }
+
+ // Test the access permissions
+ @Test
+ public void testNewFilesAreWritable() throws Exception {
+ assertTrue(xFile.isWritable());
+ }
+
+ @Test
+ public void testNewFilesAreReadable() throws Exception {
+ assertTrue(xFile.isReadable());
+ }
+
+ @Test
+ public void testNewDirsAreWritable() throws Exception {
+ assertTrue(xEmptyDirectory.isWritable());
+ }
+
+ @Test
+ public void testNewDirsAreReadable() throws Exception {
+ assertTrue(xEmptyDirectory.isReadable());
+ }
+
+ @Test
+ public void testNewDirsAreExecutable() throws Exception {
+ assertTrue(xEmptyDirectory.isExecutable());
+ }
+
+ @Test
+ public void testCannotGetExecutableOnNonexistingFile() throws Exception {
+ try {
+ xNothing.isExecutable();
+ fail("No exception thrown.");
+ } catch (FileNotFoundException ex) {
+ assertEquals(xNothing + " (No such file or directory)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotSetExecutableOnNonexistingFile() throws Exception {
+ try {
+ xNothing.setExecutable(true);
+ fail("No exception thrown.");
+ } catch (FileNotFoundException ex) {
+ assertEquals(xNothing + " (No such file or directory)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotGetWritableOnNonexistingFile() throws Exception {
+ try {
+ xNothing.isWritable();
+ fail("No exception thrown.");
+ } catch (FileNotFoundException ex) {
+ assertEquals(xNothing + " (No such file or directory)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotSetWritableOnNonexistingFile() throws Exception {
+ try {
+ xNothing.setWritable(false);
+ fail("No exception thrown.");
+ } catch (FileNotFoundException ex) {
+ assertEquals(xNothing + " (No such file or directory)", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testSetReadableOnFile() throws Exception {
+ xFile.setReadable(false);
+ assertFalse(xFile.isReadable());
+ xFile.setReadable(true);
+ assertTrue(xFile.isReadable());
+ }
+
+ @Test
+ public void testSetWritableOnFile() throws Exception {
+ xFile.setWritable(false);
+ assertFalse(xFile.isWritable());
+ xFile.setWritable(true);
+ assertTrue(xFile.isWritable());
+ }
+
+ @Test
+ public void testSetExecutableOnFile() throws Exception {
+ xFile.setExecutable(true);
+ assertTrue(xFile.isExecutable());
+ xFile.setExecutable(false);
+ assertFalse(xFile.isExecutable());
+ }
+
+ @Test
+ public void testSetExecutableOnDirectory() throws Exception {
+ setExecutable(xNonEmptyDirectory, false);
+
+ try {
+ // We can't map names->inodes in a non-executable directory:
+ xNonEmptyDirectoryFoo.isWritable(); // i.e. stat
+ fail();
+ } catch (IOException e) {
+ MoreAsserts.assertEndsWith(" (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testWritingToReadOnlyFileThrowsException() throws Exception {
+ xFile.setWritable(false);
+ try {
+ FileSystemUtils.writeContent(xFile, "hello, world!".getBytes());
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ assertEquals(xFile + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testReadingFromUnreadableFileThrowsException() throws Exception {
+ FileSystemUtils.writeContent(xFile, "hello, world!".getBytes());
+ xFile.setReadable(false);
+ try {
+ FileSystemUtils.readContent(xFile);
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ assertEquals(xFile + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateFileInReadOnlyDirectory() throws Exception {
+ Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar");
+ xNonEmptyDirectory.setWritable(false);
+
+ try {
+ FileSystemUtils.createEmptyFile(xNonEmptyDirectoryBar);
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ assertEquals(xNonEmptyDirectoryBar + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateDirectoryInReadOnlyDirectory() throws Exception {
+ Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar");
+ xNonEmptyDirectory.setWritable(false);
+
+ try {
+ xNonEmptyDirectoryBar.createDirectory();
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ assertEquals(xNonEmptyDirectoryBar + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotMoveIntoReadOnlyDirectory() throws Exception {
+ Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar");
+ xNonEmptyDirectory.setWritable(false);
+
+ try {
+ xFile.renameTo(xNonEmptyDirectoryBar);
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ MoreAsserts.assertEndsWith(" (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotMoveFromReadOnlyDirectory() throws Exception {
+ xNonEmptyDirectory.setWritable(false);
+
+ try {
+ xNonEmptyDirectoryFoo.renameTo(xNothing);
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ MoreAsserts.assertEndsWith(" (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotDeleteInReadOnlyDirectory() throws Exception {
+ xNonEmptyDirectory.setWritable(false);
+
+ try {
+ xNonEmptyDirectoryFoo.delete();
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ assertEquals(xNonEmptyDirectoryFoo + " (Permission denied)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreatSymbolicLinkInReadOnlyDirectory() throws Exception {
+ Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar");
+ xNonEmptyDirectory.setWritable(false);
+
+ if (supportsSymlinks) {
+ try {
+ createSymbolicLink(xNonEmptyDirectoryBar, xNonEmptyDirectoryFoo);
+ fail("No exception thrown.");
+ } catch (IOException e) {
+ assertEquals(xNonEmptyDirectoryBar + " (Permission denied)", e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testGetMD5DigestForEmptyFile() throws Exception {
+ Fingerprint fp = new Fingerprint();
+ fp.addBytes(new byte[0]);
+ assertEquals(BaseEncoding.base16().lowerCase().encode(xFile.getMD5Digest()),
+ fp.hexDigestAndReset());
+ }
+
+ @Test
+ public void testGetMD5Digest() throws Exception {
+ byte[] buffer = new byte[500000];
+ for (int i = 0; i < buffer.length; ++i) {
+ buffer[i] = 1;
+ }
+ FileSystemUtils.writeContent(xFile, buffer);
+ Fingerprint fp = new Fingerprint();
+ fp.addBytes(buffer);
+ assertEquals(BaseEncoding.base16().lowerCase().encode(xFile.getMD5Digest()),
+ fp.hexDigestAndReset());
+ }
+
+ @Test
+ public void testStatFailsFastOnNonExistingFiles() throws Exception {
+ try {
+ xNothing.stat();
+ fail("Expected IOException");
+ } catch(IOException e) {
+ // Do nothing.
+ }
+ }
+
+ @Test
+ public void testStatNullableFailsFastOnNonExistingFiles() throws Exception {
+ assertNull(xNothing.statNullable());
+ }
+
+ @Test
+ public void testResolveSymlinks() throws Exception {
+ if (supportsSymlinks) {
+ createSymbolicLink(xNothing, xFile);
+ FileSystemUtils.createEmptyFile(xFile);
+ assertEquals(xFile.asFragment(), testFS.resolveOneLink(xNothing));
+ assertEquals(xFile, xNothing.resolveSymbolicLinks());
+ }
+ }
+
+ @Test
+ public void testResolveNonSymlinks() throws Exception {
+ if (supportsSymlinks) {
+ assertEquals(null, testFS.resolveOneLink(xFile));
+ assertEquals(xFile, xFile.resolveSymbolicLinks());
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemUtilsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemUtilsTest.java
new file mode 100644
index 0000000..21ca39b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemUtilsTest.java
@@ -0,0 +1,878 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.appendWithoutExtension;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.commonAncestor;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.copyFile;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.copyTool;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.createDirectoryAndParents;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.deleteTree;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.deleteTreesBelowNotPrefixed;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.longestPathPrefix;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.plantLinkForest;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.relativePath;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.removeExtension;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.touchFile;
+import static com.google.devtools.build.lib.vfs.FileSystemUtils.traverseTree;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.testutil.BlazeTestUtils;
+import com.google.devtools.build.lib.testutil.ManualClock;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * This class tests the file system utilities.
+ */
+@RunWith(JUnit4.class)
+public class FileSystemUtilsTest {
+ private ManualClock clock;
+ private FileSystem fileSystem;
+ private Path workingDir;
+
+ @Before
+ public void setUp() throws Exception {
+ clock = new ManualClock();
+ fileSystem = new InMemoryFileSystem(clock);
+ workingDir = fileSystem.getPath("/workingDir");
+ }
+
+ Path topDir;
+ Path file1;
+ Path file2;
+ Path aDir;
+ Path file3;
+ Path innerDir;
+ Path link1;
+ Path dirLink;
+ Path file4;
+
+ /*
+ * Build a directory tree that looks like:
+ * top-dir/
+ * file-1
+ * file-2
+ * a-dir/
+ * file-3
+ * inner-dir/
+ * link-1 => file-4
+ * dir-link => a-dir
+ * file-4
+ */
+ private void createTestDirectoryTree() throws IOException {
+ topDir = fileSystem.getPath("/top-dir");
+ file1 = fileSystem.getPath("/top-dir/file-1");
+ file2 = fileSystem.getPath("/top-dir/file-2");
+ aDir = fileSystem.getPath("/top-dir/a-dir");
+ file3 = fileSystem.getPath("/top-dir/a-dir/file-3");
+ innerDir = fileSystem.getPath("/top-dir/a-dir/inner-dir");
+ link1 = fileSystem.getPath("/top-dir/a-dir/inner-dir/link-1");
+ dirLink = fileSystem.getPath("/top-dir/a-dir/inner-dir/dir-link");
+ file4 = fileSystem.getPath("/file-4");
+
+ topDir.createDirectory();
+ FileSystemUtils.createEmptyFile(file1);
+ FileSystemUtils.createEmptyFile(file2);
+ aDir.createDirectory();
+ FileSystemUtils.createEmptyFile(file3);
+ innerDir.createDirectory();
+ link1.createSymbolicLink(file4); // simple symlink
+ dirLink.createSymbolicLink(aDir); // creates link loop
+ FileSystemUtils.createEmptyFile(file4);
+ }
+
+ private void checkTestDirectoryTreesBelow(Path toPath) throws IOException {
+ Path copiedFile1 = toPath.getChild("file-1");
+ assertTrue(copiedFile1.exists());
+ assertTrue(copiedFile1.isFile());
+
+ Path copiedFile2 = toPath.getChild("file-2");
+ assertTrue(copiedFile2.exists());
+ assertTrue(copiedFile2.isFile());
+
+ Path copiedADir = toPath.getChild("a-dir");
+ assertTrue(copiedADir.exists());
+ assertTrue(copiedADir.isDirectory());
+ Collection<Path> aDirEntries = copiedADir.getDirectoryEntries();
+ assertEquals(2, aDirEntries.size());
+
+ Path copiedFile3 = copiedADir.getChild("file-3");
+ assertTrue(copiedFile3.exists());
+ assertTrue(copiedFile3.isFile());
+
+ Path copiedInnerDir = copiedADir.getChild("inner-dir");
+ assertTrue(copiedInnerDir.exists());
+ assertTrue(copiedInnerDir.isDirectory());
+
+ Path copiedLink1 = copiedInnerDir.getChild("link-1");
+ assertTrue(copiedLink1.exists());
+ assertTrue(copiedLink1.isSymbolicLink());
+ assertEquals(copiedLink1.resolveSymbolicLinks(), file4);
+
+ Path copiedDirLink = copiedInnerDir.getChild("dir-link");
+ assertTrue(copiedDirLink.exists());
+ assertTrue(copiedDirLink.isSymbolicLink());
+ assertEquals(copiedDirLink.resolveSymbolicLinks(), aDir);
+ }
+
+ // tests
+
+ @Test
+ public void testChangeModtime() throws IOException {
+ Path file = fileSystem.getPath("/my-file");
+ try {
+ BlazeTestUtils.changeModtime(file);
+ fail();
+ } catch (FileNotFoundException e) {
+ /* ok */
+ }
+ FileSystemUtils.createEmptyFile(file);
+ long prevMtime = file.getLastModifiedTime();
+ BlazeTestUtils.changeModtime(file);
+ assertFalse(prevMtime == file.getLastModifiedTime());
+ }
+
+ @Test
+ public void testCommonAncestor() {
+ assertEquals(topDir, commonAncestor(topDir, topDir));
+ assertEquals(topDir, commonAncestor(file1, file3));
+ assertEquals(topDir, commonAncestor(file1, dirLink));
+ }
+
+ @Test
+ public void testRelativePath() throws IOException {
+ createTestDirectoryTree();
+ assertEquals("file-1", relativePath(topDir, file1).getPathString());
+ assertEquals(".", relativePath(topDir, topDir).getPathString());
+ assertEquals("a-dir/inner-dir/dir-link", relativePath(topDir, dirLink).getPathString());
+ assertEquals("../file-4", relativePath(topDir, file4).getPathString());
+ assertEquals("../../../file-4", relativePath(innerDir, file4).getPathString());
+ }
+
+ private String longestPathPrefixStr(String path, String... prefixStrs) {
+ Set<PathFragment> prefixes = new HashSet<>();
+ for (String prefix : prefixStrs) {
+ prefixes.add(new PathFragment(prefix));
+ }
+ PathFragment longest = longestPathPrefix(new PathFragment(path), prefixes);
+ return longest != null ? longest.getPathString() : null;
+ }
+
+ @Test
+ public void testLongestPathPrefix() {
+ assertEquals("A", longestPathPrefixStr("A/b", "A", "B")); // simple parent
+ assertEquals("A", longestPathPrefixStr("A", "A", "B")); // self
+ assertEquals("A/B", longestPathPrefixStr("A/B/c", "A", "A/B")); // want longest
+ assertNull(longestPathPrefixStr("C/b", "A", "B")); // not found in other parents
+ assertNull(longestPathPrefixStr("A", "A/B", "B")); // not found in child
+ assertEquals("A/B/C", longestPathPrefixStr("A/B/C/d/e/f.h", "A/B/C", "B/C/d"));
+ }
+
+ @Test
+ public void testRemoveExtension_Strings() throws Exception {
+ assertEquals("foo", removeExtension("foo.c"));
+ assertEquals("a/foo", removeExtension("a/foo.c"));
+ assertEquals("a.b/foo", removeExtension("a.b/foo"));
+ assertEquals("foo", removeExtension("foo"));
+ assertEquals("foo", removeExtension("foo."));
+ }
+
+ @Test
+ public void testRemoveExtension_Paths() throws Exception {
+ assertPath("/foo", removeExtension(fileSystem.getPath("/foo.c")));
+ assertPath("/a/foo", removeExtension(fileSystem.getPath("/a/foo.c")));
+ assertPath("/a.b/foo", removeExtension(fileSystem.getPath("/a.b/foo")));
+ assertPath("/foo", removeExtension(fileSystem.getPath("/foo")));
+ assertPath("/foo", removeExtension(fileSystem.getPath("/foo.")));
+ }
+
+ private static void assertPath(String expected, PathFragment actual) {
+ assertEquals(expected, actual.getPathString());
+ }
+
+ private static void assertPath(String expected, Path actual) {
+ assertEquals(expected, actual.getPathString());
+ }
+
+ @Test
+ public void testReplaceExtension_Path() throws Exception {
+ assertPath("/foo/bar.baz",
+ FileSystemUtils.replaceExtension(fileSystem.getPath("/foo/bar"), ".baz"));
+ assertPath("/foo/bar.baz",
+ FileSystemUtils.replaceExtension(fileSystem.getPath("/foo/bar.cc"), ".baz"));
+ assertPath("/foo.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/foo/"), ".baz"));
+ assertPath("/foo.baz",
+ FileSystemUtils.replaceExtension(fileSystem.getPath("/foo.cc/"), ".baz"));
+ assertPath("/foo.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/foo"), ".baz"));
+ assertPath("/foo.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/foo.cc"), ".baz"));
+ assertPath("/.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/.cc"), ".baz"));
+ assertEquals(null, FileSystemUtils.replaceExtension(fileSystem.getPath("/"), ".baz"));
+ }
+
+ @Test
+ public void testReplaceExtension_PathFragment() throws Exception {
+ assertPath("foo/bar.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("foo/bar"), ".baz"));
+ assertPath("foo/bar.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("foo/bar.cc"), ".baz"));
+ assertPath("/foo/bar.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("/foo/bar"), ".baz"));
+ assertPath("/foo/bar.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("/foo/bar.cc"), ".baz"));
+ assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo/"), ".baz"));
+ assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo.cc/"), ".baz"));
+ assertPath("/foo.baz", FileSystemUtils.replaceExtension(new PathFragment("/foo/"), ".baz"));
+ assertPath("/foo.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("/foo.cc/"), ".baz"));
+ assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo"), ".baz"));
+ assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo.cc"), ".baz"));
+ assertPath("/foo.baz", FileSystemUtils.replaceExtension(new PathFragment("/foo"), ".baz"));
+ assertPath("/foo.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("/foo.cc"), ".baz"));
+ assertPath(".baz", FileSystemUtils.replaceExtension(new PathFragment(".cc"), ".baz"));
+ assertEquals(null, FileSystemUtils.replaceExtension(new PathFragment("/"), ".baz"));
+ assertEquals(null, FileSystemUtils.replaceExtension(new PathFragment(""), ".baz"));
+ assertPath("foo/bar.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("foo/bar.pony"), ".baz", ".pony"));
+ assertPath("foo/bar.baz",
+ FileSystemUtils.replaceExtension(new PathFragment("foo/bar"), ".baz", ""));
+ assertEquals(null, FileSystemUtils.replaceExtension(new PathFragment(""), ".baz", ".pony"));
+ assertEquals(null,
+ FileSystemUtils.replaceExtension(new PathFragment("foo/bar.pony"), ".baz", ".unicorn"));
+ }
+
+ @Test
+ public void testAppendWithoutExtension() throws Exception {
+ assertPath("libfoo-src.jar",
+ appendWithoutExtension(new PathFragment("libfoo.jar"), "-src"));
+ assertPath("foo/libfoo-src.jar",
+ appendWithoutExtension(new PathFragment("foo/libfoo.jar"), "-src"));
+ assertPath("java/com/google/foo/libfoo-src.jar",
+ appendWithoutExtension(new PathFragment("java/com/google/foo/libfoo.jar"), "-src"));
+ assertPath("libfoo.bar-src.jar",
+ appendWithoutExtension(new PathFragment("libfoo.bar.jar"), "-src"));
+ assertPath("libfoo-src",
+ appendWithoutExtension(new PathFragment("libfoo"), "-src"));
+ assertPath("libfoo-src.jar",
+ appendWithoutExtension(new PathFragment("libfoo.jar/"), "-src"));
+ assertPath("libfoo.src.jar",
+ appendWithoutExtension(new PathFragment("libfoo.jar"), ".src"));
+ assertEquals(null, appendWithoutExtension(new PathFragment("/"), "-src"));
+ assertEquals(null, appendWithoutExtension(new PathFragment(""), "-src"));
+ }
+
+ @Test
+ public void testReplaceSegments() {
+ assertPath(
+ "poo/bar/baz.cc",
+ FileSystemUtils.replaceSegments(new PathFragment("foo/bar/baz.cc"), "foo", "poo", true));
+ assertPath(
+ "poo/poo/baz.cc",
+ FileSystemUtils.replaceSegments(new PathFragment("foo/foo/baz.cc"), "foo", "poo", true));
+ assertPath(
+ "poo/foo/baz.cc",
+ FileSystemUtils.replaceSegments(new PathFragment("foo/foo/baz.cc"), "foo", "poo", false));
+ assertPath(
+ "foo/bar/baz.cc",
+ FileSystemUtils.replaceSegments(new PathFragment("foo/bar/baz.cc"), "boo", "poo", true));
+ }
+
+ @Test
+ public void testGetWorkingDirectory() {
+ String userDir = System.getProperty("user.dir");
+
+ assertEquals(FileSystemUtils.getWorkingDirectory(fileSystem),
+ fileSystem.getPath(System.getProperty("user.dir", "/")));
+
+ System.setProperty("user.dir", "/blah/blah/blah");
+ assertEquals(FileSystemUtils.getWorkingDirectory(fileSystem),
+ fileSystem.getPath("/blah/blah/blah"));
+
+ System.setProperty("user.dir", userDir);
+ }
+
+ @Test
+ public void testResolveRelativeToFilesystemWorkingDir() {
+ PathFragment relativePath = new PathFragment("relative/path");
+ assertEquals(workingDir.getRelative(relativePath),
+ workingDir.getRelative(relativePath));
+
+ PathFragment absolutePath = new PathFragment("/absolute/path");
+ assertEquals(fileSystem.getPath(absolutePath),
+ workingDir.getRelative(absolutePath));
+ }
+
+ @Test
+ public void testTouchFileCreatesFile() throws IOException {
+ createTestDirectoryTree();
+ Path nonExistingFile = fileSystem.getPath("/previously-non-existing");
+ assertFalse(nonExistingFile.exists());
+ touchFile(nonExistingFile);
+
+ assertTrue(nonExistingFile.exists());
+ }
+
+ @Test
+ public void testTouchFileAdjustsFileTime() throws IOException {
+ createTestDirectoryTree();
+ Path testFile = file4;
+ long oldTime = testFile.getLastModifiedTime();
+ testFile.setLastModifiedTime(42);
+ touchFile(testFile);
+
+ assertTrue(testFile.getLastModifiedTime() >= oldTime);
+ }
+
+ @Test
+ public void testCopyFile() throws IOException {
+ createTestDirectoryTree();
+ Path originalFile = file1;
+ byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 };
+ FileSystemUtils.writeContent(originalFile, content);
+
+ Path copyTarget = file2;
+
+ copyFile(originalFile, copyTarget);
+
+ assertTrue(Arrays.equals(content, FileSystemUtils.readContent(copyTarget)));
+ }
+
+ @Test
+ public void testReadContentWithLimit() throws IOException {
+ createTestDirectoryTree();
+ String str = "this is a test of readContentWithLimit method";
+ FileSystemUtils.writeContent(file1, StandardCharsets.ISO_8859_1, str);
+ assertEquals(readStringFromFile(file1, 0), "");
+ assertEquals(readStringFromFile(file1, 10), str.substring(0, 10));
+ assertEquals(readStringFromFile(file1, 1000000), str);
+ }
+
+ private String readStringFromFile(Path file, int limit) throws IOException {
+ byte[] bytes = FileSystemUtils.readContentWithLimit(file, limit);
+ return new String(bytes, StandardCharsets.ISO_8859_1);
+ }
+
+ @Test
+ public void testAppend() throws IOException {
+ createTestDirectoryTree();
+ FileSystemUtils.writeIsoLatin1(file1, "nobody says ");
+ FileSystemUtils.writeIsoLatin1(file1, "mary had");
+ FileSystemUtils.appendIsoLatin1(file1, "a little lamb");
+ assertEquals(
+ "mary had\na little lamb\n",
+ new String(FileSystemUtils.readContentAsLatin1(file1)));
+ }
+
+ @Test
+ public void testCopyFileAttributes() throws IOException {
+ createTestDirectoryTree();
+ Path originalFile = file1;
+ byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 };
+ FileSystemUtils.writeContent(originalFile, content);
+ file1.setLastModifiedTime(12345L);
+ file1.setWritable(false);
+ file1.setExecutable(false);
+
+ Path copyTarget = file2;
+ copyFile(originalFile, copyTarget);
+
+ assertEquals(12345L, file2.getLastModifiedTime());
+ assertFalse(file2.isExecutable());
+ assertFalse(file2.isWritable());
+
+ file1.setWritable(true);
+ file1.setExecutable(true);
+
+ copyFile(originalFile, copyTarget);
+
+ assertEquals(12345L, file2.getLastModifiedTime());
+ assertTrue(file2.isExecutable());
+ assertTrue(file2.isWritable());
+
+ }
+
+ @Test
+ public void testCopyFileThrowsExceptionIfTargetCantBeDeleted() throws IOException {
+ createTestDirectoryTree();
+ Path originalFile = file1;
+ byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 };
+ FileSystemUtils.writeContent(originalFile, content);
+
+ try {
+ copyFile(originalFile, aDir);
+ fail();
+ } catch (IOException ex) {
+ assertEquals("error copying file: couldn't delete destination: "
+ + aDir + " (Directory not empty)",
+ ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testCopyTool() throws IOException {
+ createTestDirectoryTree();
+ Path originalFile = file1;
+ byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 };
+ FileSystemUtils.writeContent(originalFile, content);
+
+ Path copyTarget = copyTool(topDir.getRelative("file-1"), aDir.getRelative("file-1"));
+
+ assertTrue(Arrays.equals(content, FileSystemUtils.readContent(copyTarget)));
+ assertEquals(file1.isWritable(), copyTarget.isWritable());
+ assertEquals(file1.isExecutable(), copyTarget.isExecutable());
+ assertEquals(file1.getLastModifiedTime(), copyTarget.getLastModifiedTime());
+ }
+
+ @Test
+ public void testCopyTreesBelow() throws IOException {
+ createTestDirectoryTree();
+ Path toPath = fileSystem.getPath("/copy-here");
+ toPath.createDirectory();
+
+ FileSystemUtils.copyTreesBelow(topDir, toPath);
+ checkTestDirectoryTreesBelow(toPath);
+ }
+
+ @Test
+ public void testCopyTreesBelowWithOverriding() throws IOException {
+ createTestDirectoryTree();
+ Path toPath = fileSystem.getPath("/copy-here");
+ toPath.createDirectory();
+ toPath.getChild("file-2");
+
+ FileSystemUtils.copyTreesBelow(topDir, toPath);
+ checkTestDirectoryTreesBelow(toPath);
+ }
+
+ @Test
+ public void testCopyTreesBelowToSubtree() throws IOException {
+ createTestDirectoryTree();
+ try {
+ FileSystemUtils.copyTreesBelow(topDir, aDir);
+ fail("Should not be able to copy a directory to a subdir");
+ } catch (IllegalArgumentException expected) {
+ assertEquals("/top-dir/a-dir is a subdirectory of /top-dir", expected.getMessage());
+ }
+ }
+
+ @Test
+ public void testCopyFileAsDirectoryTree() throws IOException {
+ createTestDirectoryTree();
+ try {
+ FileSystemUtils.copyTreesBelow(file1, aDir);
+ fail("Should not be able to copy a file with copyDirectory method");
+ } catch (IOException expected) {
+ assertEquals("/top-dir/file-1 (Not a directory)", expected.getMessage());
+ }
+ }
+
+ @Test
+ public void testCopyTreesBelowToFile() throws IOException {
+ createTestDirectoryTree();
+ Path copyDir = fileSystem.getPath("/my-dir");
+ Path copySubDir = fileSystem.getPath("/my-dir/subdir");
+ FileSystemUtils.createDirectoryAndParents(copySubDir);
+ try {
+ FileSystemUtils.copyTreesBelow(copyDir, file4);
+ fail("Should not be able to copy a directory to a file");
+ } catch (IOException expected) {
+ assertEquals("/file-4 (Not a directory)", expected.getMessage());
+ }
+ }
+
+ @Test
+ public void testCopyTreesBelowFromUnexistingDir() throws IOException {
+ createTestDirectoryTree();
+
+ try {
+ Path unexistingDir = fileSystem.getPath("/unexisting-dir");
+ FileSystemUtils.copyTreesBelow(unexistingDir, aDir);
+ fail("Should not be able to copy from an unexisting path");
+ } catch (FileNotFoundException expected) {
+ assertEquals("/unexisting-dir (No such file or directory)", expected.getMessage());
+ }
+ }
+
+ @Test
+ public void testTraverseTree() throws IOException {
+ createTestDirectoryTree();
+
+ Collection<Path> paths = traverseTree(topDir, new Predicate<Path>() {
+ @Override
+ public boolean apply(Path p) {
+ return !p.getPathString().contains("a-dir");
+ }
+ });
+ assertThat(paths).containsExactly(file1, file2);
+ }
+
+ @Test
+ public void testTraverseTreeDeep() throws IOException {
+ createTestDirectoryTree();
+
+ Collection<Path> paths = traverseTree(topDir,
+ Predicates.alwaysTrue());
+ assertThat(paths).containsExactly(aDir,
+ file3,
+ innerDir,
+ link1,
+ file1,
+ file2,
+ dirLink);
+ }
+
+ @Test
+ public void testTraverseTreeLinkDir() throws IOException {
+ // Use a new little tree for this test:
+ // top-dir/
+ // dir-link2 => linked-dir
+ // linked-dir/
+ // file
+ topDir = fileSystem.getPath("/top-dir");
+ Path dirLink2 = fileSystem.getPath("/top-dir/dir-link2");
+ Path linkedDir = fileSystem.getPath("/linked-dir");
+ Path linkedDirFile = fileSystem.getPath("/top-dir/dir-link2/file");
+
+ topDir.createDirectory();
+ linkedDir.createDirectory();
+ dirLink2.createSymbolicLink(linkedDir); // simple symlink
+ FileSystemUtils.createEmptyFile(linkedDirFile); // created through the link
+
+ // traverseTree doesn't follow links:
+ Collection<Path> paths = traverseTree(topDir, Predicates.alwaysTrue());
+ assertThat(paths).containsExactly(dirLink2);
+
+ paths = traverseTree(linkedDir, Predicates.alwaysTrue());
+ assertThat(paths).containsExactly(fileSystem.getPath("/linked-dir/file"));
+ }
+
+ @Test
+ public void testDeleteTreeCommandDeletesTree() throws IOException {
+ createTestDirectoryTree();
+ Path toDelete = topDir;
+ deleteTree(toDelete);
+
+ assertTrue(file4.exists());
+ assertFalse(topDir.exists());
+ assertFalse(file1.exists());
+ assertFalse(file2.exists());
+ assertFalse(aDir.exists());
+ assertFalse(file3.exists());
+ }
+
+ @Test
+ public void testDeleteTreeCommandsDeletesUnreadableDirectories() throws IOException {
+ createTestDirectoryTree();
+ Path toDelete = topDir;
+
+ try {
+ aDir.setReadable(false);
+ } catch (UnsupportedOperationException e) {
+ // For file systems that do not support setting readable attribute to
+ // false, this test is simply skipped.
+
+ return;
+ }
+
+ deleteTree(toDelete);
+ assertFalse(topDir.exists());
+ assertFalse(aDir.exists());
+
+ }
+
+ @Test
+ public void testDeleteTreeCommandDoesNotFollowLinksOut() throws IOException {
+ createTestDirectoryTree();
+ Path toDelete = topDir;
+ Path outboundLink = fileSystem.getPath("/top-dir/outbound-link");
+ outboundLink.createSymbolicLink(file4);
+
+ deleteTree(toDelete);
+
+ assertTrue(file4.exists());
+ assertFalse(topDir.exists());
+ assertFalse(file1.exists());
+ assertFalse(file2.exists());
+ assertFalse(aDir.exists());
+ assertFalse(file3.exists());
+ }
+
+ @Test
+ public void testDeleteTreesBelowNotPrefixed() throws IOException {
+ createTestDirectoryTree();
+ deleteTreesBelowNotPrefixed(topDir, new String[] { "file-"});
+ assertTrue(file1.exists());
+ assertTrue(file2.exists());
+ assertFalse(aDir.exists());
+ }
+
+ @Test
+ public void testCreateDirectories() throws IOException {
+ Path mainPath = fileSystem.getPath("/some/where/deep/in/the/hierarchy");
+ assertTrue(createDirectoryAndParents(mainPath));
+ assertTrue(mainPath.exists());
+ assertFalse(createDirectoryAndParents(mainPath));
+ }
+
+ @Test
+ public void testCreateDirectoriesWhenAncestorIsFile() throws IOException {
+ Path somewhereDeepIn = fileSystem.getPath("/somewhere/deep/in");
+ assertTrue(createDirectoryAndParents(somewhereDeepIn.getParentDirectory()));
+ FileSystemUtils.createEmptyFile(somewhereDeepIn);
+ Path theHierarchy = somewhereDeepIn.getChild("the-hierarchy");
+ try {
+ createDirectoryAndParents(theHierarchy);
+ fail();
+ } catch (IOException e) {
+ assertEquals("/somewhere/deep/in (Not a directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCreateDirectoriesWhenSymlinkToDir() throws IOException {
+ Path somewhereDeepIn = fileSystem.getPath("/somewhere/deep/in");
+ assertTrue(createDirectoryAndParents(somewhereDeepIn));
+ Path realDir = fileSystem.getPath("/real/dir");
+ assertTrue(createDirectoryAndParents(realDir));
+
+ Path theHierarchy = somewhereDeepIn.getChild("the-hierarchy");
+ theHierarchy.createSymbolicLink(realDir);
+
+ assertFalse(createDirectoryAndParents(theHierarchy));
+ }
+
+ @Test
+ public void testCreateDirectoriesWhenSymlinkEmbedded() throws IOException {
+ Path somewhereDeepIn = fileSystem.getPath("/somewhere/deep/in");
+ assertTrue(createDirectoryAndParents(somewhereDeepIn));
+ Path realDir = fileSystem.getPath("/real/dir");
+ assertTrue(createDirectoryAndParents(realDir));
+
+ Path the = somewhereDeepIn.getChild("the");
+ the.createSymbolicLink(realDir);
+
+ Path theHierarchy = somewhereDeepIn.getChild("hierarchy");
+ assertTrue(createDirectoryAndParents(theHierarchy));
+ }
+
+ PathFragment createPkg(Path rootA, Path rootB, String pkg) throws IOException {
+ if (rootA != null) {
+ createDirectoryAndParents(rootA.getRelative(pkg));
+ FileSystemUtils.createEmptyFile(rootA.getRelative(pkg).getChild("file"));
+ }
+ if (rootB != null) {
+ createDirectoryAndParents(rootB.getRelative(pkg));
+ FileSystemUtils.createEmptyFile(rootB.getRelative(pkg).getChild("file"));
+ }
+ return new PathFragment(pkg);
+ }
+
+ void assertLinksTo(Path fromRoot, Path toRoot, String relpart) throws IOException {
+ assertTrue(fromRoot.getRelative(relpart).isSymbolicLink());
+ assertEquals(toRoot.getRelative(relpart).asFragment(),
+ fromRoot.getRelative(relpart).readSymbolicLink());
+ }
+
+ void assertIsDir(Path root, String relpart) {
+ assertTrue(root.getRelative(relpart).isDirectory(Symlinks.NOFOLLOW));
+ }
+
+ void dumpTree(Path root, PrintStream out) throws IOException {
+ out.println("\n" + root);
+ for (Path p : FileSystemUtils.traverseTree(root, Predicates.alwaysTrue())) {
+ if (p.isDirectory(Symlinks.NOFOLLOW)) {
+ out.println(" " + p + "/");
+ } else if (p.isSymbolicLink()) {
+ out.println(" " + p + " => " + p.readSymbolicLink());
+ } else {
+ out.println(" " + p + " [" + p.resolveSymbolicLinks() + "]");
+ }
+ }
+ }
+
+ @Test
+ public void testPlantLinkForest() throws IOException {
+ Path rootA = fileSystem.getPath("/A");
+ Path rootB = fileSystem.getPath("/B");
+
+ ImmutableMap<PathFragment, Path> packageRootMap = ImmutableMap.<PathFragment, Path>builder()
+ .put(createPkg(rootA, rootB, "pkgA"), rootA)
+ .put(createPkg(rootA, rootB, "dir1/pkgA"), rootA)
+ .put(createPkg(rootA, rootB, "dir1/pkgB"), rootB)
+ .put(createPkg(rootA, rootB, "dir2/pkg"), rootA)
+ .put(createPkg(rootA, rootB, "dir2/pkg/pkg"), rootB)
+ .put(createPkg(rootA, rootB, "pkgB"), rootB)
+ .put(createPkg(rootA, rootB, "pkgB/dir/pkg"), rootA)
+ .put(createPkg(rootA, rootB, "pkgB/pkg"), rootA)
+ .put(createPkg(rootA, rootB, "pkgB/pkg/pkg"), rootA)
+ .build();
+ createPkg(rootA, rootB, "pkgB/dir"); // create a file in there
+
+ //dumpTree(rootA, System.err);
+ //dumpTree(rootB, System.err);
+
+ Path linkRoot = fileSystem.getPath("/linkRoot");
+ createDirectoryAndParents(linkRoot);
+ plantLinkForest(packageRootMap, linkRoot);
+
+ //dumpTree(linkRoot, System.err);
+
+ assertLinksTo(linkRoot, rootA, "pkgA");
+ assertIsDir(linkRoot, "dir1");
+ assertLinksTo(linkRoot, rootA, "dir1/pkgA");
+ assertLinksTo(linkRoot, rootB, "dir1/pkgB");
+ assertIsDir(linkRoot, "dir2");
+ assertIsDir(linkRoot, "dir2/pkg");
+ assertLinksTo(linkRoot, rootA, "dir2/pkg/file");
+ assertLinksTo(linkRoot, rootB, "dir2/pkg/pkg");
+ assertIsDir(linkRoot, "pkgB");
+ assertIsDir(linkRoot, "pkgB/dir");
+ assertLinksTo(linkRoot, rootB, "pkgB/dir/file");
+ assertLinksTo(linkRoot, rootA, "pkgB/dir/pkg");
+ assertLinksTo(linkRoot, rootA, "pkgB/pkg");
+ }
+
+ @Test
+ public void testWriteIsoLatin1() throws Exception {
+ Path file = fileSystem.getPath("/does/not/exist/yet.txt");
+ FileSystemUtils.writeIsoLatin1(file, "Line 1", "Line 2", "Line 3");
+ String expected = "Line 1\nLine 2\nLine 3\n";
+ String actual = new String(FileSystemUtils.readContentAsLatin1(file));
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testWriteLinesAs() throws Exception {
+ Path file = fileSystem.getPath("/does/not/exist/yet.txt");
+ FileSystemUtils.writeLinesAs(file, UTF_8, "\u00F6"); // an oe umlaut
+ byte[] expected = new byte[] {(byte) 0xC3, (byte) 0xB6, 0x0A};//"\u00F6\n";
+ byte[] actual = FileSystemUtils.readContent(file);
+ assertArrayEquals(expected, actual);
+ }
+
+ @Test
+ public void testGetFileSystem() throws Exception {
+ Path mountTable = fileSystem.getPath("/proc/mounts");
+ FileSystemUtils.writeIsoLatin1(mountTable,
+ "/dev/sda1 / ext2 blah 0 0",
+ "/dev/mapper/_dev_sda6 /usr/local/google ext3 blah 0 0",
+ "devshm /dev/shm tmpfs blah 0 0",
+ "/dev/fuse /fuse/mnt fuse blah 0 0",
+ "mtvhome22.nfs:/vol/mtvhome22/johndoe /home/johndoe nfs blah 0 0",
+ "/dev/foo /foo dummy_foo blah 0 0",
+ "/dev/foobar /foobar dummy_foobar blah 0 0",
+ "proc proc proc rw,noexec,nosuid,nodev 0 0");
+ Path path = fileSystem.getPath("/usr/local/google/_blaze");
+ FileSystemUtils.createDirectoryAndParents(path);
+ assertEquals("ext3", FileSystemUtils.getFileSystem(path));
+
+ // Should match the root "/"
+ path = fileSystem.getPath("/usr/local/tmp");
+ FileSystemUtils.createDirectoryAndParents(path);
+ assertEquals("ext2", FileSystemUtils.getFileSystem(path));
+
+ // Make sure we don't consider /foobar matches /foo
+ path = fileSystem.getPath("/foo");
+ FileSystemUtils.createDirectoryAndParents(path);
+ assertEquals("dummy_foo", FileSystemUtils.getFileSystem(path));
+ path = fileSystem.getPath("/foobar");
+ FileSystemUtils.createDirectoryAndParents(path);
+ assertEquals("dummy_foobar", FileSystemUtils.getFileSystem(path));
+
+ path = fileSystem.getPath("/dev/shm/blaze");
+ FileSystemUtils.createDirectoryAndParents(path);
+ assertEquals("tmpfs", FileSystemUtils.getFileSystem(path));
+
+ Path fusePath = fileSystem.getPath("/fuse/mnt/tmp");
+ FileSystemUtils.createDirectoryAndParents(fusePath);
+ assertEquals("fuse", FileSystemUtils.getFileSystem(fusePath));
+
+ // Create a symlink and make sure it gives the file system of the symlink target.
+ path = fileSystem.getPath("/usr/local/google/_blaze/out");
+ path.createSymbolicLink(fusePath);
+ assertEquals("fuse", FileSystemUtils.getFileSystem(path));
+
+ // Non existent path should return "unknown"
+ path = fileSystem.getPath("/does/not/exist");
+ assertEquals("unknown", FileSystemUtils.getFileSystem(path));
+ }
+
+ @Test
+ public void testStartsWithAnySuccess() throws Exception {
+ PathFragment a = new PathFragment("a");
+ assertTrue(FileSystemUtils.startsWithAny(a,
+ Arrays.asList(new PathFragment("b"), new PathFragment("a"))));
+ }
+
+ @Test
+ public void testStartsWithAnyNotFound() throws Exception {
+ PathFragment a = new PathFragment("a");
+ assertFalse(FileSystemUtils.startsWithAny(a,
+ Arrays.asList(new PathFragment("b"), new PathFragment("c"))));
+ }
+
+ @Test
+ public void testIterateLines() throws Exception {
+ Path file = fileSystem.getPath("/test.txt");
+ FileSystemUtils.writeContent(file, ISO_8859_1, "a\nb");
+ assertEquals(Arrays.asList("a", "b"),
+ Lists.newArrayList(FileSystemUtils.iterateLinesAsLatin1(file)));
+
+ FileSystemUtils.writeContent(file, ISO_8859_1, "a\rb");
+ assertEquals(Arrays.asList("a", "b"),
+ Lists.newArrayList(FileSystemUtils.iterateLinesAsLatin1(file)));
+
+ FileSystemUtils.writeContent(file, ISO_8859_1, "a\r\nb");
+ assertEquals(Arrays.asList("a", "b"),
+ Lists.newArrayList(FileSystemUtils.iterateLinesAsLatin1(file)));
+ }
+
+ @Test
+ public void testEnsureSymbolicLinkDoesNotMakeUnnecessaryChanges() throws Exception {
+ PathFragment target = new PathFragment("/b");
+ Path file = fileSystem.getPath("/a");
+ file.createSymbolicLink(target);
+ long prevTimeMillis = clock.currentTimeMillis();
+ clock.advanceMillis(1000);
+ FileSystemUtils.ensureSymbolicLink(file, target);
+ long timestamp = file.getLastModifiedTime(Symlinks.NOFOLLOW);
+ assertTrue(timestamp == prevTimeMillis);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemsTest.java
new file mode 100644
index 0000000..88a000f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemsTest.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+
+import com.google.devtools.build.lib.vfs.util.FileSystems;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * This class handles the tests for the FileSystems class.
+ */
+@RunWith(JUnit4.class)
+public class FileSystemsTest {
+
+ @Test
+ public void testFileSystemsCreatesOnlyOneDefaultNative() {
+ assertSame(FileSystems.initDefaultAsNative(),
+ FileSystems.initDefaultAsNative());
+ }
+
+ @Test
+ public void testFileSystemsCreatesOnlyOneDefaultJavaIo() {
+ assertSame(FileSystems.initDefaultAsJavaIo(),
+ FileSystems.initDefaultAsJavaIo());
+ }
+
+ @Test
+ public void testFileSystemsCanSwitchDefaults() {
+ assertNotSame(FileSystems.initDefaultAsNative(),
+ FileSystems.initDefaultAsJavaIo());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/GlobTest.java b/src/test/java/com/google/devtools/build/lib/vfs/GlobTest.java
new file mode 100644
index 0000000..37b7dc4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/GlobTest.java
@@ -0,0 +1,417 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ordering;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Tests {@link UnixGlob}
+ */
+@RunWith(JUnit4.class)
+public class GlobTest {
+
+ private Path tmpPath;
+ private FileSystem fs;
+ @Before
+ public void setUp() throws Exception {
+ fs = new InMemoryFileSystem();
+ tmpPath = fs.getPath("/globtmp");
+ for (String dir : ImmutableList.of("foo/bar/wiz",
+ "foo/barnacle/wiz",
+ "food/barnacle/wiz",
+ "fool/barnacle/wiz")) {
+ FileSystemUtils.createDirectoryAndParents(tmpPath.getRelative(dir));
+ }
+ FileSystemUtils.createEmptyFile(tmpPath.getRelative("foo/bar/wiz/file"));
+ }
+
+ @Test
+ public void testQuestionMarkMatch() throws Exception {
+ assertGlobMatches("foo?", /* => */"food", "fool");
+ }
+
+ @Test
+ public void testQuestionMarkNoMatch() throws Exception {
+ assertGlobMatches("food/bar?" /* => nothing */);
+ }
+
+ @Test
+ public void testStartsWithStar() throws Exception {
+ assertGlobMatches("*oo", /* => */"foo");
+ }
+
+ @Test
+ public void testStartsWithStarWithMiddleStar() throws Exception {
+ assertGlobMatches("*f*o", /* => */"foo");
+ }
+
+ @Test
+ public void testEndsWithStar() throws Exception {
+ assertGlobMatches("foo*", /* => */"foo", "food", "fool");
+ }
+
+ @Test
+ public void testEndsWithStarWithMiddleStar() throws Exception {
+ assertGlobMatches("f*oo*", /* => */"foo", "food", "fool");
+ }
+
+ @Test
+ public void testMiddleStar() throws Exception {
+ assertGlobMatches("f*o", /* => */"foo");
+ }
+
+ @Test
+ public void testTwoMiddleStars() throws Exception {
+ assertGlobMatches("f*o*o", /* => */"foo");
+ }
+
+ @Test
+ public void testSingleStarPatternWithNamedChild() throws Exception {
+ assertGlobMatches("*/bar", /* => */"foo/bar");
+ }
+
+ @Test
+ public void testSingleStarPatternWithChildGlob() throws Exception {
+ assertGlobMatches("*/bar*", /* => */
+ "foo/bar", "foo/barnacle", "food/barnacle", "fool/barnacle");
+ }
+
+ @Test
+ public void testSingleStarAsChildGlob() throws Exception {
+ assertGlobMatches("foo/*/wiz", /* => */"foo/bar/wiz", "foo/barnacle/wiz");
+ }
+
+ @Test
+ public void testNoAsteriskAndFilesDontExist() throws Exception {
+ // Note un-UNIX like semantics:
+ assertGlobMatches("ceci/n'est/pas/une/globbe" /* => nothing */);
+ }
+
+ @Test
+ public void testSingleAsteriskUnderNonexistentDirectory() throws Exception {
+ // Note un-UNIX like semantics:
+ assertGlobMatches("not-there/*" /* => nothing */);
+ }
+
+ @Test
+ public void testGlobWithNonExistentBase() throws Exception {
+ Collection<Path> globResult = UnixGlob.forPath(fs.getPath("/does/not/exist"))
+ .addPattern("*.txt")
+ .globInterruptible();
+ assertEquals(0, globResult.size());
+ }
+
+ @Test
+ public void testGlobUnderFile() throws Exception {
+ assertGlobMatches("foo/bar/wiz/file/*" /* => nothing */);
+ }
+
+ @Test
+ public void testSingleFileExclude() throws Exception {
+ assertGlobWithExcludeMatches("*", "food", "foo", "fool");
+ }
+
+ @Test
+ public void testExcludeAll() throws Exception {
+ assertGlobWithExcludeMatches("*", "*");
+ }
+
+ @Test
+ public void testExcludeAllButNoMatches() throws Exception {
+ assertGlobWithExcludeMatches("not-there", "*");
+ }
+
+ @Test
+ public void testSingleFileExcludeDoesntMatch() throws Exception {
+ assertGlobWithExcludeMatches("food", "foo", "food");
+ }
+
+ @Test
+ public void testSingleFileExcludeForDirectoryWithChildGlob()
+ throws Exception {
+ assertGlobWithExcludeMatches("foo/*", "foo", "foo/bar", "foo/barnacle");
+ }
+
+ @Test
+ public void testChildGlobWithChildExclude()
+ throws Exception {
+ assertGlobWithExcludeMatches("foo/*", "foo/*");
+ assertGlobWithExcludeMatches("foo/bar", "foo/*");
+ assertGlobWithExcludeMatches("foo/bar", "foo/bar");
+ assertGlobWithExcludeMatches("foo/bar", "*/bar");
+ assertGlobWithExcludeMatches("foo/bar", "*/*");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "*/*/*");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "foo/*/*");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "foo/bar/*");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "foo/bar/wiz");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "*/bar/wiz");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "*/*/wiz");
+ assertGlobWithExcludeMatches("foo/bar/wiz", "foo/*/wiz");
+ }
+
+ private void assertGlobMatches(String pattern, String... expecteds)
+ throws Exception {
+ assertGlobWithExcludesMatches(
+ Collections.singleton(pattern), Collections.<String>emptyList(),
+ expecteds);
+ }
+
+ private void assertGlobMatches(Collection<String> pattern,
+ String... expecteds)
+ throws Exception {
+ assertGlobWithExcludesMatches(pattern, Collections.<String>emptyList(),
+ expecteds);
+ }
+
+ private void assertGlobWithExcludeMatches(String pattern, String exclude,
+ String... expecteds)
+ throws Exception {
+ assertGlobWithExcludesMatches(
+ Collections.singleton(pattern), Collections.singleton(exclude),
+ expecteds);
+ }
+
+ private void assertGlobWithExcludesMatches(Collection<String> pattern,
+ Collection<String> excludes,
+ String... expecteds)
+ throws Exception {
+ MoreAsserts.assertSameContents(resolvePaths(expecteds),
+ new UnixGlob.Builder(tmpPath)
+ .addPatterns(pattern)
+ .addExcludes(excludes)
+ .globInterruptible());
+ }
+
+ private Set<Path> resolvePaths(String... relativePaths) {
+ Set<Path> expectedFiles = new HashSet<>();
+ for (String expected : relativePaths) {
+ Path file = expected.equals(".")
+ ? tmpPath
+ : tmpPath.getRelative(expected);
+ expectedFiles.add(file);
+ }
+ return expectedFiles;
+ }
+
+ @Test
+ public void testGlobWithoutWildcardsDoesNotCallReaddir() throws Exception {
+ UnixGlob.FilesystemCalls syscalls = new UnixGlob.FilesystemCalls() {
+ @Override
+ public FileStatus statNullable(Path path, Symlinks symlinks) {
+ return UnixGlob.DEFAULT_SYSCALLS.statNullable(path, symlinks);
+ }
+
+ @Override
+ public Collection<Dirent> readdir(Path path, Symlinks symlinks) {
+ throw new IllegalStateException();
+ }
+ };
+
+ MoreAsserts.assertSameContents(ImmutableList.of(tmpPath.getRelative("foo/bar/wiz/file")),
+ new UnixGlob.Builder(tmpPath)
+ .addPattern("foo/bar/wiz/file")
+ .setFilesystemCalls(new AtomicReference<>(syscalls))
+ .glob());
+ }
+
+ @Test
+ public void testIllegalPatterns() throws Exception {
+ assertIllegalPattern("(illegal) pattern");
+ assertIllegalPattern("[illegal pattern");
+ assertIllegalPattern("}illegal pattern");
+ assertIllegalPattern("foo**bar");
+ assertIllegalPattern("");
+ assertIllegalPattern(".");
+ assertIllegalPattern("/foo");
+ assertIllegalPattern("./foo");
+ assertIllegalPattern("foo/");
+ assertIllegalPattern("foo/./bar");
+ assertIllegalPattern("../foo/bar");
+ assertIllegalPattern("foo//bar");
+ }
+
+ /**
+ * Tests that globs can contain Java regular expression special characters
+ */
+ @Test
+ public void testSpecialRegexCharacter() throws Exception {
+ Path tmpPath2 = fs.getPath("/globtmp2");
+ FileSystemUtils.createDirectoryAndParents(tmpPath2);
+ Path aDotB = tmpPath2.getChild("a.b");
+ FileSystemUtils.createEmptyFile(aDotB);
+ FileSystemUtils.createEmptyFile(tmpPath2.getChild("aab"));
+ // Note: this contains two asterisks because otherwise a RE is not built,
+ // as an optimization.
+ assertThat(UnixGlob.forPath(tmpPath2).addPattern("*a.b*").globInterruptible()).containsExactly(
+ aDotB);
+ }
+
+ @Test
+ public void testMatchesCallWithNoCache() {
+ assertTrue(UnixGlob.matches("*a*b", "CaCb", null));
+ }
+
+ @Test
+ public void testMultiplePatterns() throws Exception {
+ assertGlobMatches(Lists.newArrayList("foo", "fool"), "foo", "fool");
+ }
+
+ @Test
+ public void testMultiplePatternsWithExcludes() throws Exception {
+ assertGlobWithExcludesMatches(Lists.newArrayList("foo", "foo?"),
+ Lists.newArrayList("fool"), "foo", "food");
+ }
+
+ @Test
+ public void testMultiplePatternsWithOverlap() throws Exception {
+ assertGlobMatchesAnyOrder(Lists.newArrayList("food", "foo?"),
+ "food", "fool");
+ assertGlobMatchesAnyOrder(Lists.newArrayList("food", "?ood", "f??d"),
+ "food");
+ assertThat(resolvePaths("food", "fool", "foo")).containsExactlyElementsIn(
+ new UnixGlob.Builder(tmpPath).addPatterns("food", "xxx", "*").glob());
+
+ }
+
+ private void assertGlobMatchesAnyOrder(ArrayList<String> patterns,
+ String... paths) throws Exception {
+ assertThat(resolvePaths(paths)).containsExactlyElementsIn(
+ new UnixGlob.Builder(tmpPath).addPatterns(patterns).globInterruptible());
+ }
+
+ /**
+ * Tests that a glob returns files in sorted order.
+ */
+ @Test
+ public void testGlobEntriesAreSorted() throws Exception {
+ Collection<Path> directoryEntries = tmpPath.getDirectoryEntries();
+ List<Path> globResult = new UnixGlob.Builder(tmpPath)
+ .addPattern("*")
+ .setExcludeDirectories(false)
+ .globInterruptible();
+ assertThat(Ordering.natural().sortedCopy(directoryEntries)).containsExactlyElementsIn(
+ globResult).inOrder();
+ }
+
+ private void assertIllegalPattern(String pattern) throws Exception {
+ try {
+ new UnixGlob.Builder(tmpPath)
+ .addPattern(pattern)
+ .globInterruptible();
+ fail();
+ } catch (IllegalArgumentException e) {
+ MoreAsserts.assertContainsRegex("in glob pattern", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testHiddenFiles() throws Exception {
+ for (String dir : ImmutableList.of(".hidden", "..also.hidden", "not.hidden")) {
+ FileSystemUtils.createDirectoryAndParents(tmpPath.getRelative(dir));
+ }
+ // Note that these are not in the result: ".", ".."
+ assertGlobMatches("*", "not.hidden", "foo", "fool", "food", ".hidden", "..also.hidden");
+ assertGlobMatches("*.hidden", "not.hidden");
+ }
+
+ @Test
+ public void testCheckCanBeInterrupted() throws Exception {
+ final Thread mainThread = Thread.currentThread();
+ final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+
+ Predicate<Path> interrupterPredicate = new Predicate<Path>() {
+ @Override
+ public boolean apply(Path input) {
+ mainThread.interrupt();
+ return true;
+ }
+ };
+
+ try {
+ new UnixGlob.Builder(tmpPath)
+ .addPattern("**")
+ .setDirectoryFilter(interrupterPredicate)
+ .setThreadPool(executor)
+ .globInterruptible();
+ fail(); // Should have received InterruptedException
+ } catch (InterruptedException e) {
+ // good
+ }
+
+ assertFalse(executor.isShutdown());
+ executor.shutdown();
+ assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void testCheckCannotBeInterrupted() throws Exception {
+ final Thread mainThread = Thread.currentThread();
+ final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ final AtomicBoolean sentInterrupt = new AtomicBoolean(false);
+
+ Predicate<Path> interrupterPredicate = new Predicate<Path>() {
+ @Override
+ public boolean apply(Path input) {
+ if (!sentInterrupt.getAndSet(true)) {
+ mainThread.interrupt();
+ }
+ return true;
+ }
+ };
+
+ List<Path> result = new UnixGlob.Builder(tmpPath)
+ .addPatterns("**", "*")
+ .setDirectoryFilter(interrupterPredicate).setThreadPool(executor).glob();
+
+ // In the non-interruptible case, the interrupt bit should be set, but the
+ // glob should return the correct set of full results.
+ assertTrue(Thread.interrupted());
+ MoreAsserts.assertSameContents(resolvePaths(".", "foo", "foo/bar", "foo/bar/wiz",
+ "foo/bar/wiz/file", "foo/barnacle", "foo/barnacle/wiz", "food", "food/barnacle",
+ "food/barnacle/wiz", "fool", "fool/barnacle", "fool/barnacle/wiz"), result);
+
+ assertFalse(executor.isShutdown());
+ executor.shutdown();
+ assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/JavaIoFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/JavaIoFileSystemTest.java
new file mode 100644
index 0000000..fdb6283
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/JavaIoFileSystemTest.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for the {@link JavaIoFileSystem}. That file system by itself is not
+ * capable of creating symlinks; use the unix one to create them, so that the
+ * test can check that the file system handles their existence correctly.
+ */
+@RunWith(JUnit4.class)
+public class JavaIoFileSystemTest extends SymlinkAwareFileSystemTest {
+
+ @Override
+ public FileSystem getFreshFileSystem() {
+ return new JavaIoFileSystem();
+ }
+
+ // The tests are just inherited from the FileSystemTest
+
+ // JavaIoFileSystem incorrectly throws a FileNotFoundException for all IO errors. This means that
+ // statIfFound incorrectly suppresses those errors.
+ @Override
+ @Test
+ public void testBadPermissionsThrowsExceptionOnStatIfFound() {}
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/ModifiedFileSetTest.java b/src/test/java/com/google/devtools/build/lib/vfs/ModifiedFileSetTest.java
new file mode 100644
index 0000000..96001df
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/ModifiedFileSetTest.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.testing.EqualsTester;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link ModifiedFileSet}.
+ */
+@RunWith(JUnit4.class)
+public class ModifiedFileSetTest {
+
+ @Test
+ public void testHashCodeAndEqualsContract() throws Exception {
+ PathFragment fragA = new PathFragment("a");
+ PathFragment fragB = new PathFragment("b");
+
+ ModifiedFileSet empty1 = ModifiedFileSet.NOTHING_MODIFIED;
+ ModifiedFileSet empty2 = ModifiedFileSet.builder().build();
+ ModifiedFileSet empty3 = ModifiedFileSet.builder().modifyAll(
+ ImmutableList.<PathFragment>of()).build();
+
+ ModifiedFileSet nonEmpty1 = ModifiedFileSet.builder().modifyAll(
+ ImmutableList.of(fragA, fragB)).build();
+ ModifiedFileSet nonEmpty2 = ModifiedFileSet.builder().modifyAll(
+ ImmutableList.of(fragB, fragA)).build();
+ ModifiedFileSet nonEmpty3 = ModifiedFileSet.builder().modify(fragA).modify(fragB).build();
+ ModifiedFileSet nonEmpty4 = ModifiedFileSet.builder().modify(fragB).modify(fragA).build();
+
+ ModifiedFileSet everythingModified = ModifiedFileSet.EVERYTHING_MODIFIED;
+
+ new EqualsTester()
+ .addEqualityGroup(empty1, empty2, empty3)
+ .addEqualityGroup(nonEmpty1, nonEmpty2, nonEmpty3, nonEmpty4)
+ .addEqualityGroup(everythingModified)
+ .testEquals();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentTest.java
new file mode 100644
index 0000000..9ab9bfa
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentTest.java
@@ -0,0 +1,481 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This class tests the functionality of the PathFragment.
+ */
+@RunWith(JUnit4.class)
+public class PathFragmentTest {
+ @Test
+ public void testMergeFourPathsWithAbsolute() {
+ assertEquals(new PathFragment("x/y/z/a/b/c/d/e"),
+ new PathFragment(new PathFragment("x/y"), new PathFragment("z/a"),
+ new PathFragment("/b/c"), // absolute!
+ new PathFragment("d/e")));
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ InMemoryFileSystem filesystem = new InMemoryFileSystem();
+
+ new EqualsTester()
+ .addEqualityGroup(new PathFragment("../relative/path"),
+ new PathFragment("../relative/path"),
+ new PathFragment(new File("../relative/path")))
+ .addEqualityGroup(new PathFragment("something/else"))
+ .addEqualityGroup(new PathFragment("/something/else"))
+ .addEqualityGroup(new PathFragment("/"),
+ new PathFragment("//////"))
+ .addEqualityGroup(new PathFragment(""))
+ .addEqualityGroup(filesystem.getRootDirectory()) // A Path object.
+ .testEquals();
+ }
+
+ @Test
+ public void testHashCodeCache() {
+ PathFragment relativePath = new PathFragment("../relative/path");
+ PathFragment rootPath = new PathFragment("/");
+
+ int oldResult = relativePath.hashCode();
+ int rootResult = rootPath.hashCode();
+ assertEquals(oldResult, relativePath.hashCode());
+ assertEquals(rootResult, rootPath.hashCode());
+ }
+
+ private void checkRelativeTo(String path, String base) {
+ PathFragment relative = new PathFragment(path).relativeTo(base);
+ assertEquals(new PathFragment(path), new PathFragment(base).getRelative(relative).normalize());
+ }
+
+ @Test
+ public void testRelativeTo() {
+ assertPath("bar/baz", new PathFragment("foo/bar/baz").relativeTo("foo"));
+ assertPath("bar/baz", new PathFragment("/foo/bar/baz").relativeTo("/foo"));
+ assertPath("baz", new PathFragment("foo/bar/baz").relativeTo("foo/bar"));
+ assertPath("baz", new PathFragment("/foo/bar/baz").relativeTo("/foo/bar"));
+ assertPath("foo", new PathFragment("/foo").relativeTo("/"));
+ assertPath("foo", new PathFragment("foo").relativeTo(""));
+ assertPath("foo/bar", new PathFragment("foo/bar").relativeTo(""));
+
+ checkRelativeTo("foo/bar/baz", "foo");
+ checkRelativeTo("/foo/bar/baz", "/foo");
+ checkRelativeTo("foo/bar/baz", "foo/bar");
+ checkRelativeTo("/foo/bar/baz", "/foo/bar");
+ checkRelativeTo("/foo", "/");
+ checkRelativeTo("foo", "");
+ checkRelativeTo("foo/bar", "");
+ }
+
+ @Test
+ public void testIsAbsolute() {
+ assertTrue(new PathFragment("/absolute/test").isAbsolute());
+ assertFalse(new PathFragment("relative/test").isAbsolute());
+ assertTrue(new PathFragment(new File("/absolute/test")).isAbsolute());
+ assertFalse(new PathFragment(new File("relative/test")).isAbsolute());
+ }
+
+ @Test
+ public void testIsNormalized() {
+ assertTrue(new PathFragment("/absolute/path").isNormalized());
+ assertTrue(new PathFragment("some//path").isNormalized());
+ assertFalse(new PathFragment("some/./path").isNormalized());
+ assertFalse(new PathFragment("../some/path").isNormalized());
+ assertFalse(new PathFragment("some/other/../path").isNormalized());
+ assertTrue(new PathFragment("some/other//tricky..path..").isNormalized());
+ assertTrue(new PathFragment("/some/other//tricky..path..").isNormalized());
+ }
+
+ @Test
+ public void testRootNodeReturnsRootString() {
+ PathFragment rootFragment = new PathFragment("/");
+ assertEquals("/", rootFragment.getPathString());
+ }
+
+ @Test
+ public void testGetPathFragmentDoesNotNormalize() {
+ String nonCanonicalPath = "/a/weird/noncanonical/../path/.";
+ assertEquals(nonCanonicalPath,
+ new PathFragment(nonCanonicalPath).getPathString());
+ }
+
+ @Test
+ public void testGetRelative() {
+ assertEquals("a/b", new PathFragment("a").getRelative("b").getPathString());
+ assertEquals("a/b/c/d", new PathFragment("a/b").getRelative("c/d").getPathString());
+ assertEquals("/a/b", new PathFragment("c/d").getRelative("/a/b").getPathString());
+ assertEquals("a", new PathFragment("a").getRelative("").getPathString());
+ assertEquals("/", new PathFragment("/").getRelative("").getPathString());
+ }
+
+ @Test
+ public void testGetChildWorks() {
+ PathFragment pf = new PathFragment("../some/path");
+ assertEquals(new PathFragment("../some/path/hi"), pf.getChild("hi"));
+ }
+
+ @Test
+ public void testGetChildRejectsInvalidBaseNames() {
+ PathFragment pf = new PathFragment("../some/path");
+ assertGetChildFails(pf, ".");
+ assertGetChildFails(pf, "..");
+ assertGetChildFails(pf, "x/y");
+ assertGetChildFails(pf, "/y");
+ assertGetChildFails(pf, "y/");
+ assertGetChildFails(pf, "");
+ }
+
+ private void assertGetChildFails(PathFragment pf, String baseName) {
+ try {
+ pf.getChild(baseName);
+ fail();
+ } catch (Exception e) { /* Expected. */ }
+ }
+
+ // Tests after here test the canonicalization
+ private void assertRegular(String expected, String actual) {
+ assertEquals(expected, new PathFragment(actual).getPathString()); // compare string forms
+ assertEquals(new PathFragment(expected), new PathFragment(actual)); // compare fragment forms
+ }
+
+ @Test
+ public void testEmptyPathToEmptyPath() {
+ assertRegular("/", "/");
+ assertRegular("", "");
+ }
+
+ @Test
+ public void testRedundantSlashes() {
+ assertRegular("/", "///");
+ assertRegular("/foo/bar", "/foo///bar");
+ assertRegular("/foo/bar", "////foo//bar");
+ }
+
+ @Test
+ public void testSimpleNameToSimpleName() {
+ assertRegular("/foo", "/foo");
+ assertRegular("foo", "foo");
+ }
+
+ @Test
+ public void testSimplePathToSimplePath() {
+ assertRegular("/foo/bar", "/foo/bar");
+ assertRegular("foo/bar", "foo/bar");
+ }
+
+ @Test
+ public void testStripsTrailingSlash() {
+ assertRegular("/foo/bar", "/foo/bar/");
+ }
+
+ @Test
+ public void testGetParentDirectory() {
+ PathFragment fooBarWiz = new PathFragment("foo/bar/wiz");
+ PathFragment fooBar = new PathFragment("foo/bar");
+ PathFragment foo = new PathFragment("foo");
+ PathFragment empty = new PathFragment("");
+ assertEquals(fooBar, fooBarWiz.getParentDirectory());
+ assertEquals(foo, fooBar.getParentDirectory());
+ assertEquals(empty, foo.getParentDirectory());
+ assertNull(empty.getParentDirectory());
+
+ PathFragment fooBarWizAbs = new PathFragment("/foo/bar/wiz");
+ PathFragment fooBarAbs = new PathFragment("/foo/bar");
+ PathFragment fooAbs = new PathFragment("/foo");
+ PathFragment rootAbs = new PathFragment("/");
+ assertEquals(fooBarAbs, fooBarWizAbs.getParentDirectory());
+ assertEquals(fooAbs, fooBarAbs.getParentDirectory());
+ assertEquals(rootAbs, fooAbs.getParentDirectory());
+ assertNull(rootAbs.getParentDirectory());
+
+ // Note, this is surprising but correct behavior:
+ assertEquals(fooBarAbs,
+ new PathFragment("/foo/bar/..").getParentDirectory());
+ }
+
+ @Test
+ public void testSegmentsCount() {
+ assertEquals(2, new PathFragment("foo/bar").segmentCount());
+ assertEquals(2, new PathFragment("/foo/bar").segmentCount());
+ assertEquals(2, new PathFragment("foo//bar").segmentCount());
+ assertEquals(2, new PathFragment("/foo//bar").segmentCount());
+ assertEquals(1, new PathFragment("foo/").segmentCount());
+ assertEquals(1, new PathFragment("/foo/").segmentCount());
+ assertEquals(1, new PathFragment("foo").segmentCount());
+ assertEquals(1, new PathFragment("/foo").segmentCount());
+ assertEquals(0, new PathFragment("/").segmentCount());
+ assertEquals(0, new PathFragment("").segmentCount());
+ }
+
+
+ @Test
+ public void testGetSegment() {
+ assertEquals("foo", new PathFragment("foo/bar").getSegment(0));
+ assertEquals("bar", new PathFragment("foo/bar").getSegment(1));
+ assertEquals("foo", new PathFragment("/foo/bar").getSegment(0));
+ assertEquals("bar", new PathFragment("/foo/bar").getSegment(1));
+ assertEquals("foo", new PathFragment("foo/").getSegment(0));
+ assertEquals("foo", new PathFragment("/foo/").getSegment(0));
+ assertEquals("foo", new PathFragment("foo").getSegment(0));
+ assertEquals("foo", new PathFragment("/foo").getSegment(0));
+ }
+
+ @Test
+ public void testBasename() throws Exception {
+ assertEquals("bar", new PathFragment("foo/bar").getBaseName());
+ assertEquals("bar", new PathFragment("/foo/bar").getBaseName());
+ assertEquals("foo", new PathFragment("foo/").getBaseName());
+ assertEquals("foo", new PathFragment("/foo/").getBaseName());
+ assertEquals("foo", new PathFragment("foo").getBaseName());
+ assertEquals("foo", new PathFragment("/foo").getBaseName());
+ assertEquals("", new PathFragment("/").getBaseName());
+ assertEquals("", new PathFragment("").getBaseName());
+ }
+
+ private static void assertPath(String expected, PathFragment actual) {
+ assertEquals(expected, actual.getPathString());
+ }
+
+ @Test
+ public void testReplaceName() throws Exception {
+ assertPath("foo/baz", new PathFragment("foo/bar").replaceName("baz"));
+ assertPath("/foo/baz", new PathFragment("/foo/bar").replaceName("baz"));
+ assertPath("foo", new PathFragment("foo/bar").replaceName(""));
+ assertPath("baz", new PathFragment("foo/").replaceName("baz"));
+ assertPath("/baz", new PathFragment("/foo/").replaceName("baz"));
+ assertPath("baz", new PathFragment("foo").replaceName("baz"));
+ assertPath("/baz", new PathFragment("/foo").replaceName("baz"));
+ assertEquals(null, new PathFragment("/").replaceName("baz"));
+ assertEquals(null, new PathFragment("/").replaceName(""));
+ assertEquals(null, new PathFragment("").replaceName("baz"));
+ assertEquals(null, new PathFragment("").replaceName(""));
+
+ assertPath("foo/bar/baz", new PathFragment("foo/bar").replaceName("bar/baz"));
+ assertPath("foo/bar/baz", new PathFragment("foo/bar").replaceName("bar/baz/"));
+
+ // Absolute path arguments will clobber the original path.
+ assertPath("/absolute", new PathFragment("foo/bar").replaceName("/absolute"));
+ assertPath("/", new PathFragment("foo/bar").replaceName("/"));
+ }
+ @Test
+ public void testSubFragment() throws Exception {
+ assertPath("/foo/bar/baz",
+ new PathFragment("/foo/bar/baz").subFragment(0, 3));
+ assertPath("foo/bar/baz",
+ new PathFragment("foo/bar/baz").subFragment(0, 3));
+ assertPath("/foo/bar",
+ new PathFragment("/foo/bar/baz").subFragment(0, 2));
+ assertPath("bar/baz",
+ new PathFragment("/foo/bar/baz").subFragment(1, 3));
+ assertPath("/foo",
+ new PathFragment("/foo/bar/baz").subFragment(0, 1));
+ assertPath("bar",
+ new PathFragment("/foo/bar/baz").subFragment(1, 2));
+ assertPath("baz", new PathFragment("/foo/bar/baz").subFragment(2, 3));
+ assertPath("/", new PathFragment("/foo/bar/baz").subFragment(0, 0));
+ assertPath("", new PathFragment("foo/bar/baz").subFragment(0, 0));
+ assertPath("", new PathFragment("foo/bar/baz").subFragment(1, 1));
+ try {
+ fail("unexpectedly succeeded: " + new PathFragment("foo/bar/baz").subFragment(3, 2));
+ } catch (IndexOutOfBoundsException e) { /* Expected. */ }
+ try {
+ fail("unexpectedly succeeded: " + new PathFragment("foo/bar/baz").subFragment(4, 4));
+ } catch (IndexOutOfBoundsException e) { /* Expected. */ }
+ }
+
+ @Test
+ public void testStartsWith() {
+ PathFragment foobar = new PathFragment("/foo/bar");
+ PathFragment foobarRelative = new PathFragment("foo/bar");
+
+ // (path, prefix) => true
+ assertTrue(foobar.startsWith(foobar));
+ assertTrue(foobar.startsWith(new PathFragment("/")));
+ assertTrue(foobar.startsWith(new PathFragment("/foo")));
+ assertTrue(foobar.startsWith(new PathFragment("/foo/")));
+ assertTrue(foobar.startsWith(new PathFragment("/foo/bar/"))); // Includes trailing slash.
+
+ // (prefix, path) => false
+ assertFalse(new PathFragment("/foo").startsWith(foobar));
+ assertFalse(new PathFragment("/").startsWith(foobar));
+
+ // (absolute, relative) => false
+ assertFalse(foobar.startsWith(foobarRelative));
+ assertFalse(foobarRelative.startsWith(foobar));
+
+ // (relative path, relative prefix) => true
+ assertTrue(foobarRelative.startsWith(foobarRelative));
+ assertTrue(foobarRelative.startsWith(new PathFragment("foo")));
+ assertTrue(foobarRelative.startsWith(new PathFragment("")));
+
+ // (path, sibling) => false
+ assertFalse(new PathFragment("/foo/wiz").startsWith(foobar));
+ assertFalse(foobar.startsWith(new PathFragment("/foo/wiz")));
+
+ // Does not normalize.
+ PathFragment foodotbar = new PathFragment("foo/./bar");
+ assertTrue(foodotbar.startsWith(foodotbar));
+ assertTrue(foodotbar.startsWith(new PathFragment("foo/.")));
+ assertTrue(foodotbar.startsWith(new PathFragment("foo/./")));
+ assertTrue(foodotbar.startsWith(new PathFragment("foo/./bar")));
+ assertFalse(foodotbar.startsWith(new PathFragment("foo/bar")));
+ }
+
+ @Test
+ public void testEndsWith() {
+ PathFragment foobar = new PathFragment("/foo/bar");
+ PathFragment foobarRelative = new PathFragment("foo/bar");
+
+ // (path, suffix) => true
+ assertTrue(foobar.endsWith(foobar));
+ assertTrue(foobar.endsWith(new PathFragment("bar")));
+ assertTrue(foobar.endsWith(new PathFragment("foo/bar")));
+ assertTrue(foobar.endsWith(new PathFragment("/foo/bar")));
+ assertFalse(foobar.endsWith(new PathFragment("/bar")));
+
+ // (prefix, path) => false
+ assertFalse(new PathFragment("/foo").endsWith(foobar));
+ assertFalse(new PathFragment("/").endsWith(foobar));
+
+ // (suffix, path) => false
+ assertFalse(new PathFragment("/bar").endsWith(foobar));
+ assertFalse(new PathFragment("bar").endsWith(foobar));
+ assertFalse(new PathFragment("").endsWith(foobar));
+
+ // (absolute, relative) => true
+ assertTrue(foobar.endsWith(foobarRelative));
+
+ // (relative, absolute) => false
+ assertFalse(foobarRelative.endsWith(foobar));
+
+ // (relative path, relative prefix) => true
+ assertTrue(foobarRelative.endsWith(foobarRelative));
+ assertTrue(foobarRelative.endsWith(new PathFragment("bar")));
+ assertTrue(foobarRelative.endsWith(new PathFragment("")));
+
+ // (path, sibling) => false
+ assertFalse(new PathFragment("/foo/wiz").endsWith(foobar));
+ assertFalse(foobar.endsWith(new PathFragment("/foo/wiz")));
+ }
+
+ static List<PathFragment> toPaths(List<String> strs) {
+ List<PathFragment> paths = Lists.newArrayList();
+ for (String s : strs) {
+ paths.add(new PathFragment(s));
+ }
+ return paths;
+ }
+
+ @Test
+ public void testCompareTo() throws Exception {
+ List<String> pathStrs = ImmutableList.of(
+ "", "/", "//", ".", "/./", "foo/.//bar", "foo", "/foo", "foo/bar", "foo/Bar", "Foo/bar");
+ List<PathFragment> paths = toPaths(pathStrs);
+ // First test that compareTo is self-consistent.
+ for (PathFragment x : paths) {
+ for (PathFragment y : paths) {
+ for (PathFragment z : paths) {
+ // Anti-symmetry
+ assertEquals(Integer.signum(x.compareTo(y)),
+ -1 * Integer.signum(y.compareTo(x)));
+ // Transitivity
+ if (x.compareTo(y) > 0 && y.compareTo(z) > 0) {
+ MoreAsserts.assertGreaterThan(0, x.compareTo(z));
+ }
+ // "Substitutability"
+ if (x.compareTo(y) == 0) {
+ assertEquals(Integer.signum(x.compareTo(z)), Integer.signum(y.compareTo(z)));
+ }
+ // Consistency with equals
+ assertEquals((x.compareTo(y) == 0), x.equals(y));
+ }
+ }
+ }
+ // Now test that compareTo does what we expect. The exact ordering here doesn't matter much,
+ // but there are three things to notice: 1. absolute < relative, 2. comparison is lexicographic
+ // 3. repeated slashes are ignored. (PathFragment("//") prints as "/").
+ Collections.shuffle(paths);
+ Collections.sort(paths);
+ List<PathFragment> expectedOrder = toPaths(ImmutableList.of(
+ "/", "//", "/./", "/foo", "", ".", "Foo/bar", "foo", "foo/.//bar", "foo/Bar", "foo/bar"));
+ assertEquals(expectedOrder, paths);
+ }
+
+ @Test
+ public void testGetSafePathString() {
+ assertEquals("/", new PathFragment("/").getSafePathString());
+ assertEquals("/abc", new PathFragment("/abc").getSafePathString());
+ assertEquals(".", new PathFragment("").getSafePathString());
+ assertEquals(".", PathFragment.EMPTY_FRAGMENT.getSafePathString());
+ assertEquals("abc/def", new PathFragment("abc/def").getSafePathString());
+ }
+
+ @Test
+ public void testNormalize() {
+ assertEquals(new PathFragment("/a/b"), new PathFragment("/a/b").normalize());
+ assertEquals(new PathFragment("/a/b"), new PathFragment("/a/./b").normalize());
+ assertEquals(new PathFragment("/b"), new PathFragment("/a/../b").normalize());
+ assertEquals(new PathFragment("a/b"), new PathFragment("a/b").normalize());
+ assertEquals(new PathFragment("../b"), new PathFragment("a/../../b").normalize());
+ assertEquals(new PathFragment(".."), new PathFragment("a/../..").normalize());
+ assertEquals(new PathFragment("b"), new PathFragment("a/../b").normalize());
+ assertEquals(new PathFragment("a/b"), new PathFragment("a/b/../b").normalize());
+ assertEquals(new PathFragment("/.."), new PathFragment("/..").normalize());
+ }
+
+ @Test
+ public void testSerializationSimple() throws Exception {
+ checkSerialization("a", 91);
+ }
+
+ @Test
+ public void testSerializationAbsolute() throws Exception {
+ checkSerialization("/foo", 94);
+ }
+
+ @Test
+ public void testSerializationNested() throws Exception {
+ checkSerialization("foo/bar/baz", 101);
+ }
+
+ private void checkSerialization(String pathFragmentString, int expectedSize) throws Exception {
+ PathFragment a = new PathFragment(pathFragmentString);
+ byte[] sa = TestUtils.serializeObject(a);
+ assertEquals(expectedSize, sa.length);
+
+ PathFragment a2 = (PathFragment) TestUtils.deserializeObject(sa);
+ assertEquals(a, a2);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentWindowsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentWindowsTest.java
new file mode 100644
index 0000000..43c94d4
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentWindowsTest.java
@@ -0,0 +1,218 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+
+/**
+ * This class tests the functionality of the PathFragment.
+ */
+@RunWith(JUnit4.class)
+public class PathFragmentWindowsTest {
+
+ @Test
+ public void testWindowsSeparator() {
+ assertEquals("bar/baz", new PathFragment("bar\\baz").toString());
+ assertEquals("C:/bar/baz", new PathFragment("c:\\bar\\baz").toString());
+ }
+
+ @Test
+ public void testIsAbsoluteWindows() {
+ assertTrue(new PathFragment("C:/").isAbsolute());
+ assertTrue(new PathFragment("C:/").isAbsolute());
+ assertTrue(new PathFragment("C:/foo").isAbsolute());
+ assertTrue(new PathFragment("d:/foo/bar").isAbsolute());
+
+ assertFalse(new PathFragment("*:/").isAbsolute());
+
+ // C: is not an absolute path, it points to the current active directory on drive C:.
+ assertFalse(new PathFragment("C:").isAbsolute());
+ assertFalse(new PathFragment("C:foo").isAbsolute());
+ }
+
+ @Test
+ public void testIsAbsoluteWindowsBackslash() {
+ assertTrue(new PathFragment(new File("C:\\blah")).isAbsolute());
+ assertTrue(new PathFragment(new File("C:\\")).isAbsolute());
+ assertTrue(new PathFragment(new File("\\blah")).isAbsolute());
+ assertTrue(new PathFragment(new File("\\")).isAbsolute());
+ }
+
+ @Test
+ public void testIsNormalizedWindows() {
+ assertTrue(new PathFragment("C:/").isNormalized());
+ assertTrue(new PathFragment("C:/absolute/path").isNormalized());
+ assertFalse(new PathFragment("C:/absolute/./path").isNormalized());
+ assertFalse(new PathFragment("C:/absolute/../path").isNormalized());
+ }
+
+ @Test
+ public void testRootNodeReturnsRootStringWindows() {
+ PathFragment rootFragment = new PathFragment("C:/");
+ assertEquals("C:/", rootFragment.getPathString());
+ }
+
+ @Test
+ public void testGetRelativeWindows() {
+ assertEquals("C:/a/b", new PathFragment("C:/a").getRelative("b").getPathString());
+ assertEquals("C:/a/b/c/d", new PathFragment("C:/a/b").getRelative("c/d").getPathString());
+ assertEquals("C:/b", new PathFragment("C:/a").getRelative("C:/b").getPathString());
+ assertEquals("C:/c/d", new PathFragment("C:/a/b").getRelative("C:/c/d").getPathString());
+ assertEquals("C:/b", new PathFragment("a").getRelative("C:/b").getPathString());
+ assertEquals("C:/c/d", new PathFragment("a/b").getRelative("C:/c/d").getPathString());
+ }
+
+ @Test
+ public void testGetRelativeMixed() {
+ assertEquals("/b", new PathFragment("C:/a").getRelative("/b").getPathString());
+ assertEquals("C:/b", new PathFragment("/a").getRelative("C:/b").getPathString());
+ }
+
+ @Test
+ public void testGetChildWorks() {
+ PathFragment pf = new PathFragment("../some/path");
+ assertEquals(new PathFragment("../some/path/hi"), pf.getChild("hi"));
+ }
+
+ // Tests after here test the canonicalization
+ private void assertRegular(String expected, String actual) {
+ assertEquals(expected, new PathFragment(actual).getPathString()); // compare string forms
+ assertEquals(new PathFragment(expected), new PathFragment(actual)); // compare fragment forms
+ }
+
+ @Test
+ public void testEmptyPathToEmptyPathWindows() {
+ assertRegular("C:/", "C:/");
+ }
+
+ @Test
+ public void testEmptyRelativePathToEmptyPathWindows() {
+ assertRegular("C:", "C:");
+ }
+
+ @Test
+ public void testWindowsVolumeUppercase() {
+ assertRegular("C:/", "c:/");
+ }
+
+ @Test
+ public void testRedundantSlashesWindows() {
+ assertRegular("C:/", "C:///");
+ assertRegular("C:/foo/bar", "C:/foo///bar");
+ assertRegular("C:/foo/bar", "C:////foo//bar");
+ }
+
+ @Test
+ public void testSimpleNameToSimpleNameWindows() {
+ assertRegular("C:/foo", "C:/foo");
+ }
+
+ @Test
+ public void testStripsTrailingSlashWindows() {
+ assertRegular("C:/foo/bar", "C:/foo/bar/");
+ }
+
+ @Test
+ public void testGetParentDirectoryWindows() {
+ PathFragment fooBarWizAbs = new PathFragment("C:/foo/bar/wiz");
+ PathFragment fooBarAbs = new PathFragment("C:/foo/bar");
+ PathFragment fooAbs = new PathFragment("C:/foo");
+ PathFragment rootAbs = new PathFragment("C:/");
+ assertEquals(fooBarAbs, fooBarWizAbs.getParentDirectory());
+ assertEquals(fooAbs, fooBarAbs.getParentDirectory());
+ assertEquals(rootAbs, fooAbs.getParentDirectory());
+ assertNull(rootAbs.getParentDirectory());
+
+ // Note, this is suprising but correct behaviour:
+ assertEquals(fooBarAbs,
+ new PathFragment("C:/foo/bar/..").getParentDirectory());
+ }
+
+ @Test
+ public void testSegmentsCountWindows() {
+ assertEquals(1, new PathFragment("C:/foo").segmentCount());
+ assertEquals(0, new PathFragment("C:/").segmentCount());
+ }
+
+ @Test
+ public void testGetSegmentWindows() {
+ assertEquals("foo", new PathFragment("C:/foo/bar").getSegment(0));
+ assertEquals("bar", new PathFragment("C:/foo/bar").getSegment(1));
+ assertEquals("foo", new PathFragment("C:/foo/").getSegment(0));
+ assertEquals("foo", new PathFragment("C:/foo").getSegment(0));
+ }
+
+ @Test
+ public void testBasenameWindows() throws Exception {
+ assertEquals("bar", new PathFragment("C:/foo/bar").getBaseName());
+ assertEquals("foo", new PathFragment("C:/foo").getBaseName());
+ // Never return the drive name as a basename.
+ assertEquals("", new PathFragment("C:/").getBaseName());
+ }
+
+ private static void assertPath(String expected, PathFragment actual) {
+ assertEquals(expected, actual.getPathString());
+ }
+
+ @Test
+ public void testReplaceNameWindows() throws Exception {
+ assertPath("C:/foo/baz", new PathFragment("C:/foo/bar").replaceName("baz"));
+ assertEquals(null, new PathFragment("C:/").replaceName("baz"));
+ }
+
+ @Test
+ public void testStartsWithWindows() {
+ assertTrue(new PathFragment("C:/foo/bar").startsWith(new PathFragment("C:/foo")));
+ assertTrue(new PathFragment("C:/foo/bar").startsWith(new PathFragment("C:/")));
+ assertTrue(new PathFragment("C:foo/bar").startsWith(new PathFragment("C:")));
+ assertTrue(new PathFragment("C:/").startsWith(new PathFragment("C:/")));
+ assertTrue(new PathFragment("C:").startsWith(new PathFragment("C:")));
+
+ // The first path is absolute, the second is not.
+ assertFalse(new PathFragment("C:/foo/bar").startsWith(new PathFragment("C:")));
+ assertFalse(new PathFragment("C:/").startsWith(new PathFragment("C:")));
+ }
+
+ @Test
+ public void testEndsWithWindows() {
+ assertTrue(new PathFragment("C:/foo/bar").endsWith(new PathFragment("bar")));
+ assertTrue(new PathFragment("C:/foo/bar").endsWith(new PathFragment("foo/bar")));
+ assertTrue(new PathFragment("C:/foo/bar").endsWith(new PathFragment("C:/foo/bar")));
+ assertTrue(new PathFragment("C:/").endsWith(new PathFragment("C:/")));
+ }
+
+ @Test
+ public void testGetSafePathStringWindows() {
+ assertEquals("C:/", new PathFragment("C:/").getSafePathString());
+ assertEquals("C:/abc", new PathFragment("C:/abc").getSafePathString());
+ assertEquals("C:/abc/def", new PathFragment("C:/abc/def").getSafePathString());
+ }
+
+ @Test
+ public void testNormalizeWindows() {
+ assertEquals(new PathFragment("C:/a/b"), new PathFragment("C:/a/b").normalize());
+ assertEquals(new PathFragment("C:/a/b"), new PathFragment("C:/a/./b").normalize());
+ assertEquals(new PathFragment("C:/b"), new PathFragment("C:/a/../b").normalize());
+ assertEquals(new PathFragment("C:/../b"), new PathFragment("C:/../b").normalize());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathTest.java
new file mode 100644
index 0000000..738e454
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/PathTest.java
@@ -0,0 +1,312 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Lists;
+import com.google.common.testing.EqualsTester;
+import com.google.common.testing.GcFinalization;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.lang.ref.WeakReference;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A test for {@link Path}.
+ */
+@RunWith(JUnit4.class)
+public class PathTest {
+ private FileSystem filesystem;
+ private Path root;
+
+ @Before
+ public void setUp() throws Exception {
+ filesystem = new InMemoryFileSystem(BlazeClock.instance());
+ root = filesystem.getRootDirectory();
+ Path first = root.getChild("first");
+ first.createDirectory();
+ }
+
+ @Test
+ public void testStartsWithWorksForSelf() {
+ assertStartsWithReturns(true, "/first/child", "/first/child");
+ }
+
+ @Test
+ public void testStartsWithWorksForChild() {
+ assertStartsWithReturns(true,
+ "/first/child", "/first/child/grandchild");
+ }
+
+ @Test
+ public void testStartsWithWorksForDeepDescendant() {
+ assertStartsWithReturns(true,
+ "/first/child", "/first/child/grandchild/x/y/z");
+ }
+
+ @Test
+ public void testStartsWithFailsForParent() {
+ assertStartsWithReturns(false, "/first/child", "/first");
+ }
+
+ @Test
+ public void testStartsWithFailsForSibling() {
+ assertStartsWithReturns(false, "/first/child", "/first/child2");
+ }
+
+ @Test
+ public void testStartsWithFailsForLinkToDescendant()
+ throws Exception {
+ Path linkTarget = filesystem.getPath("/first/linked_to");
+ FileSystemUtils.createEmptyFile(linkTarget);
+ Path second = filesystem.getPath("/second/");
+ second.createDirectory();
+ second.getChild("child_link").createSymbolicLink(linkTarget);
+ assertStartsWithReturns(false, "/first", "/second/child_link");
+ }
+
+ @Test
+ public void testStartsWithFailsForNullPrefix() {
+ try {
+ filesystem.getPath("/first").startsWith(null);
+ fail();
+ } catch (Exception e) {
+ }
+ }
+
+ private void assertStartsWithReturns(boolean expected,
+ String ancestor,
+ String descendant) {
+ Path parent = filesystem.getPath(ancestor);
+ Path child = filesystem.getPath(descendant);
+ assertEquals(expected, child.startsWith(parent));
+ }
+
+ @Test
+ public void testGetChildWorks() {
+ assertGetChildWorks("second");
+ assertGetChildWorks("...");
+ assertGetChildWorks("....");
+ }
+
+ private void assertGetChildWorks(String childName) {
+ assertEquals(filesystem.getPath("/first/" + childName),
+ filesystem.getPath("/first").getChild(childName));
+ }
+
+ @Test
+ public void testGetChildFailsForChildWithSlashes() {
+ assertGetChildFails("second/third");
+ assertGetChildFails("./third");
+ assertGetChildFails("../third");
+ assertGetChildFails("second/..");
+ assertGetChildFails("second/.");
+ assertGetChildFails("/third");
+ assertGetChildFails("third/");
+ }
+
+ private void assertGetChildFails(String childName) {
+ try {
+ filesystem.getPath("/first").getChild(childName);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testGetChildFailsForDotAndDotDot() {
+ assertGetChildFails(".");
+ assertGetChildFails("..");
+ }
+
+ @Test
+ public void testGetChildFailsForEmptyString() {
+ assertGetChildFails("");
+ }
+
+ @Test
+ public void testRelativeToWorks() {
+ assertRelativeToWorks("apple", "/fruit/apple", "/fruit");
+ assertRelativeToWorks("apple/jonagold", "/fruit/apple/jonagold", "/fruit");
+ }
+
+ @Test
+ public void testGetRelativeWithStringWorks() {
+ assertGetRelativeWorks("/first/x/y", "y");
+ assertGetRelativeWorks("/y", "/y");
+ assertGetRelativeWorks("/first/x/x", "./x");
+ assertGetRelativeWorks("/first/y", "../y");
+ assertGetRelativeWorks("/", "../../../../..");
+ }
+
+ @Test
+ public void testAsFragmentWorks() {
+ assertAsFragmentWorks("/");
+ assertAsFragmentWorks("//");
+ assertAsFragmentWorks("/first");
+ assertAsFragmentWorks("/first/x/y");
+ assertAsFragmentWorks("/first/x/y.foo");
+ }
+
+ @Test
+ public void testGetRelativeWithFragmentWorks() {
+ Path dir = filesystem.getPath("/first/x");
+ assertEquals("/first/x/y",
+ dir.getRelative(new PathFragment("y")).toString());
+ assertEquals("/first/x/x",
+ dir.getRelative(new PathFragment("./x")).toString());
+ assertEquals("/first/y",
+ dir.getRelative(new PathFragment("../y")).toString());
+
+ }
+
+ @Test
+ public void testGetRelativeWithAbsoluteFragmentWorks() {
+ Path root = filesystem.getPath("/first/x");
+ assertEquals("/x/y",
+ root.getRelative(new PathFragment("/x/y")).toString());
+ }
+
+ @Test
+ public void testGetRelativeWithAbsoluteStringWorks() {
+ Path root = filesystem.getPath("/first/x");
+ assertEquals("/x/y", root.getRelative("/x/y").toString());
+ }
+
+ @Test
+ public void testComparableSortOrder() {
+ Path zzz = filesystem.getPath("/zzz");
+ Path ZZZ = filesystem.getPath("/ZZZ");
+ Path abc = filesystem.getPath("/abc");
+ Path aBc = filesystem.getPath("/aBc");
+ Path AbC = filesystem.getPath("/AbC");
+ Path ABC = filesystem.getPath("/ABC");
+ List<Path> list = Lists.newArrayList(zzz, ZZZ, ABC, aBc, AbC, abc);
+ Collections.sort(list);
+ assertThat(list).containsExactly(ABC, AbC, ZZZ, aBc, abc, zzz).inOrder();
+ }
+
+ @Test
+ public void testParentOfRootIsRoot() {
+ assertSame(root, root.getRelative(".."));
+
+ assertSame(root.getRelative("dots"),
+ root.getRelative("broken/../../dots"));
+ }
+
+ @Test
+ public void testSingleSegmentEquivalence() {
+ assertSame(
+ root.getRelative("aSingleSegment"),
+ root.getRelative("aSingleSegment"));
+ }
+
+ @Test
+ public void testSiblingNonEquivalenceString() {
+ assertNotSame(
+ root.getRelative("aSingleSegment"),
+ root.getRelative("aDifferentSegment"));
+ }
+
+ @Test
+ public void testSiblingNonEquivalenceFragment() {
+ assertNotSame(
+ root.getRelative(new PathFragment("aSingleSegment")),
+ root.getRelative(new PathFragment("aDifferentSegment")));
+ }
+
+ @Test
+ public void testHashCodeStableAcrossGarbageCollections() {
+ Path parent = filesystem.getPath("/a");
+ PathFragment childFragment = new PathFragment("b");
+ Path child = parent.getRelative(childFragment);
+ WeakReference<Path> childRef = new WeakReference<>(child);
+ int childHashCode1 = childRef.get().hashCode();
+ assertEquals(childHashCode1, parent.getRelative(childFragment).hashCode());
+ child = null;
+ GcFinalization.awaitClear(childRef);
+ int childHashCode2 = parent.getRelative(childFragment).hashCode();
+ assertEquals(childHashCode1, childHashCode2);
+ }
+
+ @Test
+ public void testSerialization() throws Exception {
+ FileSystem oldFileSystem = Path.getFileSystemForSerialization();
+ try {
+ Path.setFileSystemForSerialization(filesystem);
+ Path root = filesystem.getPath("/");
+ Path p1 = filesystem.getPath("/foo");
+ Path p2 = filesystem.getPath("/foo/bar");
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(bos);
+
+ oos.writeObject(root);
+ oos.writeObject(p1);
+ oos.writeObject(p2);
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+ ObjectInputStream ois = new ObjectInputStream(bis);
+
+ Path dsRoot = (Path) ois.readObject();
+ Path dsP1 = (Path) ois.readObject();
+ Path dsP2 = (Path) ois.readObject();
+
+ new EqualsTester()
+ .addEqualityGroup(root, dsRoot)
+ .addEqualityGroup(p1, dsP1)
+ .addEqualityGroup(p2, dsP2)
+ .testEquals();
+
+ assertTrue(p2.startsWith(p1));
+ assertTrue(p2.startsWith(dsP1));
+ assertTrue(dsP2.startsWith(p1));
+ assertTrue(dsP2.startsWith(dsP1));
+ } finally {
+ Path.setFileSystemForSerialization(oldFileSystem);
+ }
+ }
+
+ private void assertAsFragmentWorks(String expected) {
+ assertEquals(new PathFragment(expected), filesystem.getPath(expected).asFragment());
+ }
+
+ private void assertGetRelativeWorks(String expected, String relative) {
+ assertEquals(filesystem.getPath(expected),
+ filesystem.getPath("/first/x").getRelative(relative));
+ }
+
+ private void assertRelativeToWorks(String expected, String relative, String original) {
+ assertEquals(new PathFragment(expected),
+ filesystem.getPath(relative).relativeTo(filesystem.getPath(original)));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathWindowsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathWindowsTest.java
new file mode 100644
index 0000000..c92fc2b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/PathWindowsTest.java
@@ -0,0 +1,98 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * A test for windows aspects of {@link Path}.
+ */
+@RunWith(JUnit4.class)
+public class PathWindowsTest {
+ private FileSystem filesystem;
+ private Path root;
+
+ @Before
+ public void setUp() throws Exception {
+ filesystem = new InMemoryFileSystem(BlazeClock.instance());
+ root = filesystem.getRootDirectory();
+ Path first = root.getChild("first");
+ first.createDirectory();
+ }
+
+ private void assertAsFragmentWorks(String expected) {
+ assertEquals(new PathFragment(expected), filesystem.getPath(expected).asFragment());
+ }
+
+ @Test
+ public void testWindowsPath() {
+ Path p = filesystem.getPath("C:/foo/bar");
+ assertEquals("C:/foo/bar", p.getPathString());
+ assertEquals("C:/foo/bar", p.toString());
+ }
+
+ @Test
+ public void testAsFragmentWindows() {
+ assertAsFragmentWorks("C:/");
+ assertAsFragmentWorks("C://");
+ assertAsFragmentWorks("C:/first");
+ assertAsFragmentWorks("C:/first/x/y");
+ assertAsFragmentWorks("C:/first/x/y.foo");
+ }
+
+ @Test
+ public void testGetRelativeWithFragmentWindows() {
+ Path dir = filesystem.getPath("C:/first/x");
+ assertEquals("C:/first/x/y",
+ dir.getRelative(new PathFragment("y")).toString());
+ assertEquals("C:/first/x/x",
+ dir.getRelative(new PathFragment("./x")).toString());
+ assertEquals("C:/first/y",
+ dir.getRelative(new PathFragment("../y")).toString());
+ assertEquals("C:/first/y",
+ dir.getRelative(new PathFragment("../y")).toString());
+ assertEquals("C:/y",
+ dir.getRelative(new PathFragment("../../../y")).toString());
+ }
+
+ @Test
+ public void testGetRelativeWithAbsoluteFragmentWindows() {
+ Path root = filesystem.getPath("C:/first/x");
+ assertEquals("C:/x/y",
+ root.getRelative(new PathFragment("C:/x/y")).toString());
+ }
+
+ @Test
+ public void testGetRelativeWithAbsoluteStringWorksWindows() {
+ Path root = filesystem.getPath("C:/first/x");
+ assertEquals("C:/x/y", root.getRelative("C:/x/y").toString());
+ }
+
+ @Test
+ public void testParentOfRootIsRootWindows() {
+ assertSame(root, root.getRelative(".."));
+
+ assertSame(root.getRelative("dots"),
+ root.getRelative("broken/../../dots"));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/RecursiveGlobTest.java b/src/test/java/com/google/devtools/build/lib/vfs/RecursiveGlobTest.java
new file mode 100644
index 0000000..5e0012a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/RecursiveGlobTest.java
@@ -0,0 +1,227 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.testutil.MoreAsserts.assertSameContents;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ordering;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Tests {@link UnixGlob} recursive globs.
+ */
+@RunWith(JUnit4.class)
+public class RecursiveGlobTest {
+
+ private Path tmpPath;
+ private FileSystem fileSystem;
+
+ @Before
+ public void setUp() throws Exception {
+ fileSystem = new InMemoryFileSystem(BlazeClock.instance());
+ tmpPath = fileSystem.getPath("/rglobtmp");
+ for (String dir : ImmutableList.of("foo/bar/wiz",
+ "foo/baz/wiz",
+ "foo/baz/quip/wiz",
+ "food/baz/wiz",
+ "fool/baz/wiz")) {
+ FileSystemUtils.createDirectoryAndParents(tmpPath.getRelative(dir));
+ }
+ FileSystemUtils.createEmptyFile(tmpPath.getRelative("foo/bar/wiz/file"));
+ }
+
+ @Test
+ public void testDoubleStar() throws Exception {
+ assertGlobMatches("**", ".", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", "foo/baz/quip",
+ "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file", "food", "food/baz",
+ "food/baz/wiz", "fool", "fool/baz", "fool/baz/wiz");
+ }
+
+ @Test
+ public void testDoubleDoubleStar() throws Exception {
+ assertGlobMatches("**/**", ".", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", "foo/baz/quip",
+ "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file", "food", "food/baz",
+ "food/baz/wiz", "fool", "fool/baz", "fool/baz/wiz");
+ }
+
+ @Test
+ public void testDirectoryWithDoubleStar() throws Exception {
+ assertGlobMatches("foo/**", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", "foo/baz/quip",
+ "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file");
+ }
+
+ @Test
+ public void testIllegalPatterns() throws Exception {
+ for (String prefix : Lists.newArrayList("", "*/", "**/", "ba/")) {
+ String suffix = ("/" + prefix).substring(0, prefix.length());
+ for (String pattern : Lists.newArrayList("**fo", "fo**", "**fo**", "fo**fo", "fo**fo**fo")) {
+ assertIllegalWildcard(prefix + pattern);
+ assertIllegalWildcard(pattern + suffix);
+ assertIllegalWildcard("foo", pattern + suffix);
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleStarPatternWithNamedChild() throws Exception {
+ assertGlobMatches("**/bar", "foo/bar");
+ }
+
+ @Test
+ public void testDoubleStarPatternWithChildGlob() throws Exception {
+ assertGlobMatches("**/ba*",
+ "foo/bar", "foo/baz", "food/baz", "fool/baz");
+ }
+
+ @Test
+ public void testDoubleStarAsChildGlob() throws Exception {
+ assertGlobMatches("foo/**/wiz", "foo/bar/wiz", "foo/baz/quip/wiz", "foo/baz/wiz");
+ }
+
+ @Test
+ public void testDoubleStarUnderNonexistentDirectory() throws Exception {
+ assertGlobMatches("not-there/**" /* => nothing */);
+ }
+
+ @Test
+ public void testDoubleStarGlobWithNonExistentBase() throws Exception {
+ Collection<Path> globResult = UnixGlob.forPath(fileSystem.getPath("/does/not/exist"))
+ .addPattern("**")
+ .globInterruptible();
+ assertEquals(0, globResult.size());
+ }
+
+ @Test
+ public void testDoubleStarUnderFile() throws Exception {
+ assertGlobMatches("foo/bar/wiz/file/**" /* => nothing */);
+ }
+
+ @Test
+ public void testSingleFileExclude() throws Exception {
+ assertGlobWithExcludeMatches("**", "food", ".", "foo", "foo/bar", "foo/bar/wiz", "foo/baz",
+ "foo/baz/quip", "foo/baz/quip/wiz", "foo/baz/wiz",
+ "foo/bar/wiz/file", "food/baz", "food/baz/wiz", "fool", "fool/baz",
+ "fool/baz/wiz");
+ }
+
+ @Test
+ public void testSingleFileExcludeForDirectoryWithChildGlob()
+ throws Exception {
+ assertGlobWithExcludeMatches("foo/**", "foo", "foo/bar", "foo/bar/wiz", "foo/baz",
+ "foo/baz/quip", "foo/baz/quip/wiz", "foo/baz/wiz",
+ "foo/bar/wiz/file");
+ }
+
+ @Test
+ public void testGlobExcludeForDirectoryWithChildGlob()
+ throws Exception {
+ assertGlobWithExcludeMatches("foo/**", "foo/*", "foo", "foo/bar/wiz", "foo/baz/quip",
+ "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file");
+ }
+
+ @Test
+ public void testExcludeAll() throws Exception {
+ assertGlobWithExcludesMatches(Lists.newArrayList("**"),
+ Lists.newArrayList("*", "*/*", "*/*/*", "*/*/*/*"), ".");
+ }
+
+ @Test
+ public void testManualGlobExcludeForDirectoryWithChildGlob()
+ throws Exception {
+ assertGlobWithExcludesMatches(Lists.newArrayList("foo/**"),
+ Lists.newArrayList("foo", "foo/*", "foo/*/*", "foo/*/*/*"));
+ }
+
+ private void assertGlobMatches(String pattern, String... expecteds)
+ throws Exception {
+ assertGlobWithExcludesMatches(
+ Collections.singleton(pattern), Collections.<String>emptyList(),
+ expecteds);
+ }
+
+ private void assertGlobWithExcludeMatches(String pattern, String exclude,
+ String... expecteds)
+ throws Exception {
+ assertGlobWithExcludesMatches(
+ Collections.singleton(pattern), Collections.singleton(exclude),
+ expecteds);
+ }
+
+ private void assertGlobWithExcludesMatches(Collection<String> pattern,
+ Collection<String> excludes,
+ String... expecteds) throws Exception {
+ assertSameContents(resolvePaths(expecteds),
+ new UnixGlob.Builder(tmpPath)
+ .addPatterns(pattern)
+ .addExcludes(excludes)
+ .globInterruptible());
+ }
+
+ private Set<Path> resolvePaths(String... relativePaths) {
+ Set<Path> expectedFiles = new HashSet<>();
+ for (String expected : relativePaths) {
+ Path file = expected.equals(".")
+ ? tmpPath
+ : tmpPath.getRelative(expected);
+ expectedFiles.add(file);
+ }
+ return expectedFiles;
+ }
+
+ /**
+ * Tests that a recursive glob returns files in sorted order.
+ */
+ @Test
+ public void testGlobEntriesAreSorted() throws Exception {
+ List<Path> globResult = new UnixGlob.Builder(tmpPath)
+ .addPattern("**")
+ .setExcludeDirectories(false)
+ .globInterruptible();
+
+ assertThat(Ordering.natural().sortedCopy(globResult)).containsExactlyElementsIn(globResult)
+ .inOrder();
+ }
+
+ private void assertIllegalWildcard(String pattern, String... excludePatterns)
+ throws Exception {
+ try {
+ new UnixGlob.Builder(tmpPath)
+ .addPattern(pattern)
+ .addExcludes(excludePatterns)
+ .globInterruptible();
+ fail();
+ } catch (IllegalArgumentException e) {
+ MoreAsserts.assertContainsRegex("recursive wildcard must be its own segment", e.getMessage());
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/RootedPathTest.java b/src/test/java/com/google/devtools/build/lib/vfs/RootedPathTest.java
new file mode 100644
index 0000000..46d286d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/RootedPathTest.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link RootedPath}.
+ */
+@RunWith(JUnit4.class)
+public class RootedPathTest {
+ private FileSystem filesystem;
+ private Path root;
+
+ @Before
+ public void setUp() throws Exception {
+ filesystem = new InMemoryFileSystem(BlazeClock.instance());
+ root = filesystem.getRootDirectory();
+ }
+
+ @Test
+ public void testEqualsAndHashCodeContract() throws Exception {
+ Path pkgRoot1 = root.getRelative("pkgroot1");
+ Path pkgRoot2 = root.getRelative("pkgroot2");
+ RootedPath rootedPathA1 = RootedPath.toRootedPath(pkgRoot1, new PathFragment("foo/bar"));
+ RootedPath rootedPathA2 = RootedPath.toRootedPath(pkgRoot1, new PathFragment("foo/bar"));
+ RootedPath absolutePath1 = RootedPath.toRootedPath(root, new PathFragment("pkgroot1/foo/bar"));
+ RootedPath rootedPathB1 = RootedPath.toRootedPath(pkgRoot2, new PathFragment("foo/bar"));
+ RootedPath rootedPathB2 = RootedPath.toRootedPath(pkgRoot2, new PathFragment("foo/bar"));
+ RootedPath absolutePath2 = RootedPath.toRootedPath(root, new PathFragment("pkgroot2/foo/bar"));
+ new EqualsTester()
+ .addEqualityGroup(rootedPathA1, rootedPathA2)
+ .addEqualityGroup(rootedPathB1, rootedPathB2)
+ .addEqualityGroup(absolutePath1)
+ .addEqualityGroup(absolutePath2)
+ .testEquals();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystemTest.java
new file mode 100644
index 0000000..6c8071f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystemTest.java
@@ -0,0 +1,806 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+
+/**
+ * Generic tests for any file system that implements {@link ScopeEscapableFileSystem},
+ * i.e. any file system that supports symlinks that escape its scope.
+ *
+ * Each suitable file system test should inherit from this class, thereby obtaining
+ * all the tests.
+ */
+public abstract class ScopeEscapableFileSystemTest extends SymlinkAwareFileSystemTest {
+
+ /**
+ * Trivial FileSystem implementation that can record the last path passed to each method
+ * and read/write to a unified "state" variable (which can then be checked by tests) for
+ * each data type this class manipulates.
+ *
+ * The default implementation of each method throws an exception. Each test case should
+ * selectively override the methods it expects to be invoked.
+ */
+ private static class TestDelegator extends FileSystem {
+ protected Path lastPath;
+ protected boolean booleanState;
+ protected long longState;
+ protected Object objectState;
+
+ public void setState(boolean state) { booleanState = state; }
+ public void setState(long state) { longState = state; }
+ public void setState(Object state) { objectState = state; }
+
+ public boolean booleanState() { return booleanState; }
+ public long longState() { return longState; }
+ public Object objectState() { return objectState; }
+
+ public PathFragment lastPath() {
+ Path ans = lastPath;
+ // Clear this out to protect against accidental matches when testing the same path multiple
+ // consecutive times.
+ lastPath = null;
+ return ans != null ? ans.asFragment() : null;
+ }
+
+ @Override public boolean supportsModifications() { return true; }
+ @Override public boolean supportsSymbolicLinks() { return true; }
+
+ private static RuntimeException re() {
+ return new RuntimeException("This method should not be called in this context");
+ }
+
+ @Override protected boolean isReadable(Path path) { throw re(); }
+ @Override protected boolean isWritable(Path path) { throw re(); }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { throw re(); }
+ @Override protected boolean isFile(Path path, boolean followSymlinks) { throw re(); }
+ @Override protected boolean isExecutable(Path path) { throw re(); }
+ @Override protected boolean exists(Path path, boolean followSymlinks) {throw re(); }
+ @Override protected boolean isSymbolicLink(Path path) { throw re(); }
+ @Override protected boolean createDirectory(Path path) { throw re(); }
+ @Override protected boolean delete(Path path) { throw re(); }
+
+ @Override protected long getFileSize(Path path, boolean followSymlinks) { throw re(); }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { throw re(); }
+
+ @Override protected void setWritable(Path path, boolean writable) { throw re(); }
+ @Override protected void setExecutable(Path path, boolean executable) { throw re(); }
+ @Override protected void setReadable(Path path, boolean readable) { throw re(); }
+ @Override protected void setLastModifiedTime(Path path, long newTime) { throw re(); }
+ @Override protected void renameTo(Path sourcePath, Path targetPath) { throw re(); }
+ @Override protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) {
+ throw re();
+ }
+
+ @Override protected PathFragment readSymbolicLink(Path path) { throw re(); }
+ @Override protected InputStream getInputStream(Path path) { throw re(); }
+ @Override protected Collection<Path> getDirectoryEntries(Path path) { throw re(); }
+ @Override protected OutputStream getOutputStream(Path path, boolean append) { throw re(); }
+ @Override
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
+ throw re();
+ }
+ }
+
+ protected static final PathFragment SCOPE_ROOT = new PathFragment("/fs/root");
+
+ private Path fileLink;
+ private PathFragment fileLinkTarget;
+ private Path dirLink;
+ private PathFragment dirLinkTarget;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+
+ Preconditions.checkState(testFS instanceof ScopeEscapableFileSystem,
+ "Not ScopeEscapable: " + testFS);
+ ((ScopeEscapableFileSystem) testFS).enableScopeChecking(false);
+ for (int i = 1; i <= SCOPE_ROOT.segmentCount(); i++) {
+ testFS.getPath(SCOPE_ROOT.subFragment(0, i)).createDirectory();
+ }
+
+ fileLink = testFS.getPath(SCOPE_ROOT.getRelative("link"));
+ fileLinkTarget = new PathFragment("/should/be/delegated/fileLinkTarget");
+ testFS.createSymbolicLink(fileLink, fileLinkTarget);
+
+ dirLink = testFS.getPath(SCOPE_ROOT.getRelative("dirlink"));
+ dirLinkTarget = new PathFragment("/should/be/delegated/dirLinkTarget");
+ testFS.createSymbolicLink(dirLink, dirLinkTarget);
+ }
+
+ /**
+ * Returns the file system supplied by {@link #getFreshFileSystem}, cast to
+ * a {@link ScopeEscapableFileSystem}. Also enables scope checking within
+ * the file system (which we keep disabled for inherited tests that aren't
+ * intended to test scope boundaries).
+ */
+ private ScopeEscapableFileSystem scopedFS() {
+ ScopeEscapableFileSystem fs = (ScopeEscapableFileSystem) testFS;
+ fs.enableScopeChecking(true);
+ return fs;
+ }
+
+ // Checks that the semi-resolved path passed to the delegator matches the expected value.
+ private void checkPath(TestDelegator delegator, PathFragment expectedDelegatedPath) {
+ assertTrue(expectedDelegatedPath.equals(delegator.lastPath()));
+ }
+
+ // Asserts that the condition is false and checks that the expected path was delegated.
+ private void assertFalseWithPathCheck(boolean result, TestDelegator delegator,
+ PathFragment expectedDelegatedPath) {
+ assertFalse(result);
+ checkPath(delegator, expectedDelegatedPath);
+ }
+
+ // Asserts that the condition is true and checks that the expected path was delegated.
+ private void assertTrueWithPathCheck(boolean result, TestDelegator delegator,
+ PathFragment expectedDelegatedPath) {
+ assertTrue(result);
+ checkPath(delegator, expectedDelegatedPath);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // Tests:
+ /////////////////////////////////////////////////////////////////////////////
+
+ @Test
+ public void testIsReadableCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean isReadable(Path path) {
+ lastPath = path;
+ return booleanState();
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(fileLink.isReadable(), delegator, fileLinkTarget);
+ assertFalseWithPathCheck(dirLink.getRelative("a").isReadable(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(fileLink.isReadable(), delegator, fileLinkTarget);
+ assertTrueWithPathCheck(dirLink.getRelative("a").isReadable(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testIsWritableCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean isWritable(Path path) {
+ lastPath = path;
+ return booleanState();
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(fileLink.isWritable(), delegator, fileLinkTarget);
+ assertFalseWithPathCheck(dirLink.getRelative("a").isWritable(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(fileLink.isWritable(), delegator, fileLinkTarget);
+ assertTrueWithPathCheck(dirLink.getRelative("a").isWritable(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testisExecutableCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean isExecutable(Path path) {
+ lastPath = path;
+ return booleanState();
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(fileLink.isExecutable(), delegator, fileLinkTarget);
+ assertFalseWithPathCheck(dirLink.getRelative("a").isExecutable(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(fileLink.isExecutable(), delegator, fileLinkTarget);
+ assertTrueWithPathCheck(dirLink.getRelative("a").isExecutable(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testIsDirectoryCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) {
+ lastPath = path;
+ return booleanState();
+ }
+ @Override protected boolean exists(Path path, boolean followSymlinks) { return true; }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(fileLink.isDirectory(), delegator, fileLinkTarget);
+ assertFalseWithPathCheck(dirLink.getRelative("a").isDirectory(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(fileLink.isDirectory(), delegator, fileLinkTarget);
+ assertTrueWithPathCheck(dirLink.getRelative("a").isDirectory(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testIsFileCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean isFile(Path path, boolean followSymlinks) {
+ lastPath = path;
+ return booleanState();
+ }
+ @Override protected boolean exists(Path path, boolean followSymlinks) { return true; }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(fileLink.isFile(), delegator, fileLinkTarget);
+ assertFalseWithPathCheck(dirLink.getRelative("a").isFile(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(fileLink.isFile(), delegator, fileLinkTarget);
+ assertTrueWithPathCheck(dirLink.getRelative("a").isFile(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testIsSymbolicLinkCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean isSymbolicLink(Path path) {
+ lastPath = path;
+ return booleanState();
+ }
+ @Override protected boolean exists(Path path, boolean followSymlinks) { return true; }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ // We shouldn't follow final-segment links, so they should never invoke the delegator.
+ delegator.setState(false);
+ assertTrue(fileLink.isSymbolicLink());
+ assertTrue(delegator.lastPath() == null);
+
+ assertFalseWithPathCheck(dirLink.getRelative("a").isSymbolicLink(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(dirLink.getRelative("a").isSymbolicLink(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ /**
+ * Returns a test delegator that reflects info passed to Path.exists() calls.
+ */
+ private TestDelegator newExistsDelegator() {
+ return new TestDelegator() {
+ @Override protected boolean exists(Path path, boolean followSymlinks) {
+ lastPath = path;
+ return booleanState();
+ }
+ @Override protected FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (!exists(path, followSymlinks)) {
+ throw new IOException("Expected exception on stat of non-existent file");
+ }
+ return super.stat(path, followSymlinks);
+ }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; }
+ };
+ }
+
+ @Test
+ public void testExistsCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = newExistsDelegator();
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(fileLink.exists(), delegator, fileLinkTarget);
+ assertFalseWithPathCheck(dirLink.getRelative("a").exists(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(fileLink.exists(), delegator, fileLinkTarget);
+ assertTrueWithPathCheck(dirLink.getRelative("a").exists(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCreateDirectoryCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean createDirectory(Path path) {
+ lastPath = path;
+ return booleanState();
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertFalseWithPathCheck(dirLink.getRelative("a").createDirectory(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(dirLink.getRelative("a").createDirectory(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testDeleteCallOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected boolean delete(Path path) {
+ lastPath = path;
+ return booleanState();
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ assertTrue(fileLink.delete());
+ assertTrue(delegator.lastPath() == null); // Deleting a link shouldn't require delegation.
+ assertFalseWithPathCheck(dirLink.getRelative("a").delete(), delegator,
+ dirLinkTarget.getRelative("a"));
+
+ delegator.setState(true);
+ assertTrueWithPathCheck(dirLink.getRelative("a").delete(), delegator,
+ dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallGetFileSizeOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected long getFileSize(Path path, boolean followSymlinks) {
+ lastPath = path;
+ return longState();
+ }
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ final int state1 = 10;
+ delegator.setState(state1);
+ assertEquals(state1, fileLink.getFileSize());
+ checkPath(delegator, fileLinkTarget);
+ assertEquals(state1, dirLink.getRelative("a").getFileSize());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+
+ final int state2 = 10;
+ delegator.setState(state2);
+ assertEquals(state2, fileLink.getFileSize());
+ checkPath(delegator, fileLinkTarget);
+ assertEquals(state2, dirLink.getRelative("a").getFileSize());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallGetLastModifiedTimeOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) {
+ lastPath = path;
+ return longState();
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ final int state1 = 10;
+ delegator.setState(state1);
+ assertEquals(state1, fileLink.getLastModifiedTime());
+ checkPath(delegator, fileLinkTarget);
+ assertEquals(state1, dirLink.getRelative("a").getLastModifiedTime());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+
+ final int state2 = 10;
+ delegator.setState(state2);
+ assertEquals(state2, fileLink.getLastModifiedTime());
+ checkPath(delegator, fileLinkTarget);
+ assertEquals(state2, dirLink.getRelative("a").getLastModifiedTime());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallSetReadableOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected void setReadable(Path path, boolean readable) {
+ lastPath = path;
+ setState(readable);
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ fileLink.setReadable(true);
+ assertTrue(delegator.booleanState());
+ checkPath(delegator, fileLinkTarget);
+ fileLink.setReadable(false);
+ assertFalse(delegator.booleanState());
+ checkPath(delegator, fileLinkTarget);
+
+ delegator.setState(false);
+ dirLink.getRelative("a").setReadable(true);
+ assertTrue(delegator.booleanState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ dirLink.getRelative("a").setReadable(false);
+ assertFalse(delegator.booleanState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallSetWritableOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected void setWritable(Path path, boolean writable) {
+ lastPath = path;
+ setState(writable);
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ fileLink.setWritable(true);
+ assertTrue(delegator.booleanState());
+ checkPath(delegator, fileLinkTarget);
+ fileLink.setWritable(false);
+ assertFalse(delegator.booleanState());
+ checkPath(delegator, fileLinkTarget);
+
+ delegator.setState(false);
+ dirLink.getRelative("a").setWritable(true);
+ assertTrue(delegator.booleanState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ dirLink.getRelative("a").setWritable(false);
+ assertFalse(delegator.booleanState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallSetExecutableOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected void setReadable(Path path, boolean readable) {
+ lastPath = path;
+ setState(readable);
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(false);
+ fileLink.setReadable(true);
+ assertTrue(delegator.booleanState());
+ checkPath(delegator, fileLinkTarget);
+ fileLink.setReadable(false);
+ assertFalse(delegator.booleanState());
+ checkPath(delegator, fileLinkTarget);
+
+ delegator.setState(false);
+ dirLink.getRelative("a").setReadable(true);
+ assertTrue(delegator.booleanState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ dirLink.getRelative("a").setReadable(false);
+ assertFalse(delegator.booleanState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallSetLastModifiedTimeOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected void setLastModifiedTime(Path path, long newTime) {
+ lastPath = path;
+ setState(newTime);
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(0);
+ fileLink.setLastModifiedTime(10);
+ assertEquals(10, delegator.longState());
+ checkPath(delegator, fileLinkTarget);
+ fileLink.setLastModifiedTime(15);
+ assertEquals(15, delegator.longState());
+ checkPath(delegator, fileLinkTarget);
+
+ dirLink.getRelative("a").setLastModifiedTime(20);
+ assertEquals(20, delegator.longState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ dirLink.getRelative("a").setLastModifiedTime(25);
+ assertEquals(25, delegator.longState());
+ checkPath(delegator, dirLinkTarget.getRelative("a"));
+ }
+
+ @Test
+ public void testCallRenameToOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected void renameTo(Path sourcePath, Path targetPath) {
+ lastPath = sourcePath;
+ setState(targetPath);
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ // Renaming a link should work fine.
+ delegator.setState(null);
+ fileLink.renameTo(testFS.getPath(SCOPE_ROOT).getRelative("newname"));
+ assertEquals(null, delegator.lastPath()); // Renaming a link shouldn't require delegation.
+ assertEquals(null, delegator.objectState());
+
+ // Renaming an out-of-scope path to an in-scope path should fail due to filesystem mismatch
+ // errors.
+ Path newPath = testFS.getPath(SCOPE_ROOT.getRelative("blah"));
+ try {
+ dirLink.getRelative("a").renameTo(newPath);
+ fail("This is an attempt at a cross-filesystem renaming, which should fail");
+ } catch (IOException e) {
+ // Expected.
+ }
+
+ // Renaming an out-of-scope path to another out-of-scope path can be valid.
+ newPath = dirLink.getRelative("b");
+ dirLink.getRelative("a").renameTo(newPath);
+ assertEquals(dirLinkTarget.getRelative("a"), delegator.lastPath());
+ assertEquals(dirLinkTarget.getRelative("b"), ((Path) delegator.objectState()).asFragment());
+ }
+
+ @Test
+ public void testCallCreateSymbolicLinkOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) {
+ lastPath = linkPath;
+ setState(targetFragment);
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ PathFragment newLinkTarget = new PathFragment("/something/else");
+ dirLink.getRelative("a").createSymbolicLink(newLinkTarget);
+ assertEquals(dirLinkTarget.getRelative("a"), delegator.lastPath());
+ assertSame(newLinkTarget, delegator.objectState());
+ }
+
+ @Test
+ public void testCallReadSymbolicLinkOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected PathFragment readSymbolicLink(Path path) {
+ lastPath = path;
+ return (PathFragment) objectState;
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ // Since we're not following the link, this shouldn't invoke delegation.
+ delegator.setState(new PathFragment("whatever"));
+ PathFragment p = fileLink.readSymbolicLink();
+ assertEquals(null, delegator.lastPath());
+ assertNotSame(delegator.objectState(), p);
+
+ // This should.
+ p = dirLink.getRelative("a").readSymbolicLink();
+ assertEquals(dirLinkTarget.getRelative("a"), delegator.lastPath());
+ assertSame(delegator.objectState(), p);
+ }
+
+ @Test
+ public void testCallGetInputStreamOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected InputStream getInputStream(Path path) {
+ lastPath = path;
+ return (InputStream) objectState;
+ }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(new ByteArrayInputStream("blah".getBytes()));
+ InputStream is = fileLink.getInputStream();
+ assertEquals(fileLinkTarget, delegator.lastPath());
+ assertSame(delegator.objectState(), is);
+
+ delegator.setState(new ByteArrayInputStream("blah2".getBytes()));
+ is = dirLink.getInputStream();
+ assertEquals(dirLinkTarget, delegator.lastPath());
+ assertSame(delegator.objectState(), is);
+ }
+
+ @Test
+ public void testCallGetOutputStreamOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected OutputStream getOutputStream(Path path, boolean append) {
+ lastPath = path;
+ return (OutputStream) objectState;
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(new ByteArrayOutputStream());
+ OutputStream os = fileLink.getOutputStream();
+ assertEquals(fileLinkTarget, delegator.lastPath());
+ assertSame(delegator.objectState(), os);
+
+ delegator.setState(new ByteArrayOutputStream());
+ os = dirLink.getOutputStream();
+ assertEquals(dirLinkTarget, delegator.lastPath());
+ assertSame(delegator.objectState(), os);
+ }
+
+ @Test
+ public void testCallGetDirectoryEntriesOnEscapingSymlink() throws Exception {
+ TestDelegator delegator = new TestDelegator() {
+ @Override protected Collection<Path> getDirectoryEntries(Path path) {
+ lastPath = path;
+ return ImmutableList.of((Path) objectState);
+ }
+ @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; }
+ };
+ scopedFS().setDelegator(delegator);
+
+ delegator.setState(testFS.getPath("/anything"));
+ Collection<Path> entries = dirLink.getDirectoryEntries();
+ assertEquals(dirLinkTarget, delegator.lastPath());
+ assertEquals(1, entries.size());
+ assertSame(delegator.objectState(), entries.iterator().next());
+ }
+
+ /**
+ * Asserts that "link" is an in-scope link that doesn't result in an out-of-FS
+ * delegation. If link is relative, its path is relative to SCOPE_ROOT.
+ *
+ * Note that we don't actually check that the canonicalized target path matches
+ * the link's target value. Such testing should be covered by
+ * SymlinkAwareFileSystemTest.
+ */
+ private void assertInScopeLink(String link, String target, TestDelegator d) throws IOException {
+ Path l = testFS.getPath(SCOPE_ROOT.getRelative(link));
+ testFS.createSymbolicLink(l, new PathFragment(target));
+ l.exists();
+ assertNull(d.lastPath());
+ }
+
+ /**
+ * Asserts that "link" is an out-of-scope link and that the re-delegated path
+ * matches expectedPath. If link is relative, its path is relative to SCOPE_ROOT.
+ */
+ private void assertOutOfScopeLink(String link, String target, String expectedPath,
+ TestDelegator d) throws IOException {
+ Path l = testFS.getPath(SCOPE_ROOT.getRelative(link));
+ testFS.createSymbolicLink(l, new PathFragment(target));
+ l.exists();
+ assertEquals(expectedPath, d.lastPath().getPathString());
+ }
+
+ /**
+ * Returns the scope root with the final n segments chopped off (or a 0-segment path
+ * if n > SCOPE_ROOT.segmentCount()).
+ */
+ private String chopScopeRoot(int n) {
+ return SCOPE_ROOT
+ .subFragment(0, n > SCOPE_ROOT.segmentCount() ? 0 : SCOPE_ROOT.segmentCount() - n)
+ .getPathString();
+ }
+
+ /**
+ * Tests that absolute symlinks with ".." and "." segments are delegated to
+ * the expected paths.
+ */
+ @Test
+ public void testAbsoluteSymlinksWithParentReferences() throws Exception {
+ TestDelegator d = newExistsDelegator();
+ scopedFS().setDelegator(d);
+ testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir")));
+ String scopeRoot = SCOPE_ROOT.getPathString();
+ String scopeBase = SCOPE_ROOT.getBaseName();
+
+ // Symlinks that should never escape our scope.
+ assertInScopeLink("ilink1", scopeRoot, d);
+ assertInScopeLink("ilink2", scopeRoot + "/target", d);
+ assertInScopeLink("ilink3", scopeRoot + "/dir/../target", d);
+ assertInScopeLink("ilink4", scopeRoot + "/dir/../dir/dir2/../target", d);
+ assertInScopeLink("ilink5", scopeRoot + "/./dir/.././target", d);
+ assertInScopeLink("ilink6", scopeRoot + "/../" + scopeBase + "/target", d);
+ assertInScopeLink("ilink7", "/some/path/../.." + scopeRoot + "/target", d);
+
+ // Symlinks that should escape our scope.
+ assertOutOfScopeLink("olink1", scopeRoot + "/../target", chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("olink2", "/some/other/path", "/some/other/path", d);
+ assertOutOfScopeLink("olink3", scopeRoot + "/../target", chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("olink4", chopScopeRoot(1) + "/target", chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("olink5", scopeRoot + "/../../../../target", "/target", d);
+
+ // In-scope symlink that's not the final segment in a query.
+ Path iDirLink = testFS.getPath(SCOPE_ROOT.getRelative("ilinkdir"));
+ testFS.createSymbolicLink(iDirLink, SCOPE_ROOT.getRelative("dir"));
+ iDirLink.getRelative("file").exists();
+ assertNull(d.lastPath());
+
+ // Out-of-scope symlink that's not the final segment in a query.
+ Path oDirLink = testFS.getPath(SCOPE_ROOT.getRelative("olinkdir"));
+ testFS.createSymbolicLink(oDirLink, new PathFragment("/some/other/dir"));
+ oDirLink.getRelative("file").exists();
+ assertEquals("/some/other/dir/file", d.lastPath().getPathString());
+ }
+
+ /**
+ * Tests that relative symlinks with ".." and "." segments are delegated to
+ * the expected paths.
+ */
+ @Test
+ public void testRelativeSymlinksWithParentReferences() throws Exception {
+ TestDelegator d = newExistsDelegator();
+ scopedFS().setDelegator(d);
+ testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir")));
+ testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2")));
+ testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2/dir3")));
+ String scopeRoot = SCOPE_ROOT.getPathString();
+ String scopeBase = SCOPE_ROOT.getBaseName();
+
+ // Symlinks that should never escape our scope.
+ assertInScopeLink("ilink1", "target", d);
+ assertInScopeLink("ilink2", "dir/../otherdir/target", d);
+ assertInScopeLink("dir/ilink3", "../target", d);
+ assertInScopeLink("dir/dir2/ilink4", "../../target", d);
+ assertInScopeLink("dir/dir2/ilink5", ".././../dir/./target", d);
+ assertInScopeLink("dir/dir2/ilink6", "../dir2/../../dir/dir2/dir3/../../../target", d);
+
+ // Symlinks that should escape our scope.
+ assertOutOfScopeLink("olink1", "../target", chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("dir/olink2", "../../target", chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("olink3", "../" + scopeBase + "/target", scopeRoot + "/target", d);
+ assertOutOfScopeLink("dir/dir2/olink5", "../../../target", chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("dir/dir2/olink6", "../dir2/../../dir/dir2/../../../target",
+ chopScopeRoot(1) + "/target", d);
+ assertOutOfScopeLink("dir/olink7", "../../../target", chopScopeRoot(2) + "target", d);
+ assertOutOfScopeLink("olink8", "../../../../../target", "/target", d);
+
+ // In-scope symlink that's not the final segment in a query.
+ Path iDirLink = testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2/ilinkdir"));
+ testFS.createSymbolicLink(iDirLink, new PathFragment("../../dir"));
+ iDirLink.getRelative("file").exists();
+ assertNull(d.lastPath());
+
+ // Out-of-scope symlink that's not the final segment in a query.
+ Path oDirLink = testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2/olinkdir"));
+ testFS.createSymbolicLink(oDirLink, new PathFragment("../../../other/dir"));
+ oDirLink.getRelative("file").exists();
+ assertEquals(chopScopeRoot(1) + "/other/dir/file", d.lastPath().getPathString());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/SymlinkAwareFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/SymlinkAwareFileSystemTest.java
new file mode 100644
index 0000000..a728c88
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/SymlinkAwareFileSystemTest.java
@@ -0,0 +1,717 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.vfs.FileSystem.NotASymlinkException;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * This class handles the generic tests that any filesystem must pass.
+ *
+ * <p>Each filesystem-test should inherit from this class, thereby obtaining
+ * all the tests.
+ */
+public abstract class SymlinkAwareFileSystemTest extends FileSystemTest {
+
+ protected Path xLinkToFile;
+ protected Path xLinkToLinkToFile;
+ protected Path xLinkToDirectory;
+ protected Path xDanglingLink;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+
+ // % ls -lR
+ // -rw-rw-r-- xFile
+ // drwxrwxr-x xNonEmptyDirectory
+ // -rw-rw-r-- xNonEmptyDirectory/foo
+ // drwxrwxr-x xEmptyDirectory
+ // lrwxrwxr-x xLinkToFile -> xFile
+ // lrwxrwxr-x xLinkToDirectory -> xEmptyDirectory
+ // lrwxrwxr-x xLinkToLinkToFile -> xLinkToFile
+ // lrwxrwxr-x xDanglingLink -> xNothing
+
+ xLinkToFile = absolutize("xLinkToFile");
+ xLinkToLinkToFile = absolutize("xLinkToLinkToFile");
+ xLinkToDirectory = absolutize("xLinkToDirectory");
+ xDanglingLink = absolutize("xDanglingLink");
+
+ createSymbolicLink(xLinkToFile, xFile);
+ createSymbolicLink(xLinkToLinkToFile, xLinkToFile);
+ createSymbolicLink(xLinkToDirectory, xEmptyDirectory);
+ createSymbolicLink(xDanglingLink, xNothing);
+ }
+
+ @Test
+ public void testCreateLinkToFile() throws IOException {
+ Path newPath = xEmptyDirectory.getChild("new-file");
+ FileSystemUtils.createEmptyFile(newPath);
+
+ Path linkPath = xEmptyDirectory.getChild("some-link");
+
+ createSymbolicLink(linkPath, newPath);
+
+ assertTrue(linkPath.isSymbolicLink());
+
+ assertTrue(linkPath.isFile());
+ assertFalse(linkPath.isFile(Symlinks.NOFOLLOW));
+ assertTrue(linkPath.isFile(Symlinks.FOLLOW));
+
+ assertFalse(linkPath.isDirectory());
+ assertFalse(linkPath.isDirectory(Symlinks.NOFOLLOW));
+ assertFalse(linkPath.isDirectory(Symlinks.FOLLOW));
+
+ if (supportsSymlinks) {
+ assertEquals(newPath.toString().length(), linkPath.getFileSize(Symlinks.NOFOLLOW));
+ assertEquals(newPath.getFileSize(Symlinks.NOFOLLOW), linkPath.getFileSize());
+ }
+ assertEquals(2,
+ linkPath.getParentDirectory().getDirectoryEntries().size());
+ assertThat(linkPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath,
+ linkPath);
+ }
+
+ @Test
+ public void testCreateLinkToDirectory() throws IOException {
+ Path newPath = xEmptyDirectory.getChild("new-file");
+ newPath.createDirectory();
+
+ Path linkPath = xEmptyDirectory.getChild("some-link");
+
+ createSymbolicLink(linkPath, newPath);
+
+ assertTrue(linkPath.isSymbolicLink());
+ assertFalse(linkPath.isFile());
+ assertTrue(linkPath.isDirectory());
+ assertEquals(2,
+ linkPath.getParentDirectory().getDirectoryEntries().size());
+ assertThat(linkPath.getParentDirectory().
+ getDirectoryEntries()).containsExactly(newPath, linkPath);
+ }
+
+ @Test
+ public void testFileCanonicalPath() throws IOException {
+ Path newPath = absolutize("new-file");
+ FileSystemUtils.createEmptyFile(newPath);
+ newPath = newPath.resolveSymbolicLinks();
+
+ Path link1 = absolutize("some-link");
+ Path link2 = absolutize("some-link2");
+
+ createSymbolicLink(link1, newPath);
+ createSymbolicLink(link2, link1);
+
+ assertCanonicalPathsMatch(newPath, link1, link2);
+ }
+
+ @Test
+ public void testDirectoryCanonicalPath() throws IOException {
+ Path newPath = absolutize("new-folder");
+ newPath.createDirectory();
+ newPath = newPath.resolveSymbolicLinks();
+
+ Path newFile = newPath.getChild("file");
+ FileSystemUtils.createEmptyFile(newFile);
+
+ Path link1 = absolutize("some-link");
+ Path link2 = absolutize("some-link2");
+
+ createSymbolicLink(link1, newPath);
+ createSymbolicLink(link2, link1);
+
+ Path linkFile1 = link1.getChild("file");
+ Path linkFile2 = link2.getChild("file");
+
+ assertCanonicalPathsMatch(newFile, linkFile1, linkFile2);
+ }
+
+ private void assertCanonicalPathsMatch(Path newPath, Path link1, Path link2)
+ throws IOException {
+ assertEquals(newPath, link1.resolveSymbolicLinks());
+ assertEquals(newPath, link2.resolveSymbolicLinks());
+ }
+
+ //
+ // createDirectory
+ //
+
+ @Test
+ public void testCreateDirectoryWhereDanglingSymlinkAlreadyExists() {
+ try {
+ xDanglingLink.createDirectory();
+ fail();
+ } catch (IOException e) {
+ assertEquals(xDanglingLink + " (File exists)", e.getMessage());
+ }
+ assertTrue(xDanglingLink.isSymbolicLink()); // still a symbolic link
+ assertFalse(xDanglingLink.isDirectory(Symlinks.FOLLOW)); // link still dangles
+ }
+
+ @Test
+ public void testCreateDirectoryWhereSymlinkAlreadyExists() {
+ try {
+ xLinkToDirectory.createDirectory();
+ fail();
+ } catch (IOException e) {
+ assertEquals(xLinkToDirectory + " (File exists)", e.getMessage());
+ }
+ assertTrue(xLinkToDirectory.isSymbolicLink()); // still a symbolic link
+ assertTrue(xLinkToDirectory.isDirectory(Symlinks.FOLLOW)); // link still points to dir
+ }
+
+ // createSymbolicLink(PathFragment)
+
+ @Test
+ public void testCreateSymbolicLinkFromFragment() throws IOException {
+ String[] linkTargets = {
+ "foo",
+ "foo/bar",
+ ".",
+ "..",
+ "../foo",
+ "../../foo",
+ "../../../../../../../../../../../../../../../../../../../../../foo",
+ "/foo",
+ "/foo/bar",
+ "/..",
+ "/foo/../bar",
+ };
+ Path linkPath = absolutize("link");
+ for (String linkTarget : linkTargets) {
+ PathFragment relative = new PathFragment(linkTarget);
+ linkPath.delete();
+ createSymbolicLink(linkPath, relative);
+ if (supportsSymlinks) {
+ assertEquals(linkTarget.length(), linkPath.getFileSize(Symlinks.NOFOLLOW));
+ assertEquals(relative, linkPath.readSymbolicLink());
+ }
+ }
+ }
+
+ @Test
+ public void testLinkToRootResolvesCorrectly() throws IOException {
+ Path rootPath = testFS.getPath("/");
+ Path linkPath = absolutize("link");
+ createSymbolicLink(linkPath, rootPath);
+
+ // resolveSymbolicLinks requires an existing path:
+ try {
+ linkPath.getRelative("test").resolveSymbolicLinks();
+ fail();
+ } catch (FileNotFoundException e) { /* ok */ }
+
+ // The path may not be a symlink, neither on Darwin nor on Linux.
+ Path rootChild = testFS.getPath("/sbin");
+ if (!rootChild.isDirectory()) {
+ rootChild.createDirectory();
+ }
+ assertEquals(rootChild, linkPath.getRelative("sbin").resolveSymbolicLinks());
+ }
+
+ @Test
+ public void testLinkToFragmentContainingLinkResolvesCorrectly() throws IOException {
+ Path link1 = absolutize("link1");
+ PathFragment link1target = new PathFragment("link2/foo");
+ Path link2 = absolutize("link2");
+ Path link2target = xNonEmptyDirectory;
+
+ createSymbolicLink(link1, link1target); // ln -s link2/foo link1
+ createSymbolicLink(link2, link2target); // ln -s xNonEmptyDirectory link2
+ // link1 --> xNonEmptyDirectory/foo
+ assertEquals(link1.resolveSymbolicLinks(), link2target.getRelative("foo"));
+ }
+
+ //
+ // readSymbolicLink / resolveSymbolicLinks
+ //
+
+ @Test
+ public void testRecursiveSymbolicLink() throws IOException {
+ Path link = absolutize("recursive-link");
+ createSymbolicLink(link, link);
+
+ if (supportsSymlinks) {
+ try {
+ link.resolveSymbolicLinks();
+ fail();
+ } catch (IOException e) {
+ assertEquals(link + " (Too many levels of symbolic links)",
+ e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testMutuallyRecursiveSymbolicLinks() throws IOException {
+ Path link1 = absolutize("link1");
+ Path link2 = absolutize("link2");
+ createSymbolicLink(link2, link1);
+ createSymbolicLink(link1, link2);
+
+ if (supportsSymlinks) {
+ try {
+ link1.resolveSymbolicLinks();
+ fail();
+ } catch (IOException e) {
+ assertEquals(link1 + " (Too many levels of symbolic links)", e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testResolveSymbolicLinksENOENT() {
+ if (supportsSymlinks) {
+ try {
+ xDanglingLink.resolveSymbolicLinks();
+ fail();
+ } catch (IOException e) {
+ assertEquals(xNothing + " (No such file or directory)", e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testResolveSymbolicLinksENOTDIR() throws IOException {
+ if (supportsSymlinks) {
+ Path badLinkTarget = xFile.getChild("bad"); // parent is not a directory!
+ Path badLink = absolutize("badLink");
+ createSymbolicLink(badLink, badLinkTarget);
+ try {
+ badLink.resolveSymbolicLinks();
+ fail();
+ } catch (IOException e) {
+ // ok. Ideally we would assert "(Not a directory)" in the error
+ // message, but that would require yet another stat in the
+ // implementation.
+ }
+ }
+ }
+
+ @Test
+ public void testResolveSymbolicLinksWithUplevelRefs() throws IOException {
+ if (supportsSymlinks) {
+ // Create a series of links that refer to xFile as ./xFile,
+ // ./../foo/xFile, ./../../bar/foo/xFile, etc. They should all resolve
+ // to xFile.
+ Path ancestor = xFile;
+ String prefix = "./";
+ while ((ancestor = ancestor.getParentDirectory()) != null) {
+ xLinkToFile.delete();
+ createSymbolicLink(xLinkToFile, new PathFragment(prefix + xFile.relativeTo(ancestor)));
+ assertEquals(xFile, xLinkToFile.resolveSymbolicLinks());
+
+ prefix += "../";
+ }
+ }
+ }
+
+ @Test
+ public void testReadSymbolicLink() throws IOException {
+ if (supportsSymlinks) {
+ assertEquals(xNothing.toString(),
+ xDanglingLink.readSymbolicLink().toString());
+ }
+
+ assertEquals(xFile.toString(),
+ xLinkToFile.readSymbolicLink().toString());
+
+ assertEquals(xEmptyDirectory.toString(),
+ xLinkToDirectory.readSymbolicLink().toString());
+
+ try {
+ xFile.readSymbolicLink(); // not a link
+ fail();
+ } catch (NotASymlinkException e) {
+ assertEquals(xFile.toString(), e.getMessage());
+ }
+
+ try {
+ xNothing.readSymbolicLink(); // nothing there
+ fail();
+ } catch (IOException e) {
+ assertEquals(xNothing + " (No such file or directory)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCannotCreateSymbolicLinkWithReadOnlyParent()
+ throws IOException {
+ xEmptyDirectory.setWritable(false);
+ Path xChildOfReadonlyDir = xEmptyDirectory.getChild("x");
+ if (supportsSymlinks) {
+ try {
+ xChildOfReadonlyDir.createSymbolicLink(xNothing);
+ fail();
+ } catch (IOException e) {
+ assertEquals(xChildOfReadonlyDir + " (Permission denied)", e.getMessage());
+ }
+ }
+ }
+
+ //
+ // createSymbolicLink
+ //
+
+ @Test
+ public void testCanCreateDanglingLink() throws IOException {
+ Path newPath = absolutize("non-existing-dir/new-file");
+ Path someLink = absolutize("dangling-link");
+ createSymbolicLink(someLink, newPath);
+ assertTrue(someLink.isSymbolicLink());
+ assertTrue(someLink.exists(Symlinks.NOFOLLOW)); // the link itself exists
+ assertFalse(someLink.exists()); // ...but the referent doesn't
+ if (supportsSymlinks) {
+ try {
+ someLink.resolveSymbolicLinks();
+ } catch (FileNotFoundException e) {
+ assertEquals(newPath.getParentDirectory()
+ + " (No such file or directory)", e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testCannotCreateSymbolicLinkWithoutParent() throws IOException {
+ Path xChildOfMissingDir = xNothing.getChild("x");
+ if (supportsSymlinks) {
+ try {
+ xChildOfMissingDir.createSymbolicLink(xFile);
+ fail();
+ } catch (FileNotFoundException e) {
+ MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testCreateSymbolicLinkWhereNothingExists() throws IOException {
+ createSymbolicLink(xNothing, xFile);
+ assertTrue(xNothing.isSymbolicLink());
+ }
+
+ @Test
+ public void testCreateSymbolicLinkWhereDirectoryAlreadyExists() {
+ try {
+ createSymbolicLink(xEmptyDirectory, xFile);
+ fail();
+ } catch (IOException e) { // => couldn't be created
+ assertEquals(xEmptyDirectory + " (File exists)", e.getMessage());
+ }
+ assertTrue(xEmptyDirectory.isDirectory(Symlinks.NOFOLLOW));
+ }
+
+ @Test
+ public void testCreateSymbolicLinkWhereFileAlreadyExists() {
+ try {
+ createSymbolicLink(xFile, xEmptyDirectory);
+ fail();
+ } catch (IOException e) { // => couldn't be created
+ assertEquals(xFile + " (File exists)", e.getMessage());
+ }
+ assertTrue(xFile.isFile(Symlinks.NOFOLLOW));
+ }
+
+ @Test
+ public void testCreateSymbolicLinkWhereDanglingSymlinkAlreadyExists() {
+ try {
+ createSymbolicLink(xDanglingLink, xFile);
+ fail();
+ } catch (IOException e) {
+ assertEquals(xDanglingLink + " (File exists)", e.getMessage());
+ }
+ assertTrue(xDanglingLink.isSymbolicLink()); // still a symbolic link
+ assertFalse(xDanglingLink.isDirectory()); // link still dangles
+ }
+
+ @Test
+ public void testCreateSymbolicLinkWhereSymlinkAlreadyExists() {
+ try {
+ createSymbolicLink(xLinkToDirectory, xNothing);
+ fail();
+ } catch (IOException e) {
+ assertEquals(xLinkToDirectory + " (File exists)", e.getMessage());
+ }
+ assertTrue(xLinkToDirectory.isSymbolicLink()); // still a symbolic link
+ assertTrue(xLinkToDirectory.isDirectory()); // link still points to dir
+ }
+
+ @Test
+ public void testDeleteLink() throws IOException {
+ Path newPath = xEmptyDirectory.getChild("new-file");
+ Path someLink = xEmptyDirectory.getChild("a-link");
+ FileSystemUtils.createEmptyFile(newPath);
+ createSymbolicLink(someLink, newPath);
+
+ assertEquals(xEmptyDirectory.getDirectoryEntries().size(), 2);
+
+ assertTrue(someLink.delete());
+ assertEquals(xEmptyDirectory.getDirectoryEntries().size(), 1);
+
+ assertThat(xEmptyDirectory.getDirectoryEntries()).containsExactly(newPath);
+ }
+
+ // Testing the links
+ @Test
+ public void testLinkFollowedToDirectory() throws IOException {
+ Path theDirectory = absolutize("foo/");
+ assertTrue(theDirectory.createDirectory());
+ Path newPath1 = absolutize("foo/new-file-1");
+ Path newPath2 = absolutize("foo/new-file-2");
+ Path newPath3 = absolutize("foo/new-file-3");
+
+ FileSystemUtils.createEmptyFile(newPath1);
+ FileSystemUtils.createEmptyFile(newPath2);
+ FileSystemUtils.createEmptyFile(newPath3);
+
+ Path linkPath = absolutize("link");
+ createSymbolicLink(linkPath, theDirectory);
+
+ Path resultPath1 = absolutize("link/new-file-1");
+ Path resultPath2 = absolutize("link/new-file-2");
+ Path resultPath3 = absolutize("link/new-file-3");
+ assertThat(linkPath.getDirectoryEntries()).containsExactly(resultPath1, resultPath2,
+ resultPath3);
+ }
+
+ @Test
+ public void testDanglingLinkIsNoFile() throws IOException {
+ Path newPath1 = absolutize("new-file-1");
+ Path newPath2 = absolutize("new-file-2");
+ FileSystemUtils.createEmptyFile(newPath1);
+ assertTrue(newPath2.createDirectory());
+
+ Path linkPath1 = absolutize("link1");
+ Path linkPath2 = absolutize("link2");
+ createSymbolicLink(linkPath1, newPath1);
+ createSymbolicLink(linkPath2, newPath2);
+
+ newPath1.delete();
+ newPath2.delete();
+
+ assertFalse(linkPath1.isFile());
+ assertFalse(linkPath2.isDirectory());
+ }
+
+ @Test
+ public void testWriteOnLinkChangesFile() throws IOException {
+ Path testFile = absolutize("test-file");
+ FileSystemUtils.createEmptyFile(testFile);
+ String testData = "abc19";
+
+ Path testLink = absolutize("a-link");
+ createSymbolicLink(testLink, testFile);
+
+ FileSystemUtils.writeContentAsLatin1(testLink, testData);
+ String resultData =
+ new String(FileSystemUtils.readContentAsLatin1(testFile));
+
+ assertEquals(testData,resultData);
+ }
+
+ //
+ // Symlink tests:
+ //
+
+ @Test
+ public void testExistsWithSymlinks() throws IOException {
+ Path a = absolutize("a");
+ Path b = absolutize("b");
+ FileSystemUtils.createEmptyFile(b);
+ createSymbolicLink(a, b); // ln -sf "b" "a"
+ assertTrue(a.exists()); // = exists(FOLLOW)
+ assertTrue(b.exists()); // = exists(FOLLOW)
+ assertTrue(a.exists(Symlinks.FOLLOW));
+ assertTrue(b.exists(Symlinks.FOLLOW));
+ assertTrue(a.exists(Symlinks.NOFOLLOW));
+ assertTrue(b.exists(Symlinks.NOFOLLOW));
+ b.delete(); // "a" is now a dangling link
+ assertFalse(a.exists()); // = exists(FOLLOW)
+ assertFalse(b.exists()); // = exists(FOLLOW)
+ assertFalse(a.exists(Symlinks.FOLLOW));
+ assertFalse(b.exists(Symlinks.FOLLOW));
+
+ assertTrue(a.exists(Symlinks.NOFOLLOW)); // symlink still exists
+ assertFalse(b.exists(Symlinks.NOFOLLOW));
+ }
+
+ @Test
+ public void testIsDirectoryWithSymlinks() throws IOException {
+ Path a = absolutize("a");
+ Path b = absolutize("b");
+ b.createDirectory();
+ createSymbolicLink(a, b); // ln -sf "b" "a"
+ assertTrue(a.isDirectory()); // = isDirectory(FOLLOW)
+ assertTrue(b.isDirectory()); // = isDirectory(FOLLOW)
+ assertTrue(a.isDirectory(Symlinks.FOLLOW));
+ assertTrue(b.isDirectory(Symlinks.FOLLOW));
+ assertFalse(a.isDirectory(Symlinks.NOFOLLOW)); // it's a link!
+ assertTrue(b.isDirectory(Symlinks.NOFOLLOW));
+ b.delete(); // "a" is now a dangling link
+ assertFalse(a.isDirectory()); // = isDirectory(FOLLOW)
+ assertFalse(b.isDirectory()); // = isDirectory(FOLLOW)
+ assertFalse(a.isDirectory(Symlinks.FOLLOW));
+ assertFalse(b.isDirectory(Symlinks.FOLLOW));
+ assertFalse(a.isDirectory(Symlinks.NOFOLLOW));
+ assertFalse(b.isDirectory(Symlinks.NOFOLLOW));
+ }
+
+ @Test
+ public void testIsFileWithSymlinks() throws IOException {
+ Path a = absolutize("a");
+ Path b = absolutize("b");
+ FileSystemUtils.createEmptyFile(b);
+ createSymbolicLink(a, b); // ln -sf "b" "a"
+ assertTrue(a.isFile()); // = isFile(FOLLOW)
+ assertTrue(b.isFile()); // = isFile(FOLLOW)
+ assertTrue(a.isFile(Symlinks.FOLLOW));
+ assertTrue(b.isFile(Symlinks.FOLLOW));
+ assertFalse(a.isFile(Symlinks.NOFOLLOW)); // it's a link!
+ assertTrue(b.isFile(Symlinks.NOFOLLOW));
+ b.delete(); // "a" is now a dangling link
+ assertFalse(a.isFile()); // = isFile()
+ assertFalse(b.isFile()); // = isFile()
+ assertFalse(a.isFile());
+ assertFalse(b.isFile());
+ assertFalse(a.isFile(Symlinks.NOFOLLOW));
+ assertFalse(b.isFile(Symlinks.NOFOLLOW));
+ }
+
+ @Test
+ public void testGetDirectoryEntriesOnLinkToDirectory() throws Exception {
+ Path fooAlias = xNothing.getChild("foo");
+ createSymbolicLink(xNothing, xNonEmptyDirectory);
+ Collection<Path> dirents = xNothing.getDirectoryEntries();
+ assertThat(dirents).containsExactly(fooAlias);
+ }
+
+ @Test
+ public void testFilesOfLinkedDirectories() throws Exception {
+ Path child = xEmptyDirectory.getChild("child");
+ Path aliasToChild = xLinkToDirectory.getChild("child");
+
+ assertFalse(aliasToChild.exists());
+ FileSystemUtils.createEmptyFile(child);
+ assertTrue(aliasToChild.exists());
+ assertTrue(aliasToChild.isFile());
+ assertFalse(aliasToChild.isDirectory());
+
+ validateLinkedReferenceObeysReadOnly(child, aliasToChild);
+ validateLinkedReferenceObeysExecutable(child, aliasToChild);
+ }
+
+ @Test
+ public void testDirectoriesOfLinkedDirectories() throws Exception {
+ Path childDir = xEmptyDirectory.getChild("childDir");
+ Path linkToChildDir = xLinkToDirectory.getChild("childDir");
+
+ assertFalse(linkToChildDir.exists());
+ childDir.createDirectory();
+ assertTrue(linkToChildDir.exists());
+ assertTrue(linkToChildDir.isDirectory());
+ assertFalse(linkToChildDir.isFile());
+
+ validateLinkedReferenceObeysReadOnly(childDir, linkToChildDir);
+ validateLinkedReferenceObeysExecutable(childDir, linkToChildDir);
+ }
+
+ @Test
+ public void testDirectoriesOfLinkedDirectoriesOfLinkedDirectories() throws Exception {
+ Path childDir = xEmptyDirectory.getChild("childDir");
+ Path linkToLinkToDirectory = absolutize("xLinkToLinkToDirectory");
+ createSymbolicLink(linkToLinkToDirectory, xLinkToDirectory);
+ Path linkToChildDir = linkToLinkToDirectory.getChild("childDir");
+
+ assertFalse(linkToChildDir.exists());
+ childDir.createDirectory();
+ assertTrue(linkToChildDir.exists());
+ assertTrue(linkToChildDir.isDirectory());
+ assertFalse(linkToChildDir.isFile());
+
+ validateLinkedReferenceObeysReadOnly(childDir, linkToChildDir);
+ validateLinkedReferenceObeysExecutable(childDir, linkToChildDir);
+ }
+
+ private void validateLinkedReferenceObeysReadOnly(Path path, Path link) throws IOException {
+ path.setWritable(false);
+ assertFalse(path.isWritable());
+ assertFalse(link.isWritable());
+ path.setWritable(true);
+ assertTrue(path.isWritable());
+ assertTrue(link.isWritable());
+ path.setWritable(false);
+ assertFalse(path.isWritable());
+ assertFalse(link.isWritable());
+ }
+
+ private void validateLinkedReferenceObeysExecutable(Path path, Path link) throws IOException {
+ path.setExecutable(true);
+ assertTrue(path.isExecutable());
+ assertTrue(link.isExecutable());
+ path.setExecutable(false);
+ assertFalse(path.isExecutable());
+ assertFalse(link.isExecutable());
+ path.setExecutable(true);
+ assertTrue(path.isExecutable());
+ assertTrue(link.isExecutable());
+ }
+
+ @Test
+ public void testReadingFileFromLinkedDirectory() throws Exception {
+ Path linkedTo = absolutize("linkedTo");
+ linkedTo.createDirectory();
+ Path child = linkedTo.getChild("child");
+ FileSystemUtils.createEmptyFile(child);
+
+ byte[] outputData = "This is a test".getBytes();
+ FileSystemUtils.writeContent(child, outputData);
+
+ Path link = absolutize("link");
+ createSymbolicLink(link, linkedTo);
+ Path linkedChild = link.getChild("child");
+ byte[] inputData = FileSystemUtils.readContent(linkedChild);
+ assertArrayEquals(outputData, inputData);
+ }
+
+ @Test
+ public void testCreatingFileInLinkedDirectory() throws Exception {
+ Path linkedTo = absolutize("linkedTo");
+ linkedTo.createDirectory();
+ Path child = linkedTo.getChild("child");
+
+ Path link = absolutize("link");
+ createSymbolicLink(link, linkedTo);
+ Path linkedChild = link.getChild("child");
+ byte[] outputData = "This is a test".getBytes();
+ FileSystemUtils.writeContent(linkedChild, outputData);
+
+ byte[] inputData = FileSystemUtils.readContent(child);
+ assertArrayEquals(outputData, inputData);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnionFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnionFileSystemTest.java
new file mode 100644
index 0000000..396a9f8
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/UnionFileSystemTest.java
@@ -0,0 +1,330 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Tests for the UnionFileSystem, both of generic FileSystem functionality
+ * (inherited) and tests of UnionFileSystem-specific behavior.
+ */
+@RunWith(JUnit4.class)
+public class UnionFileSystemTest extends SymlinkAwareFileSystemTest {
+ private XAttrInMemoryFs inDelegate;
+ private XAttrInMemoryFs outDelegate;
+ private XAttrInMemoryFs defaultDelegate;
+ private UnionFileSystem unionfs;
+
+ private static final String XATTR_VAL = "SOME_XATTR_VAL";
+ private static final String XATTR_KEY = "SOME_XATTR_KEY";
+
+ private void setupDelegateFileSystems() {
+ inDelegate = new XAttrInMemoryFs(BlazeClock.instance());
+ outDelegate = new XAttrInMemoryFs(BlazeClock.instance());
+ defaultDelegate = new XAttrInMemoryFs(BlazeClock.instance());
+
+ unionfs = createDefaultUnionFileSystem();
+ }
+
+ private UnionFileSystem createDefaultUnionFileSystem() {
+ return createDefaultUnionFileSystem(false);
+ }
+
+ private UnionFileSystem createDefaultUnionFileSystem(boolean readOnly) {
+ return new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(
+ new PathFragment("/in"), inDelegate,
+ new PathFragment("/out"), outDelegate),
+ defaultDelegate, readOnly);
+ }
+
+ @Override
+ protected FileSystem getFreshFileSystem() {
+ // Executed with each new test because it is called by super.setUp().
+ setupDelegateFileSystems();
+ return unionfs;
+ }
+
+ @Override
+ public void destroyFileSystem(FileSystem fileSystem) {
+ // Nothing.
+ }
+
+ // Tests of UnionFileSystem-specific behavior below.
+
+ @Test
+ public void testBasicDelegation() throws Exception {
+ unionfs = createDefaultUnionFileSystem();
+ Path fooPath = unionfs.getPath("/foo");
+ Path inPath = unionfs.getPath("/in");
+ Path outPath = unionfs.getPath("/out/in.txt");
+ assertSame(inDelegate, unionfs.getDelegate(inPath));
+ assertSame(outDelegate, unionfs.getDelegate(outPath));
+ assertSame(defaultDelegate, unionfs.getDelegate(fooPath));
+ }
+
+ @Test
+ public void testBasicXattr() throws Exception {
+ Path fooPath = unionfs.getPath("/foo");
+ Path inPath = unionfs.getPath("/in");
+ Path outPath = unionfs.getPath("/out/in.txt");
+
+ assertArrayEquals(XATTR_VAL.getBytes(UTF_8), inPath.getxattr(XATTR_KEY));
+ assertArrayEquals(XATTR_VAL.getBytes(UTF_8), outPath.getxattr(XATTR_KEY));
+ assertArrayEquals(XATTR_VAL.getBytes(UTF_8), fooPath.getxattr(XATTR_KEY));
+ assertNull(inPath.getxattr("not_key"));
+ assertNull(outPath.getxattr("not_key"));
+ assertNull(fooPath.getxattr("not_key"));
+ }
+
+ @Test
+ public void testDefaultFileSystemRequired() throws Exception {
+ try {
+ new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(), null);
+ fail("Able to create a UnionFileSystem with no default!");
+ } catch (NullPointerException expected) {
+ // OK - should fail in this case.
+ }
+ }
+
+ // Check for appropriate registration and lookup of delegate filesystems based
+ // on path prefixes, including non-canonical paths.
+ @Test
+ public void testPrefixDelegation() throws Exception {
+ unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(
+ new PathFragment("/foo"), inDelegate,
+ new PathFragment("/foo/bar"), outDelegate), defaultDelegate);
+
+ assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/foo/foo.txt")));
+ assertSame(outDelegate, unionfs.getDelegate(unionfs.getPath("/foo/bar/foo.txt")));
+ assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/foo/bar/../foo.txt")));
+ assertSame(defaultDelegate, unionfs.getDelegate(unionfs.getPath("/bar/foo.txt")));
+ assertSame(defaultDelegate, unionfs.getDelegate(unionfs.getPath("/foo/bar/../..")));
+ }
+
+ // Checks that files cannot be modified when the filesystem is created
+ // read-only, even if the delegate filesystems are read/write.
+ @Test
+ public void testModificationFlag() throws Exception {
+ assertTrue(unionfs.supportsModifications());
+ Path outPath = unionfs.getPath("/out/foo.txt");
+ assertTrue(unionfs.createDirectory(outPath.getParentDirectory()));
+ OutputStream outFile = unionfs.getOutputStream(outPath);
+ outFile.write('b');
+ outFile.close();
+
+ unionfs.setExecutable(outPath, true);
+
+ // Note that this does not destroy the underlying filesystems;
+ // UnionFileSystem is just a view.
+ unionfs = createDefaultUnionFileSystem(true);
+ assertFalse(unionfs.supportsModifications());
+
+ InputStream outFileInput = unionfs.getInputStream(outPath);
+ int outFileByte = outFileInput.read();
+ outFileInput.close();
+ assertEquals('b', outFileByte);
+
+ assertTrue(unionfs.isExecutable(outPath));
+
+ // Modifying files through the unionfs isn't permitted, even if the
+ // delegates are read/write.
+ try {
+ unionfs.setExecutable(outPath, false);
+ fail("Modification to a read-only UnionFileSystem succeeded.");
+ } catch (UnsupportedOperationException expected) {
+ // OK - should fail.
+ }
+ }
+
+ // Checks that roots of delegate filesystems are created outside of the
+ // delegate filesystems; i.e. they can be seen from the filesystem of the parent.
+ @Test
+ public void testDelegateRootDirectoryCreation() throws Exception {
+ Path foo = unionfs.getPath("/foo");
+ Path bar = unionfs.getPath("/bar");
+ Path out = unionfs.getPath("/out");
+ assertTrue(unionfs.createDirectory(foo));
+ assertTrue(unionfs.createDirectory(bar));
+ assertTrue(unionfs.createDirectory(out));
+ Path outFile = unionfs.getPath("/out/in");
+ FileSystemUtils.writeContentAsLatin1(outFile, "Out");
+
+ // FileSystemTest.setUp() silently creates the test root on the filesystem...
+ Path testDirUnderRoot = unionfs.getPath(workingDir.asFragment().subFragment(0, 1));
+ assertThat(unionfs.getDirectoryEntries(unionfs.getRootDirectory())).containsExactly(foo, bar,
+ out, testDirUnderRoot);
+ assertThat(unionfs.getDirectoryEntries(out)).containsExactly(outFile);
+
+ assertSame(unionfs.getDelegate(foo), defaultDelegate);
+ assertEquals(foo.asFragment(), unionfs.adjustPath(foo, defaultDelegate).asFragment());
+ assertSame(unionfs.getDelegate(bar), defaultDelegate);
+ assertSame(unionfs.getDelegate(outFile), outDelegate);
+ assertSame(unionfs.getDelegate(out), outDelegate);
+
+ // As a fragment (i.e. without filesystem or root info), the path name should be preserved.
+ assertEquals(outFile.asFragment(), unionfs.adjustPath(outFile, outDelegate).asFragment());
+ }
+
+ // Ensure that the right filesystem is still chosen when paths contain "..".
+ @Test
+ public void testDelegationOfUpLevelReferences() throws Exception {
+ assertSame(defaultDelegate, unionfs.getDelegate(unionfs.getPath("/in/../foo.txt")));
+ assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/out/../in")));
+ assertSame(outDelegate, unionfs.getDelegate(unionfs.getPath("/out/../in/../out/foo.txt")));
+ assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/in/./foo.txt")));
+ }
+
+ // Basic *explicit* cross-filesystem symlink check.
+ // Note: This does not work implicitly yet, as the next test illustrates.
+ @Test
+ public void testCrossDeviceSymlinks() throws Exception {
+ assertTrue(unionfs.createDirectory(unionfs.getPath("/out")));
+
+ // Create an "/in" directory directly on the output delegate to bypass the
+ // UnionFileSystem's mapping.
+ assertTrue(inDelegate.getPath("/in").createDirectory());
+ OutputStream outStream = inDelegate.getPath("/in/bar.txt").getOutputStream();
+ outStream.write('i');
+ outStream.close();
+
+ Path outFoo = unionfs.getPath("/out/foo");
+ unionfs.createSymbolicLink(outFoo, new PathFragment("../in/bar.txt"));
+ assertTrue(unionfs.stat(outFoo, false).isSymbolicLink());
+
+ try {
+ unionfs.stat(outFoo, true).isFile();
+ fail("Stat on cross-device symlink succeeded!");
+ } catch (FileNotFoundException expected) {
+ // OK
+ }
+
+ Path resolved = unionfs.resolveSymbolicLinks(outFoo);
+ assertSame(unionfs, resolved.getFileSystem());
+ InputStream barInput = resolved.getInputStream();
+ int barChar = barInput.read();
+ barInput.close();
+ assertEquals('i', barChar);
+ }
+
+ @Test
+ public void testNoDelegateLeakage() throws Exception {
+ assertSame(unionfs, unionfs.getPath("/in/foo.txt").getFileSystem());
+ assertSame(unionfs, unionfs.getPath("/in/foo/bar").getParentDirectory().getFileSystem());
+ unionfs.createDirectory(unionfs.getPath("/out"));
+ unionfs.createDirectory(unionfs.getPath("/out/foo"));
+ unionfs.createDirectory(unionfs.getPath("/out/foo/bar"));
+ assertSame(unionfs, Iterables.getOnlyElement(unionfs.getDirectoryEntries(
+ unionfs.getPath("/out/foo"))).getParentDirectory().getFileSystem());
+ }
+
+ // Prefix mappings can apply to files starting with a prefix within a directory.
+ @Test
+ public void testWithinDirectoryMapping() throws Exception {
+ unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(
+ new PathFragment("/fruit/a"), inDelegate,
+ new PathFragment("/fruit/b"), outDelegate), defaultDelegate);
+ assertTrue(unionfs.createDirectory(unionfs.getPath("/fruit")));
+ assertTrue(defaultDelegate.getPath("/fruit").isDirectory());
+ assertTrue(inDelegate.getPath("/fruit").createDirectory());
+ assertTrue(outDelegate.getPath("/fruit").createDirectory());
+
+ Path apple = unionfs.getPath("/fruit/apple");
+ Path banana = unionfs.getPath("/fruit/banana");
+ Path cherry = unionfs.getPath("/fruit/cherry");
+ unionfs.createDirectory(apple);
+ unionfs.createDirectory(banana);
+ assertSame(inDelegate, unionfs.getDelegate(apple));
+ assertSame(outDelegate, unionfs.getDelegate(banana));
+ assertSame(defaultDelegate, unionfs.getDelegate(cherry));
+
+ FileSystemUtils.writeContentAsLatin1(apple.getRelative("table"), "penny");
+ FileSystemUtils.writeContentAsLatin1(banana.getRelative("nana"), "nanana");
+ FileSystemUtils.writeContentAsLatin1(cherry, "garcia");
+
+ assertEquals("penny", new String(
+ FileSystemUtils.readContentAsLatin1(inDelegate.getPath("/fruit/apple/table"))));
+ assertEquals("nanana", new String(
+ FileSystemUtils.readContentAsLatin1(outDelegate.getPath("/fruit/banana/nana"))));
+ assertEquals("garcia", new String(
+ FileSystemUtils.readContentAsLatin1(defaultDelegate.getPath("/fruit/cherry"))));
+ }
+
+ // Write using the VFS through a UnionFileSystem and check that the file can
+ // be read back in the same location using standard Java IO.
+ // There is a similar test in UnixFileSystem, but this is essential to ensure
+ // that paths aren't being remapped in some nasty way on the underlying FS.
+ @Test
+ public void testDelegateOperationsReflectOnLocalFilesystem() throws Exception {
+ unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(
+ workingDir.getParentDirectory().asFragment(), new UnixFileSystem()),
+ defaultDelegate, false);
+ // This is a child of the current tmpdir, and doesn't exist on its own.
+ // It would be created in setup(), but of course, that didn't use a UnixFileSystem.
+ unionfs.createDirectory(workingDir);
+ Path testFile = unionfs.getPath(workingDir.getRelative("test_file").asFragment());
+ assertTrue(testFile.asFragment().startsWith(workingDir.asFragment()));
+ String testString = "This is a test file";
+ FileSystemUtils.writeContentAsLatin1(testFile, testString);
+ try {
+ assertEquals(testString, new String(FileSystemUtils.readContentAsLatin1(testFile)));
+ } finally {
+ testFile.delete();
+ assertTrue(unionfs.delete(workingDir));
+ }
+ }
+
+ // Regression test for [UnionFS: Directory creation across mapping fails.]
+ @Test
+ public void testCreateParentsAcrossMapping() throws Exception {
+ unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(
+ new PathFragment("/out/dir"), outDelegate), defaultDelegate, false);
+ Path outDir = unionfs.getPath("/out/dir/biz/bang");
+ FileSystemUtils.createDirectoryAndParents(outDir);
+ assertTrue(outDir.isDirectory());
+ }
+
+ private static class XAttrInMemoryFs extends InMemoryFileSystem {
+ public XAttrInMemoryFs(Clock clock) {
+ super(clock);
+ }
+
+ @Override
+ protected byte[] getxattr(Path path, String name, boolean followSymlinks) {
+ assertSame(this, path.getFileSystem());
+ return (name.equals(XATTR_KEY)) ? XATTR_VAL.getBytes(UTF_8) : null;
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixFileSystemTest.java
new file mode 100644
index 0000000..0ded404
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixFileSystemTest.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+
+/**
+ * Tests for the {@link UnixFileSystem} class.
+ */
+@RunWith(JUnit4.class)
+public class UnixFileSystemTest extends SymlinkAwareFileSystemTest {
+
+ @Override
+ protected FileSystem getFreshFileSystem() {
+ return new UnixFileSystem();
+ }
+
+ @Override
+ public void destroyFileSystem(FileSystem fileSystem) {
+ // Nothing.
+ }
+
+ @Override
+ protected void expectNotFound(Path path) throws IOException {
+ assertNull(path.statIfFound());
+ }
+
+ // Most tests are just inherited from FileSystemTest.
+
+ @Test
+ public void testCircularSymlinkFound() throws Exception {
+ Path linkA = absolutize("link-a");
+ Path linkB = absolutize("link-b");
+ linkA.createSymbolicLink(linkB);
+ linkB.createSymbolicLink(linkA);
+ assertFalse(linkA.exists(Symlinks.FOLLOW));
+ try {
+ linkA.statIfFound(Symlinks.FOLLOW);
+ fail();
+ } catch (IOException expected) {
+ // Expected.
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixPathEqualityTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathEqualityTest.java
new file mode 100644
index 0000000..f5f58e2
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathEqualityTest.java
@@ -0,0 +1,118 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * This tests how canonical paths and non-canonical paths are equal with each
+ * other, and also how paths from different filesystems behave with each other.
+ */
+@RunWith(JUnit4.class)
+public class UnixPathEqualityTest {
+
+ private FileSystem otherUnixFs;
+ private FileSystem unixFs;
+
+ @Before
+ public void setUp() throws Exception {
+ unixFs = new UnixFileSystem();
+ otherUnixFs = new UnixFileSystem();
+ assertTrue(unixFs != otherUnixFs);
+ }
+
+ private void assertTwoWayEquals(Object obj1, Object obj2) {
+ assertTrue(obj1.equals(obj2));
+ assertTrue(obj2.equals(obj1));
+ assertEquals(obj1.hashCode(), obj2.hashCode());
+ }
+
+ private void assertTwoWayNotEquals(Object obj1, Object obj2) {
+ assertFalse(obj1.equals(obj2));
+ assertFalse(obj2.equals(obj1));
+ }
+
+ @Test
+ public void testPathsAreEqualEvenIfNotCanonical() {
+ // This path is already canonical, so there's no difference between
+ // the canonical / nonCanonical path, as far as equals is concerned
+ Path nonCanonical = unixFs.getPath("/a/canonical/unix/path");
+ Path canonical = unixFs.getPath("/a/canonical/unix/path");
+ assertTwoWayEquals(nonCanonical, canonical);
+ }
+
+ @Test
+ public void testPathsAreNeverEqualWithStrings() {
+ // Make sure that paths aren't equal to plain old strings
+ Path nonCanonical = unixFs.getPath("/a/non/../canonical/unix/path");
+ Path canonical = unixFs.getPath("/a/non/../canonical/unix/path");
+ assertTwoWayNotEquals(nonCanonical, "/a/non/../canonical/unix/path");
+ assertTwoWayNotEquals(canonical, "/a/non/../canonical/unix/path");
+ }
+
+ @Test
+ public void testCanonicalPathsFromDifferentFileSystemsAreNeverEqual() {
+ Path canonical = unixFs.getPath("/canonical/path");
+ Path otherCanonical = otherUnixFs.getPath("/canonical/path");
+ assertTwoWayNotEquals(canonical, otherCanonical);
+ }
+
+ @Test
+ public void testNonCanonicalPathsFromDifferentFileSystemsAreNeverEqual() {
+ Path nonCanonical = unixFs.getPath("/non/canonical/path");
+ Path otherNonCanonical = otherUnixFs.getPath("/non/canonical/path");
+ assertTwoWayNotEquals(nonCanonical, otherNonCanonical);
+ }
+
+ @Test
+ public void testCrossFilesystemStartsWithReturnsFalse() {
+ assertFalse(unixFs.getPath("/a").startsWith(otherUnixFs.getPath("/b")));
+ }
+
+ @Test
+ public void testCrossFilesystemOperationsForbidden() throws Exception {
+ Path a = unixFs.getPath("/a");
+ Path b = otherUnixFs.getPath("/b");
+
+ try {
+ a.renameTo(b);
+ fail();
+ } catch (IllegalArgumentException e) {
+ assertThat(e.getMessage()).contains("different filesystems");
+ }
+
+ try {
+ a.relativeTo(b);
+ fail();
+ } catch (IllegalArgumentException e) {
+ assertThat(e.getMessage()).contains("different filesystems");
+ }
+
+ try {
+ a.createSymbolicLink(b);
+ fail();
+ } catch (IllegalArgumentException e) {
+ assertThat(e.getMessage()).contains("different filesystems");
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixPathGetParentTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathGetParentTest.java
new file mode 100644
index 0000000..0f679c3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathGetParentTest.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.vfs.util.FileSystems;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+
+/**
+ * A test for {@link Path} in the context of {@link UnixFileSystem}.
+ */
+@RunWith(JUnit4.class)
+public class UnixPathGetParentTest {
+
+ private FileSystem unixFs;
+ private Path testRoot;
+
+ @Before
+ public void setUp() throws Exception {
+ unixFs = FileSystems.initDefaultAsNative();
+ testRoot = unixFs.getPath(TestUtils.tmpDir()).getRelative("UnixPathGetParentTest");
+ FileSystemUtils.createDirectoryAndParents(testRoot);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ FileSystemUtils.deleteTree(testRoot); // (comment out during debugging)
+ }
+
+ private Path getParent(String path) {
+ return unixFs.getPath(path).getParentDirectory();
+ }
+
+ @Test
+ public void testAbsoluteRootHasNoParent() {
+ assertEquals(null, getParent("/"));
+ }
+
+ @Test
+ public void testParentOfSimpleDirectory() {
+ assertEquals("/foo", getParent("/foo/bar").getPathString());
+ }
+
+ @Test
+ public void testParentOfDotDotInMiddleOfPathname() {
+ assertEquals("/", getParent("/foo/../bar").getPathString());
+ }
+
+ @Test
+ public void testGetPathDoesNormalizationWithoutIO() throws IOException {
+ Path tmp = testRoot.getChild("tmp");
+ Path tmpWiz = tmp.getChild("wiz");
+
+ tmp.createDirectory();
+
+ // ln -sf /tmp /tmp/wiz
+ tmpWiz.createSymbolicLink(tmp);
+
+ assertEquals(testRoot, tmp.getParentDirectory());
+
+ assertEquals(tmp, tmpWiz.getParentDirectory());
+
+ // Under UNIX, inode(/tmp/wiz/..) == inode(/). However getPath() does not
+ // perform I/O, only string operations, so it disagrees:
+ assertEquals(tmp, tmp.getRelative(new PathFragment("wiz/..")));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixPathTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathTest.java
new file mode 100644
index 0000000..b593367
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathTest.java
@@ -0,0 +1,279 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.vfs.util.FileSystems;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Tests for {@link Path}.
+ */
+@RunWith(JUnit4.class)
+public class UnixPathTest {
+
+ private FileSystem unixFs;
+ private File aDirectory;
+ private File aFile;
+ private File anotherFile;
+ private File tmpDir;
+
+ protected FileSystem getUnixFileSystem() {
+ return FileSystems.initDefaultAsNative();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ unixFs = getUnixFileSystem();
+ tmpDir = new File(TestUtils.tmpDir(), "tmpDir");
+ tmpDir.mkdirs();
+ aDirectory = new File(tmpDir, "a_directory");
+ aDirectory.mkdirs();
+ aFile = new File(tmpDir, "a_file");
+ new FileWriter(aFile).close();
+ anotherFile = new File(aDirectory, "another_file.txt");
+ new FileWriter(anotherFile).close();
+ }
+
+ @Test
+ public void testExists() {
+ assertTrue(unixFs.getPath(aDirectory.getPath()).exists());
+ assertTrue(unixFs.getPath(aFile.getPath()).exists());
+ assertFalse(unixFs.getPath("/does/not/exist").exists());
+ }
+
+ @Test
+ public void testDirectoryEntriesForDirectory() throws IOException {
+ Collection<Path> entries =
+ unixFs.getPath(tmpDir.getPath()).getDirectoryEntries();
+ List<Path> expectedEntries = Arrays.asList(
+ unixFs.getPath(tmpDir.getPath() + "/a_file"),
+ unixFs.getPath(tmpDir.getPath() + "/a_directory"));
+
+ assertEquals(new HashSet<Object>(expectedEntries),
+ new HashSet<Object>(entries));
+ }
+
+ @Test
+ public void testDirectoryEntriesForFileThrowsException() {
+ try {
+ unixFs.getPath(aFile.getPath()).getDirectoryEntries();
+ fail("No exception thrown.");
+ } catch (IOException x) {
+ // The expected result.
+ }
+ }
+
+ @Test
+ public void testIsFileIsTrueForFile() {
+ assertTrue(unixFs.getPath(aFile.getPath()).isFile());
+ }
+
+ @Test
+ public void testIsFileIsFalseForDirectory() {
+ assertFalse(unixFs.getPath(aDirectory.getPath()).isFile());
+ }
+
+ @Test
+ public void testBaseName() {
+ assertEquals("base", unixFs.getPath("/foo/base").getBaseName());
+ }
+
+ @Test
+ public void testBaseNameRunsAfterDotDotInterpretation() {
+ assertEquals("base", unixFs.getPath("/base/foo/..").getBaseName());
+ }
+
+ @Test
+ public void testParentOfRootIsRoot() {
+ assertEquals(unixFs.getPath("/"), unixFs.getPath("/.."));
+ assertEquals(unixFs.getPath("/"), unixFs.getPath("/../../../../../.."));
+ assertEquals(unixFs.getPath("/foo"), unixFs.getPath("/../../../foo"));
+ }
+
+ @Test
+ public void testIsDirectory() {
+ assertTrue(unixFs.getPath(aDirectory.getPath()).isDirectory());
+ assertFalse(unixFs.getPath(aFile.getPath()).isDirectory());
+ assertFalse(unixFs.getPath("/does/not/exist").isDirectory());
+ }
+
+ @Test
+ public void testListNonExistingDirectoryThrowsException() {
+ try {
+ unixFs.getPath("/does/not/exist").getDirectoryEntries();
+ fail("No exception thrown.");
+ } catch (IOException ex) {
+ // success!
+ }
+ }
+
+ private void assertPathSet(Collection<Path> actual, String... expected) {
+ List<String> actualStrings = Lists.newArrayListWithCapacity(actual.size());
+
+ for (Path path : actual) {
+ actualStrings.add(path.getPathString());
+ }
+
+ assertThat(actualStrings).containsExactlyElementsIn(Arrays.asList(expected));
+ }
+
+ @Test
+ public void testGlob() throws Exception {
+ Collection<Path> textFiles = UnixGlob.forPath(unixFs.getPath(tmpDir.getPath()))
+ .addPattern("*/*.txt")
+ .globInterruptible();
+ assertEquals(1, textFiles.size());
+ Path onlyFile = textFiles.iterator().next();
+ assertEquals(unixFs.getPath(anotherFile.getPath()), onlyFile);
+
+ Collection<Path> onlyFiles =
+ UnixGlob.forPath(unixFs.getPath(tmpDir.getPath()))
+ .addPattern("*")
+ .setExcludeDirectories(true)
+ .globInterruptible();
+ assertPathSet(onlyFiles, aFile.getPath());
+
+ Collection<Path> directoriesToo =
+ UnixGlob.forPath(unixFs.getPath(tmpDir.getPath()))
+ .addPattern("*")
+ .setExcludeDirectories(false)
+ .globInterruptible();
+ assertPathSet(directoriesToo, aFile.getPath(), aDirectory.getPath());
+ }
+
+ @Test
+ public void testGetRelative() {
+ Path relative = unixFs.getPath("/foo").getChild("bar");
+ Path expected = unixFs.getPath("/foo/bar");
+ assertEquals(expected, relative);
+ }
+
+ @Test
+ public void testEqualsAndHash() {
+ Path path = unixFs.getPath("/foo/bar");
+ Path equalPath = unixFs.getPath("/foo/bar");
+ Path differentPath = unixFs.getPath("/foo/bar/baz");
+ Object differentType = new Object();
+
+ assertEquals(path.hashCode(), equalPath.hashCode());
+ assertEquals(path, equalPath);
+ assertFalse(path.equals(differentPath));
+ assertFalse(path.equals(differentType));
+ }
+
+ @Test
+ public void testLatin1ReadAndWrite() throws IOException {
+ char[] allLatin1Chars = new char[256];
+ for (int i = 0; i < 256; i++) {
+ allLatin1Chars[i] = (char) i;
+ }
+ Path path = unixFs.getPath(aFile.getPath());
+ String latin1String = new String(allLatin1Chars);
+ FileSystemUtils.writeContentAsLatin1(path, latin1String);
+ String fileContent = new String(FileSystemUtils.readContentAsLatin1(path));
+ assertEquals(fileContent, latin1String);
+ }
+
+ /**
+ * Verify that the encoding implemented by
+ * {@link FileSystemUtils#writeContentAsLatin1(Path, String)}
+ * really is 8859-1 (latin1).
+ */
+ @Test
+ public void testVerifyLatin1() throws IOException {
+ char[] allLatin1Chars = new char[256];
+ for( int i = 0; i < 256; i++) {
+ allLatin1Chars[i] = (char)i;
+ }
+ Path path = unixFs.getPath(aFile.getPath());
+ String latin1String = new String(allLatin1Chars);
+ FileSystemUtils.writeContentAsLatin1(path, latin1String);
+ byte[] bytes = FileSystemUtils.readContent(path);
+ assertEquals(new String(bytes, "ISO-8859-1"), latin1String);
+ }
+
+ @Test
+ public void testBytesReadAndWrite() throws IOException {
+ byte[] bytes = new byte[] { (byte) 0xdeadbeef, (byte) 0xdeadbeef>>8,
+ (byte) 0xdeadbeef>>16, (byte) 0xdeadbeef>>24 };
+ Path path = unixFs.getPath(aFile.getPath());
+ FileSystemUtils.writeContent(path, bytes);
+ byte[] content = FileSystemUtils.readContent(path);
+ assertEquals(bytes.length, content.length);
+ for (int i = 0; i < bytes.length; i++) {
+ assertEquals(bytes[i], content[i]);
+ }
+ }
+
+ @Test
+ public void testInputOutputStreams() throws IOException {
+ Path path = unixFs.getPath(aFile.getPath());
+ OutputStream out = path.getOutputStream();
+ for (int i = 0; i < 256; i++) {
+ out.write(i);
+ }
+ out.close();
+ InputStream in = path.getInputStream();
+ for (int i = 0; i < 256; i++) {
+ assertEquals(i, in.read());
+ }
+ assertEquals(-1, in.read());
+ in.close();
+ }
+
+ @Test
+ public void testAbsolutePathRoot() {
+ assertEquals("/", new Path(null).toString());
+ }
+
+ @Test
+ public void testAbsolutePath() {
+ Path segment = new Path(null, "bar.txt",
+ new Path(null, "foo", new Path(null)));
+ assertEquals("/foo/bar.txt", segment.toString());
+ }
+
+ @Test
+ public void testDerivedSegmentEquality() {
+ Path absoluteSegment = new Path(null);
+
+ Path derivedNode = absoluteSegment.getChild("derivedSegment");
+ Path otherDerivedNode = absoluteSegment.getChild("derivedSegment");
+
+ assertSame(derivedNode, otherDerivedNode);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/ZipFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/ZipFileSystemTest.java
new file mode 100644
index 0000000..9dc1276
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/ZipFileSystemTest.java
@@ -0,0 +1,233 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Lists;
+import com.google.common.io.CharStreams;
+import com.google.devtools.build.lib.testutil.BlazeTestUtils;
+import com.google.devtools.build.lib.testutil.TestConstants;
+import com.google.devtools.build.lib.vfs.util.FileSystems;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+@RunWith(JUnit4.class)
+public class ZipFileSystemTest {
+
+ /**
+ * Expected listing of sample zip files, in alpha sorted order
+ */
+ private static final String[] LISTING = {
+ "/dir1",
+ "/dir1/file1a",
+ "/dir1/file1b",
+ "/dir2",
+ "/dir2/dir3",
+ "/dir2/dir3/dir4",
+ "/dir2/dir3/dir4/file4",
+ "/dir2/file2",
+ "/file0",
+ };
+
+ private FileSystem zipFS1;
+ private FileSystem zipFS2;
+
+ @Before
+ public void setUp() throws Exception {
+ FileSystem unixFs = FileSystems.initDefaultAsNative();
+ Path testdataDir = unixFs.getPath(BlazeTestUtils.runfilesDir()).getRelative(
+ TestConstants.JAVATESTS_ROOT + "/com/google/devtools/build/lib/vfs");
+ Path zPath1 = testdataDir.getChild("sample_with_dirs.zip");
+ Path zPath2 = testdataDir.getChild("sample_without_dirs.zip");
+ zipFS1 = new ZipFileSystem(zPath1);
+ zipFS2 = new ZipFileSystem(zPath2);
+ }
+
+ private void checkExists(FileSystem fs) {
+ assertTrue(fs.getPath("/dir2/dir3/dir4").exists());
+ assertTrue(fs.getPath("/dir2/dir3/dir4/file4").exists());
+ assertFalse(fs.getPath("/dir2/dir3/dir4/bogus").exists());
+ }
+
+ @Test
+ public void testExists() {
+ checkExists(zipFS1);
+ checkExists(zipFS2);
+ }
+
+ private void checkIsFile(FileSystem fs) {
+ assertFalse(fs.getPath("/dir2/dir3/dir4").isFile());
+ assertTrue(fs.getPath("/dir2/dir3/dir4/file4").isFile());
+ assertFalse(fs.getPath("/dir2/dir3/dir4/bogus").isFile());
+ }
+
+ @Test
+ public void testIsFile() {
+ checkIsFile(zipFS1);
+ checkIsFile(zipFS2);
+ }
+
+ private void checkIsDir(FileSystem fs) {
+ assertTrue(fs.getPath("/dir2/dir3/dir4").isDirectory());
+ assertFalse(fs.getPath("/dir2/dir3/dir4/file4").isDirectory());
+ assertFalse(fs.getPath("/bogus/mobogus").isDirectory());
+ assertFalse(fs.getPath("/bogus").isDirectory());
+ }
+
+ @Test
+ public void testIsDir() {
+ checkIsDir(zipFS1);
+ checkIsDir(zipFS2);
+ }
+
+ /**
+ * Recursively add the contents of a given path, rendered as strings, into a
+ * given list.
+ */
+ private static void listChildren(Path p, List<String> list)
+ throws IOException {
+ for (Path c : p.getDirectoryEntries()) {
+ list.add(c.getPathString());
+ if (c.isDirectory()) {
+ listChildren(c, list);
+ }
+ }
+ }
+
+ private void checkListing(FileSystem fs) throws Exception {
+ List<String> list = new ArrayList<>();
+ listChildren(fs.getRootDirectory(), list);
+ Collections.sort(list);
+ assertEquals(Lists.newArrayList(LISTING), list);
+ }
+
+ @Test
+ public void testListing() throws Exception {
+ checkListing(zipFS1);
+ checkListing(zipFS2);
+
+ // Regression test for: creation of a path (i.e. a file *name*)
+ // must not affect the result of getDirectoryEntries().
+ zipFS1.getPath("/dir1/notthere");
+ checkListing(zipFS1);
+ }
+
+ private void checkFileSize(FileSystem fs, String name, long expectedSize)
+ throws IOException {
+ assertEquals(expectedSize, fs.getPath(name).getFileSize());
+ }
+
+ @Test
+ public void testCanReadRoot() {
+ Path rootDirectory = zipFS1.getRootDirectory();
+ assertTrue(rootDirectory.isDirectory());
+ }
+
+ @Test
+ public void testFileSize() throws IOException {
+ checkFileSize(zipFS1, "/dir1/file1a", 5);
+ checkFileSize(zipFS2, "/dir1/file1a", 5);
+ checkFileSize(zipFS1, "/dir2/dir3/dir4/file4", 5000);
+ checkFileSize(zipFS2, "/dir2/dir3/dir4/file4", 5000);
+ }
+
+ private void checkCantGetFileSize(FileSystem fs, String name) {
+ try {
+ fs.getPath(name).getFileSize();
+ fail();
+ } catch (IOException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testCantGetFileSize() {
+ checkCantGetFileSize(zipFS1, "/dir2/dir3/dir4/bogus");
+ checkCantGetFileSize(zipFS2, "/dir2/dir3/dir4/bogus");
+ }
+
+ private void checkOpenFile(FileSystem fs, String name, int expectedSize)
+ throws Exception {
+ InputStream is = fs.getPath(name).getInputStream();
+ List<String> lines = CharStreams.readLines(new InputStreamReader(is, "ISO-8859-1"));
+ assertEquals(expectedSize, lines.size());
+ for (int i = 0; i < expectedSize; i++) {
+ assertEquals("body", lines.get(i));
+ }
+ }
+
+ @Test
+ public void testOpenSmallFile() throws Exception {
+ checkOpenFile(zipFS1, "/dir1/file1a", 1);
+ checkOpenFile(zipFS2, "/dir1/file1a", 1);
+ }
+
+ @Test
+ public void testOpenBigFile() throws Exception {
+ checkOpenFile(zipFS1, "/dir2/dir3/dir4/file4", 1000);
+ checkOpenFile(zipFS2, "/dir2/dir3/dir4/file4", 1000);
+ }
+
+ private void checkCantOpenFile(FileSystem fs, String name) {
+ try {
+ fs.getPath(name).getInputStream();
+ fail();
+ } catch (IOException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testCantOpenFile() throws Exception {
+ checkCantOpenFile(zipFS1, "/dir2/dir3/dir4/bogus");
+ checkCantOpenFile(zipFS2, "/dir2/dir3/dir4/bogus");
+ }
+
+ private void checkCantCreateAnything(FileSystem fs, String name) {
+ Path p = fs.getPath(name);
+ try {
+ p.createDirectory();
+ fail();
+ } catch (Exception expected) {}
+ try {
+ FileSystemUtils.createEmptyFile(p);
+ fail();
+ } catch (Exception expected) {}
+ try {
+ p.createSymbolicLink(p);
+ fail();
+ } catch (Exception expected) {}
+ }
+
+ @Test
+ public void testCantCreateAnything() throws Exception {
+ checkCantCreateAnything(zipFS1, "/dir2/dir3/dir4/new");
+ checkCantCreateAnything(zipFS2, "/dir2/dir3/dir4/new");
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfoTest.java b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfoTest.java
new file mode 100644
index 0000000..dbdd64a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfoTest.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class InMemoryContentInfoTest {
+
+ private Clock clock;
+
+ @Before
+ public void setUp() throws Exception {
+ clock = BlazeClock.instance();
+ }
+
+ @Test
+ public void testDirectoryCannotAddNullChild() {
+ InMemoryDirectoryInfo directory = new InMemoryDirectoryInfo(clock);
+
+ try {
+ directory.addChild("bar", null);
+ fail("NullPointerException not thrown.");
+ } catch (NullPointerException e) {
+ // success.
+ }
+ }
+
+ @Test
+ public void testDirectoryCannotAddChildTwice() {
+ InMemoryDirectoryInfo directory = new InMemoryDirectoryInfo(clock);
+ InMemoryFileInfo otherFile = new InMemoryFileInfo(clock);
+ directory.addChild("bar", otherFile);
+
+ try {
+ directory.addChild("bar", otherFile);
+ fail("IllegalArgumentException not thrown.");
+ } catch (IllegalArgumentException e) {
+ // success.
+ }
+ }
+
+ @Test
+ public void testDirectoryRemoveNonExistingChild() {
+ InMemoryDirectoryInfo directory = new InMemoryDirectoryInfo(clock);
+ try {
+ directory.removeChild("bar");
+ fail();
+ } catch (IllegalArgumentException e) {
+ // success
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystemTest.java
new file mode 100644
index 0000000..65ea6f6
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystemTest.java
@@ -0,0 +1,414 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.inmemoryfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.ScopeEscapableFileSystemTest;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Tests for {@link InMemoryFileSystem}. Note that most tests are inherited
+ * from {@link ScopeEscapableFileSystemTest} and ancestors. This specific
+ * file focuses only on concurrency tests.
+ *
+ */
+@RunWith(JUnit4.class)
+public class InMemoryFileSystemTest extends ScopeEscapableFileSystemTest {
+
+ @Override
+ public FileSystem getFreshFileSystem() {
+ return new InMemoryFileSystem(BlazeClock.instance(), SCOPE_ROOT);
+ }
+
+ @Override
+ public void destroyFileSystem(FileSystem fileSystem) {
+ // Nothing.
+ }
+
+ private static final int NUM_THREADS_FOR_CONCURRENCY_TESTS = 10;
+ private static final String TEST_FILE_DATA = "data";
+
+ /**
+ * Writes the given data to the given file.
+ */
+ private static void writeToFile(Path path, String data) throws IOException {
+ OutputStream out = path.getOutputStream();
+ out.write(data.getBytes(Charset.defaultCharset()));
+ out.close();
+ }
+
+ /**
+ * Tests concurrent creation of a substantial tree hierarchy including
+ * files, directories, symlinks, file contents, and permissions.
+ */
+ @Test
+ public void testConcurrentTreeConstruction() throws Exception {
+ final int NUM_TO_WRITE = 10000;
+ final AtomicInteger baseSelector = new AtomicInteger();
+
+ // 1) Define the intended path structure.
+ class PathCreator extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ Path base = testFS.getPath("/base" + baseSelector.getAndIncrement());
+ base.createDirectory();
+
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ Path subdir1 = base.getRelative("subdir1_" + i);
+ subdir1.createDirectory();
+ Path subdir2 = base.getRelative("subdir2_" + i);
+ subdir2.createDirectory();
+
+ Path file = base.getRelative("somefile" + i);
+ writeToFile(file, TEST_FILE_DATA);
+
+ subdir1.setReadable(true);
+ subdir2.setReadable(false);
+ file.setReadable(true);
+
+ subdir1.setWritable(false);
+ subdir2.setWritable(true);
+ file.setWritable(false);
+
+ subdir1.setExecutable(false);
+ subdir2.setExecutable(true);
+ file.setExecutable(false);
+
+ subdir1.setLastModifiedTime(100);
+ subdir2.setLastModifiedTime(200);
+ file.setLastModifiedTime(300);
+
+ Path symlink = base.getRelative("symlink" + i);
+ symlink.createSymbolicLink(file);
+ }
+ }
+ }
+
+ // 2) Construct the tree.
+ Collection<TestThread> threads =
+ Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS);
+ for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ TestThread thread = new PathCreator();
+ thread.start();
+ threads.add(thread);
+ }
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+
+ // 3) Define the validation logic.
+ class PathValidator extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ Path base = testFS.getPath("/base" + baseSelector.getAndIncrement());
+ assertTrue(base.exists());
+ assertFalse(base.getRelative("notreal").exists());
+
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ Path subdir1 = base.getRelative("subdir1_" + i);
+ assertTrue(subdir1.exists());
+ assertTrue(subdir1.isDirectory());
+ assertTrue(subdir1.isReadable());
+ assertFalse(subdir1.isWritable());
+ assertFalse(subdir1.isExecutable());
+ assertEquals(100, subdir1.getLastModifiedTime());
+
+ Path subdir2 = base.getRelative("subdir2_" + i);
+ assertTrue(subdir2.exists());
+ assertTrue(subdir2.isDirectory());
+ assertFalse(subdir2.isReadable());
+ assertTrue(subdir2.isWritable());
+ assertTrue(subdir2.isExecutable());
+ assertEquals(200, subdir2.getLastModifiedTime());
+
+ Path file = base.getRelative("somefile" + i);
+ assertTrue(file.exists());
+ assertTrue(file.isFile());
+ assertTrue(file.isReadable());
+ assertFalse(file.isWritable());
+ assertFalse(file.isExecutable());
+ assertEquals(300, file.getLastModifiedTime());
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(file.getInputStream(), Charset.defaultCharset()));
+ assertEquals(TEST_FILE_DATA, reader.readLine());
+ assertEquals(null, reader.readLine());
+
+ Path symlink = base.getRelative("symlink" + i);
+ assertTrue(symlink.exists());
+ assertTrue(symlink.isSymbolicLink());
+ assertEquals(file.asFragment(), symlink.readSymbolicLink());
+ }
+ }
+ }
+
+ // 4) Validate the results.
+ baseSelector.set(0);
+ threads = Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS);
+ for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ TestThread thread = new PathValidator();
+ thread.start();
+ threads.add(thread);
+ }
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+ }
+
+ /**
+ * Tests concurrent creation of many files, all within the same directory.
+ */
+ @Test
+ public void testConcurrentDirectoryConstruction() throws Exception {
+ final int NUM_TO_WRITE = 10000;
+ final AtomicInteger baseSelector = new AtomicInteger();
+
+ // 1) Define the intended path structure.
+ class PathCreator extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ final int threadId = baseSelector.getAndIncrement();
+ Path base = testFS.getPath("/common_dir");
+ base.createDirectory();
+
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ Path file = base.getRelative("somefile_" + threadId + "_" + i);
+ writeToFile(file, TEST_FILE_DATA);
+ file.setReadable(i % 2 == 0);
+ file.setWritable(i % 3 == 0);
+ file.setExecutable(i % 4 == 0);
+ file.setLastModifiedTime(i);
+ Path symlink = base.getRelative("symlink_" + threadId + "_" + i);
+ symlink.createSymbolicLink(file);
+ }
+ }
+ }
+
+ // 2) Create the files.
+ Collection<TestThread> threads =
+ Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS);
+ for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ TestThread thread = new PathCreator();
+ thread.start();
+ threads.add(thread);
+ }
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+
+ // 3) Define the validation logic.
+ class PathValidator extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ final int threadId = baseSelector.getAndIncrement();
+ Path base = testFS.getPath("/common_dir");
+ assertTrue(base.exists());
+
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ Path file = base.getRelative("somefile_" + threadId + "_" + i);
+ assertTrue(file.exists());
+ assertTrue(file.isFile());
+ assertEquals(i % 2 == 0, file.isReadable());
+ assertEquals(i % 3 == 0, file.isWritable());
+ assertEquals(i % 4 == 0, file.isExecutable());
+ assertEquals(i, file.getLastModifiedTime());
+ if (file.isReadable()) {
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(file.getInputStream(), Charset.defaultCharset()));
+ assertEquals(TEST_FILE_DATA, reader.readLine());
+ assertEquals(null, reader.readLine());
+ }
+
+ Path symlink = base.getRelative("symlink_" + threadId + "_" + i);
+ assertTrue(symlink.exists());
+ assertTrue(symlink.isSymbolicLink());
+ assertEquals(file.asFragment(), symlink.readSymbolicLink());
+ }
+ }
+ }
+
+ // 4) Validate the results.
+ baseSelector.set(0);
+ threads = Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS);
+ for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ TestThread thread = new PathValidator();
+ thread.start();
+ threads.add(thread);
+ }
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+ }
+
+ /**
+ * Tests concurrent file deletion.
+ */
+ @Test
+ public void testConcurrentDeletion() throws Exception {
+ final int NUM_TO_WRITE = 10000;
+ final AtomicInteger baseSelector = new AtomicInteger();
+
+ final Path base = testFS.getPath("/base");
+ base.createDirectory();
+
+ // 1) Create a bunch of files.
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ writeToFile(base.getRelative("file" + i), TEST_FILE_DATA);
+ }
+
+ // 2) Define our deletion strategy.
+ class FileDeleter extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ for (int i = 0; i < NUM_TO_WRITE / NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ int whichFile = baseSelector.getAndIncrement();
+ Path file = base.getRelative("file" + whichFile);
+ if (whichFile % 25 != 0) {
+ assertTrue(file.delete());
+ } else {
+ // Throw another concurrent access point into the mix.
+ file.setExecutable(whichFile % 2 == 0);
+ }
+ assertFalse(base.getRelative("doesnotexist" + whichFile).delete());
+ }
+ }
+ }
+
+ // 3) Delete some files.
+ Collection<TestThread> threads =
+ Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS);
+ for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ TestThread thread = new FileDeleter();
+ thread.start();
+ threads.add(thread);
+ }
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+
+ // 4) Check the results.
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ Path file = base.getRelative("file" + i);
+ if (i % 25 != 0) {
+ assertFalse(file.exists());
+ } else {
+ assertTrue(file.exists());
+ assertEquals(i % 2 == 0, file.isExecutable());
+ }
+ }
+ }
+
+ /**
+ * Tests concurrent file renaming.
+ */
+ @Test
+ public void testConcurrentRenaming() throws Exception {
+ final int NUM_TO_WRITE = 10000;
+ final AtomicInteger baseSelector = new AtomicInteger();
+
+ final Path base = testFS.getPath("/base");
+ base.createDirectory();
+
+ // 1) Create a bunch of files.
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ writeToFile(base.getRelative("file" + i), TEST_FILE_DATA);
+ }
+
+ // 2) Define our renaming strategy.
+ class FileDeleter extends TestThread {
+ @Override
+ public void runTest() throws Exception {
+ for (int i = 0; i < NUM_TO_WRITE / NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ int whichFile = baseSelector.getAndIncrement();
+ Path file = base.getRelative("file" + whichFile);
+ if (whichFile % 25 != 0) {
+ Path newName = base.getRelative("newname" + whichFile);
+ file.renameTo(newName);
+ } else {
+ // Throw another concurrent access point into the mix.
+ file.setExecutable(whichFile % 2 == 0);
+ }
+ assertFalse(base.getRelative("doesnotexist" + whichFile).delete());
+ }
+ }
+ }
+
+ // 3) Rename some files.
+ Collection<TestThread> threads =
+ Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS);
+ for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) {
+ TestThread thread = new FileDeleter();
+ thread.start();
+ threads.add(thread);
+ }
+ for (TestThread thread : threads) {
+ thread.joinAndAssertState(0);
+ }
+
+ // 4) Check the results.
+ for (int i = 0; i < NUM_TO_WRITE; i++) {
+ Path file = base.getRelative("file" + i);
+ if (i % 25 != 0) {
+ assertFalse(file.exists());
+ assertTrue(base.getRelative("newname" + i).exists());
+ } else {
+ assertTrue(file.exists());
+ assertEquals(i % 2 == 0, file.isExecutable());
+ }
+ }
+ }
+
+ @Test
+ public void testEloop() throws Exception {
+ Path a = testFS.getPath("/a");
+ Path b = testFS.getPath("/b");
+ a.createSymbolicLink(new PathFragment("b"));
+ b.createSymbolicLink(new PathFragment("a"));
+ try {
+ a.stat();
+ } catch (IOException e) {
+ assertEquals("/a (Too many levels of symbolic links)", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEloopSelf() throws Exception {
+ Path a = testFS.getPath("/a");
+ a.createSymbolicLink(new PathFragment("a"));
+ try {
+ a.stat();
+ } catch (IOException e) {
+ assertEquals("/a (Too many levels of symbolic links)", e.getMessage());
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/sample_with_dirs.zip b/src/test/java/com/google/devtools/build/lib/vfs/sample_with_dirs.zip
new file mode 100644
index 0000000..22ff63c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/sample_with_dirs.zip
Binary files differ
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/sample_without_dirs.zip b/src/test/java/com/google/devtools/build/lib/vfs/sample_without_dirs.zip
new file mode 100644
index 0000000..f3ec5ab
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/sample_without_dirs.zip
Binary files differ
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/util/FileSystems.java b/src/test/java/com/google/devtools/build/lib/vfs/util/FileSystems.java
new file mode 100644
index 0000000..6a79a2b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/util/FileSystems.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.util;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnionFileSystem;
+import com.google.devtools.build.lib.vfs.UnixFileSystem;
+import com.google.devtools.build.lib.vfs.ZipFileSystem;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * This static file system singleton manages access to a single default
+ * {@link FileSystem} instance created within the methods of this class.
+ */
+@ThreadSafe
+public final class FileSystems {
+
+ private FileSystems() {}
+
+ private static FileSystem defaultFileSystem;
+
+ /**
+ * Initializes the default {@link FileSystem} instance as a platform native
+ * (Unix) file system, creating one iff needed, and returns the instance.
+ *
+ * <p>This method is idempotent as long as the initialization is of the same
+ * type (Native/JavaIo/Union).
+ */
+ public static synchronized FileSystem initDefaultAsNative() {
+ if (!(defaultFileSystem instanceof UnixFileSystem)) {
+ defaultFileSystem = new UnixFileSystem();
+ }
+ return defaultFileSystem;
+ }
+
+ /**
+ * Initializes the default {@link FileSystem} instance as a java.io.File
+ * file system, creating one iff needed, and returns the instance.
+ *
+ * <p>This method is idempotent as long as the initialization is of the same
+ * type (Native/JavaIo/Union).
+ */
+ public static synchronized FileSystem initDefaultAsJavaIo() {
+ if (!(defaultFileSystem instanceof JavaIoFileSystem)) {
+ defaultFileSystem = new JavaIoFileSystem();
+ }
+ return defaultFileSystem;
+ }
+
+ /**
+ * Initializes the default {@link FileSystem} instance as a
+ * {@link UnionFileSystem}, creating one iff needed,
+ * and returns the instance.
+ *
+ * <p>This method is idempotent as long as the initialization is of the same
+ * type (Native/JavaIo/Union).
+ *
+ * @param prefixMapping the desired mapping of path prefixes to delegate file systems
+ * @param rootFileSystem the default file system for paths that don't match any prefix map
+ */
+ public static synchronized FileSystem initDefaultAsUnion(
+ Map<PathFragment, FileSystem> prefixMapping, FileSystem rootFileSystem) {
+ if (!(defaultFileSystem instanceof UnionFileSystem)) {
+ defaultFileSystem = new UnionFileSystem(prefixMapping, rootFileSystem);
+ }
+ return defaultFileSystem;
+ }
+
+ /**
+ * Returns a new instance of a simple {@link FileSystem} implementation that
+ * presents the contents of a zip file as a read-only file system view.
+ */
+ public static FileSystem newZipFileSystem(Path zipFile) throws IOException {
+ return new ZipFileSystem(zipFile);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/util/FsApparatus.java b/src/test/java/com/google/devtools/build/lib/vfs/util/FsApparatus.java
new file mode 100644
index 0000000..5d93351
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/util/FsApparatus.java
@@ -0,0 +1,158 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.util;
+
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import junit.framework.AssertionFailedError;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Base class for a testing apparatus for a scratch filesystem.
+ */
+public class FsApparatus {
+
+ /* ---------- State that the apparatus initializes / operates on --------- */
+ protected FileSystem fileSystem = null;
+ protected Path workingDir = null;
+
+ public static FsApparatus newInMemory() {
+ return new FsApparatus();
+ }
+
+ // TestUtil.getTmpDir is slow, so cache the result here
+ private static final String TMP_DIR =
+ new File(TestUtils.tmpDir(), "bs").toString();
+
+
+ /**
+ * When using a Native file system, absolute paths will be treated as absolute paths on the unix
+ * file system, as opposed to paths relative to the backing temp directory. So for simplicity,
+ * you ought to only use relative paths for FsApparatus#file, FsApparatus#dir, and
+ * FsApparatus#path. Otherwise, be aware of the following issue
+ *
+ * Path p1 = scratch.path(...);
+ * Path p2 = scratch.path(p1.getPathString());
+ *
+ * We'd like the invariant that p1.equals(p2) regardless if scratch is in-memory or not, but this
+ * does not hold with our usage of Unix filesystems.
+ */
+ public static FsApparatus newNative() {
+ FileSystem fs = FileSystems.initDefaultAsNative();
+ Path wd = fs.getPath(TMP_DIR);
+
+ try {
+ FileSystemUtils.deleteTree(wd);
+ } catch (IOException e) {
+ throw new AssertionFailedError(e.getMessage());
+ }
+
+ return new FsApparatus(fs, wd);
+ }
+
+ private FsApparatus() {
+ fileSystem = new InMemoryFileSystem(BlazeClock.instance());
+ workingDir = fileSystem.getPath("/");
+ }
+
+ public FsApparatus(FileSystem fs, Path cwd) {
+ fileSystem = fs;
+ workingDir = cwd;
+ }
+
+ public FsApparatus(FileSystem fs) {
+ fileSystem = fs;
+ workingDir = fs.getPath("/");
+ }
+
+ public FileSystem fs() {
+ return fileSystem;
+ }
+
+ /**
+ * Initializes this apparatus (if it hasn't been initialized yet), and creates
+ * a scratch file in the scratch filesystem with the given {@code pathName}
+ * with {@code lines} being its content. The method returns a Path instance
+ * for the scratch file.
+ */
+ public Path file(String pathName, String... lines) throws IOException {
+ Path file = path(pathName);
+ Path parentDir = file.getParentDirectory();
+ if (!parentDir.exists()) {
+ FileSystemUtils.createDirectoryAndParents(parentDir);
+ }
+ if (file.exists()) {
+ throw new IOException("Could not create scratch file (file exists) "
+ + file);
+ }
+ String fileContent = StringUtilities.joinLines(lines);
+ FileSystemUtils.writeContentAsLatin1(file, fileContent);
+ return file;
+ }
+
+ /**
+ * Initializes this apparatus (if it hasn't been initialized yet), and creates
+ * a directory in the scratch filesystem, with the given {@code pathName}.
+ * Creates parent directories as necessary.
+ */
+ public Path dir(String pathName) throws IOException {
+ Path dir = path(pathName);
+ if (!dir.exists()) {
+ FileSystemUtils.createDirectoryAndParents(dir);
+ }
+ if (!dir.isDirectory()) {
+ throw new IOException("Exists, but is not a directory: " + dir);
+ }
+ return dir;
+ }
+
+ /**
+ * Initializes this apparatus (if it hasn't been initialized yet), and returns
+ * a path object describing a file, directory, or symlink pointed at by
+ * {@code pathName}. Note that this will not create any entity in the
+ * filesystem; i.e., the file that the object is describing may not exist in
+ * the filesystem.
+ */
+ public Path path(String pathName) {
+ return workingDir.getRelative(pathName);
+ }
+
+ /**
+ * Create a fresh directory in the system temporary directory, instead of the
+ * testing directory provided by the testing framework. This path is usually
+ * shorter than a path starting with TestUtil.getTmpDir(). We care about the
+ * length because of the path length restriction for Unix local socket files.
+ *
+ * Clients are responsible for deleting the directory after tests.
+ */
+ public Path createUnixTempDir() throws IOException {
+ if (fileSystem instanceof InMemoryFileSystem) {
+ throw new IOException("Can not create Unix temporary directories in "
+ + "an in-memory file system");
+ }
+ File file = File.createTempFile("scratch", "tmp");
+ final Path path = fileSystem.getPath(file.getAbsolutePath());
+ path.delete();
+ path.createDirectory();
+ return path;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/AllTests.java b/src/test/java/com/google/devtools/build/skyframe/AllTests.java
new file mode 100644
index 0000000..4ea3691
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/AllTests.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.devtools.build.lib.testutil.ClasspathSuite;
+
+import org.junit.runner.RunWith;
+
+/**
+ * Automatically collect the tests annotated with @RunWith in this package and all subpackages.
+ */
+@RunWith(ClasspathSuite.class)
+public class AllTests {
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/ChainedFunction.java b/src/test/java/com/google/devtools/build/skyframe/ChainedFunction.java
new file mode 100644
index 0000000..4f87bb3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/ChainedFunction.java
@@ -0,0 +1,87 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.skyframe.GraphTester.ValueComputer;
+import com.google.devtools.build.skyframe.ParallelEvaluator.SkyFunctionEnvironment;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+
+import java.util.concurrent.CountDownLatch;
+
+import javax.annotation.Nullable;
+
+/**
+ * {@link ValueComputer} that can be chained together with others of its type to synchronize the
+ * order in which builders finish.
+ */
+final class ChainedFunction implements SkyFunction {
+ @Nullable private final SkyValue value;
+ @Nullable private final CountDownLatch notifyStart;
+ @Nullable private final CountDownLatch waitToFinish;
+ @Nullable private final CountDownLatch notifyFinish;
+ private final boolean waitForException;
+ private final Iterable<SkyKey> deps;
+
+ ChainedFunction(@Nullable CountDownLatch notifyStart, @Nullable CountDownLatch waitToFinish,
+ @Nullable CountDownLatch notifyFinish, boolean waitForException,
+ @Nullable SkyValue value, Iterable<SkyKey> deps) {
+ this.notifyStart = notifyStart;
+ this.waitToFinish = waitToFinish;
+ this.notifyFinish = notifyFinish;
+ this.waitForException = waitForException;
+ Preconditions.checkState(this.waitToFinish != null || !this.waitForException, value);
+ this.value = value;
+ this.deps = deps;
+ }
+
+ @Override
+ public SkyValue compute(SkyKey key, SkyFunction.Environment env) throws GenericFunctionException,
+ InterruptedException {
+ try {
+ if (notifyStart != null) {
+ notifyStart.countDown();
+ }
+ if (waitToFinish != null) {
+ TrackingAwaiter.waitAndMaybeThrowInterrupt(waitToFinish,
+ key + " timed out waiting to finish");
+ if (waitForException) {
+ SkyFunctionEnvironment skyEnv = (SkyFunctionEnvironment) env;
+ TrackingAwaiter.waitAndMaybeThrowInterrupt(skyEnv.getExceptionLatchForTesting(),
+ key + " timed out waiting for exception");
+ }
+ }
+ for (SkyKey dep : deps) {
+ env.getValue(dep);
+ }
+ if (value == null) {
+ throw new GenericFunctionException(new SomeErrorException("oops"),
+ Transience.PERSISTENT);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return value;
+ } finally {
+ if (notifyFinish != null) {
+ notifyFinish.countDown();
+ }
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/CycleDeduperTest.java b/src/test/java/com/google/devtools/build/skyframe/CycleDeduperTest.java
new file mode 100644
index 0000000..592d95f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/CycleDeduperTest.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Simple tests for {@link CycleDeduper}. */
+@RunWith(JUnit4.class)
+public class CycleDeduperTest {
+
+ private CycleDeduper<String> cycleDeduper = new CycleDeduper<>();
+
+ @Test
+ public void simple() throws Exception {
+ assertTrue(cycleDeduper.seen(ImmutableList.of("a", "b")));
+ assertFalse(cycleDeduper.seen(ImmutableList.of("a", "b")));
+ assertFalse(cycleDeduper.seen(ImmutableList.of("b", "a")));
+
+ assertTrue(cycleDeduper.seen(ImmutableList.of("a", "b", "c")));
+ assertFalse(cycleDeduper.seen(ImmutableList.of("b", "c", "a")));
+ assertFalse(cycleDeduper.seen(ImmutableList.of("c", "a", "b")));
+ assertTrue(cycleDeduper.seen(ImmutableList.of("b", "a", "c")));
+ assertFalse(cycleDeduper.seen(ImmutableList.of("c", "b", "a")));
+ }
+
+ @Test
+ public void badCycle_Empty() throws Exception {
+ try {
+ cycleDeduper.seen(ImmutableList.<String>of());
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void badCycle_NonUniqueMembers() throws Exception {
+ try {
+ cycleDeduper.seen(ImmutableList.<String>of("a", "b", "a"));
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/CyclesReporterTest.java b/src/test/java/com/google/devtools/build/skyframe/CyclesReporterTest.java
new file mode 100644
index 0000000..35bda02
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/CyclesReporterTest.java
@@ -0,0 +1,84 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.skyframe.CyclesReporter.SingleCycleReporter;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+@RunWith(JUnit4.class)
+public class CyclesReporterTest {
+
+ private static final SkyKey DUMMY_KEY = new SkyKey(SkyFunctionName.computed("func"), "key");
+
+ @Test
+ public void nullEventHandler() {
+ CyclesReporter cyclesReporter = new CyclesReporter();
+ try {
+ cyclesReporter.reportCycles(ImmutableList.<CycleInfo>of(), DUMMY_KEY, null);
+ assertThat(false).isTrue();
+ } catch (NullPointerException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void notReportedAssertion() {
+ SingleCycleReporter singleReporter = new SingleCycleReporter() {
+ @Override
+ public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo,
+ boolean alreadyReported, EventHandler eventHandler) {
+ return false;
+ }
+ };
+
+ CycleInfo cycleInfo = new CycleInfo(ImmutableList.of(DUMMY_KEY));
+ CyclesReporter cyclesReporter = new CyclesReporter(singleReporter);
+ try {
+ cyclesReporter.reportCycles(ImmutableList.of(cycleInfo), DUMMY_KEY,
+ NullEventHandler.INSTANCE);
+ assertThat(false).isTrue();
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void smoke() {
+ final AtomicBoolean reported = new AtomicBoolean();
+ SingleCycleReporter singleReporter = new SingleCycleReporter() {
+ @Override
+ public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo,
+ boolean alreadyReported, EventHandler eventHandler) {
+ reported.set(true);
+ return true;
+ }
+ };
+
+ CycleInfo cycleInfo = new CycleInfo(ImmutableList.of(DUMMY_KEY));
+ CyclesReporter cyclesReporter = new CyclesReporter(singleReporter);
+ cyclesReporter.reportCycles(ImmutableList.of(cycleInfo), DUMMY_KEY,
+ NullEventHandler.INSTANCE);
+ assertThat(reported.get()).isTrue();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/DeterministicInMemoryGraph.java b/src/test/java/com/google/devtools/build/skyframe/DeterministicInMemoryGraph.java
new file mode 100644
index 0000000..8ea81d0
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/DeterministicInMemoryGraph.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+/** {@link NotifyingInMemoryGraph} that returns reverse deps ordered alphabetically. */
+public class DeterministicInMemoryGraph extends NotifyingInMemoryGraph {
+ public DeterministicInMemoryGraph(Listener listener) {
+ super(listener);
+ }
+
+ public DeterministicInMemoryGraph() {
+ super(Listener.NULL_LISTENER);
+ }
+
+ @Override
+ protected DeterministicValueEntry getEntry(SkyKey key) {
+ return new DeterministicValueEntry(key);
+ }
+
+ /**
+ * This class uses TreeSet to store reverse dependencies of NodeEntry. As a result all values are
+ * lexicographically sorted.
+ */
+ private class DeterministicValueEntry extends NotifyingNodeEntry {
+ private DeterministicValueEntry(SkyKey myKey) {
+ super(myKey);
+ }
+
+ final Comparator<SkyKey> valueEntryComparator = new Comparator<SkyKey>() {
+ @Override
+ public int compare(SkyKey o1, SkyKey o2) {
+ return o1.toString().compareTo(o2.toString());
+ }
+ };
+ @SuppressWarnings("unchecked")
+ @Override
+ synchronized Iterable<SkyKey> getReverseDeps() {
+ TreeSet<SkyKey> result = new TreeSet<SkyKey>(valueEntryComparator);
+ if (reverseDeps instanceof List) {
+ result.addAll((Collection<? extends SkyKey>) reverseDeps);
+ } else {
+ result.add((SkyKey) reverseDeps);
+ }
+ return result;
+ }
+
+ @Override
+ synchronized Set<SkyKey> getInProgressReverseDeps() {
+ TreeSet<SkyKey> result = new TreeSet<SkyKey>(valueEntryComparator);
+ result.addAll(buildingState.getReverseDepsToSignal());
+ return result;
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/EagerInvalidatorTest.java b/src/test/java/com/google/devtools/build/skyframe/EagerInvalidatorTest.java
new file mode 100644
index 0000000..f12754a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/EagerInvalidatorTest.java
@@ -0,0 +1,616 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.skyframe.GraphTester.CONCATENATE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.testing.GcFinalization;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.skyframe.GraphTester.StringValue;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingInvalidationState;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState;
+import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationType;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.lang.ref.WeakReference;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link InvalidatingNodeVisitor}.
+ */
+@RunWith(Enclosed.class)
+public class EagerInvalidatorTest {
+ protected InMemoryGraph graph;
+ protected GraphTester tester = new GraphTester();
+ protected InvalidationState state = newInvalidationState();
+ protected AtomicReference<InvalidatingNodeVisitor> visitor = new AtomicReference<>();
+ protected DirtyKeyTrackerImpl dirtyKeyTracker;
+
+ private IntVersion graphVersion = new IntVersion(0);
+
+ // The following three methods should be abstract, but junit4 does not allow us to run inner
+ // classes in an abstract outer class. Thus, we provide implementations. These methods will never
+ // be run because only the inner classes, annotated with @RunWith, will actually be executed.
+ EvaluationProgressReceiver.InvalidationState expectedState() {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unused") // Overridden by subclasses.
+ void invalidate(DirtiableGraph graph, EvaluationProgressReceiver invalidationReceiver,
+ SkyKey... keys) throws InterruptedException { throw new UnsupportedOperationException(); }
+
+ boolean gcExpected() { throw new UnsupportedOperationException(); }
+
+ private boolean isInvalidated(SkyKey key) {
+ NodeEntry entry = graph.get(key);
+ if (gcExpected()) {
+ return entry == null;
+ } else {
+ return entry == null || entry.isDirty();
+ }
+ }
+
+ private void assertChanged(SkyKey key) {
+ NodeEntry entry = graph.get(key);
+ if (gcExpected()) {
+ assertNull(entry);
+ } else {
+ assertTrue(entry.isChanged());
+ }
+ }
+
+ private void assertDirtyAndNotChanged(SkyKey key) {
+ NodeEntry entry = graph.get(key);
+ if (gcExpected()) {
+ assertNull(entry);
+ } else {
+ assertTrue(entry.isDirty());
+ assertFalse(entry.isChanged());
+ }
+
+ }
+
+ protected InvalidationState newInvalidationState() {
+ throw new UnsupportedOperationException("Sublcasses must override");
+ }
+
+ protected InvalidationType defaultInvalidationType() {
+ throw new UnsupportedOperationException("Sublcasses must override");
+ }
+
+ // Convenience method for eval-ing a single value.
+ protected SkyValue eval(boolean keepGoing, SkyKey key) throws InterruptedException {
+ SkyKey[] keys = { key };
+ return eval(keepGoing, keys).get(key);
+ }
+
+ protected <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, SkyKey... keys)
+ throws InterruptedException {
+ Reporter reporter = new Reporter();
+ ParallelEvaluator evaluator = new ParallelEvaluator(graph, graphVersion,
+ ImmutableMap.of(GraphTester.NODE_TYPE, tester.createDelegatingFunction()),
+ reporter, new MemoizingEvaluator.EmittedEventState(), keepGoing, 200, null,
+ new DirtyKeyTrackerImpl());
+ graphVersion = graphVersion.next();
+ return evaluator.eval(ImmutableList.copyOf(keys));
+ }
+
+ protected void invalidateWithoutError(@Nullable EvaluationProgressReceiver invalidationReceiver,
+ SkyKey... keys) throws InterruptedException {
+ invalidate(graph, invalidationReceiver, keys);
+ assertTrue(state.isEmpty());
+ }
+
+ protected void set(String name, String value) {
+ tester.set(name, new StringValue(value));
+ }
+
+ protected SkyKey skyKey(String name) {
+ return GraphTester.toSkyKeys(name)[0];
+ }
+
+ protected void assertValueValue(String name, String expectedValue) throws InterruptedException {
+ StringValue value = (StringValue) eval(false, skyKey(name));
+ assertEquals(expectedValue, value.getValue());
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ dirtyKeyTracker = new DirtyKeyTrackerImpl();
+ }
+
+ @Test
+ public void receiverWorks() throws Exception {
+ final Set<String> invalidated = Sets.newConcurrentHashSet();
+ EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ Preconditions.checkState(state == expectedState());
+ invalidated.add(((StringValue) value).getValue());
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ graph = new InMemoryGraph();
+ set("a", "a");
+ set("b", "b");
+ tester.getOrCreate("ab").addDependency("a").addDependency("b")
+ .setComputedValue(CONCATENATE);
+ assertValueValue("ab", "ab");
+
+ set("a", "c");
+ invalidateWithoutError(receiver, skyKey("a"));
+ assertThat(invalidated).containsExactly("a", "ab");
+ assertValueValue("ab", "cb");
+ set("b", "d");
+ invalidateWithoutError(receiver, skyKey("b"));
+ assertThat(invalidated).containsExactly("a", "ab", "b", "cb");
+ }
+
+ @Test
+ public void receiverIsNotNotifiedAboutValuesInError() throws Exception {
+ final Set<String> invalidated = Sets.newConcurrentHashSet();
+ EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ Preconditions.checkState(state == expectedState());
+ invalidated.add(((StringValue) value).getValue());
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ graph = new InMemoryGraph();
+ set("a", "a");
+ tester.getOrCreate("ab").addDependency("a").setHasError(true);
+ eval(false, skyKey("ab"));
+
+ invalidateWithoutError(receiver, skyKey("a"));
+ assertThat(invalidated).containsExactly("a").inOrder();
+ }
+
+ @Test
+ public void invalidateValuesNotInGraph() throws Exception {
+ final Set<String> invalidated = Sets.newConcurrentHashSet();
+ EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ Preconditions.checkState(state == InvalidationState.DIRTY);
+ invalidated.add(((StringValue) value).getValue());
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ graph = new InMemoryGraph();
+ invalidateWithoutError(receiver, skyKey("a"));
+ assertThat(invalidated).isEmpty();
+ set("a", "a");
+ assertValueValue("a", "a");
+ invalidateWithoutError(receiver, skyKey("b"));
+ assertThat(invalidated).isEmpty();
+ }
+
+ @Test
+ public void invalidatedValuesAreGCedAsExpected() throws Exception {
+ SkyKey key = GraphTester.skyKey("a");
+ HeavyValue heavyValue = new HeavyValue();
+ WeakReference<HeavyValue> weakRef = new WeakReference<>(heavyValue);
+ tester.set("a", heavyValue);
+
+ graph = new InMemoryGraph();
+ eval(false, key);
+ invalidate(graph, null, key);
+
+ tester = null;
+ heavyValue = null;
+ if (gcExpected()) {
+ GcFinalization.awaitClear(weakRef);
+ } else {
+ // Not a reliable check, but better than nothing.
+ System.gc();
+ Thread.sleep(300);
+ assertNotNull(weakRef.get());
+ }
+ }
+
+ @Test
+ public void reverseDepsConsistent() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a");
+ set("b", "b");
+ set("c", "c");
+ tester.getOrCreate("ab").addDependency("a").addDependency("b").setComputedValue(CONCATENATE);
+ tester.getOrCreate("bc").addDependency("b").addDependency("c").setComputedValue(CONCATENATE);
+ tester.getOrCreate("ab_c").addDependency("ab").addDependency("c")
+ .setComputedValue(CONCATENATE);
+ eval(false, skyKey("ab_c"), skyKey("bc"));
+
+ assertThat(graph.get(skyKey("a")).getReverseDeps()).containsExactly(skyKey("ab"));
+ assertThat(graph.get(skyKey("b")).getReverseDeps()).containsExactly(skyKey("ab"), skyKey("bc"));
+ assertThat(graph.get(skyKey("c")).getReverseDeps()).containsExactly(skyKey("ab_c"),
+ skyKey("bc"));
+
+ invalidateWithoutError(null, skyKey("ab"));
+ eval(false);
+
+ // The graph values should be gone.
+ assertTrue(isInvalidated(skyKey("ab")));
+ assertTrue(isInvalidated(skyKey("abc")));
+
+ // The reverse deps to ab and ab_c should have been removed.
+ assertThat(graph.get(skyKey("a")).getReverseDeps()).isEmpty();
+ assertThat(graph.get(skyKey("b")).getReverseDeps()).containsExactly(skyKey("bc"));
+ assertThat(graph.get(skyKey("c")).getReverseDeps()).containsExactly(skyKey("bc"));
+ }
+
+ @Test
+ public void interruptChild() throws Exception {
+ graph = new InMemoryGraph();
+ int numValues = 50; // More values than the invalidator has threads.
+ final SkyKey[] family = new SkyKey[numValues];
+ final SkyKey child = GraphTester.skyKey("child");
+ final StringValue childValue = new StringValue("child");
+ tester.set(child, childValue);
+ family[0] = child;
+ for (int i = 1; i < numValues; i++) {
+ SkyKey member = skyKey(Integer.toString(i));
+ tester.getOrCreate(member).addDependency(family[i - 1]).setComputedValue(CONCATENATE);
+ family[i] = member;
+ }
+ SkyKey parent = GraphTester.skyKey("parent");
+ tester.getOrCreate(parent).addDependency(family[numValues - 1]).setComputedValue(CONCATENATE);
+ eval(/*keepGoing=*/false, parent);
+ final Thread mainThread = Thread.currentThread();
+ final AtomicReference<SkyValue> badValue = new AtomicReference<>();
+ EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ if (value == childValue) {
+ // Interrupt on the very first invalidate
+ mainThread.interrupt();
+ } else if (!childValue.equals(value)) {
+ // All other invalidations should be of the same value.
+ // Exceptions thrown here may be silently dropped, so keep track of errors ourselves.
+ badValue.set(value);
+ }
+ try {
+ assertTrue(visitor.get().awaitInterruptionForTestingOnly(2, TimeUnit.HOURS));
+ } catch (InterruptedException e) {
+ // We may well have thrown here because by the time we try to await, the main thread is
+ // already interrupted.
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ try {
+ invalidateWithoutError(receiver, child);
+ fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ assertNull(badValue.get());
+ assertFalse(state.isEmpty());
+ final Set<SkyValue> invalidated = Sets.newConcurrentHashSet();
+ assertFalse(isInvalidated(parent));
+ SkyValue parentValue = graph.getValue(parent);
+ assertNotNull(parentValue);
+ receiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ invalidated.add(value);
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ invalidateWithoutError(receiver);
+ assertTrue(invalidated.contains(parentValue));
+ assertThat(state.getInvalidationsForTesting()).isEmpty();
+
+ // Regression test coverage:
+ // "all pending values are marked changed on interrupt".
+ assertTrue(isInvalidated(child));
+ assertChanged(child);
+ for (int i = 1; i < numValues; i++) {
+ assertDirtyAndNotChanged(family[i]);
+ }
+ assertDirtyAndNotChanged(parent);
+ }
+
+ private SkyKey[] constructLargeGraph(int size) {
+ Random random = new Random(TestUtils.getRandomSeed());
+ SkyKey[] values = new SkyKey[size];
+ for (int i = 0; i < size; i++) {
+ String iString = Integer.toString(i);
+ SkyKey iKey = GraphTester.toSkyKey(iString);
+ set(iString, iString);
+ for (int j = 0; j < i; j++) {
+ if (random.nextInt(3) == 0) {
+ tester.getOrCreate(iKey).addDependency(Integer.toString(j));
+ }
+ }
+ values[i] = iKey;
+ }
+ return values;
+ }
+
+ /** Returns a subset of {@code nodes} that are still valid and so can be invalidated. */
+ private Set<Pair<SkyKey, InvalidationType>> getValuesToInvalidate(SkyKey[] nodes) {
+ Set<Pair<SkyKey, InvalidationType>> result = new HashSet<>();
+ Random random = new Random(TestUtils.getRandomSeed());
+ for (SkyKey node : nodes) {
+ if (!isInvalidated(node)) {
+ if (result.isEmpty() || random.nextInt(3) == 0) {
+ // Add at least one node, if we can.
+ result.add(Pair.of(node, defaultInvalidationType()));
+ }
+ }
+ }
+ return result;
+ }
+
+ @Test
+ public void interruptThreadInReceiver() throws Exception {
+ Random random = new Random(TestUtils.getRandomSeed());
+ int graphSize = 1000;
+ int tries = 5;
+ graph = new InMemoryGraph();
+ SkyKey[] values = constructLargeGraph(graphSize);
+ eval(/*keepGoing=*/false, values);
+ final Thread mainThread = Thread.currentThread();
+ for (int run = 0; run < tries; run++) {
+ Set<Pair<SkyKey, InvalidationType>> valuesToInvalidate = getValuesToInvalidate(values);
+ // Find how many invalidations will actually be enqueued for invalidation in the first round,
+ // so that we can interrupt before all of them are done.
+ int validValuesToDo =
+ Sets.difference(valuesToInvalidate, state.getInvalidationsForTesting()).size();
+ for (Pair<SkyKey, InvalidationType> pair : state.getInvalidationsForTesting()) {
+ if (!isInvalidated(pair.first)) {
+ validValuesToDo++;
+ }
+ }
+ int countDownStart = validValuesToDo > 0 ? random.nextInt(validValuesToDo) : 0;
+ final CountDownLatch countDownToInterrupt = new CountDownLatch(countDownStart);
+ final EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ countDownToInterrupt.countDown();
+ if (countDownToInterrupt.getCount() == 0) {
+ mainThread.interrupt();
+ try {
+ // Wait for the main thread to be interrupted uninterruptibly, because the main thread
+ // is going to interrupt us, and we don't want to get into an interrupt fight. Only
+ // if we get interrupted without the main thread also being interrupted will this
+ // throw an InterruptedException.
+ TrackingAwaiter.waitAndMaybeThrowInterrupt(
+ visitor.get().getInterruptionLatchForTestingOnly(),
+ "Main thread was not interrupted");
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ try {
+ invalidate(graph, receiver,
+ Sets.newHashSet(
+ Iterables.transform(valuesToInvalidate,
+ Pair.<SkyKey, InvalidationType>firstFunction())).toArray(new SkyKey[0]));
+ assertThat(state.getInvalidationsForTesting()).isEmpty();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ if (state.isEmpty()) {
+ // Ran out of values to invalidate.
+ break;
+ }
+ }
+
+ eval(/*keepGoing=*/false, values);
+ }
+
+ protected void setupInvalidatableGraph() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a");
+ set("b", "b");
+ tester.getOrCreate("ab").addDependency("a").addDependency("b").setComputedValue(CONCATENATE);
+ assertValueValue("ab", "ab");
+ set("a", "c");
+ }
+
+ private static class HeavyValue implements SkyValue {
+ }
+
+ /**
+ * Test suite for the deleting invalidator.
+ */
+ @RunWith(JUnit4.class)
+ public static class DeletingInvalidatorTest extends EagerInvalidatorTest {
+ @Override
+ protected void invalidate(DirtiableGraph graph, EvaluationProgressReceiver invalidationReceiver,
+ SkyKey... keys) throws InterruptedException {
+ InvalidatingNodeVisitor invalidatingVisitor =
+ EagerInvalidator.createVisitor(/*delete=*/true, graph, ImmutableList.copyOf(keys),
+ invalidationReceiver, state, true, dirtyKeyTracker);
+ if (invalidatingVisitor != null) {
+ visitor.set(invalidatingVisitor);
+ invalidatingVisitor.run();
+ }
+ }
+
+ @Override
+ EvaluationProgressReceiver.InvalidationState expectedState() {
+ return EvaluationProgressReceiver.InvalidationState.DELETED;
+ }
+
+ @Override
+ boolean gcExpected() {
+ return true;
+ }
+
+ @Override
+ protected InvalidationState newInvalidationState() {
+ return new InvalidatingNodeVisitor.DeletingInvalidationState();
+ }
+
+ @Override
+ protected InvalidationType defaultInvalidationType() {
+ return InvalidationType.DELETED;
+ }
+
+ @Test
+ public void dirtyKeyTrackerWorksWithDeletingInvalidator() throws Exception {
+ setupInvalidatableGraph();
+ TrackingInvalidationReceiver receiver = new TrackingInvalidationReceiver();
+
+ // Dirty the node, and ensure that the tracker is aware of it:
+ InvalidatingNodeVisitor dirtyingVisitor =
+ EagerInvalidator.createVisitor(/*delete=*/false, graph, ImmutableList.of(skyKey("a")),
+ receiver, new DirtyingInvalidationState(), true, dirtyKeyTracker);
+ dirtyingVisitor.run();
+ assertThat(dirtyKeyTracker.getDirtyKeys()).containsExactly(skyKey("a"), skyKey("ab"));
+
+ // Delete the node, and ensure that the tracker is no longer tracking it:
+ InvalidatingNodeVisitor deletingVisitor =
+ EagerInvalidator.createVisitor(/*delete=*/true, graph, ImmutableList.of(skyKey("a")),
+ receiver, state, true, dirtyKeyTracker);
+ deletingVisitor.run();
+ assertThat(dirtyKeyTracker.getDirtyKeys()).containsExactly(skyKey("ab"));
+ }
+ }
+
+ /**
+ * Test suite for the dirtying invalidator.
+ */
+ @RunWith(JUnit4.class)
+ public static class DirtyingInvalidatorTest extends EagerInvalidatorTest {
+ @Override
+ protected void invalidate(DirtiableGraph graph, EvaluationProgressReceiver invalidationReceiver,
+ SkyKey... keys) throws InterruptedException {
+ InvalidatingNodeVisitor invalidatingVisitor =
+ EagerInvalidator.createVisitor(/*delete=*/false, graph, ImmutableList.copyOf(keys),
+ invalidationReceiver, state, true, dirtyKeyTracker);
+ if (invalidatingVisitor != null) {
+ visitor.set(invalidatingVisitor);
+ invalidatingVisitor.run();
+ }
+ }
+
+ @Override
+ EvaluationProgressReceiver.InvalidationState expectedState() {
+ return EvaluationProgressReceiver.InvalidationState.DIRTY;
+ }
+
+ @Override
+ boolean gcExpected() {
+ return false;
+ }
+
+ @Override
+ protected InvalidationState newInvalidationState() {
+ return new DirtyingInvalidationState();
+ }
+
+ @Override
+ protected InvalidationType defaultInvalidationType() {
+ return InvalidationType.CHANGED;
+ }
+
+ @Test
+ public void dirtyKeyTrackerWorksWithDirtyingInvalidator() throws Exception {
+ setupInvalidatableGraph();
+ TrackingInvalidationReceiver receiver = new TrackingInvalidationReceiver();
+
+ // Dirty the node, and ensure that the tracker is aware of it:
+ invalidate(graph, receiver, skyKey("a"));
+ assertThat(dirtyKeyTracker.getDirtyKeys()).hasSize(2);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/GenericFunctionException.java b/src/test/java/com/google/devtools/build/skyframe/GenericFunctionException.java
new file mode 100644
index 0000000..667f213
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/GenericFunctionException.java
@@ -0,0 +1,27 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+/**
+ * A {@link SkyFunctionException} wrapping a {@link SomeErrorException}.
+ */
+public final class GenericFunctionException extends SkyFunctionException {
+ public GenericFunctionException(SomeErrorException e, Transience transience) {
+ super(e, transience);
+ }
+
+ public GenericFunctionException(SomeErrorException e, SkyKey childKey) {
+ super(e, childKey);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/GraphTester.java b/src/test/java/com/google/devtools/build/skyframe/GraphTester.java
new file mode 100644
index 0000000..6095bb8
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/GraphTester.java
@@ -0,0 +1,340 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A helper class to create graphs and run skyframe tests over these graphs.
+ *
+ * <p>There are two types of values, computing values, which may not be set to a constant value,
+ * and leaf values, which must be set to a constant value and may not have any dependencies.
+ *
+ * <p>Note that the value builder looks into the test values created here to determine how to
+ * behave. However, skyframe will only re-evaluate the value and call the value builder if any of
+ * its dependencies has changed. That means in order to change the set of dependencies of a value,
+ * you need to also change one of its previous dependencies to force re-evaluation. Changing a
+ * computing value does not mark it as modified.
+ */
+public class GraphTester {
+
+ // TODO(bazel-team): Split this for computing and non-computing values?
+ public static final SkyFunctionName NODE_TYPE = new SkyFunctionName("Type", false);
+
+ private final Map<SkyKey, TestFunction> values = new HashMap<>();
+ private final Set<SkyKey> modifiedValues = new LinkedHashSet<>();
+
+ public TestFunction getOrCreate(String name) {
+ return getOrCreate(skyKey(name));
+ }
+
+ public TestFunction getOrCreate(SkyKey key) {
+ return getOrCreate(key, false);
+ }
+
+ public TestFunction getOrCreate(SkyKey key, boolean markAsModified) {
+ TestFunction result = values.get(key);
+ if (result == null) {
+ result = new TestFunction();
+ values.put(key, result);
+ } else if (markAsModified) {
+ modifiedValues.add(key);
+ }
+ return result;
+ }
+
+ public TestFunction set(String key, SkyValue value) {
+ return set(skyKey(key), value);
+ }
+
+ public TestFunction set(SkyKey key, SkyValue value) {
+ return getOrCreate(key, true).setConstantValue(value);
+ }
+
+ public Collection<SkyKey> getModifiedValues() {
+ return modifiedValues;
+ }
+
+ public SkyFunction getFunction() {
+ return new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey key, Environment env)
+ throws SkyFunctionException, InterruptedException {
+ TestFunction builder = values.get(key);
+ Preconditions.checkState(builder != null, "No TestFunction for " + key);
+ if (builder.builder != null) {
+ return builder.builder.compute(key, env);
+ }
+ if (builder.warning != null) {
+ env.getListener().handle(Event.warn(builder.warning));
+ }
+ if (builder.progress != null) {
+ env.getListener().handle(Event.progress(builder.progress));
+ }
+ Map<SkyKey, SkyValue> deps = new LinkedHashMap<>();
+ boolean oneMissing = false;
+ for (Pair<SkyKey, SkyValue> dep : builder.deps) {
+ SkyValue value;
+ if (dep.second == null) {
+ value = env.getValue(dep.first);
+ } else {
+ try {
+ value = env.getValueOrThrow(dep.first, SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ value = dep.second;
+ }
+ }
+ if (value == null) {
+ oneMissing = true;
+ } else {
+ deps.put(dep.first, value);
+ }
+ Preconditions.checkState(oneMissing == env.valuesMissing());
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ if (builder.hasTransientError) {
+ throw new GenericFunctionException(new SomeErrorException(key.toString()),
+ Transience.TRANSIENT);
+ }
+ if (builder.hasError) {
+ throw new GenericFunctionException(new SomeErrorException(key.toString()),
+ Transience.PERSISTENT);
+ }
+
+ if (builder.value != null) {
+ return builder.value;
+ }
+
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException(key.toString());
+ }
+
+ return builder.computer.compute(deps, env);
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return values.get(skyKey).tag;
+ }
+ };
+ }
+
+ public static SkyKey skyKey(String key) {
+ return new SkyKey(NODE_TYPE, key);
+ }
+
+ /**
+ * A value in the testing graph that is constructed in the tester.
+ */
+ public class TestFunction {
+ // TODO(bazel-team): We could use a multiset here to simulate multi-pass dependency discovery.
+ private final Set<Pair<SkyKey, SkyValue>> deps = new LinkedHashSet<>();
+ private SkyValue value;
+ private ValueComputer computer;
+ private SkyFunction builder = null;
+
+ private boolean hasTransientError;
+ private boolean hasError;
+
+ private String warning;
+ private String progress;
+
+ private String tag;
+
+ public TestFunction addDependency(String name) {
+ return addDependency(skyKey(name));
+ }
+
+ public TestFunction addDependency(SkyKey key) {
+ deps.add(Pair.<SkyKey, SkyValue>of(key, null));
+ return this;
+ }
+
+ public TestFunction removeDependency(String name) {
+ return removeDependency(skyKey(name));
+ }
+
+ public TestFunction removeDependency(SkyKey key) {
+ deps.remove(Pair.<SkyKey, SkyValue>of(key, null));
+ return this;
+ }
+
+ public TestFunction addErrorDependency(String name, SkyValue altValue) {
+ return addErrorDependency(skyKey(name), altValue);
+ }
+
+ public TestFunction addErrorDependency(SkyKey key, SkyValue altValue) {
+ deps.add(Pair.of(key, altValue));
+ return this;
+ }
+
+ public TestFunction setConstantValue(SkyValue value) {
+ Preconditions.checkState(this.computer == null);
+ this.value = value;
+ return this;
+ }
+
+ public TestFunction setComputedValue(ValueComputer computer) {
+ Preconditions.checkState(this.value == null);
+ this.computer = computer;
+ return this;
+ }
+
+ public TestFunction setBuilder(SkyFunction builder) {
+ Preconditions.checkState(this.value == null);
+ Preconditions.checkState(this.computer == null);
+ Preconditions.checkState(deps.isEmpty());
+ Preconditions.checkState(!hasTransientError);
+ Preconditions.checkState(!hasError);
+ Preconditions.checkState(warning == null);
+ Preconditions.checkState(progress == null);
+ this.builder = builder;
+ return this;
+ }
+
+ public TestFunction setHasTransientError(boolean hasError) {
+ this.hasTransientError = hasError;
+ return this;
+ }
+
+ public TestFunction setHasError(boolean hasError) {
+ // TODO(bazel-team): switch to an enum for hasError.
+ this.hasError = hasError;
+ return this;
+ }
+
+ public TestFunction setWarning(String warning) {
+ this.warning = warning;
+ return this;
+ }
+
+ public TestFunction setProgress(String info) {
+ this.progress = info;
+ return this;
+ }
+
+ public TestFunction setTag(String tag) {
+ this.tag = tag;
+ return this;
+ }
+
+ }
+
+ public static SkyKey[] toSkyKeys(String... names) {
+ SkyKey[] result = new SkyKey[names.length];
+ for (int i = 0; i < names.length; i++) {
+ result[i] = new SkyKey(GraphTester.NODE_TYPE, names[i]);
+ }
+ return result;
+ }
+
+ public static SkyKey toSkyKey(String name) {
+ return toSkyKeys(name)[0];
+ }
+
+ private class DelegatingFunction implements SkyFunction {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ return getFunction().compute(skyKey, env);
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return getFunction().extractTag(skyKey);
+ }
+ }
+
+ public DelegatingFunction createDelegatingFunction() {
+ return new DelegatingFunction();
+ }
+
+ /**
+ * Simple value class that stores strings.
+ */
+ public static class StringValue implements SkyValue {
+ private final String value;
+
+ public StringValue(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof StringValue)) {
+ return false;
+ }
+ return value.equals(((StringValue) o).value);
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "StringValue: " + getValue();
+ }
+ }
+
+ /**
+ * A callback interface to provide the value computation.
+ */
+ public interface ValueComputer {
+ /** This is called when all the declared dependencies exist. It may request new dependencies. */
+ SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env)
+ throws InterruptedException;
+ }
+
+ public static final ValueComputer COPY = new ValueComputer() {
+ @Override
+ public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) {
+ return Iterables.getOnlyElement(deps.values());
+ }
+ };
+
+ public static final ValueComputer CONCATENATE = new ValueComputer() {
+ @Override
+ public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) {
+ StringBuilder result = new StringBuilder();
+ for (SkyValue value : deps.values()) {
+ result.append(((StringValue) value).value);
+ }
+ return new StringValue(result.toString());
+ }
+ };
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/MemoizingEvaluatorTest.java b/src/test/java/com/google/devtools/build/skyframe/MemoizingEvaluatorTest.java
new file mode 100644
index 0000000..00df824
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/MemoizingEvaluatorTest.java
@@ -0,0 +1,2914 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static com.google.devtools.build.skyframe.GraphTester.CONCATENATE;
+import static com.google.devtools.build.skyframe.GraphTester.COPY;
+import static com.google.devtools.build.skyframe.GraphTester.NODE_TYPE;
+import static com.google.devtools.build.skyframe.GraphTester.skyKey;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.testing.GcFinalization;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.events.DelegatingEventHandler;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventCollector;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.testutil.JunitTestUtils;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.skyframe.GraphTester.StringValue;
+import com.google.devtools.build.skyframe.GraphTester.TestFunction;
+import com.google.devtools.build.skyframe.GraphTester.ValueComputer;
+import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.EventType;
+import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Listener;
+import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Order;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link MemoizingEvaluator}.
+ */
+@RunWith(JUnit4.class)
+public class MemoizingEvaluatorTest {
+
+ private MemoizingEvaluatorTester tester;
+ private EventCollector eventCollector;
+ private EventHandler reporter;
+ private MemoizingEvaluator.EmittedEventState emittedEventState;
+
+ // Knobs that control the size / duration of larger tests.
+ private static final int TEST_NODE_COUNT = 100;
+ private static final int TESTED_NODES = 10;
+ private static final int RUNS = 10;
+
+ @Before
+ public void initializeTester() {
+ initializeTester(null);
+ }
+
+ public void initializeTester(@Nullable TrackingInvalidationReceiver customInvalidationReceiver) {
+ emittedEventState = new MemoizingEvaluator.EmittedEventState();
+ tester = new MemoizingEvaluatorTester();
+ if (customInvalidationReceiver != null) {
+ tester.setInvalidationReceiver(customInvalidationReceiver);
+ }
+ tester.initialize();
+ }
+
+ @Before
+ public void initializeReporter() {
+ eventCollector = new EventCollector(EventKind.ALL_EVENTS);
+ reporter = new Reporter(eventCollector);
+ tester.resetPlayedEvents();
+ }
+
+ protected static SkyKey toSkyKey(String name) {
+ return new SkyKey(NODE_TYPE, name);
+ }
+
+ @Test
+ public void smoke() throws Exception {
+ tester.set("x", new StringValue("y"));
+ StringValue value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ }
+
+ @Test
+ public void invalidationWithNothingChanged() throws Exception {
+ tester.set("x", new StringValue("y")).setWarning("fizzlepop");
+ StringValue value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+
+ initializeReporter();
+ tester.invalidate();
+ value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ private abstract static class NoExtractorFunction implements SkyFunction {
+ @Override
+ public final String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ }
+
+ @Test
+ // Regression test for bug: "[skyframe-m1]: registerIfDone() crash".
+ public void bubbleRace() throws Exception {
+ // The top-level value declares dependencies on a "badValue" in error, and a "sleepyValue"
+ // which is very slow. After "badValue" fails, the builder interrupts the "sleepyValue" and
+ // attempts to re-run "top" for error bubbling. Make sure this doesn't cause a precondition
+ // failure because "top" still has an outstanding dep ("sleepyValue").
+ tester.getOrCreate("top").setBuilder(new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
+ env.getValue(toSkyKey("sleepyValue"));
+ try {
+ env.getValueOrThrow(toSkyKey("badValue"), SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ // In order to trigger this bug, we need to request a dep on an already computed value.
+ env.getValue(toSkyKey("otherValue1"));
+ }
+ if (!env.valuesMissing()) {
+ throw new AssertionError("SleepyValue should always be unavailable");
+ }
+ return null;
+ }
+ });
+ tester.getOrCreate("sleepyValue").setBuilder(new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
+ Thread.sleep(99999);
+ throw new AssertionError("I should have been interrupted");
+ }
+ });
+ tester.getOrCreate("badValue").addDependency("otherValue1").setHasError(true);
+ tester.getOrCreate("otherValue1").setConstantValue(new StringValue("otherVal1"));
+
+ EvaluationResult<SkyValue> result = tester.eval(false, "top");
+ assertTrue(result.hasError());
+ assertEquals(toSkyKey("badValue"), Iterables.getOnlyElement(result.getError().getRootCauses()));
+ assertThat(result.keyNames()).isEmpty();
+ }
+
+ @Test
+ public void deleteValues() throws Exception {
+ tester.getOrCreate("top").setComputedValue(CONCATENATE)
+ .addDependency("d1").addDependency("d2").addDependency("d3");
+ tester.set("d1", new StringValue("1"));
+ StringValue d2 = new StringValue("2");
+ tester.set("d2", d2);
+ StringValue d3 = new StringValue("3");
+ tester.set("d3", d3);
+ tester.eval(true, "top");
+
+ tester.delete("d1");
+ tester.eval(true, "d3");
+
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertEquals(
+ ImmutableSet.of(new StringValue("1"), new StringValue("123")), tester.getDeletedValues());
+ assertEquals(null, tester.getExistingValue("top"));
+ assertEquals(null, tester.getExistingValue("d1"));
+ assertEquals(d2, tester.getExistingValue("d2"));
+ assertEquals(d3, tester.getExistingValue("d3"));
+ }
+
+ @Test
+ public void deleteOldNodesTest() throws Exception {
+ tester.getOrCreate("top").setComputedValue(CONCATENATE).addDependency("d1").addDependency("d2");
+ tester.set("d1", new StringValue("one"));
+ tester.set("d2", new StringValue("two"));
+ tester.eval(true, "top");
+
+ tester.set("d2", new StringValue("three"));
+ tester.invalidate();
+ tester.eval(true, "d2");
+
+ // The graph now contains the three above nodes (and ERROR_TRANSIENCE).
+ assertThat(tester.graph.getValues().keySet()).containsExactly(
+ skyKey("top"), skyKey("d1"), skyKey("d2"), ErrorTransienceValue.key());
+
+ String[] noKeys = {};
+ tester.graph.deleteDirty(2);
+ tester.eval(true, noKeys);
+
+ // The top node's value is dirty, but less than two generations old, so it wasn't deleted.
+ assertThat(tester.graph.getValues().keySet()).containsExactly(
+ skyKey("top"), skyKey("d1"), skyKey("d2"), ErrorTransienceValue.key());
+
+ tester.graph.deleteDirty(2);
+ tester.eval(true, noKeys);
+
+ // The top node's value was dirty, and was two generations old, so it was deleted.
+ assertThat(tester.graph.getValues().keySet()).containsExactly(
+ skyKey("d1"), skyKey("d2"), ErrorTransienceValue.key());
+ }
+
+ @Test
+ public void deleteNonexistentValues() throws Exception {
+ tester.getOrCreate("d1").setConstantValue(new StringValue("1"));
+ tester.delete("d1");
+ tester.delete("d2");
+ tester.eval(true, "d1");
+ }
+
+ @Test
+ public void signalValueEnqueued() throws Exception {
+ tester.getOrCreate("top1").setComputedValue(CONCATENATE)
+ .addDependency("d1").addDependency("d2");
+ tester.getOrCreate("top2").setComputedValue(CONCATENATE).addDependency("d3");
+ tester.getOrCreate("top3");
+ assertThat(tester.getEnqueuedValues()).isEmpty();
+
+ tester.set("d1", new StringValue("1"));
+ tester.set("d2", new StringValue("2"));
+ tester.set("d3", new StringValue("3"));
+ tester.eval(true, "top1");
+ assertThat(tester.getEnqueuedValues()).containsExactlyElementsIn(
+ Arrays.asList(MemoizingEvaluatorTester.toSkyKeys("top1", "d1", "d2")));
+
+ tester.eval(true, "top2");
+ assertThat(tester.getEnqueuedValues()).containsExactlyElementsIn(
+ Arrays.asList(MemoizingEvaluatorTester.toSkyKeys("top1", "d1", "d2", "top2", "d3")));
+ }
+
+ // NOTE: Some of these tests exercising errors/warnings run through a size-2 for loop in order
+ // to ensure that we are properly recording and replyaing these messages on null builds.
+ @Test
+ public void warningViaMultiplePaths() throws Exception {
+ tester.set("d1", new StringValue("d1")).setWarning("warn-d1");
+ tester.set("d2", new StringValue("d2")).setWarning("warn-d2");
+ tester.getOrCreate("top").setComputedValue(CONCATENATE).addDependency("d1").addDependency("d2");
+ for (int i = 0; i < 2; i++) {
+ initializeReporter();
+ tester.evalAndGet("top");
+ JunitTestUtils.assertContainsEvent(eventCollector, "warn-d1");
+ JunitTestUtils.assertContainsEvent(eventCollector, "warn-d2");
+ JunitTestUtils.assertEventCount(2, eventCollector);
+ }
+ }
+
+ @Test
+ public void warningBeforeErrorOnFailFastBuild() throws Exception {
+ tester.set("dep", new StringValue("dep")).setWarning("warn-dep");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).setHasError(true).addDependency("dep");
+ for (int i = 0; i < 2; i++) {
+ initializeReporter();
+ EvaluationResult<StringValue> result = tester.eval(false, "top");
+ assertTrue(result.hasError());
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
+ assertEquals(topKey.toString(), result.getError(topKey).getException().getMessage());
+ assertTrue(result.getError(topKey).getException() instanceof SomeErrorException);
+ JunitTestUtils.assertContainsEvent(eventCollector, "warn-dep");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+ }
+
+ @Test
+ public void warningAndErrorOnFailFastBuild() throws Exception {
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.set(topKey, new StringValue("top")).setWarning("warning msg").setHasError(true);
+ for (int i = 0; i < 2; i++) {
+ initializeReporter();
+ EvaluationResult<StringValue> result = tester.eval(false, "top");
+ assertTrue(result.hasError());
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
+ assertEquals(topKey.toString(), result.getError(topKey).getException().getMessage());
+ assertTrue(result.getError(topKey).getException() instanceof SomeErrorException);
+ JunitTestUtils.assertContainsEvent(eventCollector, "warning msg");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+ }
+
+ @Test
+ public void warningAndErrorOnFailFastBuildAfterKeepGoingBuild() throws Exception {
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.set(topKey, new StringValue("top")).setWarning("warning msg").setHasError(true);
+ for (int i = 0; i < 2; i++) {
+ initializeReporter();
+ EvaluationResult<StringValue> result = tester.eval(i == 0, "top");
+ assertTrue(result.hasError());
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
+ assertEquals(topKey.toString(), result.getError(topKey).getException().getMessage());
+ assertTrue(result.getError(topKey).getException() instanceof SomeErrorException);
+ JunitTestUtils.assertContainsEvent(eventCollector, "warning msg");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+ }
+
+ @Test
+ public void twoTLTsOnOneWarningValue() throws Exception {
+ tester.set("t1", new StringValue("t1")).addDependency("dep");
+ tester.set("t2", new StringValue("t2")).addDependency("dep");
+ tester.set("dep", new StringValue("dep")).setWarning("look both ways before crossing");
+ for (int i = 0; i < 2; i++) {
+ // Make sure we see the warning exactly once.
+ initializeReporter();
+ tester.eval(/*keepGoing=*/false, "t1", "t2");
+ JunitTestUtils.assertContainsEvent(eventCollector, "look both ways before crossing");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+ }
+
+ @Test
+ public void errorValueDepOnWarningValue() throws Exception {
+ tester.getOrCreate("error-value").setHasError(true).addDependency("warning-value");
+ tester.set("warning-value", new StringValue("warning-value"))
+ .setWarning("don't chew with your mouth open");
+
+ for (int i = 0; i < 2; i++) {
+ initializeReporter();
+ tester.evalAndGetError("error-value");
+ JunitTestUtils.assertContainsEvent(eventCollector, "don't chew with your mouth open");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ initializeReporter();
+ tester.evalAndGet("warning-value");
+ JunitTestUtils.assertContainsEvent(eventCollector, "don't chew with your mouth open");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ @Test
+ public void progressMessageOnlyPrintedTheFirstTime() throws Exception {
+ // The framework keeps track of warning and error messages, but not progress messages.
+ // So here we see both the progress and warning on the first build, but only the warning
+ // on the subsequent null build.
+ tester.set("x", new StringValue("y")).setWarning("fizzlepop")
+ .setProgress("just letting you know");
+
+ StringValue value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop");
+ JunitTestUtils.assertContainsEvent(eventCollector, "just letting you know");
+ JunitTestUtils.assertEventCount(2, eventCollector);
+
+ // On the rebuild, we only replay warning messages.
+ initializeReporter();
+ value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ @Test
+ public void invalidationWithChangeAndThenNothingChanged() throws Exception {
+ tester.getOrCreate("a")
+ .addDependency("b")
+ .setComputedValue(COPY);
+ tester.set("b", new StringValue("y"));
+ StringValue original = (StringValue) tester.evalAndGet("a");
+ assertEquals("y", original.getValue());
+ tester.set("b", new StringValue("z"));
+ tester.invalidate();
+ StringValue old = (StringValue) tester.evalAndGet("a");
+ assertEquals("z", old.getValue());
+ tester.invalidate();
+ StringValue current = (StringValue) tester.evalAndGet("a");
+ assertSame(old, current);
+ }
+
+ @Test
+ public void transientErrorValueInvalidation() throws Exception {
+ // Verify that invalidating errors causes all transient error values to be rerun.
+ tester.getOrCreate("error-value").setHasTransientError(true).setProgress(
+ "just letting you know");
+
+ tester.evalAndGetError("error-value");
+ JunitTestUtils.assertContainsEvent(eventCollector, "just letting you know");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+
+ // Change the progress message.
+ tester.getOrCreate("error-value").setHasTransientError(true).setProgress(
+ "letting you know more");
+
+ // Without invalidating errors, we shouldn't show the new progress message.
+ for (int i = 0; i < 2; i++) {
+ initializeReporter();
+ tester.evalAndGetError("error-value");
+ JunitTestUtils.assertNoEvents(eventCollector);
+ }
+
+ // When invalidating errors, we should show the new progress message.
+ initializeReporter();
+ tester.invalidateTransientErrors();
+ tester.evalAndGetError("error-value");
+ JunitTestUtils.assertContainsEvent(eventCollector, "letting you know more");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ @Test
+ public void simpleDependency() throws Exception {
+ tester.getOrCreate("ab")
+ .addDependency("a")
+ .setComputedValue(COPY);
+ tester.set("a", new StringValue("me"));
+ StringValue value = (StringValue) tester.evalAndGet("ab");
+ assertEquals("me", value.getValue());
+ }
+
+ @Test
+ public void incrementalSimpleDependency() throws Exception {
+ tester.getOrCreate("ab")
+ .addDependency("a")
+ .setComputedValue(COPY);
+ tester.set("a", new StringValue("me"));
+ tester.evalAndGet("ab");
+
+ tester.set("a", new StringValue("other"));
+ tester.invalidate();
+ StringValue value = (StringValue) tester.evalAndGet("ab");
+ assertEquals("other", value.getValue());
+ }
+
+ @Test
+ public void diamondDependency() throws Exception {
+ setupDiamondDependency();
+ tester.set("d", new StringValue("me"));
+ StringValue value = (StringValue) tester.evalAndGet("a");
+ assertEquals("meme", value.getValue());
+ }
+
+ @Test
+ public void incrementalDiamondDependency() throws Exception {
+ setupDiamondDependency();
+ tester.set("d", new StringValue("me"));
+ tester.evalAndGet("a");
+
+ tester.set("d", new StringValue("other"));
+ tester.invalidate();
+ StringValue value = (StringValue) tester.evalAndGet("a");
+ assertEquals("otherother", value.getValue());
+ }
+
+ private void setupDiamondDependency() {
+ tester.getOrCreate("a")
+ .addDependency("b")
+ .addDependency("c")
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate("b")
+ .addDependency("d")
+ .setComputedValue(COPY);
+ tester.getOrCreate("c")
+ .addDependency("d")
+ .setComputedValue(COPY);
+ }
+
+ // Regression test: ParallelEvaluator notifies ValueProgressReceiver of already-built top-level
+ // values in error: we built "top" and "mid" as top-level targets; "mid" contains an error. We
+ // make sure "mid" is built as a dependency of "top" before enqueuing mid as a top-level target
+ // (by using a latch), so that the top-level enqueuing finds that mid has already been built. The
+ // progress receiver should not be notified of any value having been evaluated.
+ @Test
+ public void alreadyAnalyzedBadTarget() throws Exception {
+ final SkyKey mid = GraphTester.toSkyKey("mid");
+ final CountDownLatch valueSet = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiter = new TrackingAwaiter();
+ setGraphForTesting(new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (!key.equals(mid)) {
+ return;
+ }
+ switch (type) {
+ case ADD_REVERSE_DEP:
+ if (context == null) {
+ // Context is null when we are enqueuing this value as a top-level job.
+ trackingAwaiter.awaitLatchAndTrackExceptions(valueSet, "value not set");
+ }
+ break;
+ case SET_VALUE:
+ valueSet.countDown();
+ break;
+ default:
+ break;
+ }
+ }
+ }));
+ SkyKey top = GraphTester.skyKey("top");
+ tester.getOrCreate(top).addDependency(mid).setComputedValue(CONCATENATE);
+ tester.getOrCreate(mid).setHasError(true);
+ tester.eval(/*keepGoing=*/false, top, mid);
+ assertEquals(0L, valueSet.getCount());
+ trackingAwaiter.assertNoErrors();
+ assertThat(tester.invalidationReceiver.evaluated).isEmpty();
+ }
+
+ @Test
+ public void receiverNotToldOfVerifiedValueDependingOnCycle() throws Exception {
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey cycle = GraphTester.toSkyKey("cycle");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.set(leaf, new StringValue("leaf"));
+ tester.getOrCreate(cycle).addDependency(cycle);
+ tester.getOrCreate(top).addDependency(leaf).addDependency(cycle);
+ tester.eval(/*keepGoing=*/true, top);
+ assertThat(tester.invalidationReceiver.evaluated).containsExactly(leaf).inOrder();
+ tester.invalidationReceiver.clear();
+ tester.getOrCreate(leaf, /*markAsModified=*/true);
+ tester.invalidate();
+ tester.eval(/*keepGoing=*/true, top);
+ assertThat(tester.invalidationReceiver.evaluated).containsExactly(leaf).inOrder();
+ }
+
+ @Test
+ public void incrementalAddedDependency() throws Exception {
+ tester.getOrCreate("a")
+ .addDependency("b")
+ .setComputedValue(CONCATENATE);
+ tester.set("b", new StringValue("first"));
+ tester.set("c", new StringValue("second"));
+ tester.evalAndGet("a");
+
+ tester.getOrCreate("a").addDependency("c");
+ tester.set("b", new StringValue("now"));
+ tester.invalidate();
+ StringValue value = (StringValue) tester.evalAndGet("a");
+ assertEquals("nowsecond", value.getValue());
+ }
+
+ @Test
+ public void manyValuesDependOnSingleValue() throws Exception {
+ initializeTester();
+ String[] values = new String[TEST_NODE_COUNT];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = Integer.toString(i);
+ tester.getOrCreate(values[i])
+ .addDependency("leaf")
+ .setComputedValue(COPY);
+ }
+ tester.set("leaf", new StringValue("leaf"));
+
+ EvaluationResult<StringValue> result = tester.eval(/*keep_going=*/false, values);
+ for (int i = 0; i < values.length; i++) {
+ SkyValue actual = result.get(new SkyKey(GraphTester.NODE_TYPE, values[i]));
+ assertEquals(new StringValue("leaf"), actual);
+ }
+
+ for (int j = 0; j < TESTED_NODES; j++) {
+ tester.set("leaf", new StringValue("other" + j));
+ tester.invalidate();
+ result = tester.eval(/*keep_going=*/false, values);
+ for (int i = 0; i < values.length; i++) {
+ SkyValue actual = result.get(new SkyKey(GraphTester.NODE_TYPE, values[i]));
+ assertEquals("Run " + j + ", value " + i, new StringValue("other" + j), actual);
+ }
+ }
+ }
+
+ @Test
+ public void singleValueDependsOnManyValues() throws Exception {
+ initializeTester();
+ String[] values = new String[TEST_NODE_COUNT];
+ StringBuilder expected = new StringBuilder();
+ for (int i = 0; i < values.length; i++) {
+ values[i] = Integer.toString(i);
+ tester.set(values[i], new StringValue(values[i]));
+ expected.append(values[i]);
+ }
+ SkyKey rootKey = new SkyKey(GraphTester.NODE_TYPE, "root");
+ TestFunction value = tester.getOrCreate(rootKey)
+ .setComputedValue(CONCATENATE);
+ for (int i = 0; i < values.length; i++) {
+ value.addDependency(values[i]);
+ }
+
+ EvaluationResult<StringValue> result = tester.eval(/*keep_going=*/false, rootKey);
+ assertEquals(new StringValue(expected.toString()), result.get(rootKey));
+
+ for (int j = 0; j < 10; j++) {
+ expected.setLength(0);
+ for (int i = 0; i < values.length; i++) {
+ String s = "other" + i + " " + j;
+ tester.set(values[i], new StringValue(s));
+ expected.append(s);
+ }
+ tester.invalidate();
+
+ result = tester.eval(/*keep_going=*/false, rootKey);
+ assertEquals(new StringValue(expected.toString()), result.get(rootKey));
+ }
+ }
+
+ @Test
+ public void twoRailLeftRightDependencies() throws Exception {
+ initializeTester();
+ String[] leftValues = new String[TEST_NODE_COUNT];
+ String[] rightValues = new String[TEST_NODE_COUNT];
+ for (int i = 0; i < leftValues.length; i++) {
+ leftValues[i] = "left-" + i;
+ rightValues[i] = "right-" + i;
+ if (i == 0) {
+ tester.getOrCreate(leftValues[i])
+ .addDependency("leaf")
+ .setComputedValue(COPY);
+ tester.getOrCreate(rightValues[i])
+ .addDependency("leaf")
+ .setComputedValue(COPY);
+ } else {
+ tester.getOrCreate(leftValues[i])
+ .addDependency(leftValues[i - 1])
+ .addDependency(rightValues[i - 1])
+ .setComputedValue(new PassThroughSelected(toSkyKey(leftValues[i - 1])));
+ tester.getOrCreate(rightValues[i])
+ .addDependency(leftValues[i - 1])
+ .addDependency(rightValues[i - 1])
+ .setComputedValue(new PassThroughSelected(toSkyKey(rightValues[i - 1])));
+ }
+ }
+ tester.set("leaf", new StringValue("leaf"));
+
+ String lastLeft = "left-" + (TEST_NODE_COUNT - 1);
+ String lastRight = "right-" + (TEST_NODE_COUNT - 1);
+
+ EvaluationResult<StringValue> result = tester.eval(/*keep_going=*/false, lastLeft, lastRight);
+ assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastLeft)));
+ assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastRight)));
+
+ for (int j = 0; j < TESTED_NODES; j++) {
+ String value = "other" + j;
+ tester.set("leaf", new StringValue(value));
+ tester.invalidate();
+ result = tester.eval(/*keep_going=*/false, lastLeft, lastRight);
+ assertEquals(new StringValue(value), result.get(toSkyKey(lastLeft)));
+ assertEquals(new StringValue(value), result.get(toSkyKey(lastRight)));
+ }
+ }
+
+ @Test
+ public void noKeepGoingAfterKeepGoingCycle() throws Exception {
+ initializeTester();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey goodKey = GraphTester.toSkyKey("good");
+ StringValue goodValue = new StringValue("good");
+ tester.set(goodKey, goodValue);
+ tester.getOrCreate(topKey).addDependency(midKey);
+ tester.getOrCreate(midKey).addDependency(aKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, topKey, goodKey);
+ assertEquals(goodValue, result.get(goodKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder();
+
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/false, topKey, goodKey);
+ assertEquals(null, result.get(topKey));
+ errorInfo = result.getError(topKey);
+ cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder();
+ }
+
+ @Test
+ public void changeCycle() throws Exception {
+ initializeTester();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(COPY);
+ tester.getOrCreate(midKey).addDependency(aKey).setComputedValue(COPY);
+ tester.getOrCreate(aKey).addDependency(bKey).setComputedValue(COPY);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder();
+
+ tester.getOrCreate(bKey).removeDependency(aKey);
+ tester.set(bKey, new StringValue("bValue"));
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/false, topKey);
+ assertEquals(new StringValue("bValue"), result.get(topKey));
+ assertEquals(null, result.getError(topKey));
+ }
+
+ /** Regression test: "crash in cycle checker with dirty values". */
+ @Test
+ public void cycleAndSelfEdgeWithDirtyValue() throws Exception {
+ initializeTester();
+ SkyKey cycleKey1 = GraphTester.toSkyKey("cycleKey1");
+ SkyKey cycleKey2 = GraphTester.toSkyKey("cycleKey2");
+ tester.getOrCreate(cycleKey1).addDependency(cycleKey2).addDependency(cycleKey1)
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(cycleKey2).addDependency(cycleKey1).setComputedValue(COPY);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, cycleKey1);
+ assertEquals(null, result.get(cycleKey1));
+ ErrorInfo errorInfo = result.getError(cycleKey1);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).isEmpty();
+ tester.getOrCreate(cycleKey1, /*markAsModified=*/true);
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/true, cycleKey1, cycleKey2);
+ assertEquals(null, result.get(cycleKey1));
+ errorInfo = result.getError(cycleKey1);
+ cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).isEmpty();
+ cycleInfo =
+ Iterables.getOnlyElement(tester.graph.getExistingErrorForTesting(cycleKey2).getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(cycleKey2).inOrder();
+ }
+
+ /** Regression test: "crash in cycle checker with dirty values". */
+ @Test
+ public void cycleWithDirtyValue() throws Exception {
+ initializeTester();
+ SkyKey cycleKey1 = GraphTester.toSkyKey("cycleKey1");
+ SkyKey cycleKey2 = GraphTester.toSkyKey("cycleKey2");
+ tester.getOrCreate(cycleKey1).addDependency(cycleKey2).setComputedValue(COPY);
+ tester.getOrCreate(cycleKey2).addDependency(cycleKey1).setComputedValue(COPY);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, cycleKey1);
+ assertEquals(null, result.get(cycleKey1));
+ ErrorInfo errorInfo = result.getError(cycleKey1);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1, cycleKey2).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).isEmpty();
+ tester.getOrCreate(cycleKey1, /*markAsModified=*/true);
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/true, cycleKey1);
+ assertEquals(null, result.get(cycleKey1));
+ errorInfo = result.getError(cycleKey1);
+ cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1, cycleKey2).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).isEmpty();
+ }
+
+ /**
+ * Regression test: IllegalStateException in BuildingState.isReady(). The ParallelEvaluator used
+ * to assume during cycle-checking that all values had been built as fully as possible -- that
+ * evaluation had not been interrupted. However, we also do cycle-checking in nokeep-going mode
+ * when a value throws an error (possibly prematurely shutting down evaluation) but that error
+ * then bubbles up into a cycle.
+ *
+ * <p>We want to achieve the following state: we are checking for a cycle; the value we examine
+ * has not yet finished checking its children to see if they are dirty; but all children checked
+ * so far have been unchanged. This value is "otherTop". We first build otherTop, then mark its
+ * first child changed (without actually changing it), and then do a second build. On the second
+ * build, we also build "top", which requests a cycle that depends on an error. We wait to signal
+ * otherTop that its first child is done until the error throws and shuts down evaluation. The
+ * error then bubbles up to the cycle, and so the bubbling is aborted. Finally, cycle checking
+ * happens, and otherTop is examined, as desired.
+ */
+ @Test
+ public void cycleAndErrorAndReady() throws Exception {
+ // This value will not have finished building on the second build when the error is thrown.
+ final SkyKey otherTop = GraphTester.toSkyKey("otherTop");
+ final SkyKey errorKey = GraphTester.toSkyKey("error");
+ // Is the graph state all set up and ready for the error to be thrown?
+ final CountDownLatch valuesReady = new CountDownLatch(3);
+ // Is evaluation being shut down? This is counted down by the exceptionMarker's builder, after
+ // it has waited for the threadpool's exception latch to be released.
+ final CountDownLatch errorThrown = new CountDownLatch(1);
+ // We don't do anything on the first build.
+ final AtomicBoolean secondBuild = new AtomicBoolean(false);
+ final TrackingAwaiter trackingAwaiter = new TrackingAwaiter();
+ setGraphForTesting(new DeterministicInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (!secondBuild.get()) {
+ return;
+ }
+ if (key.equals(errorKey) && type == EventType.SET_VALUE) {
+ // If the error is about to be thrown, make sure all listeners are ready.
+ trackingAwaiter.awaitLatchAndTrackExceptions(valuesReady, "waiting values not ready");
+ return;
+ }
+ if (key.equals(otherTop) && type == EventType.SIGNAL) {
+ // otherTop is being signaled that dep1 is done. Tell the error value that it is ready,
+ // then wait until the error is thrown, so that otherTop's builder is not re-entered.
+ valuesReady.countDown();
+ trackingAwaiter.awaitLatchAndTrackExceptions(errorThrown, "error not thrown");
+ return;
+ }
+ }
+ }));
+ final SkyKey dep1 = GraphTester.toSkyKey("dep1");
+ tester.set(dep1, new StringValue("dep1"));
+ final SkyKey dep2 = GraphTester.toSkyKey("dep2");
+ tester.set(dep2, new StringValue("dep2"));
+ // otherTop should request the deps one at a time, so that it can be in the CHECK_DEPENDENCIES
+ // state even after one dep is re-evaluated.
+ tester.getOrCreate(otherTop).setBuilder(new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ env.getValue(dep1);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ env.getValue(dep2);
+ return env.valuesMissing() ? null : new StringValue("otherTop");
+ }
+ });
+ // Prime the graph with otherTop, so we can dirty it next build.
+ assertEquals(new StringValue("otherTop"), tester.evalAndGet(/*keepGoing=*/false, otherTop));
+ // Mark dep1 changed, so otherTop will be dirty and request re-evaluation of dep1.
+ tester.getOrCreate(dep1, /*markAsModified=*/true);
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ // Note that since DeterministicInMemoryGraph alphabetizes reverse deps, it is important that
+ // "cycle2" comes before "top".
+ final SkyKey cycle1Key = GraphTester.toSkyKey("cycle1");
+ final SkyKey cycle2Key = GraphTester.toSkyKey("cycle2");
+ tester.getOrCreate(topKey).addDependency(cycle1Key).setComputedValue(CONCATENATE);
+ tester.getOrCreate(cycle1Key).addDependency(errorKey).addDependency(cycle2Key)
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(errorKey).setHasError(true);
+ // Make sure cycle2Key has declared its dependence on cycle1Key before error throws.
+ tester.getOrCreate(cycle2Key).setBuilder(new ChainedFunction(/*notifyStart=*/valuesReady,
+ null, null, false, new StringValue("never returned"), ImmutableList.<SkyKey>of(cycle1Key)));
+ // Value that waits until an exception is thrown to finish building. We use it just to be
+ // informed when the threadpool is shutting down.
+ final SkyKey exceptionMarker = GraphTester.toSkyKey("exceptionMarker");
+ tester.getOrCreate(exceptionMarker).setBuilder(new ChainedFunction(
+ /*notifyStart=*/valuesReady, /*waitToFinish=*/new CountDownLatch(0),
+ /*notifyFinish=*/errorThrown,
+ /*waitForException=*/true, new StringValue("exception marker"),
+ ImmutableList.<SkyKey>of()));
+ tester.invalidate();
+ secondBuild.set(true);
+ // otherTop must be first, since we check top-level values for cycles in the order in which
+ // they appear here.
+ EvaluationResult<StringValue> result =
+ tester.eval(/*keepGoing=*/false, otherTop, topKey, exceptionMarker);
+ trackingAwaiter.assertNoErrors();
+ assertThat(result.errorMap().keySet()).containsExactly(topKey);
+ Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo();
+ assertWithMessage(result.toString()).that(cycleInfos).isNotEmpty();
+ CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos);
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey);
+ assertThat(cycleInfo.getCycle()).containsExactly(cycle1Key, cycle2Key);
+ }
+
+ @Test
+ public void limitEvaluatorThreads() throws Exception {
+ initializeTester();
+
+ int numKeys = 10;
+ final Object lock = new Object();
+ final AtomicInteger inProgressCount = new AtomicInteger();
+ final int[] maxValue = {0};
+
+ SkyKey topLevel = GraphTester.toSkyKey("toplevel");
+ TestFunction topLevelBuilder = tester.getOrCreate(topLevel);
+ for (int i = 0; i < numKeys; i++) {
+ topLevelBuilder.addDependency("subKey" + i);
+ tester.getOrCreate("subKey" + i).setComputedValue(new ValueComputer() {
+ @Override
+ public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) {
+ int val = inProgressCount.incrementAndGet();
+ synchronized (lock) {
+ if (val > maxValue[0]) {
+ maxValue[0] = val;
+ }
+ }
+ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
+
+ inProgressCount.decrementAndGet();
+ return new StringValue("abc");
+ }
+ });
+ }
+ topLevelBuilder.setConstantValue(new StringValue("xyz"));
+
+ EvaluationResult<StringValue> result = tester.eval(
+ /*keepGoing=*/true, /*numThreads=*/5, topLevel);
+ assertFalse(result.hasError());
+ assertEquals(5, maxValue[0]);
+ }
+
+ /**
+ * Regression test: error on clearMaybeDirtyValue. We do an evaluation of topKey, which registers
+ * dependencies on midKey and errorKey. midKey enqueues slowKey, and waits. errorKey throws an
+ * error, which bubbles up to topKey. If topKey does not unregister its dependence on midKey, it
+ * will have a dangling reference to midKey after unfinished values are cleaned from the graph.
+ * Note that slowKey will wait until errorKey has thrown and the threadpool has caught the
+ * exception before returning, so the Evaluator will already have stopped enqueuing new jobs, so
+ * midKey is not evaluated.
+ */
+ @Test
+ public void incompleteDirectDepsAreClearedBeforeInvalidation() throws Exception {
+ initializeTester();
+ CountDownLatch slowStart = new CountDownLatch(1);
+ CountDownLatch errorFinish = new CountDownLatch(1);
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart,
+ /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null,
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey slowKey = GraphTester.toSkyKey("slow");
+ tester.getOrCreate(slowKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish,
+ /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("slow"),
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(midKey).addDependency(errorKey)
+ .setComputedValue(CONCATENATE);
+ // slowKey starts -> errorKey finishes, written to graph -> slowKey finishes & (Visitor aborts)
+ // -> topKey builds.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertThat(result.getError().getRootCauses()).containsExactly(errorKey);
+ // Make sure midKey didn't finish building.
+ assertEquals(null, tester.graph.getExistingValueForTesting(midKey));
+ // Give slowKey a nice ordinary builder.
+ tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null)
+ .setConstantValue(new StringValue("slow"));
+ // Put midKey into the graph. It won't have a reverse dependence on topKey.
+ tester.evalAndGet(/*keepGoing=*/false, midKey);
+ tester.differencer.invalidate(ImmutableList.of(errorKey));
+ // topKey should not access midKey as if it were already registered as a dependency.
+ tester.eval(/*keepGoing=*/false, topKey);
+ }
+
+ /**
+ * Regression test: error on clearMaybeDirtyValue. Same as the previous test, but the second
+ * evaluation is keepGoing, which should cause an access of the children of topKey.
+ */
+ @Test
+ public void incompleteDirectDepsAreClearedBeforeKeepGoing() throws Exception {
+ initializeTester();
+ CountDownLatch slowStart = new CountDownLatch(1);
+ CountDownLatch errorFinish = new CountDownLatch(1);
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart,
+ /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null,
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey slowKey = GraphTester.toSkyKey("slow");
+ tester.getOrCreate(slowKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish,
+ /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("slow"),
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(midKey).addDependency(errorKey)
+ .setComputedValue(CONCATENATE);
+ // slowKey starts -> errorKey finishes, written to graph -> slowKey finishes & (Visitor aborts)
+ // -> topKey builds.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertThat(result.getError().getRootCauses()).containsExactly(errorKey);
+ // Make sure midKey didn't finish building.
+ assertEquals(null, tester.graph.getExistingValueForTesting(midKey));
+ // Give slowKey a nice ordinary builder.
+ tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null)
+ .setConstantValue(new StringValue("slow"));
+ // Put midKey into the graph. It won't have a reverse dependence on topKey.
+ tester.evalAndGet(/*keepGoing=*/false, midKey);
+ // topKey should not access midKey as if it were already registered as a dependency.
+ // We don't invalidate errors, but because topKey wasn't actually written to the graph last
+ // build, it should be rebuilt here.
+ tester.eval(/*keepGoing=*/true, topKey);
+ }
+
+ /**
+ * Regression test: tests that pass before other build actions fail yield crash in non -k builds.
+ */
+ @Test
+ public void passThenFailToBuild() throws Exception {
+ CountDownLatch blocker = new CountDownLatch(1);
+ SkyKey successKey = GraphTester.toSkyKey("success");
+ tester.getOrCreate(successKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null,
+ /*notifyFinish=*/blocker, /*waitForException=*/false, new StringValue("yippee"),
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey slowFailKey = GraphTester.toSkyKey("slow_then_fail");
+ tester.getOrCreate(slowFailKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/blocker,
+ /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null,
+ /*deps=*/ImmutableList.<SkyKey>of()));
+
+ EvaluationResult<StringValue> result = tester.eval(
+ /*keepGoing=*/false, successKey, slowFailKey);
+ assertThat(result.getError().getRootCauses()).containsExactly(slowFailKey);
+ assertThat(result.values()).containsExactly(new StringValue("yippee"));
+ }
+
+ @Test
+ public void passThenFailToBuildAlternateOrder() throws Exception {
+ CountDownLatch blocker = new CountDownLatch(1);
+ SkyKey successKey = GraphTester.toSkyKey("success");
+ tester.getOrCreate(successKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null,
+ /*notifyFinish=*/blocker, /*waitForException=*/false, new StringValue("yippee"),
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey slowFailKey = GraphTester.toSkyKey("slow_then_fail");
+ tester.getOrCreate(slowFailKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/blocker,
+ /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null,
+ /*deps=*/ImmutableList.<SkyKey>of()));
+
+ EvaluationResult<StringValue> result = tester.eval(
+ /*keepGoing=*/false, slowFailKey, successKey);
+ assertThat(result.getError().getRootCauses()).containsExactly(slowFailKey);
+ assertThat(result.values()).containsExactly(new StringValue("yippee"));
+ }
+
+ @Test
+ public void incompleteDirectDepsForDirtyValue() throws Exception {
+ initializeTester();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.set(topKey, new StringValue("initial"));
+ // Put topKey into graph so it will be dirtied on next run.
+ assertEquals(new StringValue("initial"), tester.evalAndGet(/*keepGoing=*/false, topKey));
+ CountDownLatch slowStart = new CountDownLatch(1);
+ CountDownLatch errorFinish = new CountDownLatch(1);
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart,
+ /*notifyFinish=*/errorFinish,
+ /*waitForException=*/false, /*value=*/null, /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey slowKey = GraphTester.toSkyKey("slow");
+ tester.getOrCreate(slowKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish,
+ /*notifyFinish=*/null, /*waitForException=*/true,
+ new StringValue("slow"), /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
+ tester.set(topKey, null);
+ tester.getOrCreate(topKey).addDependency(midKey).addDependency(errorKey)
+ .setComputedValue(CONCATENATE);
+ tester.invalidate();
+ // slowKey starts -> errorKey finishes, written to graph -> slowKey finishes & (Visitor aborts)
+ // -> topKey builds.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertThat(result.getError().getRootCauses()).containsExactly(errorKey);
+ // Make sure midKey didn't finish building.
+ assertEquals(null, tester.graph.getExistingValueForTesting(midKey));
+ // Give slowKey a nice ordinary builder.
+ tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null)
+ .setConstantValue(new StringValue("slow"));
+ // Put midKey into the graph. It won't have a reverse dependence on topKey.
+ tester.evalAndGet(/*keepGoing=*/false, midKey);
+ // topKey should not access midKey as if it were already registered as a dependency.
+ // We don't invalidate errors, but since topKey wasn't actually written to the graph before, it
+ // will be rebuilt.
+ tester.eval(/*keepGoing=*/true, topKey);
+ }
+
+ @Test
+ public void continueWithErrorDep() throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.set("after", new StringValue("after"));
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE).addDependency("after");
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, parentKey);
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("recoveredafter", result.get(parentKey).getValue());
+ tester.set("after", new StringValue("before"));
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/true, parentKey);
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("recoveredbefore", result.get(parentKey).getValue());
+ }
+
+ @Test
+ public void continueWithErrorDepTurnedGood() throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.set("after", new StringValue("after"));
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE).addDependency("after");
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, parentKey);
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("recoveredafter", result.get(parentKey).getValue());
+ tester.set(errorKey, new StringValue("reformed")).setHasError(false);
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/true, parentKey);
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("reformedafter", result.get(parentKey).getValue());
+ }
+
+ @Test
+ public void errorDepAlreadyThereThenTurnedGood() throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setHasError(true);
+ // Prime the graph by putting the error value in it beforehand.
+ assertThat(tester.evalAndGetError(errorKey).getRootCauses()).containsExactly(errorKey);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, parentKey);
+ // Request the parent.
+ assertThat(result.getError(parentKey).getRootCauses()).containsExactly(parentKey).inOrder();
+ // Change the error value to no longer throw.
+ tester.set(errorKey, new StringValue("reformed")).setHasError(false);
+ tester.getOrCreate(parentKey, /*markAsModified=*/false).setHasError(false)
+ .setComputedValue(COPY);
+ tester.differencer.invalidate(ImmutableList.of(errorKey));
+ tester.invalidate();
+ // Request the parent again. This time it should succeed.
+ result = tester.eval(/*keepGoing=*/false, parentKey);
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("reformed", result.get(parentKey).getValue());
+ // Confirm that the parent no longer depends on the error transience value -- make it
+ // unbuildable again, but without invalidating it, and invalidate transient errors. The parent
+ // should not be rebuilt.
+ tester.getOrCreate(parentKey, /*markAsModified=*/false).setHasError(true);
+ tester.invalidateTransientErrors();
+ result = tester.eval(/*keepGoing=*/false, parentKey);
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("reformed", result.get(parentKey).getValue());
+ }
+
+ /**
+ * Regression test for 2014 bug: error transience value is registered before newly requested deps.
+ * A value requests a child, gets it back immediately, and then throws, causing the error
+ * transience value to be registered as a dep. The following build, the error is invalidated via
+ * that child.
+ */
+ @Test
+ public void doubleDepOnErrorTransienceValue() throws Exception {
+ initializeTester();
+ SkyKey leafKey = GraphTester.toSkyKey("leaf");
+ tester.set(leafKey, new StringValue("leaf"));
+ // Prime the graph by putting leaf in beforehand.
+ assertEquals(new StringValue("leaf"), tester.evalAndGet(/*keepGoing=*/false, leafKey));
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(leafKey).setHasError(true);
+ // Build top -- it has an error.
+ assertThat(tester.evalAndGetError(topKey).getRootCauses()).containsExactly(topKey).inOrder();
+ // Invalidate top via leaf, and rebuild.
+ tester.set(leafKey, new StringValue("leaf2"));
+ tester.invalidate();
+ assertThat(tester.evalAndGetError(topKey).getRootCauses()).containsExactly(topKey).inOrder();
+ }
+
+ /** Regression test for crash bug. */
+ @Test
+ public void errorTransienceDepCleared() throws Exception {
+ initializeTester();
+ final SkyKey top = GraphTester.toSkyKey("top");
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ tester.set(leaf, new StringValue("leaf"));
+ tester.getOrCreate(top).addDependency(leaf).setHasTransientError(true);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top);
+ assertTrue(result.toString(), result.hasError());
+ tester.getOrCreate(leaf, /*markAsModified=*/true);
+ tester.invalidate();
+ SkyKey irrelevant = GraphTester.toSkyKey("irrelevant");
+ tester.set(irrelevant, new StringValue("irrelevant"));
+ tester.eval(/*keepGoing=*/true, irrelevant);
+ tester.invalidateTransientErrors();
+ result = tester.eval(/*keepGoing=*/true, top);
+ assertTrue(result.toString(), result.hasError());
+ }
+
+ @Test
+ public void incompleteValueAlreadyThereNotUsed() throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(COPY);
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(midKey, new StringValue("don't use this"))
+ .setComputedValue(COPY);
+ // Prime the graph by evaluating the mid-level value. It shouldn't be stored in the graph
+ // because
+ // it was only called during the bubbling-up phase.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, midKey);
+ assertEquals(null, result.get(midKey));
+ assertThat(result.getError().getRootCauses()).containsExactly(errorKey);
+ // In a keepGoing build, midKey should be re-evaluated.
+ assertEquals("recovered",
+ ((StringValue) tester.evalAndGet(/*keepGoing=*/true, parentKey)).getValue());
+ }
+
+ /**
+ * "top" requests a dependency group in which the first value, called "error", throws an
+ * exception, so "mid" and "mid2", which depend on "slow", never get built.
+ */
+ @Test
+ public void errorInDependencyGroup() throws Exception {
+ initializeTester();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ CountDownLatch slowStart = new CountDownLatch(1);
+ CountDownLatch errorFinish = new CountDownLatch(1);
+ final SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart,
+ /*notifyFinish=*/errorFinish, /*waitForException=*/false,
+ // ChainedFunction throws when value is null.
+ /*value=*/null, /*deps=*/ImmutableList.<SkyKey>of()));
+ SkyKey slowKey = GraphTester.toSkyKey("slow");
+ tester.getOrCreate(slowKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish,
+ /*notifyFinish=*/null, /*waitForException=*/true,
+ new StringValue("slow"), /*deps=*/ImmutableList.<SkyKey>of()));
+ final SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
+ final SkyKey mid2Key = GraphTester.toSkyKey("mid2");
+ tester.getOrCreate(mid2Key).addDependency(slowKey).setComputedValue(COPY);
+ tester.set(topKey, null);
+ tester.getOrCreate(topKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ env.getValues(ImmutableList.of(errorKey, midKey, mid2Key));
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new StringValue("top");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+
+ // Assert that build fails and "error" really is in error.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertTrue(result.hasError());
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey);
+
+ // Ensure that evaluation succeeds if errorKey does not throw an error.
+ tester.getOrCreate(errorKey).setBuilder(null);
+ tester.set(errorKey, new StringValue("ok"));
+ tester.invalidate();
+ assertEquals(new StringValue("top"), tester.evalAndGet("top"));
+ }
+
+ /**
+ * Regression test -- if value top requests {depA, depB}, depC, with depA and depC there and depB
+ * absent, and then throws an exception, the stored deps should be depA, depC (in different
+ * groups), not {depA, depC} (same group).
+ */
+ @Test
+ public void valueInErrorWithGroups() throws Exception {
+ initializeTester();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ final SkyKey groupDepA = GraphTester.toSkyKey("groupDepA");
+ final SkyKey groupDepB = GraphTester.toSkyKey("groupDepB");
+ SkyKey depC = GraphTester.toSkyKey("depC");
+ tester.set(groupDepA, new StringValue("depC"));
+ tester.set(groupDepB, new StringValue(""));
+ tester.getOrCreate(depC).setHasError(true);
+ tester.getOrCreate(topKey).setBuilder(new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ StringValue val = ((StringValue) env.getValues(
+ ImmutableList.of(groupDepA, groupDepB)).get(groupDepA));
+ if (env.valuesMissing()) {
+ return null;
+ }
+ String nextDep = val.getValue();
+ try {
+ env.getValueOrThrow(GraphTester.toSkyKey(nextDep), SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ throw new GenericFunctionException(e, Transience.PERSISTENT);
+ }
+ return env.valuesMissing() ? null : new StringValue("top");
+ }
+ });
+
+ EvaluationResult<StringValue> evaluationResult = tester.eval(
+ /*keepGoing=*/true, groupDepA, depC);
+ assertTrue(evaluationResult.hasError());
+ assertEquals("depC", evaluationResult.get(groupDepA).getValue());
+ assertThat(evaluationResult.getError(depC).getRootCauses()).containsExactly(depC).inOrder();
+ evaluationResult = tester.eval(/*keepGoing=*/false, topKey);
+ assertTrue(evaluationResult.hasError());
+ assertThat(evaluationResult.getError(topKey).getRootCauses()).containsExactly(topKey).inOrder();
+
+ tester.set(groupDepA, new StringValue("groupDepB"));
+ tester.getOrCreate(depC, /*markAsModified=*/true);
+ tester.invalidate();
+ evaluationResult = tester.eval(/*keepGoing=*/false, topKey);
+ assertFalse(evaluationResult.toString(), evaluationResult.hasError());
+ assertEquals("top", evaluationResult.get(topKey).getValue());
+ }
+
+ @Test
+ public void errorOnlyEmittedOnce() throws Exception {
+ initializeTester();
+ tester.set("x", new StringValue("y")).setWarning("fizzlepop");
+ StringValue value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+
+ tester.invalidate();
+ value = (StringValue) tester.evalAndGet("x");
+ assertEquals("y", value.getValue());
+ // No new events emitted.
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ /**
+ * We are checking here that we are resilient to a race condition in which a value that is
+ * checking its children for dirtiness is signaled by all of its children, putting it in a ready
+ * state, before the thread has terminated. Optionally, one of its children may throw an error,
+ * shutting down the threadpool. This is similar to
+ * {@link ParallelEvaluatorTest#slowChildCleanup}: a child about to throw signals its parent and
+ * the parent's builder restarts itself before the exception is thrown. Here, the signaling
+ * happens while dirty dependencies are being checked, as opposed to during actual evaluation, but
+ * the principle is the same. We control the timing by blocking "top"'s registering itself on its
+ * deps.
+ */
+ private void dirtyChildEnqueuesParentDuringCheckDependencies(final boolean throwError)
+ throws Exception {
+ // Value to be built. It will be signaled to rebuild before it has finished checking its deps.
+ final SkyKey top = GraphTester.toSkyKey("top");
+ // Dep that blocks before it acknowledges being added as a dep by top, so the firstKey value has
+ // time to signal top.
+ final SkyKey slowAddingDep = GraphTester.toSkyKey("dep");
+ // Don't perform any blocking on the first build.
+ final AtomicBoolean delayTopSignaling = new AtomicBoolean(false);
+ final CountDownLatch topSignaled = new CountDownLatch(1);
+ final CountDownLatch topRestartedBuild = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiter = new TrackingAwaiter();
+ setGraphForTesting(new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (!delayTopSignaling.get()) {
+ return;
+ }
+ if (key.equals(top) && type == EventType.SIGNAL && order == Order.AFTER) {
+ // top is signaled by firstKey (since slowAddingDep is blocking), so slowAddingDep is now
+ // free to acknowledge top as a parent.
+ topSignaled.countDown();
+ return;
+ }
+ if (key.equals(slowAddingDep) && type == EventType.ADD_REVERSE_DEP
+ && context.equals(top) && order == Order.BEFORE) {
+ // If top is trying to declare a dep on slowAddingDep, wait until firstKey has signaled
+ // top. Then this add dep will return DONE and top will be signaled, making it ready, so
+ // it will be enqueued.
+ trackingAwaiter.awaitLatchAndTrackExceptions(topSignaled,
+ "first key didn't signal top in time");
+ }
+ }
+ }));
+ // Value that is modified on the second build. Its thread won't finish until it signals top,
+ // which will wait for the signal before it enqueues its next dep. We prevent the thread from
+ // finishing by having the listener to which it reports its warning block until top's builder
+ // starts.
+ final SkyKey firstKey = GraphTester.skyKey("first");
+ tester.set(firstKey, new StringValue("biding"));
+ tester.set(slowAddingDep, new StringValue("dep"));
+ final AtomicInteger numTopInvocations = new AtomicInteger(0);
+ tester.getOrCreate(top).setBuilder(new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey key, SkyFunction.Environment env) {
+ numTopInvocations.incrementAndGet();
+ if (delayTopSignaling.get()) {
+ // The reporter will be given firstKey's warning to emit when it is requested as a dep
+ // below, if firstKey is already built, so we release the reporter's latch beforehand.
+ topRestartedBuild.countDown();
+ }
+ // top's builder just requests both deps in a group.
+ env.getValuesOrThrow(ImmutableList.of(firstKey, slowAddingDep), SomeErrorException.class);
+ return env.valuesMissing() ? null : new StringValue("top");
+ }
+ });
+ reporter = new DelegatingEventHandler(reporter) {
+ @Override
+ public void handle(Event e) {
+ super.handle(e);
+ if (e.getKind() == EventKind.WARNING) {
+ if (!throwError) {
+ trackingAwaiter.awaitLatchAndTrackExceptions(topRestartedBuild,
+ "top's builder did not start in time");
+ }
+ }
+ }
+ };
+ // First build : just prime the graph.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top);
+ assertFalse(result.hasError());
+ assertEquals(new StringValue("top"), result.get(top));
+ assertEquals(2, numTopInvocations.get());
+ // Now dirty the graph, and maybe have firstKey throw an error.
+ String warningText = "warning text";
+ tester.getOrCreate(firstKey, /*markAsModified=*/true).setHasError(throwError)
+ .setWarning(warningText);
+ tester.invalidate();
+ delayTopSignaling.set(true);
+ result = tester.eval(/*keepGoing=*/false, top);
+ trackingAwaiter.assertNoErrors();
+ if (throwError) {
+ assertTrue(result.hasError());
+ assertThat(result.keyNames()).isEmpty(); // No successfully evaluated values.
+ ErrorInfo errorInfo = result.getError(top);
+ assertThat(errorInfo.getRootCauses()).containsExactly(firstKey);
+ assertEquals("on the incremental build, top's builder should have only been used in error "
+ + "bubbling", 3, numTopInvocations.get());
+ } else {
+ assertEquals(new StringValue("top"), result.get(top));
+ assertFalse(result.hasError());
+ assertEquals("on the incremental build, top's builder should have only been executed once in "
+ + "normal evaluation", 3, numTopInvocations.get());
+ }
+ JunitTestUtils.assertContainsEvent(eventCollector, warningText);
+ assertEquals(0, topSignaled.getCount());
+ assertEquals(0, topRestartedBuild.getCount());
+ }
+
+ @Test
+ public void dirtyChildEnqueuesParentDuringCheckDependencies_ThrowDoesntEnqueue()
+ throws Exception {
+ dirtyChildEnqueuesParentDuringCheckDependencies(/*throwError=*/true);
+ }
+
+ @Test
+ public void dirtyChildEnqueuesParentDuringCheckDependencies_NoThrow() throws Exception {
+ dirtyChildEnqueuesParentDuringCheckDependencies(/*throwError=*/false);
+ }
+
+ /**
+ * The same dep is requested in two groups, but its value determines what the other dep in the
+ * second group is. When it changes, the other dep in the second group should not be requested.
+ */
+ @Test
+ public void sameDepInTwoGroups() throws Exception {
+ initializeTester();
+
+ // leaf4 should not built in the second build.
+ final SkyKey leaf4 = GraphTester.toSkyKey("leaf4");
+ final AtomicBoolean shouldNotBuildLeaf4 = new AtomicBoolean(false);
+ setGraphForTesting(new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (shouldNotBuildLeaf4.get() && key.equals(leaf4)) {
+ throw new IllegalStateException("leaf4 should not have been considered this build: "
+ + type + ", " + order + ", " + context);
+ }
+ }
+ }));
+ tester.set(leaf4, new StringValue("leaf4"));
+
+ // Create leaf0, leaf1 and leaf2 values with values "leaf2", "leaf3", "leaf4" respectively.
+ // These will be requested as one dependency group. In the second build, leaf2 will have the
+ // value "leaf5".
+ final List<SkyKey> leaves = new ArrayList<>();
+ for (int i = 0; i <= 2; i++) {
+ SkyKey leaf = GraphTester.toSkyKey("leaf" + i);
+ leaves.add(leaf);
+ tester.set(leaf, new StringValue("leaf" + (i + 2)));
+ }
+
+ // Create "top" value. It depends on all leaf values in two overlapping dependency groups.
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ final SkyValue topValue = new StringValue("top");
+ tester.getOrCreate(topKey).setBuilder(new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ // Request the first group, [leaf0, leaf1, leaf2].
+ // In the first build, it has values ["leaf2", "leaf3", "leaf4"].
+ // In the second build it has values ["leaf2", "leaf3", "leaf5"]
+ Map<SkyKey, SkyValue> values = env.getValues(leaves);
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ // Request the second group. In the first build it's [leaf2, leaf4].
+ // In the second build it's [leaf2, leaf5]
+ env.getValues(ImmutableList.of(leaves.get(2),
+ GraphTester.toSkyKey(((StringValue) values.get(leaves.get(2))).getValue())));
+ if (env.valuesMissing()) {
+ return null;
+ }
+
+ return topValue;
+ }
+ });
+
+ // First build: assert we can evaluate "top".
+ assertEquals(topValue, tester.evalAndGet(/*keepGoing=*/false, topKey));
+
+ // Second build: replace "leaf4" by "leaf5" in leaf2's value. Assert leaf4 is not requested.
+ final SkyKey leaf5 = GraphTester.toSkyKey("leaf5");
+ tester.set(leaf5, new StringValue("leaf5"));
+ tester.set(leaves.get(2), new StringValue("leaf5"));
+ tester.invalidate();
+ shouldNotBuildLeaf4.set(true);
+ assertEquals(topValue, tester.evalAndGet(/*keepGoing=*/false, topKey));
+ }
+
+ @Test
+ public void dirtyAndChanged() throws Exception {
+ initializeTester();
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey mid = GraphTester.toSkyKey("mid");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(mid).setComputedValue(COPY);
+ tester.getOrCreate(mid).addDependency(leaf).setComputedValue(COPY);
+ tester.set(leaf, new StringValue("leafy"));
+ // For invalidation.
+ tester.set("dummy", new StringValue("dummy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafy", topValue.getValue());
+ tester.set(leaf, new StringValue("crunchy"));
+ tester.invalidate();
+ // For invalidation.
+ tester.evalAndGet("dummy");
+ tester.getOrCreate(mid, /*markAsModified=*/true);
+ tester.invalidate();
+ topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("crunchy", topValue.getValue());
+ }
+
+ /**
+ * Test whether a value that was already marked changed will be incorrectly marked dirty, not
+ * changed, if another thread tries to mark it just dirty. To exercise this, we need to have a
+ * race condition where both threads see that the value is not dirty yet, then the "changed"
+ * thread marks the value changed before the "dirty" thread marks the value dirty. To accomplish
+ * this, we use a countdown latch to make the "dirty" thread wait until the "changed" thread is
+ * done, and another countdown latch to make both of them wait until they have both checked if the
+ * value is currently clean.
+ */
+ @Test
+ public void dirtyAndChangedValueIsChanged() throws Exception {
+ final SkyKey parent = GraphTester.toSkyKey("parent");
+ final AtomicBoolean blockingEnabled = new AtomicBoolean(false);
+ final CountDownLatch waitForChanged = new CountDownLatch(1);
+ // changed thread checks value entry once (to see if it is changed). dirty thread checks twice,
+ // to see if it is changed, and if it is dirty.
+ final CountDownLatch threadsStarted = new CountDownLatch(3);
+ final TrackingAwaiter trackingAwaiter = new TrackingAwaiter();
+ setGraphForTesting(new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (!blockingEnabled.get()) {
+ return;
+ }
+ if (!key.equals(parent)) {
+ return;
+ }
+ if (type == EventType.IS_CHANGED && order == Order.BEFORE) {
+ threadsStarted.countDown();
+ }
+ // Dirtiness only checked by dirty thread.
+ if (type == EventType.IS_DIRTY && order == Order.BEFORE) {
+ threadsStarted.countDown();
+ }
+ if (type == EventType.MARK_DIRTY) {
+ trackingAwaiter.awaitLatchAndTrackExceptions(threadsStarted,
+ "Both threads did not query if value isChanged in time");
+ boolean isChanged = (Boolean) context;
+ if (order == Order.BEFORE && !isChanged) {
+ trackingAwaiter.awaitLatchAndTrackExceptions(waitForChanged,
+ "'changed' thread did not mark value changed in time");
+ return;
+ }
+ if (order == Order.AFTER && isChanged) {
+ waitForChanged.countDown();
+ }
+ }
+ }
+ }));
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ tester.set(leaf, new StringValue("leaf"));
+ tester.getOrCreate(parent).addDependency(leaf).setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result;
+ result = tester.eval(/*keepGoing=*/false, parent);
+ assertEquals("leaf", result.get(parent).getValue());
+ // Invalidate leaf, but don't actually change it. It will transitively dirty parent
+ // concurrently with parent directly dirtying itself.
+ tester.getOrCreate(leaf, /*markAsModified=*/true);
+ SkyKey other2 = GraphTester.toSkyKey("other2");
+ tester.set(other2, new StringValue("other2"));
+ // Invalidate parent, actually changing it.
+ tester.getOrCreate(parent, /*markAsModified=*/true).addDependency(other2);
+ tester.invalidate();
+ blockingEnabled.set(true);
+ result = tester.eval(/*keepGoing=*/false, parent);
+ assertEquals("leafother2", result.get(parent).getValue());
+ trackingAwaiter.assertNoErrors();
+ assertEquals(0, waitForChanged.getCount());
+ assertEquals(0, threadsStarted.getCount());
+ }
+
+ @Test
+ public void singleValueDependsOnManyDirtyValues() throws Exception {
+ initializeTester();
+ SkyKey[] values = new SkyKey[TEST_NODE_COUNT];
+ StringBuilder expected = new StringBuilder();
+ for (int i = 0; i < values.length; i++) {
+ String valueName = Integer.toString(i);
+ values[i] = GraphTester.toSkyKey(valueName);
+ tester.set(values[i], new StringValue(valueName));
+ expected.append(valueName);
+ }
+ SkyKey topKey = new SkyKey(GraphTester.NODE_TYPE, "top");
+ TestFunction value = tester.getOrCreate(topKey)
+ .setComputedValue(CONCATENATE);
+ for (int i = 0; i < values.length; i++) {
+ value.addDependency(values[i]);
+ }
+
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertEquals(new StringValue(expected.toString()), result.get(topKey));
+
+ for (int j = 0; j < RUNS; j++) {
+ for (int i = 0; i < values.length; i++) {
+ tester.getOrCreate(values[i], /*markAsModified=*/true);
+ }
+ // This value has an error, but we should never discover it because it is not marked changed
+ // and all of its dependencies re-evaluate to the same thing.
+ tester.getOrCreate(topKey, /*markAsModified=*/false).setHasError(true);
+ tester.invalidate();
+
+ result = tester.eval(/*keep_going=*/false, topKey);
+ assertEquals(new StringValue(expected.toString()), result.get(topKey));
+ }
+ }
+
+ /**
+ * Tests scenario where we have dirty values in the graph, and then one of them is deleted since
+ * its evaluation did not complete before an error was thrown. Can either test the graph via an
+ * evaluation of that deleted value, or an invalidation of a child, and can either remove the
+ * thrown error or throw it again on that evaluation.
+ */
+ private void dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(
+ boolean reevaluateMissingValue, boolean removeError) throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.set(errorKey, new StringValue("biding time"));
+ SkyKey slowKey = GraphTester.toSkyKey("slow");
+ tester.set(slowKey, new StringValue("slow"));
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
+ SkyKey lastKey = GraphTester.toSkyKey("last");
+ tester.set(lastKey, new StringValue("last"));
+ SkyKey motherKey = GraphTester.toSkyKey("mother");
+ tester.getOrCreate(motherKey).addDependency(errorKey)
+ .addDependency(midKey).addDependency(lastKey).setComputedValue(CONCATENATE);
+ SkyKey fatherKey = GraphTester.toSkyKey("father");
+ tester.getOrCreate(fatherKey).addDependency(errorKey)
+ .addDependency(midKey).addDependency(lastKey).setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, motherKey, fatherKey);
+ assertEquals("biding timeslowlast", result.get(motherKey).getValue());
+ assertEquals("biding timeslowlast", result.get(fatherKey).getValue());
+ tester.set(slowKey, null);
+ // Each parent depends on errorKey, midKey, lastKey. We keep slowKey waiting until errorKey is
+ // finished. So there is no way lastKey can be enqueued by either parent. Thus, the parent that
+ // is cleaned has not interacted with lastKey this build. Still, lastKey's reverse dep on that
+ // parent should be removed.
+ CountDownLatch errorFinish = new CountDownLatch(1);
+ tester.set(errorKey, null);
+ tester.getOrCreate(errorKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null,
+ /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null,
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ tester.getOrCreate(slowKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/errorFinish,
+ /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("leaf2"),
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ tester.invalidate();
+ // errorKey finishes, written to graph -> leafKey maybe starts+finishes & (Visitor aborts)
+ // -> one of mother or father builds. The other one should be cleaned, and no references to it
+ // left in the graph.
+ result = tester.eval(/*keepGoing=*/false, motherKey, fatherKey);
+ assertTrue(result.hasError());
+ // Only one of mother or father should be in the graph.
+ assertTrue(result.getError(motherKey) + ", " + result.getError(fatherKey),
+ (result.getError(motherKey) == null) != (result.getError(fatherKey) == null));
+ SkyKey parentKey = (reevaluateMissingValue == (result.getError(motherKey) == null))
+ ? motherKey : fatherKey;
+ // Give slowKey a nice ordinary builder.
+ tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null)
+ .setConstantValue(new StringValue("leaf2"));
+ if (removeError) {
+ tester.getOrCreate(errorKey, /*markAsModified=*/true).setBuilder(null)
+ .setConstantValue(new StringValue("reformed"));
+ }
+ String lastString = "last";
+ if (!reevaluateMissingValue) {
+ // Mark the last key modified if we're not trying the absent value again. This invalidation
+ // will test if lastKey still has a reference to the absent value.
+ lastString = "last2";
+ tester.set(lastKey, new StringValue(lastString));
+ }
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/false, parentKey);
+ if (removeError) {
+ assertEquals("reformedleaf2" + lastString, result.get(parentKey).getValue());
+ } else {
+ assertNotNull(result.getError(parentKey));
+ }
+ }
+
+ /**
+ * The following four tests (dirtyChildrenProperlyRemovedWith*) test the consistency of the graph
+ * after a failed build in which a dirty value should have been deleted from the graph. The
+ * consistency is tested via either evaluating the missing value, or the re-evaluating the present
+ * value, and either clearing the error or keeping it. To evaluate the present value, we
+ * invalidate the error value to force re-evaluation. Related to bug "skyframe m1: graph may not
+ * be properly cleaned on interrupt or failure".
+ */
+ @Test
+ public void dirtyChildrenProperlyRemovedWithInvalidateRemoveError() throws Exception {
+ dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/false,
+ /*removeError=*/true);
+ }
+
+ @Test
+ public void dirtyChildrenProperlyRemovedWithInvalidateKeepError() throws Exception {
+ dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/false,
+ /*removeError=*/false);
+ }
+
+ @Test
+ public void dirtyChildrenProperlyRemovedWithReevaluateRemoveError() throws Exception {
+ dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/true,
+ /*removeError=*/true);
+ }
+
+ @Test
+ public void dirtyChildrenProperlyRemovedWithReevaluateKeepError() throws Exception {
+ dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/true,
+ /*removeError=*/false);
+ }
+
+ /**
+ * Regression test: enqueue so many values that some of them won't have started processing, and
+ * then either interrupt processing or have a child throw an error. In the latter case, this also
+ * tests that a value that hasn't started processing can still have a child error bubble up to it.
+ * In both cases, it tests that the graph is properly cleaned of the dirty values and references
+ * to them.
+ */
+ private void manyDirtyValuesClearChildrenOnFail(boolean interrupt) throws Exception {
+ SkyKey leafKey = GraphTester.toSkyKey("leaf");
+ tester.set(leafKey, new StringValue("leafy"));
+ SkyKey lastKey = GraphTester.toSkyKey("last");
+ tester.set(lastKey, new StringValue("last"));
+ final List<SkyKey> tops = new ArrayList<>();
+ // Request far more top-level values than there are threads, so some of them will block until
+ // the
+ // leaf child is enqueued for processing.
+ for (int i = 0; i < 10000; i++) {
+ SkyKey topKey = GraphTester.toSkyKey("top" + i);
+ tester.getOrCreate(topKey).addDependency(leafKey).addDependency(lastKey)
+ .setComputedValue(CONCATENATE);
+ tops.add(topKey);
+ }
+ tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0]));
+ final CountDownLatch notifyStart = new CountDownLatch(1);
+ tester.set(leafKey, null);
+ if (interrupt) {
+ // leaf will wait for an interrupt if desired. We cannot use the usual ChainedFunction
+ // because we need to actually throw the interrupt.
+ final AtomicBoolean shouldSleep = new AtomicBoolean(true);
+ tester.getOrCreate(leafKey, /*markAsModified=*/true).setBuilder(
+ new NoExtractorFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
+ notifyStart.countDown();
+ if (shouldSleep.get()) {
+ // Should be interrupted within 5 seconds.
+ Thread.sleep(5000);
+ throw new AssertionError("leaf was not interrupted");
+ }
+ return new StringValue("crunchy");
+ }
+ });
+ tester.invalidate();
+ TestThread evalThread = new TestThread() {
+ @Override
+ public void runTest() {
+ try {
+ tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0]));
+ Assert.fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ }
+ };
+ evalThread.start();
+ assertTrue(notifyStart.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ evalThread.interrupt();
+ evalThread.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ // Free leafKey to compute next time.
+ shouldSleep.set(false);
+ } else {
+ // Non-interrupt case. Just throw an error in the child.
+ tester.getOrCreate(leafKey, /*markAsModified=*/true).setHasError(true);
+ tester.invalidate();
+ // The error thrown may non-deterministically bubble up to a parent that has not yet started
+ // processing, but has been enqueued for processing.
+ tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0]));
+ tester.getOrCreate(leafKey, /*markAsModified=*/true).setHasError(false);
+ tester.set(leafKey, new StringValue("crunchy"));
+ }
+ // lastKey was not touched during the previous build, but its reverse deps on its parents should
+ // still be accurate.
+ tester.set(lastKey, new StringValue("new last"));
+ tester.invalidate();
+ EvaluationResult<StringValue> result =
+ tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0]));
+ for (SkyKey topKey : tops) {
+ assertEquals(topKey.toString(), "crunchynew last", result.get(topKey).getValue());
+ }
+ }
+
+ /**
+ * Regression test: make sure that if an evaluation fails before a dirty value starts evaluation
+ * (in particular, before it is reset), the graph remains consistent.
+ */
+ @Test
+ public void manyDirtyValuesClearChildrenOnError() throws Exception {
+ manyDirtyValuesClearChildrenOnFail(/*interrupt=*/false);
+ }
+
+ /**
+ * Regression test: Make sure that if an evaluation is interrupted before a dirty value starts
+ * evaluation (in particular, before it is reset), the graph remains consistent.
+ */
+ @Test
+ public void manyDirtyValuesClearChildrenOnInterrupt() throws Exception {
+ manyDirtyValuesClearChildrenOnFail(/*interrupt=*/true);
+ }
+
+ /**
+ * Regression test for case where the user requests that we delete nodes that are already in the
+ * queue to be dirtied. We should handle that gracefully and not complain.
+ */
+ @Test
+ public void deletingDirtyNodes() throws Exception {
+ final Thread thread = Thread.currentThread();
+ final AtomicBoolean interruptInvalidation = new AtomicBoolean(false);
+ initializeTester(new TrackingInvalidationReceiver() {
+ private final AtomicBoolean firstInvalidation = new AtomicBoolean(true);
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ if (interruptInvalidation.get() && !firstInvalidation.getAndSet(false)) {
+ thread.interrupt();
+ }
+ super.invalidated(value, state);
+ }
+ });
+ SkyKey key = null;
+ // Create a long chain of nodes. Most of them will not actually be dirtied, but the last one to
+ // be dirtied will enqueue its parent for dirtying, so it will be in the queue for the next run.
+ for (int i = 0; i < TEST_NODE_COUNT; i++) {
+ key = GraphTester.toSkyKey("node" + i);
+ if (i > 0) {
+ tester.getOrCreate(key).addDependency("node" + (i - 1)).setComputedValue(COPY);
+ } else {
+ tester.set(key, new StringValue("node0"));
+ }
+ }
+ // Seed the graph.
+ assertEquals("node0", ((StringValue) tester.evalAndGet(/*keepGoing=*/false, key)).getValue());
+ // Start the dirtying process.
+ tester.set("node0", new StringValue("new"));
+ tester.invalidate();
+ interruptInvalidation.set(true);
+ try {
+ tester.eval(/*keepGoing=*/false, key);
+ fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ interruptInvalidation.set(false);
+ // Now delete all the nodes. The node that was going to be dirtied is also deleted, which we
+ // should handle.
+ tester.graph.delete(Predicates.<SkyKey>alwaysTrue());
+ assertEquals("new", ((StringValue) tester.evalAndGet(/*keepGoing=*/false, key)).getValue());
+ }
+
+ @Test
+ public void changePruning() throws Exception {
+ initializeTester();
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey mid = GraphTester.toSkyKey("mid");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(mid).setComputedValue(COPY);
+ tester.getOrCreate(mid).addDependency(leaf).setComputedValue(COPY);
+ tester.set(leaf, new StringValue("leafy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafy", topValue.getValue());
+ // Mark leaf changed, but don't actually change it.
+ tester.getOrCreate(leaf, /*markAsModified=*/true);
+ // mid will give an error if re-evaluated, but it shouldn't be because it is not marked changed,
+ // and its dirty child will evaluate to the same element.
+ tester.getOrCreate(mid, /*markAsModified=*/false).setHasError(true);
+ tester.invalidate();
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top);
+ assertFalse(result.hasError());
+ topValue = result.get(top);
+ assertEquals("leafy", topValue.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ }
+
+ @Test
+ public void changePruningWithDoneValue() throws Exception {
+ initializeTester();
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey mid = GraphTester.toSkyKey("mid");
+ SkyKey top = GraphTester.toSkyKey("top");
+ SkyKey suffix = GraphTester.toSkyKey("suffix");
+ StringValue suffixValue = new StringValue("suffix");
+ tester.set(suffix, suffixValue);
+ tester.getOrCreate(top).addDependency(mid).addDependency(suffix).setComputedValue(CONCATENATE);
+ tester.getOrCreate(mid).addDependency(leaf).addDependency(suffix).setComputedValue(CONCATENATE);
+ SkyValue leafyValue = new StringValue("leafy");
+ tester.set(leaf, leafyValue);
+ StringValue value = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafysuffixsuffix", value.getValue());
+ // Mark leaf changed, but don't actually change it.
+ tester.getOrCreate(leaf, /*markAsModified=*/true);
+ // mid will give an error if re-evaluated, but it shouldn't be because it is not marked changed,
+ // and its dirty child will evaluate to the same element.
+ tester.getOrCreate(mid, /*markAsModified=*/false).setHasError(true);
+ tester.invalidate();
+ value = (StringValue) tester.evalAndGet("leaf");
+ assertEquals("leafy", value.getValue());
+ assertThat(tester.getDirtyValues()).containsExactly(new StringValue("leafysuffix"),
+ new StringValue("leafysuffixsuffix"));
+ assertThat(tester.getDeletedValues()).isEmpty();
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top);
+ assertFalse(result.hasError());
+ value = result.get(top);
+ assertEquals("leafysuffixsuffix", value.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ }
+
+ @Test
+ public void changedChildChangesDepOfParent() throws Exception {
+ initializeTester();
+ final SkyKey buildFile = GraphTester.toSkyKey("buildFile");
+ ValueComputer authorDrink = new ValueComputer() {
+ @Override
+ public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) {
+ String author = ((StringValue) deps.get(buildFile)).getValue();
+ StringValue beverage;
+ switch (author) {
+ case "hemingway":
+ beverage = (StringValue) env.getValue(GraphTester.toSkyKey("absinthe"));
+ break;
+ case "joyce":
+ beverage = (StringValue) env.getValue(GraphTester.toSkyKey("whiskey"));
+ break;
+ default:
+ throw new IllegalStateException(author);
+ }
+ if (beverage == null) {
+ return null;
+ }
+ return new StringValue(author + " drank " + beverage.getValue());
+ }
+ };
+
+ tester.set(buildFile, new StringValue("hemingway"));
+ SkyKey absinthe = GraphTester.toSkyKey("absinthe");
+ tester.set(absinthe, new StringValue("absinthe"));
+ SkyKey whiskey = GraphTester.toSkyKey("whiskey");
+ tester.set(whiskey, new StringValue("whiskey"));
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(buildFile).setComputedValue(authorDrink);
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("hemingway drank absinthe", topValue.getValue());
+ tester.set(buildFile, new StringValue("joyce"));
+ // Don't evaluate absinthe successfully anymore.
+ tester.getOrCreate(absinthe).setHasError(true);
+ tester.invalidate();
+ topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("joyce drank whiskey", topValue.getValue());
+ assertThat(tester.getDirtyValues()).containsExactly(new StringValue("hemingway"),
+ new StringValue("hemingway drank absinthe"));
+ assertThat(tester.getDeletedValues()).isEmpty();
+ }
+
+ @Test
+ public void dirtyDepIgnoresChildren() throws Exception {
+ initializeTester();
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey mid = GraphTester.toSkyKey("mid");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.set(mid, new StringValue("ignore"));
+ tester.getOrCreate(top).addDependency(mid).setComputedValue(COPY);
+ tester.getOrCreate(mid).addDependency(leaf);
+ tester.set(leaf, new StringValue("leafy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("ignore", topValue.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ // Change leaf.
+ tester.set(leaf, new StringValue("crunchy"));
+ tester.invalidate();
+ topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("ignore", topValue.getValue());
+ assertThat(tester.getDirtyValues()).containsExactly(new StringValue("leafy"));
+ assertThat(tester.getDeletedValues()).isEmpty();
+ tester.set(leaf, new StringValue("smushy"));
+ tester.invalidate();
+ topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("ignore", topValue.getValue());
+ assertThat(tester.getDirtyValues()).containsExactly(new StringValue("crunchy"));
+ assertThat(tester.getDeletedValues()).isEmpty();
+ }
+
+ private static final SkyFunction INTERRUPT_BUILDER = new SkyFunction() {
+
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ throw new InterruptedException();
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ /**
+ * Utility function to induce a graph clean of whatever value is requested, by trying to build
+ * this value and interrupting the build as soon as this value's function evaluation starts.
+ */
+ private void failBuildAndRemoveValue(final SkyKey value) {
+ tester.set(value, null);
+ // Evaluator will think leaf was interrupted because it threw, so it will be cleaned from graph.
+ tester.getOrCreate(value, /*markAsModified=*/true).setBuilder(INTERRUPT_BUILDER);
+ tester.invalidate();
+ try {
+ tester.eval(/*keepGoing=*/false, value);
+ Assert.fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ tester.getOrCreate(value, /*markAsModified=*/false).setBuilder(null);
+ }
+
+ /**
+ * Make sure that when a dirty value is building, the fact that a child may no longer exist in the
+ * graph doesn't cause problems.
+ */
+ @Test
+ public void dirtyBuildAfterFailedBuild() throws Exception {
+ initializeTester();
+ final SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(leaf).setComputedValue(COPY);
+ tester.set(leaf, new StringValue("leafy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafy", topValue.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ failBuildAndRemoveValue(leaf);
+ // Leaf should no longer exist in the graph. Check that this doesn't cause problems.
+ tester.set(leaf, null);
+ tester.set(leaf, new StringValue("crunchy"));
+ tester.invalidate();
+ topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("crunchy", topValue.getValue());
+ }
+
+ /**
+ * Regression test: error when clearing reverse deps on dirty value about to be rebuilt, because
+ * child values were deleted and recreated in interim, forgetting they had reverse dep on dirty
+ * value in the first place.
+ */
+ @Test
+ public void changedBuildAfterFailedThenSuccessfulBuild() throws Exception {
+ initializeTester();
+ final SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(leaf).setComputedValue(COPY);
+ tester.set(leaf, new StringValue("leafy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafy", topValue.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ failBuildAndRemoveValue(leaf);
+ tester.set(leaf, new StringValue("crunchy"));
+ tester.invalidate();
+ tester.eval(/*keepGoing=*/false, leaf);
+ // Leaf no longer has reverse dep on top. Check that this doesn't cause problems, even if the
+ // top value is evaluated unconditionally.
+ tester.getOrCreate(top, /*markAsModified=*/true);
+ tester.invalidate();
+ topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("crunchy", topValue.getValue());
+ }
+
+ /**
+ * Regression test: child value that has been deleted since it and its parent were marked dirty no
+ * longer knows it has a reverse dep on its parent.
+ *
+ * <p>Start with:
+ * <pre>
+ * top0 ... top1000
+ * \ | /
+ * leaf
+ * </pre>
+ * Then fail to build leaf. Now the entry for leaf should have no "memory" that it was ever
+ * depended on by tops. Now build tops, but fail again.
+ */
+ @Test
+ public void manyDirtyValuesClearChildrenOnSecondFail() throws Exception {
+ final SkyKey leafKey = GraphTester.toSkyKey("leaf");
+ tester.set(leafKey, new StringValue("leafy"));
+ SkyKey lastKey = GraphTester.toSkyKey("last");
+ tester.set(lastKey, new StringValue("last"));
+ final List<SkyKey> tops = new ArrayList<>();
+ // Request far more top-level values than there are threads, so some of them will block until
+ // the leaf child is enqueued for processing.
+ for (int i = 0; i < 10000; i++) {
+ SkyKey topKey = GraphTester.toSkyKey("top" + i);
+ tester.getOrCreate(topKey).addDependency(leafKey).addDependency(lastKey)
+ .setComputedValue(CONCATENATE);
+ tops.add(topKey);
+ }
+ tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0]));
+ failBuildAndRemoveValue(leafKey);
+ // Request the tops. Since leaf was deleted from the graph last build, it no longer knows that
+ // its parents depend on it. When leaf throws, at least one of its parents (hopefully) will not
+ // have re-informed leaf that the parent depends on it, exposing the bug, since the parent
+ // should then not try to clean the reverse dep from leaf.
+ tester.set(leafKey, null);
+ // Evaluator will think leaf was interrupted because it threw, so it will be cleaned from graph.
+ tester.getOrCreate(leafKey, /*markAsModified=*/true).setBuilder(INTERRUPT_BUILDER);
+ tester.invalidate();
+ try {
+ tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0]));
+ Assert.fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void failedDirtyBuild() throws Exception {
+ initializeTester();
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addErrorDependency(leaf, new StringValue("recover"))
+ .setComputedValue(COPY);
+ tester.set(leaf, new StringValue("leafy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafy", topValue.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ // Change leaf.
+ tester.getOrCreate(leaf, /*markAsModified=*/true).setHasError(true);
+ tester.getOrCreate(top, /*markAsModified=*/false).setHasError(true);
+ tester.invalidate();
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top);
+ assertNull("value should not have completed evaluation", result.get(top));
+ assertWithMessage(
+ "The error thrown by leaf should have been swallowed by the error thrown by top")
+ .that(result.getError().getRootCauses()).containsExactly(top);
+ }
+
+ @Test
+ public void failedDirtyBuildInBuilder() throws Exception {
+ initializeTester();
+ SkyKey leaf = GraphTester.toSkyKey("leaf");
+ SkyKey secondError = GraphTester.toSkyKey("secondError");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(leaf)
+ .addErrorDependency(secondError, new StringValue("recover")).setComputedValue(CONCATENATE);
+ tester.set(secondError, new StringValue("secondError")).addDependency(leaf);
+ tester.set(leaf, new StringValue("leafy"));
+ StringValue topValue = (StringValue) tester.evalAndGet("top");
+ assertEquals("leafysecondError", topValue.getValue());
+ assertThat(tester.getDirtyValues()).isEmpty();
+ assertThat(tester.getDeletedValues()).isEmpty();
+ // Invalidate leaf.
+ tester.getOrCreate(leaf, /*markAsModified=*/true);
+ tester.set(leaf, new StringValue("crunchy"));
+ tester.getOrCreate(secondError, /*markAsModified=*/true).setHasError(true);
+ tester.getOrCreate(top, /*markAsModified=*/false).setHasError(true);
+ tester.invalidate();
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top);
+ assertNull("value should not have completed evaluation", result.get(top));
+ assertWithMessage(
+ "The error thrown by leaf should have been swallowed by the error thrown by top")
+ .that(result.getError().getRootCauses()).containsExactly(top);
+ }
+
+ @Test
+ public void dirtyErrorTransienceValue() throws Exception {
+ initializeTester();
+ SkyKey error = GraphTester.toSkyKey("error");
+ tester.getOrCreate(error).setHasError(true);
+ assertNotNull(tester.evalAndGetError(error));
+ tester.invalidateTransientErrors();
+ SkyKey secondError = GraphTester.toSkyKey("secondError");
+ tester.getOrCreate(secondError).setHasError(true);
+ // secondError declares a new dependence on ErrorTransienceValue, but not until it has already
+ // thrown an error.
+ assertNotNull(tester.evalAndGetError(secondError));
+ }
+
+ @Test
+ public void dirtyDependsOnErrorTurningGood() throws Exception {
+ initializeTester();
+ SkyKey error = GraphTester.toSkyKey("error");
+ tester.getOrCreate(error).setHasError(true);
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(error).setComputedValue(COPY);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(error);
+ tester.getOrCreate(error).setHasError(false);
+ StringValue val = new StringValue("reformed");
+ tester.set(error, val);
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/false, topKey);
+ assertEquals(val, result.get(topKey));
+ assertFalse(result.hasError());
+ }
+
+ /** Regression test for crash bug. */
+ @Test
+ public void dirtyWithOwnErrorDependsOnTransientErrorTurningGood() throws Exception {
+ initializeTester();
+ final SkyKey error = GraphTester.toSkyKey("error");
+ tester.getOrCreate(error).setHasTransientError(true);
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyFunction errorFunction = new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException,
+ InterruptedException {
+ try {
+ return env.getValueOrThrow(error, SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ throw new GenericFunctionException(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ tester.getOrCreate(topKey).setBuilder(errorFunction);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ tester.invalidateTransientErrors();
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
+ tester.getOrCreate(error).setHasTransientError(false);
+ StringValue reformed = new StringValue("reformed");
+ tester.set(error, reformed);
+ tester.getOrCreate(topKey).setBuilder(null).addDependency(error).setComputedValue(COPY);
+ tester.invalidate();
+ tester.invalidateTransientErrors();
+ result = tester.eval(/*keepGoing=*/false, topKey);
+ assertEquals(reformed, result.get(topKey));
+ assertFalse(result.hasError());
+ }
+
+ /**
+ * Make sure that when an error is thrown, it is given for handling only to parents that have
+ * already registered a dependence on the value that threw the error.
+ *
+ * <pre>
+ * topBubbleKey topErrorFirstKey
+ * | \ /
+ * midKey errorKey
+ * |
+ * slowKey
+ * </pre>
+ *
+ * On the second build, errorKey throws, and the threadpool aborts before midKey finishes.
+ * topBubbleKey therefore has not yet requested errorKey this build. If errorKey bubbles up to it,
+ * topBubbleKey must be able to handle that. (The evaluator can deal with this either by not
+ * allowing errorKey to bubble up to topBubbleKey, or by dealing with that case.)
+ */
+ @Test
+ public void errorOnlyBubblesToRequestingParents() throws Exception {
+ // We need control over the order of reverse deps, so use a deterministic graph.
+ setGraphForTesting(new DeterministicInMemoryGraph());
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.set(errorKey, new StringValue("biding time"));
+ SkyKey slowKey = GraphTester.toSkyKey("slow");
+ tester.set(slowKey, new StringValue("slow"));
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
+ SkyKey topErrorFirstKey = GraphTester.toSkyKey("2nd top alphabetically");
+ tester.getOrCreate(topErrorFirstKey).addDependency(errorKey).setComputedValue(CONCATENATE);
+ SkyKey topBubbleKey = GraphTester.toSkyKey("1st top alphabetically");
+ tester.getOrCreate(topBubbleKey).addDependency(midKey).addDependency(errorKey)
+ .setComputedValue(CONCATENATE);
+ // First error-free evaluation, to put all values in graph.
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false,
+ topErrorFirstKey, topBubbleKey);
+ assertEquals("biding time", result.get(topErrorFirstKey).getValue());
+ assertEquals("slowbiding time", result.get(topBubbleKey).getValue());
+ // Set up timing of child values: slowKey waits to finish until errorKey has thrown an
+ // exception that has been caught by the threadpool.
+ tester.set(slowKey, null);
+ CountDownLatch errorFinish = new CountDownLatch(1);
+ tester.set(errorKey, null);
+ tester.getOrCreate(errorKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null,
+ /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null,
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ tester.getOrCreate(slowKey).setBuilder(
+ new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/errorFinish,
+ /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("leaf2"),
+ /*deps=*/ImmutableList.<SkyKey>of()));
+ tester.invalidate();
+ // errorKey finishes, written to graph -> slowKey maybe starts+finishes & (Visitor aborts)
+ // -> some top key builds.
+ result = tester.eval(/*keepGoing=*/false, topErrorFirstKey, topBubbleKey);
+ assertTrue(result.hasError());
+ assertNotNull(result.getError(topErrorFirstKey));
+ }
+
+ @Test
+ public void dirtyWithRecoveryErrorDependsOnErrorTurningGood() throws Exception {
+ initializeTester();
+ final SkyKey error = GraphTester.toSkyKey("error");
+ tester.getOrCreate(error).setHasError(true);
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyFunction recoveryErrorFunction = new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ try {
+ env.getValueOrThrow(error, SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ throw new GenericFunctionException(e, Transience.PERSISTENT);
+ }
+ return null;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ tester.getOrCreate(topKey).setBuilder(recoveryErrorFunction);
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey);
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
+ tester.getOrCreate(error).setHasError(false);
+ StringValue reformed = new StringValue("reformed");
+ tester.set(error, reformed);
+ tester.getOrCreate(topKey).setBuilder(null).addDependency(error).setComputedValue(COPY);
+ tester.invalidate();
+ result = tester.eval(/*keepGoing=*/false, topKey);
+ assertEquals(reformed, result.get(topKey));
+ assertFalse(result.hasError());
+ }
+
+
+ @Test
+ public void absentParent() throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.set(errorKey, new StringValue("biding time"));
+ SkyKey absentParentKey = GraphTester.toSkyKey("absentParent");
+ tester.getOrCreate(absentParentKey).addDependency(errorKey).setComputedValue(CONCATENATE);
+ assertEquals(new StringValue("biding time"),
+ tester.evalAndGet(/*keepGoing=*/false, absentParentKey));
+ tester.getOrCreate(errorKey, /*markAsModified=*/true).setHasError(true);
+ SkyKey newParent = GraphTester.toSkyKey("newParent");
+ tester.getOrCreate(newParent).addDependency(errorKey).setComputedValue(CONCATENATE);
+ tester.invalidate();
+ EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, newParent);
+ ErrorInfo error = result.getError(newParent);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ }
+
+ // Tests that we have a sane implementation of error transience.
+ @Test
+ public void errorTransienceBug() throws Exception {
+ tester.getOrCreate("key").setHasTransientError(true);
+ assertNotNull(tester.evalAndGetError("key").getException());
+ StringValue value = new StringValue("hi");
+ tester.getOrCreate("key").setHasTransientError(false).setConstantValue(value);
+ tester.invalidateTransientErrors();
+ assertEquals(value, tester.evalAndGet("key"));
+ // This works because the version of the ValueEntry for the ErrorTransience value is always
+ // increased on each InMemoryMemoizingEvaluator#evaluate call. But that's not the only way to
+ // implement error transience; another valid implementation would be to unconditionally mark
+ // values depending on the ErrorTransience value as being changed (rather than merely dirtied)
+ // during invalidation.
+ }
+
+ @Test
+ public void transientErrorTurningGoodHasNoError() throws Exception {
+ initializeTester();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasTransientError(true);
+ ErrorInfo errorInfo = tester.evalAndGetError(errorKey);
+ assertNotNull(errorInfo);
+ assertThat(errorInfo.getRootCauses()).containsExactly(errorKey);
+ // Re-evaluates to same thing when errors are invalidated
+ tester.invalidateTransientErrors();
+ errorInfo = tester.evalAndGetError(errorKey);
+ assertNotNull(errorInfo);
+ StringValue value = new StringValue("reformed");
+ assertThat(errorInfo.getRootCauses()).containsExactly(errorKey);
+ tester.getOrCreate(errorKey, /*markAsModified=*/false).setHasTransientError(false)
+ .setConstantValue(value);
+ tester.invalidateTransientErrors();
+ StringValue stringValue = (StringValue) tester.evalAndGet(/*keepGoing=*/true, errorKey);
+ assertSame(stringValue, value);
+ // Value builder will now throw, but we should never get to it because it isn't dirty.
+ tester.getOrCreate(errorKey, /*markAsModified=*/false).setHasTransientError(true);
+ tester.invalidateTransientErrors();
+ stringValue = (StringValue) tester.evalAndGet(/*keepGoing=*/true, errorKey);
+ assertSame(stringValue, value);
+ }
+
+ @Test
+ public void deleteInvalidatedValue() throws Exception {
+ initializeTester();
+ SkyKey top = GraphTester.toSkyKey("top");
+ SkyKey toDelete = GraphTester.toSkyKey("toDelete");
+ // Must be a concatenation -- COPY doesn't actually copy.
+ tester.getOrCreate(top).addDependency(toDelete).setComputedValue(CONCATENATE);
+ tester.set(toDelete, new StringValue("toDelete"));
+ SkyValue value = tester.evalAndGet("top");
+ SkyKey forceInvalidation = GraphTester.toSkyKey("forceInvalidation");
+ tester.set(forceInvalidation, new StringValue("forceInvalidation"));
+ tester.getOrCreate(toDelete, /*markAsModified=*/true);
+ tester.invalidate();
+ tester.eval(/*keepGoing=*/false, forceInvalidation);
+ tester.delete("toDelete");
+ WeakReference<SkyValue> ref = new WeakReference<>(value);
+ value = null;
+ tester.eval(/*keepGoing=*/false, forceInvalidation);
+ tester.invalidate(); // So that invalidation receiver doesn't hang on to reference.
+ GcFinalization.awaitClear(ref);
+ }
+
+ /**
+ * General stress/fuzz test of the evaluator with failure. Construct a large graph, and then throw
+ * exceptions during building at various points.
+ */
+ @Test
+ public void twoRailLeftRightDependenciesWithFailure() throws Exception {
+ initializeTester();
+ SkyKey[] leftValues = new SkyKey[TEST_NODE_COUNT];
+ SkyKey[] rightValues = new SkyKey[TEST_NODE_COUNT];
+ for (int i = 0; i < TEST_NODE_COUNT; i++) {
+ leftValues[i] = GraphTester.toSkyKey("left-" + i);
+ rightValues[i] = GraphTester.toSkyKey("right-" + i);
+ if (i == 0) {
+ tester.getOrCreate(leftValues[i])
+ .addDependency("leaf")
+ .setComputedValue(COPY);
+ tester.getOrCreate(rightValues[i])
+ .addDependency("leaf")
+ .setComputedValue(COPY);
+ } else {
+ tester.getOrCreate(leftValues[i])
+ .addDependency(leftValues[i - 1])
+ .addDependency(rightValues[i - 1])
+ .setComputedValue(new PassThroughSelected(leftValues[i - 1]));
+ tester.getOrCreate(rightValues[i])
+ .addDependency(leftValues[i - 1])
+ .addDependency(rightValues[i - 1])
+ .setComputedValue(new PassThroughSelected(rightValues[i - 1]));
+ }
+ }
+ tester.set("leaf", new StringValue("leaf"));
+
+ String lastLeft = "left-" + (TEST_NODE_COUNT - 1);
+ String lastRight = "right-" + (TEST_NODE_COUNT - 1);
+
+ for (int i = 0; i < TESTED_NODES; i++) {
+ try {
+ tester.getOrCreate(leftValues[i], /*markAsModified=*/true).setHasError(true);
+ tester.invalidate();
+ EvaluationResult<StringValue> result = tester.eval(
+ /*keep_going=*/false, lastLeft, lastRight);
+ assertTrue(result.hasError());
+ tester.differencer.invalidate(ImmutableList.of(leftValues[i]));
+ tester.invalidate();
+ result = tester.eval(/*keep_going=*/false, lastLeft, lastRight);
+ assertTrue(result.hasError());
+ tester.getOrCreate(leftValues[i], /*markAsModified=*/true).setHasError(false);
+ tester.invalidate();
+ result = tester.eval(/*keep_going=*/false, lastLeft, lastRight);
+ assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastLeft)));
+ assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastRight)));
+ } catch (Exception e) {
+ System.err.println("twoRailLeftRightDependenciesWithFailure exception on run " + i);
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void valueInjection() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("new_value");
+ SkyValue val = new StringValue("val");
+
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("new_value"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingEntry() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.getOrCreate(key).setConstantValue(new StringValue("old_val"));
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingDirtyEntry() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.getOrCreate(key).setConstantValue(new StringValue("old_val"));
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ tester.eval(/*keepGoing=*/false, new SkyKey[0]); // Create the value.
+
+ tester.differencer.invalidate(ImmutableList.of(key));
+ tester.eval(/*keepGoing=*/false, new SkyKey[0]); // Mark value as dirty.
+
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ tester.eval(/*keepGoing=*/false, new SkyKey[0]); // Inject again.
+ assertEquals(val, tester.evalAndGet("value"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingEntryMarkedForInvalidation() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.getOrCreate(key).setConstantValue(new StringValue("old_val"));
+ tester.differencer.invalidate(ImmutableList.of(key));
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingEntryMarkedForDeletion() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.getOrCreate(key).setConstantValue(new StringValue("old_val"));
+ tester.graph.delete(Predicates.<SkyKey>alwaysTrue());
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingEqualEntryMarkedForInvalidation() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+
+ tester.differencer.invalidate(ImmutableList.of(key));
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingEqualEntryMarkedForDeletion() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+
+ tester.graph.delete(Predicates.<SkyKey>alwaysTrue());
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet("value"));
+ }
+
+ @Test
+ public void valueInjectionOverValueWithDeps() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+ StringValue prevVal = new StringValue("foo");
+
+ tester.getOrCreate("other").setConstantValue(prevVal);
+ tester.getOrCreate(key).addDependency("other").setComputedValue(COPY);
+ assertEquals(prevVal, tester.evalAndGet("value"));
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ try {
+ tester.evalAndGet("value");
+ Assert.fail("injection over value with deps should have failed");
+ } catch (IllegalStateException e) {
+ assertEquals("existing entry for Type:value has deps: [Type:other]", e.getMessage());
+ }
+ }
+
+ @Test
+ public void valueInjectionOverEqualValueWithDeps() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.getOrCreate("other").setConstantValue(val);
+ tester.getOrCreate(key).addDependency("other").setComputedValue(COPY);
+ assertEquals(val, tester.evalAndGet("value"));
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ try {
+ tester.evalAndGet("value");
+ Assert.fail("injection over value with deps should have failed");
+ } catch (IllegalStateException e) {
+ assertEquals("existing entry for Type:value has deps: [Type:other]", e.getMessage());
+ }
+ }
+
+ @Test
+ public void valueInjectionOverValueWithErrors() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("value");
+ SkyValue val = new StringValue("val");
+
+ tester.getOrCreate(key).setHasError(true);
+ tester.evalAndGetError(key);
+
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ assertEquals(val, tester.evalAndGet(false, key));
+ }
+
+ @Test
+ public void valueInjectionInvalidatesReverseDeps() throws Exception {
+ SkyKey childKey = GraphTester.toSkyKey("child");
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ StringValue oldVal = new StringValue("old_val");
+
+ tester.getOrCreate(childKey).setConstantValue(oldVal);
+ tester.getOrCreate(parentKey).addDependency("child").setComputedValue(COPY);
+
+ EvaluationResult<SkyValue> result = tester.eval(false, parentKey);
+ assertFalse(result.hasError());
+ assertEquals(oldVal, result.get(parentKey));
+
+ SkyValue val = new StringValue("val");
+ tester.differencer.inject(ImmutableMap.of(childKey, val));
+ assertEquals(val, tester.evalAndGet("child"));
+ // Injecting a new child should have invalidated the parent.
+ Assert.assertNull(tester.getExistingValue("parent"));
+
+ tester.eval(false, childKey);
+ assertEquals(val, tester.getExistingValue("child"));
+ Assert.assertNull(tester.getExistingValue("parent"));
+ assertEquals(val, tester.evalAndGet("parent"));
+ }
+
+ @Test
+ public void valueInjectionOverExistingEqualEntryDoesNotInvalidate() throws Exception {
+ SkyKey childKey = GraphTester.toSkyKey("child");
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ SkyValue val = new StringValue("same_val");
+
+ tester.getOrCreate(parentKey).addDependency("child").setComputedValue(COPY);
+ tester.getOrCreate(childKey).setConstantValue(new StringValue("same_val"));
+ assertEquals(val, tester.evalAndGet("parent"));
+
+ tester.differencer.inject(ImmutableMap.of(childKey, val));
+ assertEquals(val, tester.getExistingValue("child"));
+ // Since we are injecting an equal value, the parent should not have been invalidated.
+ assertEquals(val, tester.getExistingValue("parent"));
+ }
+
+ @Test
+ public void valueInjectionInterrupt() throws Exception {
+ SkyKey key = GraphTester.toSkyKey("key");
+ SkyValue val = new StringValue("val");
+
+ tester.differencer.inject(ImmutableMap.of(key, val));
+ Thread.currentThread().interrupt();
+ try {
+ tester.evalAndGet("key");
+ fail();
+ } catch (InterruptedException expected) {
+ // Expected.
+ }
+ SkyValue newVal = tester.evalAndGet("key");
+ assertEquals(val, newVal);
+ }
+
+ @Test
+ public void persistentErrorsNotRerun() throws Exception {
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey transientErrorKey = GraphTester.toSkyKey("transientError");
+ SkyKey persistentErrorKey1 = GraphTester.toSkyKey("persistentError1");
+ SkyKey persistentErrorKey2 = GraphTester.toSkyKey("persistentError2");
+
+ tester.getOrCreate(topKey)
+ .addErrorDependency(transientErrorKey, new StringValue("doesn't matter"))
+ .addErrorDependency(persistentErrorKey1, new StringValue("doesn't matter"))
+ .setHasError(true);
+ tester.getOrCreate(persistentErrorKey1).setHasError(true);
+ tester.getOrCreate(transientErrorKey)
+ .addErrorDependency(persistentErrorKey2, new StringValue("doesn't matter"))
+ .setHasTransientError(true);
+ tester.getOrCreate(persistentErrorKey2).setHasError(true);
+
+ tester.evalAndGetError(topKey);
+ assertThat(tester.getEnqueuedValues()).containsExactly(
+ topKey, transientErrorKey, persistentErrorKey1, persistentErrorKey2);
+
+ tester.invalidate();
+ tester.invalidateTransientErrors();
+ tester.evalAndGetError(topKey);
+ // TODO(bazel-team): We can do better here once we implement change pruning for errors.
+ assertThat(tester.getEnqueuedValues()).containsExactly(topKey, transientErrorKey);
+ }
+
+ @Test
+ public void cachedChildErrorDepWithSiblingDepOnNoKeepGoingEval() throws Exception {
+ SkyKey parent1Key = GraphTester.toSkyKey("parent1");
+ SkyKey parent2Key = GraphTester.toSkyKey("parent2");
+ final SkyKey errorKey = GraphTester.toSkyKey("error");
+ final SkyKey otherKey = GraphTester.toSkyKey("other");
+ SkyFunction parentBuilder = new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ env.getValue(errorKey);
+ env.getValue(otherKey);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new StringValue("parent");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ };
+ tester.getOrCreate(parent1Key).setBuilder(parentBuilder);
+ tester.getOrCreate(parent2Key).setBuilder(parentBuilder);
+ tester.getOrCreate(errorKey).setConstantValue(new StringValue("no error yet"));
+ tester.getOrCreate(otherKey).setConstantValue(new StringValue("other"));
+ tester.eval(/*keepGoing=*/true, parent1Key);
+ tester.eval(/*keepGoing=*/false, parent2Key);
+ tester.getOrCreate(errorKey, /*markAsModified=*/true).setHasError(true);
+ tester.invalidate();
+ tester.eval(/*keepGoing=*/true, parent1Key);
+ tester.eval(/*keepGoing=*/false, parent2Key);
+ }
+
+ private void setGraphForTesting(NotifyingInMemoryGraph notifyingInMemoryGraph) {
+ InMemoryMemoizingEvaluator memoizingEvaluator = (InMemoryMemoizingEvaluator) tester.graph;
+ memoizingEvaluator.setGraphForTesting(notifyingInMemoryGraph);
+ }
+
+ private static final class PassThroughSelected implements ValueComputer {
+ private final SkyKey key;
+
+ public PassThroughSelected(SkyKey key) {
+ this.key = key;
+ }
+
+ @Override
+ public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) {
+ return Preconditions.checkNotNull(deps.get(key));
+ }
+ }
+
+ /**
+ * A graph tester that is specific to the memoizing evaluator, with some convenience methods.
+ */
+ private class MemoizingEvaluatorTester extends GraphTester {
+ private RecordingDifferencer differencer;
+ private MemoizingEvaluator graph;
+ private SequentialBuildDriver driver;
+ private TrackingInvalidationReceiver invalidationReceiver = new TrackingInvalidationReceiver();
+
+ public void initialize() {
+ this.differencer = new RecordingDifferencer();
+ this.graph = new InMemoryMemoizingEvaluator(
+ ImmutableMap.of(NODE_TYPE, createDelegatingFunction()), differencer,
+ invalidationReceiver, emittedEventState, true);
+ this.driver = new SequentialBuildDriver(graph);
+ }
+
+ public void setInvalidationReceiver(TrackingInvalidationReceiver customInvalidationReceiver) {
+ Preconditions.checkState(graph == null, "graph already initialized");
+ invalidationReceiver = customInvalidationReceiver;
+ }
+
+ public void invalidate() {
+ differencer.invalidate(getModifiedValues());
+ getModifiedValues().clear();
+ invalidationReceiver.clear();
+ }
+
+ public void invalidateTransientErrors() {
+ differencer.invalidateTransientErrors();
+ }
+
+ public void delete(String key) {
+ graph.delete(Predicates.equalTo(GraphTester.skyKey(key)));
+ }
+
+ public void resetPlayedEvents() {
+ emittedEventState.clear();
+ }
+
+ public Set<SkyValue> getDirtyValues() {
+ return invalidationReceiver.dirty;
+ }
+
+ public Set<SkyValue> getDeletedValues() {
+ return invalidationReceiver.deleted;
+ }
+
+ public Set<SkyKey> getEnqueuedValues() {
+ return invalidationReceiver.enqueued;
+ }
+
+ public <T extends SkyValue> EvaluationResult<T> eval(
+ boolean keepGoing, int numThreads, SkyKey... keys) throws InterruptedException {
+ assertThat(getModifiedValues()).isEmpty();
+ return driver.evaluate(ImmutableList.copyOf(keys), keepGoing, numThreads, reporter);
+ }
+
+ public <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, SkyKey... keys)
+ throws InterruptedException {
+ return eval(keepGoing, 100, keys);
+ }
+
+ public <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, String... keys)
+ throws InterruptedException {
+ return eval(keepGoing, toSkyKeys(keys));
+ }
+
+ public SkyValue evalAndGet(boolean keepGoing, String key)
+ throws InterruptedException {
+ return evalAndGet(keepGoing, new SkyKey(NODE_TYPE, key));
+ }
+
+ public SkyValue evalAndGet(String key) throws InterruptedException {
+ return evalAndGet(/*keepGoing=*/false, key);
+ }
+
+ public SkyValue evalAndGet(boolean keepGoing, SkyKey key)
+ throws InterruptedException {
+ EvaluationResult<StringValue> evaluationResult = eval(keepGoing, key);
+ SkyValue result = evaluationResult.get(key);
+ assertNotNull(evaluationResult.toString(), result);
+ return result;
+ }
+
+ public ErrorInfo evalAndGetError(SkyKey key) throws InterruptedException {
+ EvaluationResult<StringValue> evaluationResult = eval(/*keepGoing=*/true, key);
+ ErrorInfo result = evaluationResult.getError(key);
+ assertNotNull(evaluationResult.toString(), result);
+ return result;
+ }
+
+ public ErrorInfo evalAndGetError(String key) throws InterruptedException {
+ return evalAndGetError(new SkyKey(NODE_TYPE, key));
+ }
+
+ @Nullable
+ public SkyValue getExistingValue(String key) {
+ return graph.getExistingValueForTesting(new SkyKey(NODE_TYPE, key));
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/NodeEntryTest.java b/src/test/java/com/google/devtools/build/skyframe/NodeEntryTest.java
new file mode 100644
index 0000000..378b097
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/NodeEntryTest.java
@@ -0,0 +1,668 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper;
+import com.google.devtools.build.skyframe.NodeEntry.DependencyState;
+import com.google.devtools.build.skyframe.SkyFunctionException.ReifiedSkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link NodeEntry}.
+ */
+@RunWith(JUnit4.class)
+public class NodeEntryTest {
+
+ private static final SkyFunctionName NODE_TYPE = new SkyFunctionName("Type", false);
+ private static final NestedSet<TaggedEvents> NO_EVENTS =
+ NestedSetBuilder.<TaggedEvents>emptySet(Order.STABLE_ORDER);
+
+ private static SkyKey key(String name) {
+ return new SkyKey(NODE_TYPE, name);
+ }
+
+ @Test
+ public void createEntry() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ assertFalse(entry.isDirty());
+ assertFalse(entry.isChanged());
+ assertThat(entry.getTemporaryDirectDeps()).isEmpty();
+ }
+
+ @Test
+ public void signalEntry() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep1 = key("dep1");
+ addTemporaryDirectDep(entry, dep1);
+ assertFalse(entry.isReady());
+ assertTrue(entry.signalDep());
+ assertTrue(entry.isReady());
+ assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep1);
+ SkyKey dep2 = key("dep2");
+ SkyKey dep3 = key("dep3");
+ addTemporaryDirectDep(entry, dep2);
+ addTemporaryDirectDep(entry, dep3);
+ assertFalse(entry.isReady());
+ assertFalse(entry.signalDep());
+ assertFalse(entry.isReady());
+ assertTrue(entry.signalDep());
+ assertTrue(entry.isReady());
+ assertThat(setValue(entry, new SkyValue() {},
+ /*errorInfo=*/null, /*graphVersion=*/0L)).isEmpty();
+ assertTrue(entry.isDone());
+ assertEquals(new IntVersion(0L), entry.getVersion());
+ assertThat(entry.getDirectDeps()).containsExactly(dep1, dep2, dep3);
+ }
+
+ @Test
+ public void reverseDeps() {
+ NodeEntry entry = new NodeEntry();
+ SkyKey mother = key("mother");
+ SkyKey father = key("father");
+ assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(mother));
+ assertEquals(DependencyState.ADDED_DEP, entry.addReverseDepAndCheckIfDone(null));
+ assertEquals(DependencyState.ADDED_DEP, entry.addReverseDepAndCheckIfDone(father));
+ assertThat(setValue(entry, new SkyValue() {},
+ /*errorInfo=*/null, /*graphVersion=*/0L)).containsExactly(mother, father);
+ assertThat(entry.getReverseDeps()).containsExactly(mother, father);
+ assertTrue(entry.isDone());
+ entry.removeReverseDep(mother);
+ assertFalse(Iterables.contains(entry.getReverseDeps(), mother));
+ }
+
+ @Test
+ public void errorValue() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException(
+ new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT),
+ key("cause"));
+ ErrorInfo errorInfo = new ErrorInfo(exception);
+ assertThat(setValue(entry, /*value=*/null, errorInfo, /*graphVersion=*/0L)).isEmpty();
+ assertTrue(entry.isDone());
+ assertNull(entry.getValue());
+ assertEquals(errorInfo, entry.getErrorInfo());
+ }
+
+ @Test
+ public void errorAndValue() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException(
+ new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT),
+ key("cause"));
+ ErrorInfo errorInfo = new ErrorInfo(exception);
+ setValue(entry, new SkyValue() {}, errorInfo, /*graphVersion=*/0L);
+ assertTrue(entry.isDone());
+ assertEquals(errorInfo, entry.getErrorInfo());
+ }
+
+ @Test
+ public void crashOnNullErrorAndValue() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ try {
+ setValue(entry, /*value=*/null, /*errorInfo=*/null, /*graphVersion=*/0L);
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnTooManySignals() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ try {
+ entry.signalDep();
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnDifferentValue() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ try {
+ // Value() {} and Value() {} are not .equals().
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/1L);
+ fail();
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void dirtyLifecycle() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/false);
+ assertTrue(entry.isDirty());
+ assertFalse(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ assertThat(entry.getTemporaryDirectDeps()).isEmpty();
+ SkyKey parent = key("parent");
+ entry.addReverseDepAndCheckIfDone(parent);
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep);
+ assertTrue(entry.isReady());
+ assertThat(setValue(entry, new SkyValue() {}, /*errorInfo=*/null,
+ /*graphVersion=*/1L)).containsExactly(parent);
+ }
+
+ @Test
+ public void changedLifecycle() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/true);
+ assertTrue(entry.isDirty());
+ assertTrue(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ SkyKey parent = key("parent");
+ entry.addReverseDepAndCheckIfDone(parent);
+ assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState());
+ assertTrue(entry.isReady());
+ assertThat(entry.getTemporaryDirectDeps()).isEmpty();
+ assertThat(setValue(entry, new SkyValue() {}, /*errorInfo=*/null,
+ /*graphVersion=*/1L)).containsExactly(parent);
+ assertEquals(new IntVersion(1L), entry.getVersion());
+ }
+
+ @Test
+ public void markDirtyThenChanged() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ addTemporaryDirectDep(entry, key("dep"));
+ entry.signalDep();
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/false);
+ assertTrue(entry.isDirty());
+ assertFalse(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ entry.markDirty(/*isChanged=*/true);
+ assertTrue(entry.isDirty());
+ assertTrue(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ }
+
+
+ @Test
+ public void markChangedThenDirty() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ addTemporaryDirectDep(entry, key("dep"));
+ entry.signalDep();
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/true);
+ assertTrue(entry.isDirty());
+ assertTrue(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ entry.markDirty(/*isChanged=*/false);
+ assertTrue(entry.isDirty());
+ assertTrue(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ }
+
+ @Test
+ public void crashOnTwiceMarkedChanged() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/true);
+ try {
+ entry.markDirty(/*isChanged=*/true);
+ fail("Cannot mark entry changed twice");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnTwiceMarkedDirty() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ addTemporaryDirectDep(entry, key("dep"));
+ entry.signalDep();
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ entry.markDirty(/*isChanged=*/false);
+ try {
+ entry.markDirty(/*isChanged=*/false);
+ fail("Cannot mark entry dirty twice");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnAddReverseDepTwice() {
+ NodeEntry entry = new NodeEntry();
+ SkyKey parent = key("parent");
+ assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(parent));
+ try {
+ entry.addReverseDepAndCheckIfDone(parent);
+ entry.getReverseDeps();
+ fail("Cannot add same dep twice");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnAddReverseDepTwiceAfterDone() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ SkyKey parent = key("parent");
+ assertEquals(DependencyState.DONE, entry.addReverseDepAndCheckIfDone(parent));
+ try {
+ entry.addReverseDepAndCheckIfDone(parent);
+ // We only check for duplicates when we request all the reverse deps.
+ entry.getReverseDeps();
+ fail("Cannot add same dep twice");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnAddReverseDepBeforeAfterDone() {
+ NodeEntry entry = new NodeEntry();
+ SkyKey parent = key("parent");
+ assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(parent));
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ try {
+ entry.addReverseDepAndCheckIfDone(parent);
+ // We only check for duplicates when we request all the reverse deps.
+ entry.getReverseDeps();
+ fail("Cannot add same dep twice");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void crashOnAddDirtyReverseDep() {
+ NodeEntry entry = new NodeEntry();
+ SkyKey parent = key("parent");
+ assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(parent));
+ try {
+ entry.addReverseDepAndCheckIfDone(parent);
+ // We only check for duplicates when we request all the reverse deps.
+ entry.getReverseDeps();
+ fail("Cannot add same dep twice in one build, even if dirty");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void pruneBeforeBuild() {
+ NodeEntry entry = new NodeEntry();
+ SkyKey dep = key("dep");
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/false);
+ assertTrue(entry.isDirty());
+ assertFalse(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ SkyKey parent = key("parent");
+ entry.addReverseDepAndCheckIfDone(parent);
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep(new IntVersion(0L));
+ assertEquals(BuildingState.DirtyState.VERIFIED_CLEAN, entry.getDirtyState());
+ assertThat(entry.markClean()).containsExactly(parent);
+ assertTrue(entry.isDone());
+ assertEquals(new IntVersion(0L), entry.getVersion());
+ }
+
+ private static class IntegerValue implements SkyValue {
+ private final int value;
+
+ IntegerValue(int value) {
+ this.value = value;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ return (that instanceof IntegerValue) && (((IntegerValue) that).value == value);
+ }
+
+ @Override
+ public int hashCode() {
+ return value;
+ }
+ }
+
+ @Test
+ public void pruneAfterBuild() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L);
+ entry.markDirty(/*isChanged=*/false);
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep(new IntVersion(1L));
+ assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState());
+ assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep);
+ setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/1L);
+ assertTrue(entry.isDone());
+ assertEquals(new IntVersion(0L), entry.getVersion());
+ }
+
+
+ @Test
+ public void noPruneWhenDetailsChange() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L);
+ assertFalse(entry.isDirty());
+ assertTrue(entry.isDone());
+ entry.markDirty(/*isChanged=*/false);
+ assertTrue(entry.isDirty());
+ assertFalse(entry.isChanged());
+ assertFalse(entry.isDone());
+ assertTrue(entry.isReady());
+ SkyKey parent = key("parent");
+ entry.addReverseDepAndCheckIfDone(parent);
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep(new IntVersion(1L));
+ assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState());
+ assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep);
+ ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException(
+ new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT),
+ key("cause"));
+ setValue(entry, new IntegerValue(5), new ErrorInfo(exception),
+ /*graphVersion=*/1L);
+ assertTrue(entry.isDone());
+ assertEquals("Version increments when setValue changes", new IntVersion(1), entry.getVersion());
+ }
+
+ @Test
+ public void pruneErrorValue() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException(
+ new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT),
+ key("cause"));
+ ErrorInfo errorInfo = new ErrorInfo(exception);
+ setValue(entry, /*value=*/null, errorInfo, /*graphVersion=*/0L);
+ entry.markDirty(/*isChanged=*/false);
+ entry.addReverseDepAndCheckIfDone(null); // Restart evaluation.
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep(new IntVersion(1L));
+ assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState());
+ assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep);
+ setValue(entry, /*value=*/null, errorInfo, /*graphVersion=*/1L);
+ assertTrue(entry.isDone());
+ assertEquals(new IntVersion(0L), entry.getVersion());
+ }
+
+ @Test
+ public void getDependencyGroup() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ SkyKey dep2 = key("dep2");
+ SkyKey dep3 = key("dep3");
+ addTemporaryDirectDeps(entry, dep, dep2);
+ addTemporaryDirectDep(entry, dep3);
+ entry.signalDep();
+ entry.signalDep();
+ entry.signalDep();
+ setValue(entry, /*value=*/new IntegerValue(5), null, 0L);
+ entry.markDirty(/*isChanged=*/false);
+ entry.addReverseDepAndCheckIfDone(null); // Restart evaluation.
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep, dep2).inOrder();
+ addTemporaryDirectDeps(entry, dep, dep2);
+ entry.signalDep(new IntVersion(0L));
+ entry.signalDep(new IntVersion(0L));
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep3).inOrder();
+ }
+
+ @Test
+ public void maintainDependencyGroupAfterRemoval() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ SkyKey dep2 = key("dep2");
+ SkyKey dep3 = key("dep3");
+ SkyKey dep4 = key("dep4");
+ SkyKey dep5 = key("dep5");
+ addTemporaryDirectDeps(entry, dep, dep2, dep3);
+ addTemporaryDirectDep(entry, dep4);
+ addTemporaryDirectDep(entry, dep5);
+ entry.signalDep();
+ entry.signalDep();
+ // Oops! Evaluation terminated with an error, but we're going to set this entry's value anyway.
+ entry.removeUnfinishedDeps(ImmutableSet.of(dep2, dep3, dep5));
+ ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException(
+ new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT),
+ key("key"));
+ setValue(entry, null, new ErrorInfo(exception), 0L);
+ entry.markDirty(/*isChanged=*/false);
+ entry.addReverseDepAndCheckIfDone(null); // Restart evaluation.
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep(new IntVersion(0L));
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep4).inOrder();
+ }
+
+ @Test
+ public void noPruneWhenDepsChange() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ SkyKey dep = key("dep");
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L);
+ entry.markDirty(/*isChanged=*/false);
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder();
+ addTemporaryDirectDep(entry, dep);
+ assertTrue(entry.signalDep(new IntVersion(1L)));
+ assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState());
+ assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep);
+ addTemporaryDirectDep(entry, key("dep2"));
+ assertTrue(entry.signalDep(new IntVersion(1L)));
+ setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/1L);
+ assertTrue(entry.isDone());
+ assertEquals("Version increments when deps change", new IntVersion(1L), entry.getVersion());
+ }
+
+ @Test
+ public void checkDepsOneByOne() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(null); // Start evaluation.
+ List<SkyKey> deps = new ArrayList<>();
+ for (int ii = 0; ii < 10; ii++) {
+ SkyKey dep = key(Integer.toString(ii));
+ deps.add(dep);
+ addTemporaryDirectDep(entry, dep);
+ entry.signalDep();
+ }
+ setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L);
+ entry.markDirty(/*isChanged=*/false);
+ entry.addReverseDepAndCheckIfDone(null); // Start new evaluation.
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ for (int ii = 0; ii < 10; ii++) {
+ assertThat(entry.getNextDirtyDirectDeps()).containsExactly(deps.get(ii)).inOrder();
+ addTemporaryDirectDep(entry, deps.get(ii));
+ assertTrue(entry.signalDep(new IntVersion(0L)));
+ if (ii < 9) {
+ assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState());
+ } else {
+ assertEquals(BuildingState.DirtyState.VERIFIED_CLEAN, entry.getDirtyState());
+ }
+ }
+ }
+
+ @Test
+ public void signalOnlyNewParents() {
+ NodeEntry entry = new NodeEntry();
+ entry.addReverseDepAndCheckIfDone(key("parent"));
+ setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L);
+ entry.markDirty(/*isChanged=*/true);
+ SkyKey newParent = key("new parent");
+ entry.addReverseDepAndCheckIfDone(newParent);
+ assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState());
+ assertThat(setValue(entry, new SkyValue() {}, /*errorInfo=*/null,
+ /*graphVersion=*/1L)).containsExactly(newParent);
+ }
+
+ @Test
+ public void testClone() {
+ NodeEntry entry = new NodeEntry();
+ IntVersion version = new IntVersion(0);
+ IntegerValue originalValue = new IntegerValue(42);
+ SkyKey originalChild = key("child");
+ addTemporaryDirectDep(entry, originalChild);
+ entry.signalDep();
+ entry.setValue(originalValue, version);
+ entry.addReverseDepAndCheckIfDone(key("parent1"));
+ NodeEntry clone1 = entry.cloneNodeEntry();
+ entry.addReverseDepAndCheckIfDone(key("parent2"));
+ NodeEntry clone2 = entry.cloneNodeEntry();
+ entry.removeReverseDep(key("parent1"));
+ entry.removeReverseDep(key("parent2"));
+ IntegerValue updatedValue = new IntegerValue(52);
+ clone2.markDirty(true);
+ clone2.addReverseDepAndCheckIfDone(null);
+ SkyKey newChild = key("newchild");
+ addTemporaryDirectDep(clone2, newChild);
+ clone2.signalDep();
+ clone2.setValue(updatedValue, version.next());
+
+ assertThat(entry.getVersion()).isEqualTo(version);
+ assertThat(clone1.getVersion()).isEqualTo(version);
+ assertThat(clone2.getVersion()).isEqualTo(version.next());
+
+ assertThat(entry.getValue()).isEqualTo(originalValue);
+ assertThat(clone1.getValue()).isEqualTo(originalValue);
+ assertThat(clone2.getValue()).isEqualTo(updatedValue);
+
+ assertThat(entry.getDirectDeps()).containsExactly(originalChild);
+ assertThat(clone1.getDirectDeps()).containsExactly(originalChild);
+ assertThat(clone2.getDirectDeps()).containsExactly(newChild);
+
+ assertThat(entry.getReverseDeps()).hasSize(0);
+ assertThat(clone1.getReverseDeps()).containsExactly(key("parent1"));
+ assertThat(clone2.getReverseDeps()).containsExactly(key("parent1"), key("parent2"));
+ }
+
+ private static Set<SkyKey> setValue(NodeEntry entry, SkyValue value,
+ @Nullable ErrorInfo errorInfo, long graphVersion) {
+ return entry.setValue(ValueWithMetadata.normal(value, errorInfo, NO_EVENTS),
+ new IntVersion(graphVersion));
+ }
+
+ private static void addTemporaryDirectDep(NodeEntry entry, SkyKey key) {
+ GroupedListHelper<SkyKey> helper = new GroupedListHelper<>();
+ helper.add(key);
+ entry.addTemporaryDirectDeps(helper);
+ }
+
+ private static void addTemporaryDirectDeps(NodeEntry entry, SkyKey... keys) {
+ GroupedListHelper<SkyKey> helper = new GroupedListHelper<>();
+ helper.startGroup();
+ for (SkyKey key : keys) {
+ helper.add(key);
+ }
+ helper.endGroup();
+ entry.addTemporaryDirectDeps(helper);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/NotifyingInMemoryGraph.java b/src/test/java/com/google/devtools/build/skyframe/NotifyingInMemoryGraph.java
new file mode 100644
index 0000000..aa88278
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/NotifyingInMemoryGraph.java
@@ -0,0 +1,128 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.Set;
+
+/**
+ * Class that allows clients to be notified on each access of the graph. Clients can simply track
+ * accesses, or they can block to achieve desired synchronization.
+ */
+public class NotifyingInMemoryGraph extends InMemoryGraph {
+ private final Listener graphListener;
+
+ public NotifyingInMemoryGraph(Listener graphListener) {
+ this.graphListener = graphListener;
+ }
+
+ @Override
+ public NodeEntry createIfAbsent(SkyKey key) {
+ graphListener.accept(key, EventType.CREATE_IF_ABSENT, Order.BEFORE, null);
+ NodeEntry newval = getEntry(key);
+ NodeEntry oldval = getNodeMap().putIfAbsent(key, newval);
+ return oldval == null ? newval : oldval;
+ }
+
+ // Subclasses should override if they wish to subclass NotifyingNodeEntry.
+ protected NotifyingNodeEntry getEntry(SkyKey key) {
+ return new NotifyingNodeEntry(key);
+ }
+
+ /** Receiver to be informed when an event for a given key occurs. */
+ public interface Listener {
+ @ThreadSafe
+ void accept(SkyKey key, EventType type, Order order, Object context);
+
+ public static Listener NULL_LISTENER = new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {}
+ };
+ }
+
+ /**
+ * Graph/value entry events that the receiver can be informed of. When writing tests, feel free to
+ * add additional events here if needed.
+ */
+ public enum EventType {
+ CREATE_IF_ABSENT,
+ ADD_REVERSE_DEP,
+ SIGNAL,
+ SET_VALUE,
+ MARK_DIRTY,
+ IS_CHANGED,
+ IS_DIRTY
+ }
+
+ public enum Order {
+ BEFORE,
+ AFTER
+ }
+
+ protected class NotifyingNodeEntry extends NodeEntry {
+ private final SkyKey myKey;
+
+ protected NotifyingNodeEntry(SkyKey key) {
+ myKey = key;
+ }
+
+ // Note that these methods are not synchronized. Necessary synchronization happens when calling
+ // the super() methods.
+ @Override
+ DependencyState addReverseDepAndCheckIfDone(SkyKey reverseDep) {
+ graphListener.accept(myKey, EventType.ADD_REVERSE_DEP, Order.BEFORE, reverseDep);
+ DependencyState result = super.addReverseDepAndCheckIfDone(reverseDep);
+ graphListener.accept(myKey, EventType.ADD_REVERSE_DEP, Order.AFTER, reverseDep);
+ return result;
+ }
+
+ @Override
+ boolean signalDep(Version childVersion) {
+ graphListener.accept(myKey, EventType.SIGNAL, Order.BEFORE, childVersion);
+ boolean result = super.signalDep(childVersion);
+ graphListener.accept(myKey, EventType.SIGNAL, Order.AFTER, childVersion);
+ return result;
+ }
+
+ @Override
+ public Set<SkyKey> setValue(SkyValue value, Version version) {
+ graphListener.accept(myKey, EventType.SET_VALUE, Order.BEFORE, value);
+ Set<SkyKey> result = super.setValue(value, version);
+ graphListener.accept(myKey, EventType.SET_VALUE, Order.AFTER, value);
+ return result;
+ }
+
+ @Override
+ Pair<? extends Iterable<SkyKey>, ? extends SkyValue> markDirty(boolean isChanged) {
+ graphListener.accept(myKey, EventType.MARK_DIRTY, Order.BEFORE, isChanged);
+ Pair<? extends Iterable<SkyKey>, ? extends SkyValue> result = super.markDirty(isChanged);
+ graphListener.accept(myKey, EventType.MARK_DIRTY, Order.AFTER, isChanged);
+ return result;
+ }
+
+ @Override
+ boolean isChanged() {
+ graphListener.accept(myKey, EventType.IS_CHANGED, Order.BEFORE, this);
+ return super.isChanged();
+ }
+
+ @Override
+ public boolean isDirty() {
+ graphListener.accept(myKey, EventType.IS_DIRTY, Order.BEFORE, this);
+ return super.isDirty();
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/ParallelEvaluatorTest.java b/src/test/java/com/google/devtools/build/skyframe/ParallelEvaluatorTest.java
new file mode 100644
index 0000000..5394437
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/ParallelEvaluatorTest.java
@@ -0,0 +1,2260 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static com.google.devtools.build.skyframe.GraphTester.CONCATENATE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventCollector;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.OutputFilter.RegexOutputFilter;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.testutil.JunitTestUtils;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.skyframe.GraphTester.StringValue;
+import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.EventType;
+import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Listener;
+import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Order;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link ParallelEvaluator}.
+ */
+@RunWith(JUnit4.class)
+public class ParallelEvaluatorTest {
+ protected ProcessableGraph graph;
+ protected IntVersion graphVersion = new IntVersion(0);
+ protected GraphTester tester = new GraphTester();
+
+ private EventCollector eventCollector;
+ private EventHandler reporter;
+
+ private EvaluationProgressReceiver revalidationReceiver;
+
+ @Before
+ public void initializeReporter() {
+ eventCollector = new EventCollector(EventKind.ALL_EVENTS);
+ reporter = new Reporter(eventCollector);
+ }
+
+ private ParallelEvaluator makeEvaluator(ProcessableGraph graph,
+ ImmutableMap<SkyFunctionName, ? extends SkyFunction> builders, boolean keepGoing) {
+ Version oldGraphVersion = graphVersion;
+ graphVersion = graphVersion.next();
+ return new ParallelEvaluator(graph, oldGraphVersion,
+ builders, reporter, new MemoizingEvaluator.EmittedEventState(), keepGoing,
+ 150, revalidationReceiver, new DirtyKeyTrackerImpl());
+ }
+
+ /** Convenience method for eval-ing a single value. */
+ protected SkyValue eval(boolean keepGoing, SkyKey key) throws InterruptedException {
+ return eval(keepGoing, ImmutableList.of(key)).get(key);
+ }
+
+ protected ErrorInfo evalValueInError(SkyKey key) throws InterruptedException {
+ return eval(true, ImmutableList.of(key)).getError(key);
+ }
+
+ protected <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, SkyKey... keys)
+ throws InterruptedException {
+ return eval(keepGoing, ImmutableList.copyOf(keys));
+ }
+
+ protected <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, Iterable<SkyKey> keys)
+ throws InterruptedException {
+ ParallelEvaluator evaluator = makeEvaluator(graph,
+ ImmutableMap.of(GraphTester.NODE_TYPE, tester.createDelegatingFunction()),
+ keepGoing);
+ return evaluator.eval(keys);
+ }
+
+ protected GraphTester.TestFunction set(String name, String value) {
+ return tester.set(name, new StringValue(value));
+ }
+
+ @Test
+ public void smoke() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a");
+ set("b", "b");
+ tester.getOrCreate("ab").addDependency("a").addDependency("b").setComputedValue(CONCATENATE);
+ StringValue value = (StringValue) eval(false, GraphTester.toSkyKey("ab"));
+ assertEquals("ab", value.getValue());
+ JunitTestUtils.assertNoEvents(eventCollector);
+ }
+
+ /**
+ * Test interruption handling when a long-running SkyFunction gets interrupted.
+ */
+ @Test
+ public void interruptedFunction() throws Exception {
+ runInterruptionTest(new SkyFunctionFactory() {
+ @Override
+ public SkyFunction create(final Semaphore threadStarted, final String[] errorMessage) {
+ return new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) throws InterruptedException {
+ // Signal the waiting test thread that the evaluator thread has really started.
+ threadStarted.release();
+
+ // Simulate a SkyFunction that runs for 10 seconds (this number was chosen arbitrarily).
+ // The main thread should interrupt it shortly after it got started.
+ Thread.sleep(10 * 1000);
+
+ // Set an error message to indicate that the expected interruption didn't happen.
+ // We can't use Assert.fail(String) on an async thread.
+ errorMessage[0] = "SkyFunction should have been interrupted";
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ };
+ }
+ });
+ }
+
+ /**
+ * Test interruption handling when the Evaluator is in-between running SkyFunctions.
+ *
+ * <p>This is the point in time after a SkyFunction requested a dependency which is not yet built
+ * so the builder returned null to the Evaluator, and the latter is about to schedule evaluation
+ * of the missing dependency but gets interrupted before the dependency's SkyFunction could start.
+ */
+ @Test
+ public void interruptedEvaluatorThread() throws Exception {
+ runInterruptionTest(new SkyFunctionFactory() {
+ @Override
+ public SkyFunction create(final Semaphore threadStarted, final String[] errorMessage) {
+ return new SkyFunction() {
+ // No need to synchronize access to this field; we always request just one more
+ // dependency, so it's only one SkyFunction running at any time.
+ private int valueIdCounter = 0;
+
+ @Override
+ public SkyValue compute(SkyKey key, Environment env) {
+ // Signal the waiting test thread that the Evaluator thread has really started.
+ threadStarted.release();
+
+ // Keep the evaluator busy until the test's thread gets scheduled and can
+ // interrupt the Evaluator's thread.
+ env.getValue(GraphTester.toSkyKey("a" + valueIdCounter++));
+
+ // This method never throws InterruptedException, therefore it's the responsibility
+ // of the Evaluator to detect the interrupt and avoid calling subsequent SkyFunctions.
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ };
+ }
+ });
+ }
+
+ private void runPartialResultOnInterruption(boolean buildFastFirst) throws Exception {
+ graph = new InMemoryGraph();
+ // Two runs for fastKey's builder and one for the start of waitKey's builder.
+ final CountDownLatch allValuesReady = new CountDownLatch(3);
+ final SkyKey waitKey = GraphTester.toSkyKey("wait");
+ final SkyKey fastKey = GraphTester.toSkyKey("fast");
+ SkyKey leafKey = GraphTester.toSkyKey("leaf");
+ tester.getOrCreate(waitKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
+ allValuesReady.countDown();
+ Thread.sleep(10000);
+ throw new AssertionError("Should have been interrupted");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ tester.getOrCreate(fastKey).setBuilder(new ChainedFunction(null, null, allValuesReady, false,
+ new StringValue("fast"), ImmutableList.of(leafKey)));
+ tester.set(leafKey, new StringValue("leaf"));
+ if (buildFastFirst) {
+ eval(/*keepGoing=*/false, fastKey);
+ }
+ final Set<SkyKey> receivedValues = Sets.newConcurrentHashSet();
+ revalidationReceiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {}
+
+ @Override
+ public void enqueueing(SkyKey key) {}
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ receivedValues.add(skyKey);
+ }
+ };
+ TestThread evalThread = new TestThread() {
+ @Override
+ public void runTest() throws Exception {
+ try {
+ eval(/*keepGoing=*/true, waitKey, fastKey);
+ fail();
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ }
+ };
+ evalThread.start();
+ assertTrue(allValuesReady.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ evalThread.interrupt();
+ evalThread.join(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ assertFalse(evalThread.isAlive());
+ if (buildFastFirst) {
+ // If leafKey was already built, it is not reported to the receiver.
+ assertThat(receivedValues).containsExactly(fastKey);
+ } else {
+ // On first time being built, leafKey is registered too.
+ assertThat(receivedValues).containsExactly(fastKey, leafKey);
+ }
+ }
+
+ @Test
+ public void partialResultOnInterruption() throws Exception {
+ runPartialResultOnInterruption(/*buildFastFirst=*/false);
+ }
+
+ @Test
+ public void partialCachedResultOnInterruption() throws Exception {
+ runPartialResultOnInterruption(/*buildFastFirst=*/true);
+ }
+
+ /**
+ * Factory for SkyFunctions for interruption testing (see {@link #runInterruptionTest}).
+ */
+ private interface SkyFunctionFactory {
+ /**
+ * Creates a SkyFunction suitable for a specific test scenario.
+ *
+ * @param threadStarted a latch which the returned SkyFunction must
+ * {@link Semaphore#release() release} once it started (otherwise the test won't work)
+ * @param errorMessage a single-element array; the SkyFunction can put a error message in it
+ * to indicate that an assertion failed (calling {@code fail} from async thread doesn't
+ * work)
+ */
+ SkyFunction create(final Semaphore threadStarted, final String[] errorMessage);
+ }
+
+ /**
+ * Test that we can handle the Evaluator getting interrupted at various points.
+ *
+ * <p>This method creates an Evaluator with the specified SkyFunction for GraphTested.NODE_TYPE,
+ * then starts a thread, requests evaluation and asserts that evaluation started. It then
+ * interrupts the Evaluator thread and asserts that it acknowledged the interruption.
+ *
+ * @param valueBuilderFactory creates a SkyFunction which may or may not handle interruptions
+ * (depending on the test)
+ */
+ private void runInterruptionTest(SkyFunctionFactory valueBuilderFactory) throws Exception {
+ final Semaphore threadStarted = new Semaphore(0);
+ final Semaphore threadInterrupted = new Semaphore(0);
+ final String[] wasError = new String[] { null };
+ final ParallelEvaluator evaluator = makeEvaluator(new InMemoryGraph(),
+ ImmutableMap.of(GraphTester.NODE_TYPE, valueBuilderFactory.create(threadStarted, wasError)),
+ false);
+
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ evaluator.eval(ImmutableList.of(GraphTester.toSkyKey("a")));
+
+ // There's no real need to set an error here. If the thread is not interrupted then
+ // threadInterrupted is not released and the test thread will fail to acquire it.
+ wasError[0] = "evaluation should have been interrupted";
+ } catch (InterruptedException e) {
+ // This is the interrupt we are waiting for. It should come straight from the
+ // evaluator (more precisely, the AbstractQueueVisitor).
+ // Signal the waiting test thread that the interrupt was acknowledged.
+ threadInterrupted.release();
+ }
+ }
+ });
+
+ // Start the thread and wait for a semaphore. This ensures that the thread was really started.
+ t.start();
+ assertTrue(threadStarted.tryAcquire(TestUtils.WAIT_TIMEOUT_MILLISECONDS,
+ TimeUnit.MILLISECONDS));
+
+ // Interrupt the thread and wait for a semaphore. This ensures that the thread was really
+ // interrupted and this fact was acknowledged.
+ t.interrupt();
+ assertTrue(threadInterrupted.tryAcquire(TestUtils.WAIT_TIMEOUT_MILLISECONDS,
+ TimeUnit.MILLISECONDS));
+
+ // The SkyFunction may have reported an error.
+ if (wasError[0] != null) {
+ fail(wasError[0]);
+ }
+
+ // Wait for the thread to finish.
+ t.join(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ }
+
+ @Test
+ public void unrecoverableError() throws Exception {
+ class CustomRuntimeException extends RuntimeException {}
+ final CustomRuntimeException expected = new CustomRuntimeException();
+
+ final SkyFunction builder = new SkyFunction() {
+ @Override
+ @Nullable
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws SkyFunctionException, InterruptedException {
+ throw expected;
+ }
+
+ @Override
+ @Nullable
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ };
+
+ final ParallelEvaluator evaluator = makeEvaluator(new InMemoryGraph(),
+ ImmutableMap.of(GraphTester.NODE_TYPE, builder),
+ false);
+
+ SkyKey valueToEval = GraphTester.toSkyKey("a");
+ try {
+ evaluator.eval(ImmutableList.of(valueToEval));
+ } catch (RuntimeException re) {
+ assertTrue(re.getMessage()
+ .contains("Unrecoverable error while evaluating node '" + valueToEval.toString() + "'"));
+ assertTrue(re.getCause() instanceof CustomRuntimeException);
+ }
+ }
+
+ @Test
+ public void simpleWarning() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a").setWarning("warning on 'a'");
+ StringValue value = (StringValue) eval(false, GraphTester.toSkyKey("a"));
+ assertEquals("a", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "warning on 'a'");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ @Test
+ public void warningMatchesRegex() throws Exception {
+ graph = new InMemoryGraph();
+ ((Reporter) reporter).setOutputFilter(RegexOutputFilter.forRegex("a"));
+ set("example", "a value").setWarning("warning message");
+ SkyKey a = GraphTester.toSkyKey("example");
+ tester.getOrCreate(a).setTag("a");
+ StringValue value = (StringValue) eval(false, a);
+ assertEquals("a value", value.getValue());
+ JunitTestUtils.assertContainsEvent(eventCollector, "warning message");
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ @Test
+ public void warningMatchesRegexOnlyTag() throws Exception {
+ graph = new InMemoryGraph();
+ ((Reporter) reporter).setOutputFilter(RegexOutputFilter.forRegex("a"));
+ set("a", "a value").setWarning("warning on 'a'");
+ SkyKey a = GraphTester.toSkyKey("a");
+ tester.getOrCreate(a).setTag("b");
+ StringValue value = (StringValue) eval(false, a);
+ assertEquals("a value", value.getValue());
+ JunitTestUtils.assertEventCount(0, eventCollector); }
+
+ @Test
+ public void warningDoesNotMatchRegex() throws Exception {
+ graph = new InMemoryGraph();
+ ((Reporter) reporter).setOutputFilter(RegexOutputFilter.forRegex("b"));
+ set("a", "a").setWarning("warning on 'a'");
+ SkyKey a = GraphTester.toSkyKey("a");
+ tester.getOrCreate(a).setTag("a");
+ StringValue value = (StringValue) eval(false, a);
+ assertEquals("a", value.getValue());
+ JunitTestUtils.assertEventCount(0, eventCollector);
+ }
+
+ /** Regression test: events from already-done value not replayed. */
+ @Test
+ public void eventFromDoneChildRecorded() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a").setWarning("warning on 'a'");
+ SkyKey a = GraphTester.toSkyKey("a");
+ SkyKey top = GraphTester.toSkyKey("top");
+ tester.getOrCreate(top).addDependency(a).setComputedValue(CONCATENATE);
+ // Build a so that it is already in the graph.
+ eval(false, a);
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ eventCollector.clear();
+ // Build top. The warning from a should be reprinted.
+ eval(false, top);
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ eventCollector.clear();
+ // Build top again. The warning should have been stored in the value.
+ eval(false, top);
+ JunitTestUtils.assertEventCount(1, eventCollector);
+ }
+
+ @Test
+ public void shouldCreateErrorValueWithRootCause() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a");
+ SkyKey parentErrorKey = GraphTester.toSkyKey("parent");
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(parentErrorKey).addDependency("a").addDependency(errorKey)
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(errorKey).setHasError(true);
+ ErrorInfo error = evalValueInError(parentErrorKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ }
+
+ @Test
+ public void shouldBuildOneTarget() throws Exception {
+ graph = new InMemoryGraph();
+ set("a", "a");
+ set("b", "b");
+ SkyKey parentErrorKey = GraphTester.toSkyKey("parent");
+ SkyKey errorFreeKey = GraphTester.toSkyKey("ab");
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(parentErrorKey).addDependency(errorKey).addDependency("a")
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.getOrCreate(errorFreeKey).addDependency("a").addDependency("b")
+ .setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(true, parentErrorKey, errorFreeKey);
+ ErrorInfo error = result.getError(parentErrorKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ StringValue abValue = result.get(errorFreeKey);
+ assertEquals("ab", abValue.getValue());
+ }
+
+ @Test
+ public void catastropheHaltsBuild_KeepGoing_KeepEdges() throws Exception {
+ catastrophicBuild(true, true);
+ }
+
+ @Test
+ public void catastropheHaltsBuild_KeepGoing_NoKeepEdges() throws Exception {
+ catastrophicBuild(true, false);
+ }
+
+ @Test
+ public void catastropheInBuild_NoKeepGoing_KeepEdges() throws Exception {
+ catastrophicBuild(false, true);
+ }
+
+ private void catastrophicBuild(boolean keepGoing, boolean keepEdges) throws Exception {
+ graph = new InMemoryGraph(keepEdges);
+
+ SkyKey catastropheKey = GraphTester.toSkyKey("catastrophe");
+ SkyKey otherKey = GraphTester.toSkyKey("someKey");
+
+ tester.getOrCreate(catastropheKey).setBuilder(new SkyFunction() {
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ throw new SkyFunctionException(new SomeErrorException("bad"),
+ Transience.PERSISTENT) {
+ @Override
+ public boolean isCatastrophic() {
+ return true;
+ }
+ };
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+
+ tester.getOrCreate(otherKey).setBuilder(new SkyFunction() {
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
+ new CountDownLatch(1).await();
+ throw new RuntimeException("can't get here");
+ }
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(catastropheKey).setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(keepGoing, topKey, otherKey);
+ if (!keepGoing) {
+ ErrorInfo error = result.getError(topKey);
+ assertThat(error.getRootCauses()).containsExactly(catastropheKey);
+ } else {
+ assertTrue(result.hasError());
+ assertThat(result.errorMap()).isEmpty();
+ }
+ }
+
+ @Test
+ public void parentFailureDoesntAffectChild() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).setHasError(true);
+ SkyKey childKey = GraphTester.toSkyKey("child");
+ set("child", "onions");
+ tester.getOrCreate(parentKey).addDependency(childKey).setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, parentKey, childKey);
+ // Child is guaranteed to complete successfully before parent can run (and fail),
+ // since parent depends on it.
+ StringValue childValue = result.get(childKey);
+ Assert.assertNotNull(childValue);
+ assertEquals("onions", childValue.getValue());
+ ErrorInfo error = result.getError(parentKey);
+ Assert.assertNotNull(error);
+ assertThat(error.getRootCauses()).containsExactly(parentKey);
+ }
+
+ @Test
+ public void newParentOfErrorShouldHaveError() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setHasError(true);
+ ErrorInfo error = evalValueInError(errorKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addDependency("error").setComputedValue(CONCATENATE);
+ error = evalValueInError(parentKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ }
+
+ @Test
+ public void errorTwoLevelsDeep() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.getOrCreate("mid").addDependency(errorKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(parentKey).addDependency("mid").setComputedValue(CONCATENATE);
+ ErrorInfo error = evalValueInError(parentKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ }
+
+ /**
+ * A recreation of BuildViewTest#testHasErrorRaceCondition. Also similar to errorTwoLevelsDeep,
+ * except here we request multiple toplevel values.
+ */
+ @Test
+ public void errorPropagationToTopLevelValues() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey badKey = GraphTester.toSkyKey("bad");
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(midKey).addDependency(badKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(badKey).setHasError(true);
+ EvaluationResult<SkyValue> result = eval(/*keepGoing=*/false, topKey, midKey);
+ assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey);
+ // Do it again with keepGoing. We should also see an error for the top key this time.
+ result = eval(/*keepGoing=*/true, topKey, midKey);
+ assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey);
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(badKey);
+ }
+
+ @Test
+ public void valueNotUsedInFailFastErrorRecovery() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey recoveryKey = GraphTester.toSkyKey("midRecovery");
+ SkyKey badKey = GraphTester.toSkyKey("bad");
+
+ tester.getOrCreate(topKey).addDependency(recoveryKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(recoveryKey).addErrorDependency(badKey, new StringValue("i recovered"))
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(badKey).setHasError(true);
+
+ EvaluationResult<SkyValue> result = eval(/*keepGoing=*/true, ImmutableList.of(recoveryKey));
+ assertThat(result.errorMap()).isEmpty();
+ assertTrue(result.hasError());
+ assertEquals(new StringValue("i recovered"), result.get(recoveryKey));
+
+ result = eval(/*keepGoing=*/false, ImmutableList.of(topKey));
+ assertTrue(result.hasError());
+ assertThat(result.keyNames()).isEmpty();
+ assertEquals(1, result.errorMap().size());
+ assertNotNull(result.getError(topKey).getException());
+ }
+
+ /**
+ * Regression test: "clearing incomplete values on --keep_going build is racy".
+ * Tests that if a value is requested on the first (non-keep-going) build and its child throws
+ * an error, when the second (keep-going) build runs, there is not a race that keeps it as a
+ * reverse dep of its children.
+ */
+ @Test
+ public void raceClearingIncompleteValues() throws Exception {
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ final SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey badKey = GraphTester.toSkyKey("bad");
+ final AtomicBoolean waitForSecondCall = new AtomicBoolean(false);
+ final TrackingAwaiter trackingAwaiter = new TrackingAwaiter();
+ final CountDownLatch otherThreadWinning = new CountDownLatch(1);
+ final AtomicReference<Thread> firstThread = new AtomicReference<>();
+ graph = new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (!waitForSecondCall.get()) {
+ return;
+ }
+ if (key.equals(midKey)) {
+ if (type == EventType.CREATE_IF_ABSENT) {
+ // The first thread to create midKey will not be the first thread to add a reverse dep
+ // to it.
+ firstThread.compareAndSet(null, Thread.currentThread());
+ return;
+ }
+ if (type == EventType.ADD_REVERSE_DEP) {
+ if (order == Order.BEFORE && Thread.currentThread().equals(firstThread.get())) {
+ // If this thread created midKey, block until the other thread adds a dep on it.
+ trackingAwaiter.awaitLatchAndTrackExceptions(otherThreadWinning,
+ "other thread didn't pass this one");
+ } else if (order == Order.AFTER && !Thread.currentThread().equals(firstThread.get())) {
+ // This thread has added a dep. Allow the other thread to proceed.
+ otherThreadWinning.countDown();
+ }
+ }
+ }
+ }
+ });
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(midKey).addDependency(badKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(badKey).setHasError(true);
+ EvaluationResult<SkyValue> result = eval(/*keepGoing=*/false, topKey, midKey);
+ assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey);
+ waitForSecondCall.set(true);
+ result = eval(/*keepGoing=*/true, topKey, midKey);
+ trackingAwaiter.assertNoErrors();
+ assertNotNull(firstThread.get());
+ assertEquals(0, otherThreadWinning.getCount());
+ assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey);
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(badKey);
+ }
+
+ @Test
+ public void multipleRootCauses() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ SkyKey errorKey2 = GraphTester.toSkyKey("error2");
+ SkyKey errorKey3 = GraphTester.toSkyKey("error3");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.getOrCreate(errorKey2).setHasError(true);
+ tester.getOrCreate(errorKey3).setHasError(true);
+ tester.getOrCreate("mid").addDependency(errorKey).addDependency(errorKey2)
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(parentKey)
+ .addDependency("mid").addDependency(errorKey2).addDependency(errorKey3)
+ .setComputedValue(CONCATENATE);
+ ErrorInfo error = evalValueInError(parentKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey, errorKey2, errorKey3);
+ }
+
+ @Test
+ public void rootCauseWithNoKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.getOrCreate("mid").addDependency(errorKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(parentKey).addDependency("mid").setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(false, ImmutableList.of(parentKey));
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(parentKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(errorKey);
+ }
+
+ @Test
+ public void errorBubblesToParentsOfTopLevelValue() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ final SkyKey errorKey = GraphTester.toSkyKey("error");
+ final CountDownLatch latch = new CountDownLatch(1);
+ tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, /*waitToFinish=*/latch, null,
+ false, /*value=*/null, ImmutableList.<SkyKey>of()));
+ tester.getOrCreate(parentKey).setBuilder(new ChainedFunction(/*notifyStart=*/latch, null, null,
+ false, new StringValue("unused"), ImmutableList.of(errorKey)));
+ EvaluationResult<StringValue> result = eval( /*keepGoing=*/false,
+ ImmutableList.of(parentKey, errorKey));
+ assertEquals(result.toString(), 2, result.errorMap().size());
+ }
+
+ @Test
+ public void noKeepGoingAfterKeepGoingFails() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addDependency(errorKey);
+ ErrorInfo error = evalValueInError(parentKey);
+ assertThat(error.getRootCauses()).containsExactly(errorKey);
+ SkyKey[] list = { parentKey };
+ EvaluationResult<StringValue> result = eval(false, list);
+ ErrorInfo errorInfo = result.getError();
+ assertEquals(errorKey, Iterables.getOnlyElement(errorInfo.getRootCauses()));
+ assertEquals(errorKey.toString(), errorInfo.getException().getMessage());
+ }
+
+ @Test
+ public void twoErrors() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey firstError = GraphTester.toSkyKey("error1");
+ SkyKey secondError = GraphTester.toSkyKey("error2");
+ CountDownLatch firstStart = new CountDownLatch(1);
+ CountDownLatch secondStart = new CountDownLatch(1);
+ tester.getOrCreate(firstError).setBuilder(new ChainedFunction(firstStart, secondStart,
+ /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null,
+ ImmutableList.<SkyKey>of()));
+ tester.getOrCreate(secondError).setBuilder(new ChainedFunction(secondStart, firstStart,
+ /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null,
+ ImmutableList.<SkyKey>of()));
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, firstError, secondError);
+ assertTrue(result.toString(), result.hasError());
+ // With keepGoing=false, the eval call will terminate with exactly one error (the first one
+ // thrown). But the first one thrown here is non-deterministic since we synchronize the
+ // builders so that they run at roughly the same time.
+ assertThat(ImmutableSet.of(firstError, secondError)).contains(
+ Iterables.getOnlyElement(result.errorMap().keySet()));
+ }
+
+ @Test
+ public void simpleCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ ErrorInfo errorInfo = eval(false, ImmutableList.of(aKey)).getError();
+ assertEquals(null, errorInfo.getException());
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertTrue(cycleInfo.getPathToCycle().isEmpty());
+ }
+
+ @Test
+ public void cycleWithHead() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(topKey).addDependency(midKey);
+ tester.getOrCreate(midKey).addDependency(aKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ ErrorInfo errorInfo = eval(false, ImmutableList.of(topKey)).getError();
+ assertEquals(null, errorInfo.getException());
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder();
+ }
+
+ @Test
+ public void selfEdgeWithHead() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ tester.getOrCreate(topKey).addDependency(midKey);
+ tester.getOrCreate(midKey).addDependency(aKey);
+ tester.getOrCreate(aKey).addDependency(aKey);
+ ErrorInfo errorInfo = eval(false, ImmutableList.of(topKey)).getError();
+ assertEquals(null, errorInfo.getException());
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder();
+ }
+
+ @Test
+ public void cycleWithKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey goodKey = GraphTester.toSkyKey("good");
+ StringValue goodValue = new StringValue("good");
+ tester.set(goodKey, goodValue);
+ tester.getOrCreate(topKey).addDependency(midKey);
+ tester.getOrCreate(midKey).addDependency(aKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ EvaluationResult<StringValue> result = eval(true, topKey, goodKey);
+ assertEquals(goodValue, result.get(goodKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder();
+ }
+
+ @Test
+ public void twoCycles() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ SkyKey dKey = GraphTester.toSkyKey("d");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(aKey).addDependency(cKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ tester.getOrCreate(cKey).addDependency(dKey);
+ tester.getOrCreate(dKey).addDependency(cKey);
+ EvaluationResult<StringValue> result = eval(false, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ Iterable<CycleInfo> cycles = CycleInfo.prepareCycles(topKey,
+ ImmutableList.of(new CycleInfo(ImmutableList.of(aKey, bKey)),
+ new CycleInfo(ImmutableList.of(cKey, dKey))));
+ assertThat(cycles).contains(getOnlyElement(errorInfo.getCycleInfo()));
+ }
+
+
+ @Test
+ public void twoCyclesKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ SkyKey dKey = GraphTester.toSkyKey("d");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(aKey).addDependency(cKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey);
+ tester.getOrCreate(cKey).addDependency(dKey);
+ tester.getOrCreate(dKey).addDependency(cKey);
+ EvaluationResult<StringValue> result = eval(true, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo aCycle = new CycleInfo(ImmutableList.of(topKey), ImmutableList.of(aKey, bKey));
+ CycleInfo cCycle = new CycleInfo(ImmutableList.of(topKey), ImmutableList.of(cKey, dKey));
+ assertThat(errorInfo.getCycleInfo()).containsExactly(aCycle, cCycle);
+ }
+
+ @Test
+ public void triangleBelowHeadCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(aKey);
+ tester.getOrCreate(aKey).addDependency(bKey).addDependency(cKey);
+ tester.getOrCreate(bKey).addDependency(cKey);
+ tester.getOrCreate(cKey).addDependency(topKey);
+ EvaluationResult<StringValue> result = eval(true, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo topCycle = new CycleInfo(ImmutableList.of(topKey, aKey, cKey));
+ assertThat(errorInfo.getCycleInfo()).containsExactly(topCycle);
+ }
+
+ @Test
+ public void longCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(aKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(cKey);
+ tester.getOrCreate(cKey).addDependency(topKey);
+ EvaluationResult<StringValue> result = eval(true, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo topCycle = new CycleInfo(ImmutableList.of(topKey, aKey, bKey, cKey));
+ assertThat(errorInfo.getCycleInfo()).containsExactly(topCycle);
+ }
+
+ @Test
+ public void cycleWithTail() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(aKey).addDependency(cKey);
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(aKey).addDependency(cKey);
+ tester.getOrCreate(cKey);
+ tester.set(cKey, new StringValue("cValue"));
+ EvaluationResult<StringValue> result = eval(false, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ ErrorInfo errorInfo = result.getError(topKey);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey).inOrder();
+ }
+
+ /** Regression test: "value cannot be ready in a cycle". */
+ @Test
+ public void selfEdgeWithExtraChildrenUnderCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(cKey).addDependency(bKey);
+ tester.getOrCreate(cKey).addDependency(aKey);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey));
+ assertEquals(null, result.get(aKey));
+ ErrorInfo errorInfo = result.getError(aKey);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(aKey).inOrder();
+ }
+
+ /** Regression test: "value cannot be ready in a cycle". */
+ @Test
+ public void cycleWithExtraChildrenUnderCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ SkyKey dKey = GraphTester.toSkyKey("d");
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(cKey).addDependency(dKey);
+ tester.getOrCreate(cKey).addDependency(aKey);
+ tester.getOrCreate(dKey).addDependency(bKey);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey));
+ assertEquals(null, result.get(aKey));
+ ErrorInfo errorInfo = result.getError(aKey);
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(bKey, dKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(aKey).inOrder();
+ }
+
+ /** Regression test: "value cannot be ready in a cycle". */
+ @Test
+ public void cycleAboveIndependentCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ SkyKey cKey = GraphTester.toSkyKey("c");
+ tester.getOrCreate(aKey).addDependency(bKey);
+ tester.getOrCreate(bKey).addDependency(cKey);
+ tester.getOrCreate(cKey).addDependency(aKey).addDependency(bKey);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey));
+ assertEquals(null, result.get(aKey));
+ assertThat(result.getError(aKey).getCycleInfo()).containsExactly(
+ new CycleInfo(ImmutableList.of(aKey, bKey, cKey)),
+ new CycleInfo(ImmutableList.of(aKey), ImmutableList.of(bKey, cKey)));
+ }
+
+ public void valueAboveCycleAndExceptionReportsException() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey aKey = GraphTester.toSkyKey("a");
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ SkyKey bKey = GraphTester.toSkyKey("b");
+ tester.getOrCreate(aKey).addDependency(bKey).addDependency(errorKey);
+ tester.getOrCreate(bKey).addDependency(bKey);
+ tester.getOrCreate(errorKey).setHasError(true);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey));
+ assertEquals(null, result.get(aKey));
+ assertNotNull(result.getError(aKey).getException());
+ CycleInfo cycleInfo = Iterables.getOnlyElement(result.getError(aKey).getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(bKey).inOrder();
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(aKey).inOrder();
+ }
+
+ @Test
+ public void errorValueStored() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ EvaluationResult<StringValue> result = eval(false, ImmutableList.of(errorKey));
+ assertThat(result.keyNames()).isEmpty();
+ assertThat(result.errorMap().keySet()).containsExactly(errorKey);
+ ErrorInfo errorInfo = result.getError();
+ assertThat(errorInfo.getRootCauses()).containsExactly(errorKey);
+ // Update value. But builder won't rebuild it.
+ tester.getOrCreate(errorKey).setHasError(false);
+ tester.set(errorKey, new StringValue("no error?"));
+ result = eval(false, ImmutableList.of(errorKey));
+ assertThat(result.keyNames()).isEmpty();
+ assertThat(result.errorMap().keySet()).containsExactly(errorKey);
+ errorInfo = result.getError();
+ assertThat(errorInfo.getRootCauses()).containsExactly(errorKey);
+ }
+
+ /**
+ * Regression test: "OOM in Skyframe cycle detection".
+ * We only store the first 20 cycles found below any given root value.
+ */
+ @Test
+ public void manyCycles() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ for (int i = 0; i < 100; i++) {
+ SkyKey dep = GraphTester.toSkyKey(Integer.toString(i));
+ tester.getOrCreate(topKey).addDependency(dep);
+ tester.getOrCreate(dep).addDependency(dep);
+ }
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ assertManyCycles(result.getError(topKey), topKey, /*selfEdge=*/false);
+ }
+
+ /**
+ * Regression test: "OOM in Skyframe cycle detection".
+ * We filter out multiple paths to a cycle that go through the same child value.
+ */
+ @Test
+ public void manyPathsToCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey cycleKey = GraphTester.toSkyKey("cycle");
+ tester.getOrCreate(topKey).addDependency(midKey);
+ tester.getOrCreate(cycleKey).addDependency(cycleKey);
+ for (int i = 0; i < 100; i++) {
+ SkyKey dep = GraphTester.toSkyKey(Integer.toString(i));
+ tester.getOrCreate(midKey).addDependency(dep);
+ tester.getOrCreate(dep).addDependency(cycleKey);
+ }
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(topKey));
+ assertEquals(null, result.get(topKey));
+ CycleInfo cycleInfo = Iterables.getOnlyElement(result.getError(topKey).getCycleInfo());
+ assertEquals(1, cycleInfo.getCycle().size());
+ assertEquals(3, cycleInfo.getPathToCycle().size());
+ assertThat(cycleInfo.getPathToCycle().subList(0, 2)).containsExactly(topKey, midKey).inOrder();
+ }
+
+ /**
+ * Checks that errorInfo has many self-edge cycles, and that one of them is a self-edge of
+ * topKey, if {@code selfEdge} is true.
+ */
+ private static void assertManyCycles(ErrorInfo errorInfo, SkyKey topKey, boolean selfEdge) {
+ MoreAsserts.assertGreaterThan(1, Iterables.size(errorInfo.getCycleInfo()));
+ MoreAsserts.assertLessThan(50, Iterables.size(errorInfo.getCycleInfo()));
+ boolean foundSelfEdge = false;
+ for (CycleInfo cycle : errorInfo.getCycleInfo()) {
+ assertEquals(1, cycle.getCycle().size()); // Self-edge.
+ if (!Iterables.isEmpty(cycle.getPathToCycle())) {
+ assertThat(cycle.getPathToCycle()).containsExactly(topKey).inOrder();
+ } else {
+ assertThat(cycle.getCycle()).containsExactly(topKey).inOrder();
+ foundSelfEdge = true;
+ }
+ }
+ assertEquals(errorInfo + ", " + topKey, selfEdge, foundSelfEdge);
+ }
+
+ @Test
+ public void manyUnprocessedValuesInCycle() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey lastSelfKey = GraphTester.toSkyKey("lastSelf");
+ SkyKey firstSelfKey = GraphTester.toSkyKey("firstSelf");
+ SkyKey midSelfKey = GraphTester.toSkyKey("midSelf");
+ // We add firstSelf first so that it is processed last in cycle detection (LIFO), meaning that
+ // none of the dep values have to be cleared from firstSelf.
+ tester.getOrCreate(firstSelfKey).addDependency(firstSelfKey);
+ for (int i = 0; i < 100; i++) {
+ SkyKey firstDep = GraphTester.toSkyKey("first" + i);
+ SkyKey midDep = GraphTester.toSkyKey("mid" + i);
+ SkyKey lastDep = GraphTester.toSkyKey("last" + i);
+ tester.getOrCreate(firstSelfKey).addDependency(firstDep);
+ tester.getOrCreate(midSelfKey).addDependency(midDep);
+ tester.getOrCreate(lastSelfKey).addDependency(lastDep);
+ if (i == 90) {
+ // Most of the deps will be cleared from midSelf.
+ tester.getOrCreate(midSelfKey).addDependency(midSelfKey);
+ }
+ tester.getOrCreate(firstDep).addDependency(firstDep);
+ tester.getOrCreate(midDep).addDependency(midDep);
+ tester.getOrCreate(lastDep).addDependency(lastDep);
+ }
+ // All the deps will be cleared from lastSelf.
+ tester.getOrCreate(lastSelfKey).addDependency(lastSelfKey);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true,
+ ImmutableList.of(lastSelfKey, firstSelfKey, midSelfKey));
+ assertWithMessage(result.toString()).that(result.keyNames()).isEmpty();
+ assertThat(result.errorMap().keySet()).containsExactly(lastSelfKey, firstSelfKey, midSelfKey);
+
+ // Check lastSelfKey.
+ ErrorInfo errorInfo = result.getError(lastSelfKey);
+ assertEquals(errorInfo.toString(), 1, Iterables.size(errorInfo.getCycleInfo()));
+ CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo());
+ assertThat(cycleInfo.getCycle()).containsExactly(lastSelfKey);
+ assertThat(cycleInfo.getPathToCycle()).isEmpty();
+
+ // Check firstSelfKey. It should not have discovered its own self-edge, because there were too
+ // many other values before it in the queue.
+ assertManyCycles(result.getError(firstSelfKey), firstSelfKey, /*selfEdge=*/false);
+
+ // Check midSelfKey. It should have discovered its own self-edge.
+ assertManyCycles(result.getError(midSelfKey), midSelfKey, /*selfEdge=*/true);
+ }
+
+ @Test
+ public void errorValueStoredWithKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ EvaluationResult<StringValue> result = eval(true, ImmutableList.of(errorKey));
+ assertThat(result.keyNames()).isEmpty();
+ assertThat(result.errorMap().keySet()).containsExactly(errorKey);
+ ErrorInfo errorInfo = result.getError();
+ assertThat(errorInfo.getRootCauses()).containsExactly(errorKey);
+ // Update value. But builder won't rebuild it.
+ tester.getOrCreate(errorKey).setHasError(false);
+ tester.set(errorKey, new StringValue("no error?"));
+ result = eval(true, ImmutableList.of(errorKey));
+ assertThat(result.keyNames()).isEmpty();
+ assertThat(result.errorMap().keySet()).containsExactly(errorKey);
+ errorInfo = result.getError();
+ assertThat(errorInfo.getRootCauses()).containsExactly(errorKey);
+ }
+
+ @Test
+ public void continueWithErrorDep() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.set("after", new StringValue("after"));
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE).addDependency("after");
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(parentKey));
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("recoveredafter", result.get(parentKey).getValue());
+ result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey));
+ assertThat(result.keyNames()).isEmpty();
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(parentKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(errorKey);
+ }
+
+ @Test
+ public void breakWithErrorDep() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.set("after", new StringValue("after"));
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE).addDependency("after");
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey));
+ assertThat(result.keyNames()).isEmpty();
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(parentKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(errorKey);
+ result = eval(/*keepGoing=*/true, ImmutableList.of(parentKey));
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("recoveredafter", result.get(parentKey).getValue());
+ }
+
+ @Test
+ public void breakWithInterruptibleErrorDep() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE);
+ // When the error value throws, the propagation will cause an interrupted exception in parent.
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey));
+ assertThat(result.keyNames()).isEmpty();
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(parentKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(errorKey);
+ assertFalse(Thread.interrupted());
+ result = eval(/*keepGoing=*/true, ImmutableList.of(parentKey));
+ assertThat(result.errorMap()).isEmpty();
+ assertEquals("recovered", result.get(parentKey).getValue());
+ }
+
+ @Test
+ public void transformErrorDep() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ SkyKey parentErrorKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setHasError(true);
+ EvaluationResult<StringValue> result = eval(
+ /*keepGoing=*/false, ImmutableList.of(parentErrorKey));
+ assertThat(result.keyNames()).isEmpty();
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(parentErrorKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(parentErrorKey);
+ }
+
+ @Test
+ public void transformErrorDepKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ SkyKey parentErrorKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setHasError(true);
+ EvaluationResult<StringValue> result = eval(
+ /*keepGoing=*/true, ImmutableList.of(parentErrorKey));
+ assertThat(result.keyNames()).isEmpty();
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(parentErrorKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(parentErrorKey);
+ }
+
+ @Test
+ public void transformErrorDepOneLevelDownKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.set("after", new StringValue("after"));
+ SkyKey parentErrorKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered"));
+ tester.set(parentErrorKey, new StringValue("parent value"));
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(parentErrorKey).addDependency("after")
+ .setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(topKey));
+ assertThat(ImmutableList.<String>copyOf(result.<String>keyNames())).containsExactly("top");
+ assertEquals("parent valueafter", result.get(topKey).getValue());
+ assertThat(result.errorMap()).isEmpty();
+ }
+
+ @Test
+ public void transformErrorDepOneLevelDownNoKeepGoing() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ tester.getOrCreate(errorKey).setHasError(true);
+ tester.set("after", new StringValue("after"));
+ SkyKey parentErrorKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered"));
+ tester.set(parentErrorKey, new StringValue("parent value"));
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(parentErrorKey).addDependency("after")
+ .setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(topKey));
+ assertThat(result.keyNames()).isEmpty();
+ Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet());
+ assertEquals(topKey, error.getKey());
+ assertThat(error.getValue().getRootCauses()).containsExactly(errorKey);
+ }
+
+ /**
+ * Make sure that multiple unfinished children can be cleared from a cycle value.
+ */
+ @Test
+ public void cycleWithMultipleUnfinishedChildren() throws Exception {
+ graph = new InMemoryGraph();
+ tester = new GraphTester();
+ SkyKey cycleKey = GraphTester.toSkyKey("cycle");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ SkyKey selfEdge1 = GraphTester.toSkyKey("selfEdge1");
+ SkyKey selfEdge2 = GraphTester.toSkyKey("selfEdge2");
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
+ // selfEdge* come before cycleKey, so cycleKey's path will be checked first (LIFO), and the
+ // cycle with mid will be detected before the selfEdge* cycles are.
+ tester.getOrCreate(midKey).addDependency(selfEdge1).addDependency(selfEdge2)
+ .addDependency(cycleKey)
+ .setComputedValue(CONCATENATE);
+ tester.getOrCreate(cycleKey).addDependency(midKey);
+ tester.getOrCreate(selfEdge1).addDependency(selfEdge1);
+ tester.getOrCreate(selfEdge2).addDependency(selfEdge2);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableSet.of(topKey));
+ assertThat(result.errorMap().keySet()).containsExactly(topKey);
+ Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo();
+ CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos);
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey);
+ assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey);
+ }
+
+ /**
+ * Regression test: "value in cycle depends on error".
+ * The mid value will have two parents -- top and cycle. Error bubbles up from mid to cycle, and
+ * we should detect cycle.
+ */
+ private void cycleAndErrorInBubbleUp(boolean keepGoing) throws Exception {
+ graph = new DeterministicInMemoryGraph();
+ tester = new GraphTester();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ SkyKey cycleKey = GraphTester.toSkyKey("cycle");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(midKey).addDependency(errorKey).addDependency(cycleKey)
+ .setComputedValue(CONCATENATE);
+
+ // We need to ensure that cycle value has finished his work, and we have recorded dependencies
+ CountDownLatch cycleFinish = new CountDownLatch(1);
+ tester.getOrCreate(cycleKey).setBuilder(new ChainedFunction(null,
+ null, cycleFinish, false, new StringValue(""), ImmutableSet.<SkyKey>of(midKey)));
+ tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, cycleFinish,
+ null, /*waitForException=*/false, null, ImmutableSet.<SkyKey>of()));
+
+ EvaluationResult<StringValue> result = eval(keepGoing, ImmutableSet.of(topKey));
+ assertThat(result.errorMap().keySet()).containsExactly(topKey);
+ Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo();
+ if (keepGoing) {
+ // The error thrown will only be recorded in keep_going mode.
+ assertThat(result.getError().getRootCauses()).containsExactly(errorKey);
+ }
+ assertThat(cycleInfos).isNotEmpty();
+ CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos);
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey);
+ assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey);
+ }
+
+ @Test
+ public void cycleAndErrorInBubbleUpNoKeepGoing() throws Exception {
+ cycleAndErrorInBubbleUp(false);
+ }
+
+ @Test
+ public void cycleAndErrorInBubbleUpKeepGoing() throws Exception {
+ cycleAndErrorInBubbleUp(true);
+ }
+
+ /**
+ * Regression test: "value in cycle depends on error".
+ * We add another value that won't finish building before the threadpool shuts down, to check that
+ * the cycle detection can handle unfinished values.
+ */
+ @Test
+ public void cycleAndErrorAndOtherInBubbleUp() throws Exception {
+ graph = new DeterministicInMemoryGraph();
+ tester = new GraphTester();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ SkyKey cycleKey = GraphTester.toSkyKey("cycle");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
+ // We should add cycleKey first and errorKey afterwards. Otherwise there is a chance that
+ // during error propagation cycleKey will not be processed, and we will not detect the cycle.
+ tester.getOrCreate(midKey).addDependency(errorKey).addDependency(cycleKey)
+ .setComputedValue(CONCATENATE);
+ SkyKey otherTop = GraphTester.toSkyKey("otherTop");
+ CountDownLatch topStartAndCycleFinish = new CountDownLatch(2);
+ // In nokeep_going mode, otherTop will wait until the threadpool has received an exception,
+ // then request its own dep. This guarantees that there is a value that is not finished when
+ // cycle detection happens.
+ tester.getOrCreate(otherTop).setBuilder(new ChainedFunction(topStartAndCycleFinish,
+ new CountDownLatch(0), null, /*waitForException=*/true, new StringValue("never returned"),
+ ImmutableSet.<SkyKey>of(GraphTester.toSkyKey("dep that never builds"))));
+
+ tester.getOrCreate(cycleKey).setBuilder(new ChainedFunction(null, null,
+ topStartAndCycleFinish, /*waitForException=*/false, new StringValue(""),
+ ImmutableSet.<SkyKey>of(midKey)));
+ // error waits until otherTop starts and cycle finishes, to make sure otherTop will request
+ // its dep before the threadpool shuts down.
+ tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, topStartAndCycleFinish,
+ null, /*waitForException=*/false, null,
+ ImmutableSet.<SkyKey>of()));
+ EvaluationResult<StringValue> result =
+ eval(/*keepGoing=*/false, ImmutableSet.of(topKey, otherTop));
+ assertThat(result.errorMap().keySet()).containsExactly(topKey);
+ Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo();
+ assertThat(cycleInfos).isNotEmpty();
+ CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos);
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey);
+ assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey);
+ }
+
+ /**
+ * Regression test: "value in cycle depends on error".
+ * Here, we add an additional top-level key in error, just to mix it up.
+ */
+ private void cycleAndErrorAndError(boolean keepGoing) throws Exception {
+ graph = new DeterministicInMemoryGraph();
+ tester = new GraphTester();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ SkyKey cycleKey = GraphTester.toSkyKey("cycle");
+ SkyKey midKey = GraphTester.toSkyKey("mid");
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(midKey).addDependency(errorKey).addDependency(cycleKey)
+ .setComputedValue(CONCATENATE);
+ SkyKey otherTop = GraphTester.toSkyKey("otherTop");
+ CountDownLatch topStartAndCycleFinish = new CountDownLatch(2);
+ // In nokeep_going mode, otherTop will wait until the threadpool has received an exception,
+ // then throw its own exception. This guarantees that its exception will not be the one
+ // bubbling up, but that there is a top-level value with an exception by the time the bubbling
+ // up starts.
+ tester.getOrCreate(otherTop).setBuilder(new ChainedFunction(topStartAndCycleFinish,
+ new CountDownLatch(0), null, /*waitForException=*/!keepGoing, null,
+ ImmutableSet.<SkyKey>of()));
+ // error waits until otherTop starts and cycle finishes, to make sure otherTop will request
+ // its dep before the threadpool shuts down.
+ tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, topStartAndCycleFinish,
+ null, /*waitForException=*/false, null,
+ ImmutableSet.<SkyKey>of()));
+ tester.getOrCreate(cycleKey).setBuilder(new ChainedFunction(null, null,
+ topStartAndCycleFinish, /*waitForException=*/false, new StringValue(""),
+ ImmutableSet.<SkyKey>of(midKey)));
+ EvaluationResult<StringValue> result =
+ eval(keepGoing, ImmutableSet.of(topKey, otherTop));
+ if (keepGoing) {
+ assertThat(result.errorMap().keySet()).containsExactly(otherTop, topKey);
+ assertThat(result.getError(otherTop).getRootCauses()).containsExactly(otherTop);
+ // The error thrown will only be recorded in keep_going mode.
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey);
+ }
+ Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo();
+ assertThat(cycleInfos).isNotEmpty();
+ CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos);
+ assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey);
+ assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey);
+ }
+
+ @Test
+ public void cycleAndErrorAndErrorNoKeepGoing() throws Exception {
+ cycleAndErrorAndError(false);
+ }
+
+ @Test
+ public void cycleAndErrorAndErrorKeepGoing() throws Exception {
+ cycleAndErrorAndError(true);
+ }
+
+ @Test
+ public void testFunctionCrashTrace() throws Exception {
+ final SkyFunctionName childType = new SkyFunctionName("child", false);
+ final SkyFunctionName parentType = new SkyFunctionName("parent", false);
+
+ class ChildFunction implements SkyFunction {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ throw new IllegalStateException("I WANT A PONY!!!");
+ }
+
+ @Override public String extractTag(SkyKey skyKey) { return null; }
+ }
+
+ class ParentFunction implements SkyFunction {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ SkyValue dep = env.getValue(new SkyKey(childType, "billy the kid"));
+ if (dep == null) {
+ return null;
+ }
+ throw new IllegalStateException(); // Should never get here.
+ }
+
+ @Override public String extractTag(SkyKey skyKey) { return null; }
+ }
+
+ ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.of(
+ childType, new ChildFunction(),
+ parentType, new ParentFunction());
+ ParallelEvaluator evaluator = makeEvaluator(new InMemoryGraph(),
+ skyFunctions, false);
+
+ try {
+ evaluator.eval(ImmutableList.of(new SkyKey(parentType, "octodad")));
+ fail();
+ } catch (RuntimeException e) {
+ assertEquals("I WANT A PONY!!!", e.getCause().getMessage());
+ assertEquals("Unrecoverable error while evaluating node 'child:billy the kid' "
+ + "(requested by nodes 'parent:octodad')", e.getMessage());
+ }
+ }
+
+ private static class SomeOtherErrorException extends Exception {
+ public SomeOtherErrorException(String msg) {
+ super(msg);
+ }
+ }
+
+ private void unexpectedErrorDep(boolean keepGoing) throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ final SomeOtherErrorException exception = new SomeOtherErrorException("error exception");
+ tester.getOrCreate(errorKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ throw new SkyFunctionException(exception, Transience.PERSISTENT) {};
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+ });
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ tester.getOrCreate(topKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(keepGoing, ImmutableList.of(topKey));
+ assertThat(result.keyNames()).isEmpty();
+ assertSame(exception, result.getError(topKey).getException());
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey);
+ }
+
+ /**
+ * This and the following three tests are in response a bug: "Skyframe error propagation model is
+ * problematic". They ensure that exceptions a child throws that a value does not specify it can
+ * handle in getValueOrThrow do not cause a crash.
+ */
+ @Test
+ public void unexpectedErrorDepKeepGoing() throws Exception {
+ unexpectedErrorDep(true);
+ }
+
+ @Test
+ public void unexpectedErrorDepNoKeepGoing() throws Exception {
+ unexpectedErrorDep(false);
+ }
+
+ private void unexpectedErrorDepOneLevelDown(final boolean keepGoing) throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey errorKey = GraphTester.toSkyKey("my_error_value");
+ final SomeErrorException exception = new SomeErrorException("error exception");
+ final SomeErrorException topException = new SomeErrorException("top exception");
+ final StringValue topValue = new StringValue("top");
+ tester.getOrCreate(errorKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException {
+ throw new GenericFunctionException(exception, Transience.PERSISTENT);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+ });
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ final SkyKey parentKey = GraphTester.toSkyKey("parent");
+ tester.getOrCreate(parentKey).addDependency(errorKey).setComputedValue(CONCATENATE);
+ tester.getOrCreate(topKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException {
+ try {
+ if (env.getValueOrThrow(parentKey, SomeErrorException.class) == null) {
+ return null;
+ }
+ } catch (SomeErrorException e) {
+ assertEquals(e.toString(), exception, e);
+ }
+ if (keepGoing) {
+ return topValue;
+ } else {
+ throw new GenericFunctionException(topException, Transience.PERSISTENT);
+ }
+ }
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ throw new UnsupportedOperationException();
+ }
+ });
+ tester.getOrCreate(topKey).addErrorDependency(errorKey, new StringValue("recovered"))
+ .setComputedValue(CONCATENATE);
+ EvaluationResult<StringValue> result = eval(keepGoing, ImmutableList.of(topKey));
+ if (!keepGoing) {
+ assertThat(result.keyNames()).isEmpty();
+ assertEquals(topException, result.getError(topKey).getException());
+ assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
+ assertTrue(result.hasError());
+ } else {
+ // result.hasError() is set to true even if the top-level value returned has recovered from
+ // an error.
+ assertTrue(result.hasError());
+ assertSame(topValue, result.get(topKey));
+ }
+ }
+
+ @Test
+ public void unexpectedErrorDepOneLevelDownKeepGoing() throws Exception {
+ unexpectedErrorDepOneLevelDown(true);
+ }
+
+ @Test
+ public void unexpectedErrorDepOneLevelDownNoKeepGoing() throws Exception {
+ unexpectedErrorDepOneLevelDown(false);
+ }
+
+ /**
+ * Exercises various situations involving groups of deps that overlap -- request one group, then
+ * request another group that has a dep in common with the first group.
+ *
+ * @param sameFirst whether the dep in common in the two groups should be the first dep.
+ * @param twoCalls whether the two groups should be requested in two different builder calls.
+ * @param valuesOrThrow whether the deps should be requested using getValuesOrThrow.
+ */
+ private void sameDepInTwoGroups(final boolean sameFirst, final boolean twoCalls,
+ final boolean valuesOrThrow) throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey topKey = GraphTester.toSkyKey("top");
+ final List<SkyKey> leaves = new ArrayList<>();
+ for (int i = 1; i <= 3; i++) {
+ SkyKey leaf = GraphTester.toSkyKey("leaf" + i);
+ leaves.add(leaf);
+ tester.set(leaf, new StringValue("leaf" + i));
+ }
+ final SkyKey leaf4 = GraphTester.toSkyKey("leaf4");
+ tester.set(leaf4, new StringValue("leaf" + 4));
+ tester.getOrCreate(topKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException,
+ InterruptedException {
+ if (valuesOrThrow) {
+ env.getValuesOrThrow(leaves, SomeErrorException.class);
+ } else {
+ env.getValues(leaves);
+ }
+ if (twoCalls && env.valuesMissing()) {
+ return null;
+ }
+ SkyKey first = sameFirst ? leaves.get(0) : leaf4;
+ SkyKey second = sameFirst ? leaf4 : leaves.get(2);
+ List<SkyKey> secondRequest = ImmutableList.of(first, second);
+ if (valuesOrThrow) {
+ env.getValuesOrThrow(secondRequest, SomeErrorException.class);
+ } else {
+ env.getValues(secondRequest);
+ }
+ if (env.valuesMissing()) {
+ return null;
+ }
+ return new StringValue("top");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ eval(/*keepGoing=*/false, topKey);
+ assertEquals(new StringValue("top"), eval(/*keepGoing=*/false, topKey));
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Same_Two_Throw() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/true, /*valuesOrThrow=*/true);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Same_Two_Deps() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/true, /*valuesOrThrow=*/false);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Same_One_Throw() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/false, /*valuesOrThrow=*/true);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Same_One_Deps() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/false, /*valuesOrThrow=*/false);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Different_Two_Throw() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/true, /*valuesOrThrow=*/true);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Different_Two_Deps() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/true, /*valuesOrThrow=*/false);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Different_One_Throw() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/false, /*valuesOrThrow=*/true);
+ }
+
+ @Test
+ public void sameDepInTwoGroups_Different_One_Deps() throws Exception {
+ sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/false, /*valuesOrThrow=*/false);
+ }
+
+ private void getValuesOrThrowWithErrors(boolean keepGoing) throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ final SkyKey errorDep = GraphTester.toSkyKey("errorChild");
+ final SomeErrorException childExn = new SomeErrorException("child error");
+ tester.getOrCreate(errorDep).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ throw new GenericFunctionException(childExn, Transience.PERSISTENT);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ final List<SkyKey> deps = new ArrayList<>();
+ for (int i = 1; i <= 3; i++) {
+ SkyKey dep = GraphTester.toSkyKey("child" + i);
+ deps.add(dep);
+ tester.set(dep, new StringValue("child" + i));
+ }
+ final SomeErrorException parentExn = new SomeErrorException("parent error");
+ tester.getOrCreate(parentKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ try {
+ SkyValue value = env.getValueOrThrow(errorDep, SomeErrorException.class);
+ if (value == null) {
+ return null;
+ }
+ } catch (SomeErrorException e) {
+ // Recover from the child error.
+ }
+ env.getValues(deps);
+ if (env.valuesMissing()) {
+ return null;
+ }
+ throw new GenericFunctionException(parentExn, Transience.PERSISTENT);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ EvaluationResult<StringValue> evaluationResult = eval(keepGoing, ImmutableList.of(parentKey));
+ assertTrue(evaluationResult.hasError());
+ assertEquals(keepGoing ? parentExn : childExn, evaluationResult.getError().getException());
+ }
+
+ @Test
+ public void getValuesOrThrowWithErrors_NoKeepGoing() throws Exception {
+ getValuesOrThrowWithErrors(/*keepGoing=*/false);
+ }
+
+ @Test
+ public void getValuesOrThrowWithErrors_KeepGoing() throws Exception {
+ getValuesOrThrowWithErrors(/*keepGoing=*/true);
+ }
+
+ @Test
+ public void duplicateCycles() throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey grandparentKey = GraphTester.toSkyKey("grandparent");
+ SkyKey parentKey1 = GraphTester.toSkyKey("parent1");
+ SkyKey parentKey2 = GraphTester.toSkyKey("parent2");
+ SkyKey loopKey1 = GraphTester.toSkyKey("loop1");
+ SkyKey loopKey2 = GraphTester.toSkyKey("loop2");
+ tester.getOrCreate(loopKey1).addDependency(loopKey2);
+ tester.getOrCreate(loopKey2).addDependency(loopKey1);
+ tester.getOrCreate(parentKey1).addDependency(loopKey1);
+ tester.getOrCreate(parentKey2).addDependency(loopKey2);
+ tester.getOrCreate(grandparentKey).addDependency(parentKey1);
+ tester.getOrCreate(grandparentKey).addDependency(parentKey2);
+
+ ErrorInfo errorInfo = evalValueInError(grandparentKey);
+ List<ImmutableList<SkyKey>> cycles = Lists.newArrayList();
+ for (CycleInfo cycleInfo : errorInfo.getCycleInfo()) {
+ cycles.add(cycleInfo.getCycle());
+ }
+ // Skyframe doesn't automatically dedupe cycles that are the same except for entry point.
+ assertEquals(2, cycles.size());
+ int numUniqueCycles = 0;
+ CycleDeduper<SkyKey> cycleDeduper = new CycleDeduper<SkyKey>();
+ for (ImmutableList<SkyKey> cycle : cycles) {
+ if (cycleDeduper.seen(cycle)) {
+ numUniqueCycles++;
+ }
+ }
+ assertEquals(1, numUniqueCycles);
+ }
+
+ @Test
+ public void signalValueEnqueuedAndEvaluated() throws Exception {
+ final Set<SkyKey> enqueuedValues = Sets.newConcurrentHashSet();
+ final Set<SkyKey> evaluatedValues = Sets.newConcurrentHashSet();
+ EvaluationProgressReceiver progressReceiver = new EvaluationProgressReceiver() {
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ enqueuedValues.add(skyKey);
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ evaluatedValues.add(skyKey);
+ }
+ };
+
+ EventHandler reporter = new EventHandler() {
+ @Override
+ public void handle(Event e) {
+ throw new IllegalStateException();
+ }
+ };
+
+ MemoizingEvaluator aug = new InMemoryMemoizingEvaluator(
+ ImmutableMap.of(GraphTester.NODE_TYPE, tester.getFunction()), new RecordingDifferencer(),
+ progressReceiver);
+ SequentialBuildDriver driver = new SequentialBuildDriver(aug);
+
+ tester.getOrCreate("top1").setComputedValue(CONCATENATE)
+ .addDependency("d1").addDependency("d2");
+ tester.getOrCreate("top2").setComputedValue(CONCATENATE).addDependency("d3");
+ tester.getOrCreate("top3");
+ assertThat(enqueuedValues).isEmpty();
+ assertThat(evaluatedValues).isEmpty();
+
+ tester.set("d1", new StringValue("1"));
+ tester.set("d2", new StringValue("2"));
+ tester.set("d3", new StringValue("3"));
+
+ driver.evaluate(ImmutableList.of(GraphTester.toSkyKey("top1")), false, 200, reporter);
+ assertThat(enqueuedValues)
+ .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top1", "d1", "d2")));
+ assertThat(evaluatedValues)
+ .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top1", "d1", "d2")));
+ enqueuedValues.clear();
+ evaluatedValues.clear();
+
+ driver.evaluate(ImmutableList.of(GraphTester.toSkyKey("top2")), false, 200, reporter);
+ assertThat(enqueuedValues)
+ .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top2", "d3")));
+ assertThat(evaluatedValues)
+ .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top2", "d3")));
+ enqueuedValues.clear();
+ evaluatedValues.clear();
+
+ driver.evaluate(ImmutableList.of(GraphTester.toSkyKey("top1")), false, 200, reporter);
+ assertThat(enqueuedValues).isEmpty();
+ assertThat(evaluatedValues)
+ .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top1")));
+ }
+
+ public void runDepOnErrorHaltsNoKeepGoingBuildEagerly(boolean childErrorCached,
+ final boolean handleChildError) throws Exception {
+ graph = new InMemoryGraph();
+ SkyKey parentKey = GraphTester.toSkyKey("parent");
+ final SkyKey childKey = GraphTester.toSkyKey("child");
+ tester.getOrCreate(childKey).setHasError(/*hasError=*/true);
+ // The parent should be built exactly twice: once during normal evaluation and once
+ // during error bubbling.
+ final AtomicInteger numParentInvocations = new AtomicInteger(0);
+ tester.getOrCreate(parentKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ int invocations = numParentInvocations.incrementAndGet();
+ if (handleChildError) {
+ try {
+ SkyValue value = env.getValueOrThrow(childKey, SomeErrorException.class);
+ // On the first invocation, either the child error should already be cached and not
+ // propagated, or it should be computed freshly and not propagated. On the second build
+ // (error bubbling), the child error should be propagated.
+ assertTrue("bogus non-null value " + value, value == null);
+ assertEquals("parent incorrectly re-computed during normal evaluation", 1, invocations);
+ assertFalse("child error not propagated during error bubbling",
+ env.inErrorBubblingForTesting());
+ return value;
+ } catch (SomeErrorException e) {
+ assertTrue("child error propagated during normal evaluation",
+ env.inErrorBubblingForTesting());
+ assertEquals(2, invocations);
+ return null;
+ }
+ } else {
+ if (invocations == 1) {
+ assertFalse("parent's first computation should be during normal evaluation",
+ env.inErrorBubblingForTesting());
+ return env.getValue(childKey);
+ } else {
+ assertEquals(2, invocations);
+ assertTrue("parent incorrectly re-computed during normal evaluation",
+ env.inErrorBubblingForTesting());
+ return env.getValue(childKey);
+ }
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ if (childErrorCached) {
+ // Ensure that the child is already in the graph.
+ evalValueInError(childKey);
+ }
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey));
+ assertEquals(2, numParentInvocations.get());
+ assertTrue(result.hasError());
+ assertEquals(childKey, result.getError().getRootCauseOfException());
+ }
+
+ @Test
+ public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorCachedAndHandled()
+ throws Exception {
+ runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/true,
+ /*handleChildError=*/true);
+ }
+
+ @Test
+ public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorCachedAndNotHandled()
+ throws Exception {
+ runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/true,
+ /*handleChildError=*/false);
+ }
+
+ @Test
+ public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorFreshAndHandled() throws Exception {
+ runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/false,
+ /*handleChildError=*/true);
+ }
+
+ @Test
+ public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorFreshAndNotHandled()
+ throws Exception {
+ runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/false,
+ /*handleChildError=*/false);
+ }
+
+ @Test
+ public void raceConditionWithNoKeepGoingErrors_InflightError() throws Exception {
+ final CountDownLatch errorCommitted = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiterForError = new TrackingAwaiter();
+ final CountDownLatch otherDone = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiterForOther = new TrackingAwaiter();
+ final SkyKey errorKey = GraphTester.toSkyKey("errorKey");
+ final SkyKey otherKey = GraphTester.toSkyKey("otherKey");
+ tester.getOrCreate(errorKey).setHasError(true);
+ final AtomicInteger numOtherInvocations = new AtomicInteger(0);
+ tester.getOrCreate(otherKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ int invocations = numOtherInvocations.incrementAndGet();
+ if (invocations == 1) {
+ trackingAwaiterForError.awaitLatchAndTrackExceptions(errorCommitted,
+ "error didn't get committed to the graph in time");
+ }
+ try {
+ SkyValue value = env.getValueOrThrow(errorKey, SomeErrorException.class);
+ assertTrue("bogus non-null value " + value, value == null);
+ assertEquals(1, invocations);
+ otherDone.countDown();
+ throw new GenericFunctionException(new SomeErrorException("other"),
+ Transience.PERSISTENT);
+ } catch (SomeErrorException e) {
+ assertEquals(2, invocations);
+ return null;
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ graph = new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (key.equals(errorKey) && type == EventType.SET_VALUE && order == Order.AFTER) {
+ errorCommitted.countDown();
+ trackingAwaiterForOther.awaitLatchAndTrackExceptions(otherDone,
+ "otherKey's SkyFunction didn't finish in time");
+ }
+ }
+ });
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false,
+ ImmutableList.of(errorKey, otherKey));
+ assertEquals(null, graph.get(otherKey));
+ assertTrue(result.hasError());
+ assertEquals(errorKey, result.getError().getRootCauseOfException());
+ }
+
+ @Test
+ public void raceConditionWithNoKeepGoingErrors_FutureError() throws Exception {
+ final CountDownLatch errorCommitted = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiterForError = new TrackingAwaiter();
+ final CountDownLatch otherStarted = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiterForOther = new TrackingAwaiter();
+ final CountDownLatch otherParentSignaled = new CountDownLatch(1);
+ final TrackingAwaiter trackingAwaiterForOtherParent = new TrackingAwaiter();
+ final SkyKey errorParentKey = GraphTester.toSkyKey("errorParentKey");
+ final SkyKey errorKey = GraphTester.toSkyKey("errorKey");
+ final SkyKey otherParentKey = GraphTester.toSkyKey("otherParentKey");
+ final SkyKey otherKey = GraphTester.toSkyKey("otherKey");
+ final AtomicInteger numOtherParentInvocations = new AtomicInteger(0);
+ final AtomicInteger numErrorParentInvocations = new AtomicInteger(0);
+ tester.getOrCreate(otherParentKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ int invocations = numOtherParentInvocations.incrementAndGet();
+ assertEquals("otherParentKey should not be restarted", 1, invocations);
+ return env.getValue(otherKey);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ tester.getOrCreate(otherKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ otherStarted.countDown();
+ trackingAwaiterForError.awaitLatchAndTrackExceptions(errorCommitted,
+ "error didn't get committed to the graph in time");
+ return new StringValue("other");
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ tester.getOrCreate(errorKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ trackingAwaiterForOther.awaitLatchAndTrackExceptions(otherStarted,
+ "other didn't start in time");
+ throw new GenericFunctionException(new SomeErrorException("error"),
+ Transience.PERSISTENT);
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ tester.getOrCreate(errorParentKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ int invocations = numErrorParentInvocations.incrementAndGet();
+ try {
+ SkyValue value = env.getValueOrThrow(errorKey, SomeErrorException.class);
+ assertTrue("bogus non-null value " + value, value == null);
+ if (invocations == 1) {
+ return null;
+ } else {
+ assertFalse(env.inErrorBubblingForTesting());
+ fail("RACE CONDITION: errorParentKey was restarted!");
+ return null;
+ }
+ } catch (SomeErrorException e) {
+ assertTrue("child error propagated during normal evaluation",
+ env.inErrorBubblingForTesting());
+ assertEquals(2, invocations);
+ return null;
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ graph = new NotifyingInMemoryGraph(new Listener() {
+ @Override
+ public void accept(SkyKey key, EventType type, Order order, Object context) {
+ if (key.equals(errorKey) && type == EventType.SET_VALUE && order == Order.AFTER) {
+ errorCommitted.countDown();
+ trackingAwaiterForOtherParent.awaitLatchAndTrackExceptions(otherParentSignaled,
+ "otherParent didn't get signaled in time");
+ // We try to give some time for ParallelEvaluator to incorrectly re-evaluate
+ // 'otherParentKey'. This test case is testing for a real race condition and the 10ms time
+ // was chosen experimentally to give a true positive rate of 99.8% (without a sleep it
+ // has a 1% true positive rate). There's no good way to do this without sleeping. We
+ // *could* introspect ParallelEvaulator's AbstractQueueVisitor to see if the re-evaluation
+ // has been enqueued, but that's relying on pretty low-level implementation details.
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+ }
+ if (key.equals(otherParentKey) && type == EventType.SIGNAL && order == Order.AFTER) {
+ otherParentSignaled.countDown();
+ }
+ }
+ });
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/false,
+ ImmutableList.of(otherParentKey, errorParentKey));
+ assertTrue(result.hasError());
+ assertEquals(errorKey, result.getError().getRootCauseOfException());
+ }
+
+ @Test
+ public void cachedErrorsFromKeepGoingUsedOnNoKeepGoing() throws Exception {
+ graph = new DeterministicInMemoryGraph();
+ tester = new GraphTester();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ SkyKey parent1Key = GraphTester.toSkyKey("parent1");
+ SkyKey parent2Key = GraphTester.toSkyKey("parent2");
+ tester.getOrCreate(parent1Key).addDependency(errorKey).setConstantValue(
+ new StringValue("parent1"));
+ tester.getOrCreate(parent2Key).addDependency(errorKey).setConstantValue(
+ new StringValue("parent2"));
+ tester.getOrCreate(errorKey).setHasError(true);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(parent1Key));
+ assertTrue(result.hasError());
+ assertEquals(errorKey, result.getError().getRootCauseOfException());
+ result = eval(/*keepGoing=*/false, ImmutableList.of(parent2Key));
+ assertTrue(result.hasError());
+ assertEquals(errorKey, result.getError(parent2Key).getRootCauseOfException());
+ }
+
+ @Test
+ public void cachedTopLevelErrorsShouldHaltNoKeepGoingBuildEarly() throws Exception {
+ graph = new DeterministicInMemoryGraph();
+ tester = new GraphTester();
+ SkyKey errorKey = GraphTester.toSkyKey("error");
+ tester.getOrCreate(errorKey).setHasError(true);
+ EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(errorKey));
+ assertTrue(result.hasError());
+ assertEquals(errorKey, result.getError().getRootCauseOfException());
+ SkyKey rogueKey = GraphTester.toSkyKey("rogue");
+ tester.getOrCreate(rogueKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ // This SkyFunction could do an arbitrarily bad computation, e.g. loop-forever. So we want
+ // to make sure that it is never run when we want to fail-fast anyway.
+ fail("eval call should have already terminated");
+ return null;
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ result = eval(/*keepGoing=*/false, ImmutableList.of(errorKey, rogueKey));
+ assertTrue(result.hasError());
+ assertEquals(errorKey, result.getError(errorKey).getRootCauseOfException());
+ assertFalse(result.errorMap().containsKey(rogueKey));
+ }
+
+ private void runUnhandledTransitiveErrors(boolean keepGoing,
+ final boolean explicitlyPropagateError) throws Exception {
+ graph = new DeterministicInMemoryGraph();
+ tester = new GraphTester();
+ SkyKey grandparentKey = GraphTester.toSkyKey("grandparent");
+ final SkyKey parentKey = GraphTester.toSkyKey("parent");
+ final SkyKey childKey = GraphTester.toSkyKey("child");
+ final AtomicBoolean errorPropagated = new AtomicBoolean(false);
+ tester.getOrCreate(grandparentKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ try {
+ return env.getValueOrThrow(parentKey, SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ errorPropagated.set(true);
+ throw new GenericFunctionException(e, Transience.PERSISTENT);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ tester.getOrCreate(parentKey).setBuilder(new SkyFunction() {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+ if (explicitlyPropagateError) {
+ try {
+ return env.getValueOrThrow(childKey, SomeErrorException.class);
+ } catch (SomeErrorException e) {
+ throw new GenericFunctionException(e, childKey);
+ }
+ } else {
+ return env.getValue(childKey);
+ }
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ });
+ tester.getOrCreate(childKey).setHasError(/*hasError=*/true);
+ EvaluationResult<StringValue> result = eval(keepGoing, ImmutableList.of(grandparentKey));
+ assertTrue(result.hasError());
+ assertTrue(errorPropagated.get());
+ assertEquals(grandparentKey, result.getError().getRootCauseOfException());
+ }
+
+ @Test
+ public void unhandledTransitiveErrorsDuringErrorBubbling_ImplicitPropagation() throws Exception {
+ runUnhandledTransitiveErrors(/*keepGoing=*/false, /*explicitlyPropagateError=*/false);
+ }
+
+ @Test
+ public void unhandledTransitiveErrorsDuringErrorBubbling_ExplicitPropagation() throws Exception {
+ runUnhandledTransitiveErrors(/*keepGoing=*/false, /*explicitlyPropagateError=*/true);
+ }
+
+ @Test
+ public void unhandledTransitiveErrorsDuringNormalEvaluation_ImplicitPropagation()
+ throws Exception {
+ runUnhandledTransitiveErrors(/*keepGoing=*/true, /*explicitlyPropagateError=*/false);
+ }
+
+ @Test
+ public void unhandledTransitiveErrorsDuringNormalEvaluation_ExplicitPropagation()
+ throws Exception {
+ runUnhandledTransitiveErrors(/*keepGoing=*/true, /*explicitlyPropagateError=*/true);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/ReverseDepsUtilTest.java b/src/test/java/com/google/devtools/build/skyframe/ReverseDepsUtilTest.java
new file mode 100644
index 0000000..9183775
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/ReverseDepsUtilTest.java
@@ -0,0 +1,155 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Test for {@code ReverseDepsUtil}.
+ */
+@RunWith(Parameterized.class)
+public class ReverseDepsUtilTest {
+
+ private static final SkyFunctionName NODE_TYPE = new SkyFunctionName("Type", false);
+ private final int numElements;
+
+ @Parameters
+ public static List<Object[]> paramenters() {
+ List<Object[]> params = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ params.add(new Object[]{i});
+ }
+ return params;
+ }
+
+ public ReverseDepsUtilTest(int numElements) {
+ this.numElements = numElements;
+ }
+
+ private static final ReverseDepsUtil<Example> REVERSE_DEPS_UTIL = new ReverseDepsUtil<Example>() {
+ @Override
+ void setReverseDepsObject(Example container, Object object) {
+ container.reverseDeps = object;
+ }
+
+ @Override
+ void setSingleReverseDep(Example container, boolean singleObject) {
+ container.single = singleObject;
+ }
+
+ @Override
+ void setReverseDepsToRemove(Example container, List<SkyKey> object) {
+ container.reverseDepsToRemove = object;
+ }
+
+ @Override
+ Object getReverseDepsObject(Example container) {
+ return container.reverseDeps;
+ }
+
+ @Override
+ boolean isSingleReverseDep(Example container) {
+ return container.single;
+ }
+
+ @Override
+ List<SkyKey> getReverseDepsToRemove(Example container) {
+ return container.reverseDepsToRemove;
+ }
+ };
+
+ private class Example {
+
+ Object reverseDeps = ImmutableList.of();
+ boolean single;
+ List<SkyKey> reverseDepsToRemove;
+ }
+
+ @Test
+ public void testAddAndRemove() {
+ for (int numRemovals = 0; numRemovals <= numElements; numRemovals++) {
+ Example example = new Example();
+ for (int j = 0; j < numElements; j++) {
+ REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, j)));
+ }
+ // Not a big test but at least check that it does not blow up.
+ assertThat(REVERSE_DEPS_UTIL.toString(example)).isNotEmpty();
+ assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements);
+ for (int i = 0; i < numRemovals; i++) {
+ REVERSE_DEPS_UTIL.removeReverseDep(example, new SkyKey(NODE_TYPE, i));
+ }
+ assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements - numRemovals);
+ assertThat(example.reverseDepsToRemove).isNull();
+ }
+ }
+
+ // Same as testAdditionAndRemoval but we add all the reverse deps in one call.
+ @Test
+ public void testAddAllAndRemove() {
+ for (int numRemovals = 0; numRemovals <= numElements; numRemovals++) {
+ Example example = new Example();
+ List<SkyKey> toAdd = new ArrayList<>();
+ for (int j = 0; j < numElements; j++) {
+ toAdd.add(new SkyKey(NODE_TYPE, j));
+ }
+ REVERSE_DEPS_UTIL.addReverseDeps(example, toAdd);
+ assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements);
+ for (int i = 0; i < numRemovals; i++) {
+ REVERSE_DEPS_UTIL.removeReverseDep(example, new SkyKey(NODE_TYPE, i));
+ }
+ assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements - numRemovals);
+ assertThat(example.reverseDepsToRemove).isNull();
+ }
+ }
+
+ @Test
+ public void testDuplicateCheckOnGetReverseDeps() {
+ Example example = new Example();
+ for (int i = 0; i < numElements; i++) {
+ REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, i)));
+ }
+ // Should only fail when we call getReverseDeps().
+ REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, 0)));
+ try {
+ REVERSE_DEPS_UTIL.getReverseDeps(example);
+ assertThat(numElements).is(0);
+ } catch (Exception expected) { }
+ }
+
+ @Test
+ public void testMaybeCheck() {
+ Example example = new Example();
+ for (int i = 0; i < numElements; i++) {
+ REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, i)));
+ // This should always succeed, since the next element is still not present.
+ REVERSE_DEPS_UTIL.maybeCheckReverseDepNotPresent(example, new SkyKey(NODE_TYPE, i + 1));
+ }
+ try {
+ REVERSE_DEPS_UTIL.maybeCheckReverseDepNotPresent(example, new SkyKey(NODE_TYPE, 0));
+ // Should only fail if empty or above the checking threshold.
+ assertThat(numElements == 0 || numElements >= ReverseDepsUtil.MAYBE_CHECK_THRESHOLD).isTrue();
+ } catch (Exception expected) { }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/SomeErrorException.java b/src/test/java/com/google/devtools/build/skyframe/SomeErrorException.java
new file mode 100644
index 0000000..b25cbd3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/SomeErrorException.java
@@ -0,0 +1,20 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+public class SomeErrorException extends Exception {
+ public SomeErrorException(String msg) {
+ super(msg);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/TrackingAwaiter.java b/src/test/java/com/google/devtools/build/skyframe/TrackingAwaiter.java
new file mode 100644
index 0000000..3757583
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/TrackingAwaiter.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.Pair;
+
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/** Safely await {@link CountDownLatch}es in tests, storing any exceptions that happen. */
+public class TrackingAwaiter {
+ private final ConcurrentLinkedQueue<Pair<String, Throwable>> exceptionsThrown =
+ new ConcurrentLinkedQueue<>();
+
+ /**
+ * This method fixes a race condition with simply calling {@link CountDownLatch#await}. If this
+ * thread is interrupted before {@code latch.await} is called, then {@code latch.await} will throw
+ * an {@link InterruptedException} without checking the value of the latch at all. This leads to a
+ * race condition in which this thread will throw an InterruptedException if it is slow calling
+ * {@code latch.await}, but it will succeed normally otherwise.
+ *
+ * <p>To avoid this, we wait for the latch uninterruptibly. In the end, if the latch has in fact
+ * been released, we do nothing, although the interrupted bit is set, so that the caller can
+ * decide to throw an InterruptedException if it wants to. If the latch was not released, then
+ * this was not a race condition, but an honest-to-goodness interrupt, and we propagate the
+ * exception onward.
+ */
+ public static void waitAndMaybeThrowInterrupt(CountDownLatch latch, String errorMessage)
+ throws InterruptedException {
+ if (Uninterruptibles.awaitUninterruptibly(latch, TestUtils.WAIT_TIMEOUT_SECONDS,
+ TimeUnit.SECONDS)) {
+ // Latch was released. We can ignore the interrupt state.
+ return;
+ }
+ if (!Thread.currentThread().isInterrupted()) {
+ // Nobody interrupted us, but latch wasn't released. Failure.
+ throw new AssertionError(errorMessage);
+ } else {
+ // We were interrupted before the latch was released. Propagate this interruption.
+ throw new InterruptedException();
+ }
+ }
+
+ /** Threadpools can swallow exceptions. Make sure they don't get lost. */
+ public void awaitLatchAndTrackExceptions(CountDownLatch latch, String errorMessage) {
+ try {
+ waitAndMaybeThrowInterrupt(latch, errorMessage);
+ } catch (Throwable e) {
+ // We would expect e to be InterruptedException or AssertionError, but we leave it open so
+ // that any throwable gets recorded.
+ exceptionsThrown.add(Pair.of(errorMessage, e));
+ // Caller will assert exceptionsThrown is empty at end of test and fail, even if this is
+ // swallowed.
+ Throwables.propagate(e);
+ }
+ }
+
+ public void assertNoErrors() {
+ assertThat(exceptionsThrown).isEmpty();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/skyframe/TrackingInvalidationReceiver.java b/src/test/java/com/google/devtools/build/skyframe/TrackingInvalidationReceiver.java
new file mode 100644
index 0000000..e93098d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/skyframe/TrackingInvalidationReceiver.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.skyframe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+/**
+ * A testing utility to keep track of evaluation.
+ */
+public class TrackingInvalidationReceiver implements EvaluationProgressReceiver {
+ public final Set<SkyValue> dirty = Sets.newConcurrentHashSet();
+ public final Set<SkyValue> deleted = Sets.newConcurrentHashSet();
+ public final Set<SkyKey> enqueued = Sets.newConcurrentHashSet();
+ public final Set<SkyKey> evaluated = Sets.newConcurrentHashSet();
+
+ @Override
+ public void invalidated(SkyValue value, InvalidationState state) {
+ switch (state) {
+ case DELETED:
+ dirty.remove(value);
+ deleted.add(value);
+ break;
+ case DIRTY:
+ dirty.add(value);
+ Preconditions.checkState(!deleted.contains(value));
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ enqueued.add(skyKey);
+ }
+
+ @Override
+ public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ evaluated.add(skyKey);
+ switch (state) {
+ default:
+ dirty.remove(value);
+ deleted.remove(value);
+ break;
+ }
+ }
+
+ public void clear() {
+ dirty.clear();
+ deleted.clear();
+ enqueued.clear();
+ evaluated.clear();
+ }
+}
diff --git a/src/test/java/com/google/devtools/common/options/AllTests.java b/src/test/java/com/google/devtools/common/options/AllTests.java
new file mode 100644
index 0000000..14d6abb
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/AllTests.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import com.google.devtools.build.lib.testutil.ClasspathSuite;
+
+import org.junit.runner.RunWith;
+
+/**
+ * Test suite for options parsing framework.
+ */
+@RunWith(ClasspathSuite.class)
+public class AllTests {
+}
diff --git a/src/test/java/com/google/devtools/common/options/AssignmentConverterTest.java b/src/test/java/com/google/devtools/common/options/AssignmentConverterTest.java
new file mode 100644
index 0000000..ecaef4c
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/AssignmentConverterTest.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.Maps;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Map;
+
+/**
+ * Test for {@link Converters.AssignmentConverter} and
+ * {@link Converters.OptionalAssignmentConverter}.
+ */
+public abstract class AssignmentConverterTest {
+
+ protected Converter<Map.Entry<String, String>> converter = null;
+
+ protected abstract void setConverter();
+
+ protected Map.Entry<String, String> convert(String input) throws Exception {
+ return converter.convert(input);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ setConverter();
+ }
+
+ @Test
+ public void assignment() throws Exception {
+ assertEquals(Maps.immutableEntry("A", "1"), convert("A=1"));
+ assertEquals(Maps.immutableEntry("A", "ABC"), convert("A=ABC"));
+ assertEquals(Maps.immutableEntry("A", ""), convert("A="));
+ }
+
+ @Test
+ public void missingName() throws Exception {
+ try {
+ convert("=VALUE");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+
+ @Test
+ public void emptyString() throws Exception {
+ try {
+ convert("");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+
+
+ @RunWith(JUnit4.class)
+ public static class MandatoryAssignmentConverterTest extends AssignmentConverterTest {
+
+ @Override
+ protected void setConverter() {
+ converter = new Converters.AssignmentConverter();
+ }
+
+ @Test
+ public void missingValue() throws Exception {
+ try {
+ convert("NAME");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+ }
+
+ @RunWith(JUnit4.class)
+ public static class OptionalAssignmentConverterTest extends AssignmentConverterTest {
+
+ @Override
+ protected void setConverter() {
+ converter = new Converters.OptionalAssignmentConverter();
+ }
+
+ @Test
+ public void missingValue() throws Exception {
+ assertEquals(Maps.immutableEntry("NAME", null), convert("NAME"));
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/common/options/CommaSeparatedOptionListConverterTest.java b/src/test/java/com/google/devtools/common/options/CommaSeparatedOptionListConverterTest.java
new file mode 100644
index 0000000..7308a91
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/CommaSeparatedOptionListConverterTest.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A test for {@link Converters.CommaSeparatedOptionListConverter}.
+ */
+@RunWith(JUnit4.class)
+public class CommaSeparatedOptionListConverterTest {
+
+ private Converter<List<String>> converter =
+ new Converters.CommaSeparatedOptionListConverter();
+
+ @Test
+ public void emptyStringYieldsEmptyList() throws Exception {
+ assertEquals(Collections.emptyList(), converter.convert(""));
+ }
+
+ @Test
+ public void commaTwoEmptyStrings() throws Exception {
+ assertEquals(Arrays.asList("", ""), converter.convert(","));
+ }
+
+ @Test
+ public void leadingCommaYieldsLeadingSpace() throws Exception {
+ assertEquals(Arrays.asList("", "leading", "comma"),
+ converter.convert(",leading,comma"));
+ }
+
+ @Test
+ public void trailingCommaYieldsTrailingSpace() throws Exception {
+ assertEquals(Arrays.asList("trailing", "comma", ""),
+ converter.convert("trailing,comma,"));
+ }
+
+ @Test
+ public void singleWord() throws Exception {
+ assertEquals(Arrays.asList("lonely"), converter.convert("lonely"));
+ }
+
+ @Test
+ public void multiWords() throws Exception {
+ assertEquals(Arrays.asList("one", "two", "three"),
+ converter.convert("one,two,three"));
+ }
+
+ @Test
+ public void spaceIsIgnored() throws Exception {
+ assertEquals(Arrays.asList("one two three"),
+ converter.convert("one two three"));
+ }
+
+ @Test
+ public void valueisUnmodifiable() throws Exception {
+ try {
+ converter.convert("value").add("other");
+ fail("could modify value");
+ } catch (UnsupportedOperationException expected) {}
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/common/options/EnumConverterTest.java b/src/test/java/com/google/devtools/common/options/EnumConverterTest.java
new file mode 100644
index 0000000..5154695
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/EnumConverterTest.java
@@ -0,0 +1,117 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import static com.google.devtools.common.options.OptionsParser.newOptionsParser;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.List;
+
+/**
+ * A test for {@link EnumConverter}.
+ */
+@RunWith(JUnit4.class)
+public class EnumConverterTest {
+
+ private enum CompilationMode {
+ DBG, OPT
+ }
+
+ private static class CompilationModeConverter
+ extends EnumConverter<CompilationMode> {
+
+ public CompilationModeConverter() {
+ super(CompilationMode.class, "compilation mode");
+ }
+ }
+
+ @Test
+ public void converterForEnumWithTwoValues() throws Exception {
+ CompilationModeConverter converter = new CompilationModeConverter();
+ assertEquals(converter.convert("dbg"), CompilationMode.DBG);
+ assertEquals(converter.convert("opt"), CompilationMode.OPT);
+ try {
+ converter.convert("none");
+ fail();
+ } catch(OptionsParsingException e) {
+ assertEquals(e.getMessage(),
+ "Not a valid compilation mode: 'none' (should be dbg or opt)");
+ }
+ assertEquals("dbg or opt", converter.getTypeDescription());
+ }
+
+ private enum Fruit {
+ Apple, Banana, Cherry
+ }
+
+ private static class FruitConverter extends EnumConverter<Fruit> {
+
+ public FruitConverter() {
+ super(Fruit.class, "fruit");
+ }
+ }
+
+ @Test
+ public void typeDescriptionForEnumWithThreeValues() throws Exception {
+ FruitConverter converter = new FruitConverter();
+ // We always use lowercase in the user-visible messages:
+ assertEquals("apple, banana or cherry",
+ converter.getTypeDescription());
+ }
+
+ @Test
+ public void converterIsCaseInsensitive() throws Exception {
+ FruitConverter converter = new FruitConverter();
+ assertSame(Fruit.Banana, converter.convert("bAnANa"));
+ }
+
+ // Regression test: lists of enum using a subclass of EnumConverter don't work
+ private static class AlphabetEnumConverter extends EnumConverter<AlphabetEnum> {
+ public AlphabetEnumConverter() {
+ super(AlphabetEnum.class, "alphabet enum");
+ }
+ }
+
+ private static enum AlphabetEnum {
+ ALPHA, BRAVO, CHARLY, DELTA, ECHO
+ }
+
+ public static class EnumListTestOptions extends OptionsBase {
+ @Option(name = "goo",
+ allowMultiple = true,
+ converter = AlphabetEnumConverter.class,
+ defaultValue = "null")
+ public List<AlphabetEnum> goo;
+ }
+
+ @Test
+ public void enumList() throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(EnumListTestOptions.class);
+ parser.parse("--goo=ALPHA", "--goo=BRAVO");
+ EnumListTestOptions options = parser.getOptions(EnumListTestOptions.class);
+ assertNotNull(options.goo);
+ assertEquals(2, options.goo.size());
+ assertEquals(AlphabetEnum.ALPHA, options.goo.get(0));
+ assertEquals(AlphabetEnum.BRAVO, options.goo.get(1));
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/common/options/GenericTypeHelperTest.java b/src/test/java/com/google/devtools/common/options/GenericTypeHelperTest.java
new file mode 100644
index 0000000..e3a02ac
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/GenericTypeHelperTest.java
@@ -0,0 +1,74 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests {@link GenericTypeHelper}.
+ */
+@RunWith(JUnit4.class)
+public class GenericTypeHelperTest {
+
+ private static interface DoSomething<T> {
+ T doIt();
+ }
+
+ private static class StringSomething implements DoSomething<String> {
+ @Override
+ public String doIt() {
+ return null;
+ }
+ }
+
+ private static class EnumSomething<T> implements DoSomething<T> {
+ @Override
+ public T doIt() {
+ return null;
+ }
+ }
+
+ private static class AlphabetSomething extends EnumSomething<String> {
+ }
+
+ private static class AlphabetTwoSomething extends AlphabetSomething {
+ }
+
+ private static void assertDoIt(Class<?> expected,
+ Class<? extends DoSomething<?>> implementingClass) throws Exception {
+ assertEquals(expected,
+ GenericTypeHelper.getActualReturnType(implementingClass,
+ implementingClass.getMethod("doIt")));
+ }
+
+ @Test
+ public void getConverterType() throws Exception {
+ assertDoIt(String.class, StringSomething.class);
+ }
+
+ @Test
+ public void getConverterTypeForGenericExtension() throws Exception {
+ assertDoIt(String.class, AlphabetSomething.class);
+ }
+
+ @Test
+ public void getConverterTypeForGenericExtensionSecondGrade() throws Exception {
+ assertDoIt(String.class, AlphabetTwoSomething.class);
+ }
+}
diff --git a/src/test/java/com/google/devtools/common/options/LogLevelConverterTest.java b/src/test/java/com/google/devtools/common/options/LogLevelConverterTest.java
new file mode 100644
index 0000000..4dfa209
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/LogLevelConverterTest.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.common.options.Converters.LogLevelConverter;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.logging.Level;
+
+/**
+ * A test for {@link LogLevelConverter}.
+ */
+@RunWith(JUnit4.class)
+public class LogLevelConverterTest {
+
+ private LogLevelConverter converter = new LogLevelConverter();
+
+ @Test
+ public void convertsIntsToLevels() throws OptionsParsingException {
+ int levelId = 0;
+ for (Level level : LogLevelConverter.LEVELS) {
+ assertEquals(level, converter.convert(Integer.toString(levelId++)));
+ }
+ }
+
+ @Test
+ public void throwsExceptionWhenInputIsNotANumber() {
+ try {
+ converter.convert("oops - not a number.");
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("Not a log level: oops - not a number.", e.getMessage());
+ }
+ }
+
+ @Test
+ public void throwsExceptionWhenInputIsInvalidInteger() {
+ for (int example : new int[] {-1, 100, 50000}) {
+ try {
+ converter.convert(Integer.toString(example));
+ fail();
+ } catch (OptionsParsingException e) {
+ String expected = "Not a log level: " + Integer.toString(example);
+ assertEquals(expected, e.getMessage());
+ }
+ }
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/common/options/OptionsParserTest.java b/src/test/java/com/google/devtools/common/options/OptionsParserTest.java
new file mode 100644
index 0000000..190d855
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/OptionsParserTest.java
@@ -0,0 +1,1026 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.common.options.OptionsParser.newOptionsParser;
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.common.options.Converters.CommaSeparatedOptionListConverter;
+import com.google.devtools.common.options.OptionsParser.OptionValueDescription;
+import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests {@link OptionsParser}.
+ */
+@RunWith(JUnit4.class)
+public class OptionsParserTest {
+
+ public static class ExampleFoo extends OptionsBase {
+
+ @Option(name = "foo",
+ category = "one",
+ defaultValue = "defaultFoo")
+ public String foo;
+
+ @Option(name = "bar",
+ category = "two",
+ defaultValue = "42")
+ public int bar;
+
+ @Option(name = "bing",
+ category = "one",
+ defaultValue = "",
+ allowMultiple = true)
+ public List<String> bing;
+
+ @Option(name = "bang",
+ category = "one",
+ defaultValue = "",
+ converter = StringConverter.class,
+ allowMultiple = true)
+ public List<String> bang;
+
+ @Option(name = "nodoc",
+ category = "undocumented",
+ defaultValue = "",
+ allowMultiple = false)
+ public String nodoc;
+ }
+
+ public static class ExampleBaz extends OptionsBase {
+
+ @Option(name = "baz",
+ category = "one",
+ defaultValue = "defaultBaz")
+ public String baz;
+ }
+
+ public static class StringConverter implements Converter<String> {
+ @Override
+ public String convert(String input) {
+ return input;
+ }
+ @Override
+ public String getTypeDescription() {
+ return "a string";
+ }
+ }
+
+ @Test
+ public void parseWithMultipleOptionsInterfaces()
+ throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ parser.parse("--baz=oops", "--bar", "17");
+ ExampleFoo foo = parser.getOptions(ExampleFoo.class);
+ assertEquals("defaultFoo", foo.foo);
+ assertEquals(17, foo.bar);
+ ExampleBaz baz = parser.getOptions(ExampleBaz.class);
+ assertEquals("oops", baz.baz);
+ }
+
+ @Test
+ public void parserWithUnknownOption() {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ try {
+ parser.parse("--unknown", "option");
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("--unknown", e.getInvalidArgument());
+ assertEquals("Unrecognized option: --unknown", e.getMessage());
+ }
+ assertEquals(Collections.<String>emptyList(), parser.getResidue());
+ }
+
+ @Test
+ public void parserWithSingleDashOption() throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ try {
+ parser.parse("-baz=oops", "-bar", "17");
+ fail();
+ } catch (OptionsParsingException expected) {}
+
+ parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ parser.setAllowSingleDashLongOptions(true);
+ parser.parse("-baz=oops", "-bar", "17");
+ ExampleFoo foo = parser.getOptions(ExampleFoo.class);
+ assertEquals("defaultFoo", foo.foo);
+ assertEquals(17, foo.bar);
+ ExampleBaz baz = parser.getOptions(ExampleBaz.class);
+ assertEquals("oops", baz.baz);
+ }
+
+ @Test
+ public void parsingFailsWithUnknownOptions() {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ List<String> unknownOpts = asList("--unknown", "option", "--more_unknowns");
+ try {
+ parser.parse(unknownOpts);
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("--unknown", e.getInvalidArgument());
+ assertEquals("Unrecognized option: --unknown", e.getMessage());
+ assertNotNull(parser.getOptions(ExampleFoo.class));
+ assertNotNull(parser.getOptions(ExampleBaz.class));
+ }
+ }
+
+ @Test
+ public void parseKnownAndUnknownOptions() {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ List<String> opts = asList("--bar", "17", "--unknown", "option");
+ try {
+ parser.parse(opts);
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("--unknown", e.getInvalidArgument());
+ assertEquals("Unrecognized option: --unknown", e.getMessage());
+ assertNotNull(parser.getOptions(ExampleFoo.class));
+ assertNotNull(parser.getOptions(ExampleBaz.class));
+ }
+ }
+
+ public static class CategoryTest extends OptionsBase {
+ @Option(name = "swiss_bank_account_number",
+ category = "undocumented", // Not printed in usage messages!
+ defaultValue = "123456789")
+ public int swissBankAccountNumber;
+
+ @Option(name = "student_bank_account_number",
+ category = "one",
+ defaultValue = "987654321")
+ public int studentBankAccountNumber;
+ }
+
+ @Test
+ public void getOptionsAndGetResidueWithNoCallToParse() {
+ // With no call to parse(), all options are at default values, and there's
+ // no reside.
+ assertEquals("defaultFoo",
+ newOptionsParser(ExampleFoo.class).
+ getOptions(ExampleFoo.class).foo);
+ assertEquals(Collections.<String>emptyList(),
+ newOptionsParser(ExampleFoo.class).getResidue());
+ }
+
+ @Test
+ public void parserCanBeCalledRepeatedly() throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class);
+ parser.parse("--foo", "foo1");
+ assertEquals("foo1", parser.getOptions(ExampleFoo.class).foo);
+ parser.parse();
+ assertEquals("foo1", parser.getOptions(ExampleFoo.class).foo); // no change
+ parser.parse("--foo", "foo2");
+ assertEquals("foo2", parser.getOptions(ExampleFoo.class).foo); // updated
+ }
+
+ @Test
+ public void multipleOccuringOption() throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class);
+ parser.parse("--bing", "abcdef", "--foo", "foo1", "--bing", "123456" );
+ assertThat(parser.getOptions(ExampleFoo.class).bing).containsExactly("abcdef", "123456");
+ }
+
+ @Test
+ public void multipleOccurringOptionWithConverter() throws OptionsParsingException {
+ // --bang is the same as --bing except that it has a "converter" specified.
+ // This test also tests option values with embedded commas and spaces.
+ OptionsParser parser = newOptionsParser(ExampleFoo.class);
+ parser.parse("--bang", "abc,def ghi", "--foo", "foo1", "--bang", "123456" );
+ assertThat(parser.getOptions(ExampleFoo.class).bang).containsExactly("abc,def ghi", "123456");
+ }
+
+ @Test
+ public void parserIgnoresOptionsAfterMinusMinus()
+ throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ parser.parse("--foo", "well", "--baz", "here", "--", "--bar", "ignore");
+ ExampleFoo foo = parser.getOptions(ExampleFoo.class);
+ ExampleBaz baz = parser.getOptions(ExampleBaz.class);
+ assertEquals("well", foo.foo);
+ assertEquals("here", baz.baz);
+ assertEquals(42, foo.bar); // the default!
+ assertEquals(asList("--bar", "ignore"), parser.getResidue());
+ }
+
+ @Test
+ public void parserThrowsExceptionIfResidueIsNotAllowed() {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class);
+ parser.setAllowResidue(false);
+ try {
+ parser.parse("residue", "is", "not", "OK");
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("Unrecognized arguments: residue is not OK", e.getMessage());
+ }
+ }
+
+ @Test
+ public void multipleCallsToParse() throws Exception {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class);
+ parser.setAllowResidue(true);
+ parser.parse("--foo", "one", "--bar", "43", "unknown1");
+ parser.parse("--foo", "two", "unknown2");
+ ExampleFoo foo = parser.getOptions(ExampleFoo.class);
+ assertEquals("two", foo.foo); // second call takes precedence
+ assertEquals(43, foo.bar);
+ assertEquals(Arrays.asList("unknown1", "unknown2"), parser.getResidue());
+ }
+
+ // Regression test for a subtle bug! The toString of each options interface
+ // instance was printing out key=value pairs for all flags in the
+ // OptionsParser, not just those belonging to the specific interface type.
+ @Test
+ public void toStringDoesntIncludeFlagsForOtherOptionsInParserInstance()
+ throws Exception {
+ OptionsParser parser = newOptionsParser(ExampleFoo.class, ExampleBaz.class);
+ parser.parse("--foo", "foo", "--bar", "43", "--baz", "baz");
+
+ String fooString = parser.getOptions(ExampleFoo.class).toString();
+ if (!fooString.contains("foo=foo") ||
+ !fooString.contains("bar=43") ||
+ !fooString.contains("ExampleFoo") ||
+ fooString.contains("baz=baz")) {
+ fail("ExampleFoo.toString() is incorrect: " + fooString);
+ }
+
+ String bazString = parser.getOptions(ExampleBaz.class).toString();
+ if (!bazString.contains("baz=baz") ||
+ !bazString.contains("ExampleBaz") ||
+ bazString.contains("foo=foo") ||
+ bazString.contains("bar=43")) {
+ fail("ExampleBaz.toString() is incorrect: " + bazString);
+ }
+ }
+
+ // Regression test for another subtle bug! The toString was printing all the
+ // explicitly-specified options, even if they were at their default values,
+ // causing toString equivalence to diverge from equals().
+ @Test
+ public void toStringIsIndependentOfExplicitCommandLineOptions() throws Exception {
+ ExampleFoo foo1 = Options.parse(ExampleFoo.class).getOptions();
+ ExampleFoo foo2 = Options.parse(ExampleFoo.class, "--bar", "42").getOptions();
+ assertEquals(foo1, foo2);
+ assertEquals(foo1.toString(), foo2.toString());
+
+ Map<String, Object> expectedMap = new ImmutableMap.Builder<String, Object>().
+ put("bing", Collections.emptyList()).
+ put("bar", 42).
+ put("nodoc", "").
+ put("bang", Collections.emptyList()).
+ put("foo", "defaultFoo").build();
+
+ assertEquals(expectedMap, foo1.asMap());
+ assertEquals(expectedMap, foo2.asMap());
+ }
+
+ // Regression test for yet another subtle bug! The inherited options weren't
+ // being printed by toString. One day, a real rain will come and wash all
+ // this scummy code off the streets.
+ public static class DerivedBaz extends ExampleBaz {
+ @Option(name = "derived", defaultValue = "defaultDerived")
+ public String derived;
+ }
+
+ @Test
+ public void toStringPrintsInheritedOptionsToo_Duh() throws Exception {
+ DerivedBaz derivedBaz = Options.parse(DerivedBaz.class).getOptions();
+ String derivedBazString = derivedBaz.toString();
+ if (!derivedBazString.contains("derived=defaultDerived") ||
+ !derivedBazString.contains("baz=defaultBaz")) {
+ fail("DerivedBaz.toString() is incorrect: " + derivedBazString);
+ }
+ }
+
+ // Tests for new default value override mechanism
+ public static class CustomOptions extends OptionsBase {
+ @Option(name = "simple",
+ category = "custom",
+ defaultValue = "simple default")
+ public String simple;
+
+ @Option(name = "multipart_name",
+ category = "custom",
+ defaultValue = "multipart default")
+ public String multipartName;
+ }
+
+ public void assertDefaultStringsForCustomOptions() throws OptionsParsingException {
+ CustomOptions options = Options.parse(CustomOptions.class).getOptions();
+ assertEquals("simple default", options.simple);
+ assertEquals("multipart default", options.multipartName);
+ }
+
+ public static class NullTestOptions extends OptionsBase {
+ @Option(name = "simple",
+ defaultValue = "null")
+ public String simple;
+ }
+
+ @Test
+ public void defaultNullStringGivesNull() throws Exception {
+ NullTestOptions options = Options.parse(NullTestOptions.class).getOptions();
+ assertNull(options.simple);
+ }
+
+ public static class ImplicitDependencyOptions extends OptionsBase {
+ @Option(name = "first",
+ implicitRequirements = "--second=second",
+ defaultValue = "null")
+ public String first;
+
+ @Option(name = "second",
+ implicitRequirements = "--third=third",
+ defaultValue = "null")
+ public String second;
+
+ @Option(name = "third",
+ defaultValue = "null")
+ public String third;
+ }
+
+ @Test
+ public void implicitDependencyHasImplicitDependency() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ImplicitDependencyOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--first=first"));
+ assertEquals("first", parser.getOptions(ImplicitDependencyOptions.class).first);
+ assertEquals("second", parser.getOptions(ImplicitDependencyOptions.class).second);
+ assertEquals("third", parser.getOptions(ImplicitDependencyOptions.class).third);
+ }
+
+ public static class BadImplicitDependencyOptions extends OptionsBase {
+ @Option(name = "first",
+ implicitRequirements = "xxx",
+ defaultValue = "null")
+ public String first;
+ }
+
+ @Test
+ public void badImplicitDependency() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(BadImplicitDependencyOptions.class);
+ try {
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--first=first"));
+ } catch (AssertionError e) {
+ /* Expected error. */
+ return;
+ }
+ fail();
+ }
+
+ public static class BadExpansionOptions extends OptionsBase {
+ @Option(name = "first",
+ expansion = { "xxx" },
+ defaultValue = "null")
+ public Void first;
+ }
+
+ @Test
+ public void badExpansionOptions() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(BadExpansionOptions.class);
+ try {
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--first"));
+ } catch (AssertionError e) {
+ /* Expected error. */
+ return;
+ }
+ fail();
+ }
+
+ public static class ExpansionOptions extends OptionsBase {
+ @Option(name = "first",
+ expansion = { "--second=first" },
+ defaultValue = "null")
+ public Void first;
+
+ @Option(name = "second",
+ defaultValue = "null")
+ public String second;
+ }
+
+ @Test
+ public void overrideExpansionWithExplicit() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ExpansionOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--first", "--second=second"));
+ ExpansionOptions options = parser.getOptions(ExpansionOptions.class);
+ assertEquals("second", options.second);
+ assertEquals(0, parser.getWarnings().size());
+ }
+
+ @Test
+ public void overrideExplicitWithExpansion() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ExpansionOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--second=second", "--first"));
+ ExpansionOptions options = parser.getOptions(ExpansionOptions.class);
+ assertEquals("first", options.second);
+ }
+
+ @Test
+ public void overrideWithHigherPriority() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NullTestOptions.class);
+ parser.parse(OptionPriority.RC_FILE, null, Arrays.asList("--simple=a"));
+ assertEquals("a", parser.getOptions(NullTestOptions.class).simple);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--simple=b"));
+ assertEquals("b", parser.getOptions(NullTestOptions.class).simple);
+ }
+
+ @Test
+ public void overrideWithLowerPriority() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NullTestOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--simple=a"));
+ assertEquals("a", parser.getOptions(NullTestOptions.class).simple);
+ parser.parse(OptionPriority.RC_FILE, null, Arrays.asList("--simple=b"));
+ assertEquals("a", parser.getOptions(NullTestOptions.class).simple);
+ }
+
+ @Test
+ public void getOptionValueDescriptionWithNonExistingOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NullTestOptions.class);
+ try {
+ parser.getOptionValueDescription("notexisting");
+ fail();
+ } catch (IllegalArgumentException e) {
+ /* Expected exception. */
+ }
+ }
+
+ @Test
+ public void getOptionValueDescriptionWithoutValue() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NullTestOptions.class);
+ assertNull(parser.getOptionValueDescription("simple"));
+ }
+
+ @Test
+ public void getOptionValueDescriptionWithValue() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NullTestOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, "my description",
+ Arrays.asList("--simple=abc"));
+ OptionValueDescription result = parser.getOptionValueDescription("simple");
+ assertNotNull(result);
+ assertEquals("simple", result.getName());
+ assertEquals("abc", result.getValue());
+ assertEquals(OptionPriority.COMMAND_LINE, result.getPriority());
+ assertEquals("my description", result.getSource());
+ assertNull(result.getImplicitDependant());
+ assertFalse(result.isImplicitDependency());
+ assertNull(result.getExpansionParent());
+ assertFalse(result.isExpansion());
+ }
+
+ public static class ImplicitDependencyWarningOptions extends OptionsBase {
+ @Option(name = "first",
+ implicitRequirements = "--second=second",
+ defaultValue = "null")
+ public String first;
+
+ @Option(name = "second",
+ defaultValue = "null")
+ public String second;
+
+ @Option(name = "third",
+ implicitRequirements = "--second=third",
+ defaultValue = "null")
+ public String third;
+ }
+
+ @Test
+ public void warningForImplicitOverridingExplicitOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ImplicitDependencyWarningOptions.class);
+ parser.parse("--second=second", "--first=first");
+ assertThat(parser.getWarnings())
+ .containsExactly("Option 'second' is implicitly defined by "
+ + "option 'first'; the implicitly set value overrides the previous one");
+ }
+
+ @Test
+ public void warningForExplicitOverridingImplicitOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ImplicitDependencyWarningOptions.class);
+ parser.parse("--first=first");
+ assertThat(parser.getWarnings()).isEmpty();
+ parser.parse("--second=second");
+ assertThat(parser.getWarnings())
+ .containsExactly("A new value for option 'second' overrides a"
+ + " previous implicit setting of that option by option 'first'");
+ }
+
+ @Test
+ public void warningForExplicitOverridingImplicitOptionInSameCall() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ImplicitDependencyWarningOptions.class);
+ parser.parse("--first=first", "--second=second");
+ assertThat(parser.getWarnings())
+ .containsExactly("Option 'second' is implicitly defined by "
+ + "option 'first'; the implicitly set value overrides the previous one");
+ }
+
+ @Test
+ public void warningForImplicitOverridingImplicitOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ImplicitDependencyWarningOptions.class);
+ parser.parse("--first=first");
+ assertThat(parser.getWarnings()).isEmpty();
+ parser.parse("--third=third");
+ assertThat(parser.getWarnings())
+ .containsExactly("Option 'second' is implicitly defined by both "
+ + "option 'first' and option 'third'");
+ }
+
+ public static class WarningOptions extends OptionsBase {
+ @Deprecated
+ @Option(name = "first",
+ defaultValue = "null")
+ public Void first;
+
+ @Deprecated
+ @Option(name = "second",
+ allowMultiple = true,
+ defaultValue = "null")
+ public List<String> second;
+
+ @Deprecated
+ @Option(name = "third",
+ expansion = "--fourth=true",
+ abbrev = 't',
+ defaultValue = "null")
+ public Void third;
+
+ @Option(name = "fourth",
+ defaultValue = "false")
+ public boolean fourth;
+ }
+
+ @Test
+ public void deprecationWarning() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(WarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--first"));
+ assertEquals(Arrays.asList("Option 'first' is deprecated"), parser.getWarnings());
+ }
+
+ @Test
+ public void deprecationWarningForListOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(WarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--second=a"));
+ assertEquals(Arrays.asList("Option 'second' is deprecated"), parser.getWarnings());
+ }
+
+ @Test
+ public void deprecationWarningForExpansionOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(WarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--third"));
+ assertEquals(Arrays.asList("Option 'third' is deprecated"), parser.getWarnings());
+ assertTrue(parser.getOptions(WarningOptions.class).fourth);
+ }
+
+ @Test
+ public void deprecationWarningForAbbreviatedExpansionOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(WarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("-t"));
+ assertEquals(Arrays.asList("Option 'third' is deprecated"), parser.getWarnings());
+ assertTrue(parser.getOptions(WarningOptions.class).fourth);
+ }
+
+ public static class NewWarningOptions extends OptionsBase {
+ @Option(name = "first",
+ defaultValue = "null",
+ deprecationWarning = "it's gone")
+ public Void first;
+
+ @Option(name = "second",
+ allowMultiple = true,
+ defaultValue = "null",
+ deprecationWarning = "sorry, no replacement")
+ public List<String> second;
+
+ @Option(name = "third",
+ expansion = "--fourth=true",
+ defaultValue = "null",
+ deprecationWarning = "use --forth instead")
+ public Void third;
+
+ @Option(name = "fourth",
+ defaultValue = "false")
+ public boolean fourth;
+ }
+
+ @Test
+ public void newDeprecationWarning() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NewWarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--first"));
+ assertEquals(Arrays.asList("Option 'first' is deprecated: it's gone"), parser.getWarnings());
+ }
+
+ @Test
+ public void newDeprecationWarningForListOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NewWarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--second=a"));
+ assertEquals(Arrays.asList("Option 'second' is deprecated: sorry, no replacement"),
+ parser.getWarnings());
+ }
+
+ @Test
+ public void newDeprecationWarningForExpansionOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(NewWarningOptions.class);
+ parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--third"));
+ assertEquals(Arrays.asList("Option 'third' is deprecated: use --forth instead"),
+ parser.getWarnings());
+ assertTrue(parser.getOptions(NewWarningOptions.class).fourth);
+ }
+
+ public static class ExpansionWarningOptions extends OptionsBase {
+ @Option(name = "first",
+ expansion = "--second=other",
+ defaultValue = "null")
+ public Void first;
+
+ @Option(name = "second",
+ defaultValue = "null")
+ public String second;
+ }
+
+ @Test
+ public void warningForExpansionOverridingExplicitOption() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ExpansionWarningOptions.class);
+ parser.parse("--second=second", "--first");
+ assertThat(parser.getWarnings())
+ .containsExactly("The option 'first' was expanded and now overrides a "
+ + "previous explicitly specified option 'second'");
+ }
+
+ public static class InvalidOptionConverter extends OptionsBase {
+ @Option(name = "foo",
+ converter = StringConverter.class,
+ defaultValue = "1")
+ public Integer foo;
+ }
+
+ @Test
+ public void errorForInvalidOptionConverter() throws Exception {
+ try {
+ OptionsParser.newOptionsParser(InvalidOptionConverter.class);
+ } catch (AssertionError e) {
+ // Expected exception
+ return;
+ }
+ fail();
+ }
+
+ public static class InvalidListOptionConverter extends OptionsBase {
+ @Option(name = "foo",
+ converter = StringConverter.class,
+ defaultValue = "1",
+ allowMultiple = true)
+ public List<Integer> foo;
+ }
+
+ @Test
+ public void errorForInvalidListOptionConverter() throws Exception {
+ try {
+ OptionsParser.newOptionsParser(InvalidListOptionConverter.class);
+ } catch (AssertionError e) {
+ // Expected exception
+ return;
+ }
+ fail();
+ }
+
+ // This test is here to make sure that nobody accidentally changes the
+ // order of the enum values and breaks the implicit assumptions elsewhere
+ // in the code.
+ @Test
+ public void optionPrioritiesAreCorrectlyOrdered() throws Exception {
+ assertEquals(5, OptionPriority.values().length);
+ assertEquals(-1, OptionPriority.DEFAULT.compareTo(OptionPriority.COMPUTED_DEFAULT));
+ assertEquals(-1, OptionPriority.COMPUTED_DEFAULT.compareTo(OptionPriority.RC_FILE));
+ assertEquals(-1, OptionPriority.RC_FILE.compareTo(OptionPriority.COMMAND_LINE));
+ assertEquals(-1, OptionPriority.COMMAND_LINE.compareTo(OptionPriority.SOFTWARE_REQUIREMENT));
+ }
+
+ public static class IntrospectionExample extends OptionsBase {
+ @Option(name = "alpha",
+ category = "one",
+ defaultValue = "alpha")
+ public String alpha;
+
+ @Option(name = "beta",
+ category = "one",
+ defaultValue = "beta")
+ public String beta;
+
+ @Option(name = "gamma",
+ category = "undocumented",
+ defaultValue = "gamma")
+ public String gamma;
+
+ @Option(name = "delta",
+ category = "undocumented",
+ defaultValue = "delta")
+ public String delta;
+
+ @Option(name = "echo",
+ category = "hidden",
+ defaultValue = "echo")
+ public String echo;
+ }
+
+ @Test
+ public void asListOfUnparsedOptions() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, "source",
+ Arrays.asList("--alpha=one", "--gamma=two", "--echo=three"));
+ List<UnparsedOptionValueDescription> result = parser.asListOfUnparsedOptions();
+ assertNotNull(result);
+ assertEquals(3, result.size());
+
+ assertEquals("alpha", result.get(0).getName());
+ assertEquals(true, result.get(0).isDocumented());
+ assertEquals(false, result.get(0).isHidden());
+ assertEquals("one", result.get(0).getUnparsedValue());
+ assertEquals("source", result.get(0).getSource());
+ assertEquals(OptionPriority.COMMAND_LINE, result.get(0).getPriority());
+
+ assertEquals("gamma", result.get(1).getName());
+ assertEquals(false, result.get(1).isDocumented());
+ assertEquals(false, result.get(1).isHidden());
+ assertEquals("two", result.get(1).getUnparsedValue());
+ assertEquals("source", result.get(1).getSource());
+ assertEquals(OptionPriority.COMMAND_LINE, result.get(1).getPriority());
+
+ assertEquals("echo", result.get(2).getName());
+ assertEquals(false, result.get(2).isDocumented());
+ assertEquals(true, result.get(2).isHidden());
+ assertEquals("three", result.get(2).getUnparsedValue());
+ assertEquals("source", result.get(2).getSource());
+ assertEquals(OptionPriority.COMMAND_LINE, result.get(2).getPriority());
+ }
+
+ @Test
+ public void asListOfExplicitOptions() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, "source",
+ Arrays.asList("--alpha=one", "--gamma=two"));
+ List<UnparsedOptionValueDescription> result = parser.asListOfExplicitOptions();
+ assertNotNull(result);
+ assertEquals(2, result.size());
+
+ assertEquals("alpha", result.get(0).getName());
+ assertEquals(true, result.get(0).isDocumented());
+ assertEquals("one", result.get(0).getUnparsedValue());
+ assertEquals("source", result.get(0).getSource());
+ assertEquals(OptionPriority.COMMAND_LINE, result.get(0).getPriority());
+
+ assertEquals("gamma", result.get(1).getName());
+ assertEquals(false, result.get(1).isDocumented());
+ assertEquals("two", result.get(1).getUnparsedValue());
+ assertEquals("source", result.get(1).getSource());
+ assertEquals(OptionPriority.COMMAND_LINE, result.get(1).getPriority());
+ }
+
+ private void assertOptionValue(String expectedName, Object expectedValue,
+ OptionPriority expectedPriority, String expectedSource,
+ OptionValueDescription actual) {
+ assertNotNull(actual);
+ assertEquals(expectedName, actual.getName());
+ assertEquals(expectedValue, actual.getValue());
+ assertEquals(expectedPriority, actual.getPriority());
+ assertEquals(expectedSource, actual.getSource());
+ }
+
+ @Test
+ public void asListOfEffectiveOptions() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, "source",
+ Arrays.asList("--alpha=one", "--gamma=two"));
+ List<OptionValueDescription> result = parser.asListOfEffectiveOptions();
+ assertNotNull(result);
+ assertEquals(5, result.size());
+ HashMap<String,OptionValueDescription> map = new HashMap<String,OptionValueDescription>();
+ for (OptionValueDescription description : result) {
+ map.put(description.getName(), description);
+ }
+
+ assertOptionValue("alpha", "one", OptionPriority.COMMAND_LINE, "source",
+ map.get("alpha"));
+ assertOptionValue("beta", "beta", OptionPriority.DEFAULT, null,
+ map.get("beta"));
+ assertOptionValue("gamma", "two", OptionPriority.COMMAND_LINE, "source",
+ map.get("gamma"));
+ assertOptionValue("delta", "delta", OptionPriority.DEFAULT, null,
+ map.get("delta"));
+ assertOptionValue("echo", "echo", OptionPriority.DEFAULT, null,
+ map.get("echo"));
+ }
+
+ // Regression tests for bug:
+ // "--option from blazerc unexpectedly overrides --option from command line"
+ public static class ListExample extends OptionsBase {
+ @Option(name = "alpha",
+ converter = StringConverter.class,
+ allowMultiple = true,
+ defaultValue = "null")
+ public List<String> alpha;
+ }
+
+ @Test
+ public void overrideListOptions() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(ListExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, "a", Arrays.asList("--alpha=two"));
+ parser.parse(OptionPriority.RC_FILE, "b", Arrays.asList("--alpha=one"));
+ assertEquals(Arrays.asList("one", "two"), parser.getOptions(ListExample.class).alpha);
+ }
+
+ public static class CommaSeparatedOptionsExample extends OptionsBase {
+ @Option(name = "alpha",
+ converter = CommaSeparatedOptionListConverter.class,
+ allowMultiple = true,
+ defaultValue = "null")
+ public List<String> alpha;
+ }
+
+ @Test
+ public void commaSeparatedOptionsWithAllowMultiple() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(CommaSeparatedOptionsExample.class);
+ parser.parse(OptionPriority.COMMAND_LINE, "a", Arrays.asList("--alpha=one",
+ "--alpha=two,three"));
+ assertEquals(Arrays.asList("one", "two", "three"),
+ parser.getOptions(CommaSeparatedOptionsExample.class).alpha);
+ }
+
+ public static class IllegalListTypeExample extends OptionsBase {
+ @Option(name = "alpha",
+ converter = CommaSeparatedOptionListConverter.class,
+ allowMultiple = true,
+ defaultValue = "null")
+ public List<Integer> alpha;
+ }
+
+ @Test
+ public void illegalListType() throws Exception {
+ try {
+ OptionsParser.newOptionsParser(IllegalListTypeExample.class);
+ } catch (AssertionError e) {
+ // Expected exception
+ return;
+ }
+ fail();
+ }
+
+ public static class Yesterday extends OptionsBase {
+
+ @Option(name = "a",
+ defaultValue = "a")
+ public String a;
+
+ @Option(name = "b",
+ defaultValue = "b")
+ public String b;
+
+ @Option(name = "c",
+ defaultValue = "null",
+ expansion = {"--a=0"})
+ public Void c;
+
+ @Option(name = "d",
+ defaultValue = "null",
+ allowMultiple = true)
+ public List<String> d;
+
+ @Option(name = "e",
+ defaultValue = "null",
+ implicitRequirements = { "--a==1" })
+ public String e;
+
+ @Option(name = "f",
+ defaultValue = "null",
+ implicitRequirements = { "--b==1" })
+ public String f;
+
+ @Option(name = "g",
+ abbrev = 'h',
+ defaultValue = "false")
+ public boolean g;
+ }
+
+ public static List<String> canonicalize(Class<? extends OptionsBase> optionsClass, String... args)
+ throws OptionsParsingException {
+ return OptionsParser.canonicalize(ImmutableList.<Class<? extends OptionsBase>>of(optionsClass),
+ Arrays.asList(args));
+ }
+
+ @Test
+ public void canonicalizeEasy() throws Exception {
+ assertEquals(Arrays.asList("--a=x"), canonicalize(Yesterday.class, "--a=x"));
+ }
+
+ @Test
+ public void canonicalizeSkipDuplicate() throws Exception {
+ assertEquals(Arrays.asList("--a=x"), canonicalize(Yesterday.class, "--a=y", "--a=x"));
+ }
+
+ @Test
+ public void canonicalizeExpands() throws Exception {
+ assertEquals(Arrays.asList("--a=0"), canonicalize(Yesterday.class, "--c"));
+ }
+
+ @Test
+ public void canonicalizeExpansionOverridesExplicit() throws Exception {
+ assertEquals(Arrays.asList("--a=0"), canonicalize(Yesterday.class, "--a=x", "--c"));
+ }
+
+ @Test
+ public void canonicalizeExplicitOverridesExpansion() throws Exception {
+ assertEquals(Arrays.asList("--a=x"), canonicalize(Yesterday.class, "--c", "--a=x"));
+ }
+
+ @Test
+ public void canonicalizeSorts() throws Exception {
+ assertEquals(Arrays.asList("--a=x", "--b=y"), canonicalize(Yesterday.class, "--b=y", "--a=x"));
+ }
+
+ @Test
+ public void canonicalizeImplicitDepsAtEnd() throws Exception {
+ assertEquals(Arrays.asList("--a=x", "--e=y"), canonicalize(Yesterday.class, "--e=y", "--a=x"));
+ }
+
+ @Test
+ public void canonicalizeImplicitDepsSkipsDuplicate() throws Exception {
+ assertEquals(Arrays.asList("--e=y"), canonicalize(Yesterday.class, "--e=x", "--e=y"));
+ }
+
+ @Test
+ public void canonicalizeDoesNotSortImplicitDeps() throws Exception {
+ assertEquals(Arrays.asList("--a=x", "--f=z", "--e=y"),
+ canonicalize(Yesterday.class, "--f=z", "--e=y", "--a=x"));
+ }
+
+ @Test
+ public void canonicalizeDoesNotSkipAllowMultiple() throws Exception {
+ assertEquals(Arrays.asList("--d=a", "--d=b"),
+ canonicalize(Yesterday.class, "--d=a", "--d=b"));
+ }
+
+ @Test
+ public void canonicalizeReplacesAbbrevWithName() throws Exception {
+ assertEquals(Arrays.asList("--g=1"),
+ canonicalize(Yesterday.class, "-h"));
+ }
+
+ public static class LongValueExample extends OptionsBase {
+ @Option(name = "longval",
+ defaultValue = "2147483648")
+ public long longval;
+
+ @Option(name = "intval",
+ defaultValue = "2147483647")
+ public int intval;
+ }
+
+ @Test
+ public void parseLong() throws OptionsParsingException {
+ OptionsParser parser = newOptionsParser(LongValueExample.class);
+ parser.parse("");
+ LongValueExample result = parser.getOptions(LongValueExample.class);
+ assertEquals(2147483648L, result.longval);
+ assertEquals(2147483647, result.intval);
+
+ parser.parse("--longval", Long.toString(Long.MIN_VALUE));
+ result = parser.getOptions(LongValueExample.class);
+ assertEquals(Long.MIN_VALUE, result.longval);
+
+ try {
+ parser.parse("--intval=2147483648");
+ fail();
+ } catch (OptionsParsingException e) {
+ }
+
+ parser.parse("--longval", "100");
+ result = parser.getOptions(LongValueExample.class);
+ assertEquals(100, result.longval);
+ }
+}
diff --git a/src/test/java/com/google/devtools/common/options/OptionsTest.java b/src/test/java/com/google/devtools/common/options/OptionsTest.java
new file mode 100644
index 0000000..700e26b
--- /dev/null
+++ b/src/test/java/com/google/devtools/common/options/OptionsTest.java
@@ -0,0 +1,500 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.common.options;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+/**
+ * Test for {@link Options}.
+ */
+@RunWith(JUnit4.class)
+public class OptionsTest {
+
+ private static final String[] NO_ARGS = {};
+
+ public static class HttpOptions extends OptionsBase {
+
+ @Option(name = "host",
+ defaultValue = "www.google.com",
+ help = "The URL at which the server will be running.")
+ public String host;
+
+ @Option(name = "port",
+ abbrev = 'p',
+ defaultValue = "80",
+ help = "The port at which the server will be running.")
+ public int port;
+
+ @Option(name = "debug",
+ abbrev = 'd',
+ defaultValue = "false",
+ help = "debug")
+ public boolean isDebugging;
+
+ @Option(name = "tristate",
+ abbrev = 't',
+ defaultValue = "auto",
+ help = "tri-state option returning auto by default")
+ public TriState triState;
+
+ @Option(name = "special",
+ defaultValue = "null",
+ expansion = { "--host=special.google.com", "--port=8080"})
+ public Void special;
+ }
+
+ @Test
+ public void paragraphFill() throws Exception {
+ // TODO(bazel-team): don't include trailing space after last word in line.
+ String input = "The quick brown fox jumps over the lazy dog.";
+
+ assertEquals(" The quick \n brown fox \n jumps over \n the lazy \n"
+ + " dog.",
+ OptionsUsage.paragraphFill(input, 2, 13));
+ assertEquals(" The quick brown \n fox jumps over \n the lazy dog.",
+ OptionsUsage.paragraphFill(input, 3, 19));
+
+ String input2 = "The quick brown fox jumps\nAnother paragraph.";
+ assertEquals(" The quick brown fox \n jumps\n Another paragraph.",
+ OptionsUsage.paragraphFill(input2, 2, 23));
+ }
+
+ @Test
+ public void getsDefaults() throws OptionsParsingException {
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, NO_ARGS);
+ String[] remainingArgs = options.getRemainingArgs();
+ HttpOptions webFlags = options.getOptions();
+
+ assertEquals("www.google.com", webFlags.host);
+ assertEquals(80, webFlags.port);
+ assertEquals(false, webFlags.isDebugging);
+ assertEquals(TriState.AUTO, webFlags.triState);
+ assertEquals(0, remainingArgs.length);
+ }
+
+ @Test
+ public void objectMethods() throws OptionsParsingException {
+ String[] args = { "--host", "foo", "--port", "80" };
+ HttpOptions left =
+ Options.parse(HttpOptions.class, args).getOptions();
+ HttpOptions likeLeft =
+ Options.parse(HttpOptions.class, args).getOptions();
+ String [] rightArgs = {"--host", "other", "--port", "90" };
+ HttpOptions right =
+ Options.parse(HttpOptions.class, rightArgs).getOptions();
+
+ String toString = left.toString();
+ // Don't rely on Set.toString iteration order:
+ assertTrue(toString.startsWith(
+ "com.google.devtools.common.options.OptionsTest"
+ + "$HttpOptions{"));
+ assertTrue(toString.contains("host=foo"));
+ assertTrue(toString.contains("port=80"));
+ assertTrue(toString.endsWith("}"));
+
+ assertTrue(left.equals(left));
+ assertTrue(left.toString().equals(likeLeft.toString()));
+ assertTrue(left.equals(likeLeft));
+ assertTrue(likeLeft.equals(left));
+ assertFalse(left.equals(right));
+ assertFalse(right.equals(left));
+ assertFalse(left.equals(null));
+ assertFalse(likeLeft.equals(null));
+ assertEquals(likeLeft.hashCode(), likeLeft.hashCode());
+ assertEquals(left.hashCode(), likeLeft.hashCode());
+ // Strictly speaking this is not required for hashCode to be correct,
+ // but a good hashCode should be different at least for some values. So,
+ // we're making sure that at least this particular pair of inputs yields
+ // different values.
+ assertFalse(left.hashCode() == right.hashCode());
+ }
+
+ @Test
+ public void equals() throws OptionsParsingException {
+ String[] args = { "--host", "foo", "--port", "80" };
+ HttpOptions options1 = Options.parse(HttpOptions.class, args).getOptions();
+
+ String[] args2 = { "-p", "80", "--host", "foo" };
+ HttpOptions options2 = Options.parse(HttpOptions.class, args2).getOptions();
+ assertEquals("order/abbreviations shouldn't matter", options1, options2);
+
+ assertEquals("explicitly setting a default shouldn't matter",
+ Options.parse(HttpOptions.class, "--port", "80").getOptions(),
+ Options.parse(HttpOptions.class).getOptions());
+
+ assertThat(Options.parse(HttpOptions.class, "--port", "3").getOptions())
+ .isNotEqualTo(Options.parse(HttpOptions.class).getOptions());
+ }
+
+ @Test
+ public void getsFlagsProvidedInArguments()
+ throws OptionsParsingException {
+ String[] args = {"--host", "google.com",
+ "-p", "8080", // short form
+ "--debug"};
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, args);
+ String[] remainingArgs = options.getRemainingArgs();
+ HttpOptions webFlags = options.getOptions();
+
+ assertEquals("google.com", webFlags.host);
+ assertEquals(8080, webFlags.port);
+ assertEquals(true, webFlags.isDebugging);
+ assertEquals(0, remainingArgs.length);
+ }
+
+ @Test
+ public void getsFlagsProvidedWithEquals() throws OptionsParsingException {
+ String[] args = {"--host=google.com",
+ "--port=8080",
+ "--debug"};
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, args);
+ String[] remainingArgs = options.getRemainingArgs();
+ HttpOptions webFlags = options.getOptions();
+
+ assertEquals("google.com", webFlags.host);
+ assertEquals(8080, webFlags.port);
+ assertEquals(true, webFlags.isDebugging);
+ assertEquals(0, remainingArgs.length);
+ }
+
+ @Test
+ public void booleanNo() throws OptionsParsingException {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[]{"--nodebug", "--notristate"});
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(false, webFlags.isDebugging);
+ assertEquals(TriState.NO, webFlags.triState);
+ }
+
+ @Test
+ public void booleanNoUnderscore() throws OptionsParsingException {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[]{"--no_debug", "--no_tristate"});
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(false, webFlags.isDebugging);
+ assertEquals(TriState.NO, webFlags.triState);
+ }
+
+ @Test
+ public void booleanAbbrevMinus() throws OptionsParsingException {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[]{"-d-", "-t-"});
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(false, webFlags.isDebugging);
+ assertEquals(TriState.NO, webFlags.triState);
+ }
+
+ @Test
+ public void boolean0() throws OptionsParsingException {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[]{"--debug=0", "--tristate=0"});
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(false, webFlags.isDebugging);
+ assertEquals(TriState.NO, webFlags.triState);
+ }
+
+ @Test
+ public void boolean1() throws OptionsParsingException {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[]{"--debug=1", "--tristate=1"});
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(true, webFlags.isDebugging);
+ assertEquals(TriState.YES, webFlags.triState);
+ }
+
+ @Test
+ public void retainsStuffThatsNotOptions() throws OptionsParsingException {
+ String[] args = {"these", "aint", "options"};
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, args);
+ String[] remainingArgs = options.getRemainingArgs();
+ assertEquals(asList(args), asList(remainingArgs));
+ }
+
+ @Test
+ public void retainsStuffThatsNotComplexOptions()
+ throws OptionsParsingException {
+ String[] args = {"--host", "google.com",
+ "notta",
+ "--port=8080",
+ "option",
+ "--debug=true"};
+ String[] notoptions = {"notta", "option" };
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, args);
+ String[] remainingArgs = options.getRemainingArgs();
+ assertEquals(asList(notoptions), asList(remainingArgs));
+ }
+
+ @Test
+ public void wontParseUnknownOptions() {
+ String [] args = { "--unknown", "--other=23", "--options" };
+ try {
+ Options.parse(HttpOptions.class, args);
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("Unrecognized option: --unknown", e.getMessage());
+ }
+ }
+
+ @Test
+ public void requiresOptionValue() {
+ String[] args = {"--port"};
+ try {
+ Options.parse(HttpOptions.class, args);
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("Expected value after --port", e.getMessage());
+ }
+ }
+
+ @Test
+ public void handlesDuplicateOptions_full() throws Exception {
+ String[] args = {"--port=80", "--port", "81"};
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, args);
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(81, webFlags.port);
+ }
+
+ @Test
+ public void handlesDuplicateOptions_abbrev() throws Exception {
+ String[] args = {"--port=80", "-p", "81"};
+ Options<HttpOptions> options = Options.parse(HttpOptions.class, args);
+ HttpOptions webFlags = options.getOptions();
+ assertEquals(81, webFlags.port);
+ }
+
+ @Test
+ public void duplicateOptionsOkWithSameValues() throws Exception {
+ // These would throw OptionsParsingException if they failed.
+ Options.parse(HttpOptions.class,"--port=80", "--port", "80");
+ Options.parse(HttpOptions.class, "--port=80", "-p", "80");
+ }
+
+ @Test
+ public void isPickyAboutBooleanValues() {
+ try {
+ Options.parse(HttpOptions.class, new String[]{"--debug=not_a_boolean"});
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("While parsing option --debug=not_a_boolean: "
+ + "\'not_a_boolean\' is not a boolean", e.getMessage());
+ }
+ }
+
+ @Test
+ public void isPickyAboutBooleanNos() {
+ try {
+ Options.parse(HttpOptions.class, new String[]{"--nodebug=1"});
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("Unexpected value after boolean option: --nodebug=1", e.getMessage());
+ }
+ }
+
+ @Test
+ public void usageForBuiltinTypes() {
+ String usage = Options.getUsage(HttpOptions.class);
+ // We can't rely on the option ordering.
+ assertTrue(usage.contains(
+ " --[no]debug [-d] (a boolean; default: \"false\")\n" +
+ " debug"));
+ assertTrue(usage.contains(
+ " --host (a string; default: \"www.google.com\")\n" +
+ " The URL at which the server will be running."));
+ assertTrue(usage.contains(
+ " --port [-p] (an integer; default: \"80\")\n" +
+ " The port at which the server will be running."));
+ assertTrue(usage.contains(
+ " --special\n" +
+ " Expands to: --host=special.google.com --port=8080"));
+ assertTrue(usage.contains(
+ " --[no]tristate [-t] (a tri-state (auto, yes, no); default: \"auto\")\n" +
+ " tri-state option returning auto by default"));
+ }
+
+ public static class NullTestOptions extends OptionsBase {
+ @Option(name = "host",
+ defaultValue = "null",
+ help = "The URL at which the server will be running.")
+ public String host;
+
+ @Option(name = "none",
+ defaultValue = "null",
+ expansion = {"--host=www.google.com"},
+ help = "An expanded option.")
+ public Void none;
+ }
+
+ @Test
+ public void usageForNullDefault() {
+ String usage = Options.getUsage(NullTestOptions.class);
+ assertTrue(usage.contains(
+ " --host (a string; default: see description)\n" +
+ " The URL at which the server will be running."));
+ assertTrue(usage.contains(
+ " --none\n" +
+ " An expanded option.\n" +
+ " Expands to: --host=www.google.com"));
+ }
+
+ public static class MyURLConverter implements Converter<URL> {
+
+ @Override
+ public URL convert(String input) throws OptionsParsingException {
+ try {
+ return new URL(input);
+ } catch (MalformedURLException e) {
+ throw new OptionsParsingException("Could not convert '" + input + "': "
+ + e.getMessage());
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a url";
+ }
+
+ }
+
+ public static class UsesCustomConverter extends OptionsBase {
+
+ @Option(name = "url",
+ defaultValue = "http://www.google.com/",
+ converter = MyURLConverter.class)
+ public URL url;
+
+ }
+
+ @Test
+ public void customConverter() throws Exception {
+ Options<UsesCustomConverter> options =
+ Options.parse(UsesCustomConverter.class, new String[0]);
+ URL expected = new URL("http://www.google.com/");
+ assertEquals(expected, options.getOptions().url);
+ }
+
+ @Test
+ public void customConverterThrowsException() throws Exception {
+ String[] args = {"--url=a_malformed:url"};
+ try {
+ Options.parse(UsesCustomConverter.class, args);
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("While parsing option --url=a_malformed:url: "
+ + "Could not convert 'a_malformed:url': "
+ + "no protocol: a_malformed:url", e.getMessage());
+ }
+ }
+
+ @Test
+ public void usageWithCustomConverter() {
+ assertEquals(
+ " --url (a url; default: \"http://www.google.com/\")\n",
+ Options.getUsage(UsesCustomConverter.class));
+ }
+
+ @Test
+ public void unknownBooleanOption() {
+ try {
+ Options.parse(HttpOptions.class, new String[]{"--no-debug"});
+ fail();
+ } catch (OptionsParsingException e) {
+ assertEquals("Unrecognized option: --no-debug", e.getMessage());
+ }
+ }
+
+ public static class J extends OptionsBase {
+ @Option(name = "j", defaultValue = "null")
+ public String string;
+ }
+ @Test
+ public void nullDefaultForReferenceTypeOption() throws Exception {
+ J options = Options.parse(J.class, NO_ARGS).getOptions();
+ assertNull(options.string);
+ }
+
+ public static class K extends OptionsBase {
+ @Option(name = "1", defaultValue = "null")
+ public int int1;
+ }
+ @Test
+ public void nullDefaultForPrimitiveTypeOption() throws Exception {
+ // defaultValue() = "null" is not treated specially for primitive types, so
+ // we get an NumberFormatException from the converter (not a
+ // ClassCastException from casting null to int), just as we would for any
+ // other non-integer-literal string default.
+ try {
+ Options.parse(K.class, NO_ARGS).getOptions();
+ fail();
+ } catch (IllegalStateException e) {
+ assertEquals("OptionsParsingException while retrieving default for "
+ + "int1: 'null' is not an int",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void nullIsntInterpretedSpeciallyExceptAsADefaultValue()
+ throws Exception {
+ HttpOptions options =
+ Options.parse(HttpOptions.class,
+ new String[] { "--host", "null" }).getOptions();
+ assertEquals("null", options.host);
+ }
+
+ @Test
+ public void nonDecimalRadicesForIntegerOptions() throws Exception {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[] { "--port", "0x51"});
+ assertEquals(81, options.getOptions().port);
+ }
+
+ @Test
+ public void expansionOptionSimple() throws Exception {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[] {"--special"});
+ assertEquals("special.google.com", options.getOptions().host);
+ assertEquals(8080, options.getOptions().port);
+ }
+
+ @Test
+ public void expansionOptionOverride() throws Exception {
+ Options<HttpOptions> options =
+ Options.parse(HttpOptions.class, new String[] {"--port=90", "--special", "--host=foo"});
+ assertEquals("foo", options.getOptions().host);
+ assertEquals(8080, options.getOptions().port);
+ }
+
+ @Test
+ public void expansionOptionEquals() throws Exception {
+ Options<HttpOptions> options1 =
+ Options.parse(HttpOptions.class, new String[] { "--host=special.google.com", "--port=8080"});
+ Options<HttpOptions> options2 =
+ Options.parse(HttpOptions.class, new String[] { "--special" });
+ assertEquals(options1.getOptions(), options2.getOptions());
+ }
+}
diff --git a/src/tools/xcode-common/BUILD b/src/tools/xcode-common/BUILD
new file mode 100644
index 0000000..f6c45f3
--- /dev/null
+++ b/src/tools/xcode-common/BUILD
@@ -0,0 +1,14 @@
+package(default_visibility = ["//src/main/java:__subpackages__"])
+
+# TODO(bazel-team): Split this into multiple rules.
+java_library(
+ name = "xcode-common",
+ srcs = glob([
+ "java/com/google/devtools/build/xcode/util/*.java",
+ "java/com/google/devtools/build/xcode/common/*.java",
+ ]),
+ deps = [
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/BuildOptionsUtil.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/BuildOptionsUtil.java
new file mode 100644
index 0000000..9f1a967
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/BuildOptionsUtil.java
@@ -0,0 +1,25 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+/**
+ * Utility code related to build settings (referred to as <em>build configuration</em> within
+ * Xcode).
+ */
+public class BuildOptionsUtil {
+ private BuildOptionsUtil() {}
+
+ public static final String DEFAULT_OPTIONS_NAME = "Bazel";
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/InvalidFamilyNameException.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/InvalidFamilyNameException.java
new file mode 100644
index 0000000..930b504
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/InvalidFamilyNameException.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+/**
+ * An exception that indicates the name of a device family was not recognized or is somehow invalid.
+ */
+public class InvalidFamilyNameException extends IllegalArgumentException {
+ public InvalidFamilyNameException(String message) {
+ super(message);
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/PathTransformer.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/PathTransformer.java
new file mode 100644
index 0000000..08b3f6d
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/PathTransformer.java
@@ -0,0 +1,46 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+import java.nio.file.Path;
+
+/** Defines operations common to file paths. */
+public interface PathTransformer<P> {
+ /** Returns the containing directory of the given path. */
+ P parent(P path);
+
+ /** Returns the result of joining a path component string to the given path. */
+ P join(P path, String segment);
+
+ /** Returns the name of the file at the given path, i.e. the last path component. */
+ String name(P path);
+
+ static final PathTransformer<Path> FOR_JAVA_PATH = new PathTransformer<Path>() {
+ @Override
+ public Path parent(Path path) {
+ return path.getParent();
+ }
+
+ @Override
+ public Path join(Path path, String segment) {
+ return path.resolve(segment);
+ }
+
+ @Override
+ public String name(Path path) {
+ return path.getFileName().toString();
+ }
+ };
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/Platform.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/Platform.java
new file mode 100644
index 0000000..6e7a059
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/Platform.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.xcode.util.Containing;
+
+import java.util.Locale;
+import java.util.Set;
+
+/**
+ * An enum that can be used to distinguish between an iOS simulator and device.
+ */
+public enum Platform {
+ DEVICE("iPhoneOS"), SIMULATOR("iPhoneSimulator");
+
+ private static final Set<String> SIMULATOR_ARCHS = ImmutableSet.of("i386", "x86_64");
+
+ private final String nameInPlist;
+
+ Platform(String nameInPlist) {
+ this.nameInPlist = Preconditions.checkNotNull(nameInPlist);
+ }
+
+ /**
+ * Returns the name of the "platform" as it appears in the CFBundleSupportedPlatforms plist
+ * setting.
+ */
+ public String getNameInPlist() {
+ return nameInPlist;
+ }
+
+ /**
+ * Returns the name of the "platform" as it appears in the plist when it appears in all-lowercase.
+ */
+ public String getLowerCaseNameInPlist() {
+ return nameInPlist.toLowerCase(Locale.US);
+ }
+
+ /**
+ * Returns the platform for the arch.
+ */
+ public static Platform forArch(String arch) {
+ return Containing.item(SIMULATOR_ARCHS, arch) ? SIMULATOR : DEVICE;
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/RepeatedFamilyNameException.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/RepeatedFamilyNameException.java
new file mode 100644
index 0000000..39be4e6
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/RepeatedFamilyNameException.java
@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+/**
+ * An exception that indicates a family name appeared twice in a sequence when only one is expected.
+ */
+public class RepeatedFamilyNameException extends IllegalArgumentException {
+ public RepeatedFamilyNameException(String message) {
+ super(message);
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/TargetDeviceFamily.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/TargetDeviceFamily.java
new file mode 100644
index 0000000..3d369a1
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/TargetDeviceFamily.java
@@ -0,0 +1,132 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Possible values in the {@code TARGETED_DEVICE_FAMILY} build setting.
+ */
+public enum TargetDeviceFamily {
+ IPAD, IPHONE;
+
+ /**
+ * Contains the values of the UIDeviceFamily plist info setting for each valid set of
+ * TargetDeviceFamilies.
+ */
+ public static final Map<Set<TargetDeviceFamily>, List<Integer>> UI_DEVICE_FAMILY_VALUES =
+ ImmutableMap.<Set<TargetDeviceFamily>, List<Integer>>builder()
+ .put(ImmutableSet.of(TargetDeviceFamily.IPHONE), ImmutableList.of(1))
+ .put(ImmutableSet.of(TargetDeviceFamily.IPAD), ImmutableList.of(2))
+ .put(ImmutableSet.of(TargetDeviceFamily.IPHONE, TargetDeviceFamily.IPAD),
+ ImmutableList.of(1, 2))
+ .build();
+
+ /**
+ * Returns the name of the family as it appears in build rules.
+ */
+ public String getNameInRule() {
+ return BY_NAME_IN_RULE.get(this);
+ }
+
+ /**
+ * Returns the name of family which should be used in the bundlemerge control proto.
+ */
+ public String getBundleMergeName() {
+ return BY_BUNDLE_MERGE_NAME.get(this);
+ }
+
+ private static final ImmutableBiMap<TargetDeviceFamily, String> BY_NAME_IN_RULE =
+ ImmutableBiMap.<TargetDeviceFamily, String>of(IPAD, "ipad", IPHONE, "iphone");
+
+ private static final ImmutableBiMap<TargetDeviceFamily, String> BY_BUNDLE_MERGE_NAME =
+ ImmutableBiMap.<TargetDeviceFamily, String>of(IPAD, "IPAD", IPHONE, "IPHONE");
+
+ private static Set<TargetDeviceFamily> fromNames(
+ Iterable<String> names, Map<String, TargetDeviceFamily> mapping) {
+ Set<TargetDeviceFamily> families = EnumSet.noneOf(TargetDeviceFamily.class);
+ for (String name : names) {
+ TargetDeviceFamily family = mapping.get(name);
+ if (family == null) {
+ throw new InvalidFamilyNameException(name);
+ }
+ if (!families.add(family)) {
+ throw new RepeatedFamilyNameException(name);
+ }
+ }
+ return families;
+ }
+
+ /**
+ * Converts a sequence containing the strings returned by {@link #getBundleMergeName()} to a set
+ * of instances of this enum.
+ *
+ * <p>If there are multiple items in the returned set, they are in enumeration order.
+ *
+ * @param names the names of the families
+ * @throws InvalidFamilyNameException if some family name in the sequence was not recognized
+ * @throws RepeatedFamilyNameException if some family name appeared in the sequence twice
+ */
+ public static Set<TargetDeviceFamily> fromBundleMergeNames(Iterable<String> names) {
+ return fromNames(names, BY_BUNDLE_MERGE_NAME.inverse());
+ }
+
+ /**
+ * Converts a sequence containing the strings returned by {@link #getNameInRule()} to a set of
+ * instances of this enum.
+ *
+ * <p>If there are multiple items in the returned set, they are in enumeration order.
+ *
+ * @param names the names of the families
+ * @throws InvalidFamilyNameException if some family name in the sequence was not recognized
+ * @throws RepeatedFamilyNameException if some family name appeared in the sequence twice
+ */
+ public static Set<TargetDeviceFamily> fromNamesInRule(Iterable<String> names) {
+ return fromNames(names, BY_NAME_IN_RULE.inverse());
+ }
+
+ /**
+ * Converts the {@code TARGETED_DEVICE_FAMILY} setting in build settings to a set of
+ * {@code TargetedDevice}s.
+ */
+ public static Set<TargetDeviceFamily> fromBuildSetting(String targetedDevice) {
+ ImmutableSet.Builder<TargetDeviceFamily> result = ImmutableSet.builder();
+ for (String numericSetting : Splitter.on(",").split(targetedDevice)) {
+ numericSetting = numericSetting.trim();
+ switch (numericSetting) {
+ case "1":
+ result.add(IPHONE);
+ break;
+ case "2":
+ result.add(IPAD);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Expect comma-separated list containing only '1' and/or '2' for "
+ + "TARGETED_DEVICE_FAMILY: " + targetedDevice);
+ }
+ }
+ return result.build();
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/XcodeprojPath.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/XcodeprojPath.java
new file mode 100644
index 0000000..eedf326
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/common/XcodeprojPath.java
@@ -0,0 +1,121 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.common;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.util.Equaling;
+import com.google.devtools.build.xcode.util.Value;
+
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * Represents the path to an xcodeproj directory. Contains utilities for getting related
+ * information, including the project.pbxproj file and the project <em>name</em>, which is the
+ * .xcodeproj directory name without the ".xcodeproj" extension.
+ *
+ * @param <T> The type of the backing path, such as {@link java.nio.file.Path}.
+ */
+public class XcodeprojPath<T extends Comparable<T>>
+ extends Value<XcodeprojPath<T>>
+ implements Comparable<XcodeprojPath<T>> {
+ public static final String PBXPROJ_FILE_NAME = "project.pbxproj";
+ public static final String XCODEPROJ_DIRECTORY_SUFFIX = ".xcodeproj";
+
+ /**
+ * An object that knows how to create {@code XcodeprojPath}s from paths of some other type.
+ */
+ public static class Converter<T extends Comparable<T>> {
+ private final PathTransformer<T> transformer;
+
+ public Converter(PathTransformer<T> transformer) {
+ this.transformer = checkNotNull(transformer);
+ }
+
+ /**
+ * Converts a path to an XcodeprojPath. The given path may point to the {@code project.pbxproj}
+ * file or the {@code *.xcodeproj} directory.
+ */
+ public XcodeprojPath<T> fromPath(T path) {
+ if (Equaling.of(PBXPROJ_FILE_NAME, transformer.name(path))) {
+ path = transformer.parent(path);
+ }
+ return new XcodeprojPath<T>(path, transformer);
+ }
+
+ /**
+ * Converts normal paths to {@code ProjectFilePath}s using {@link #fromPath(Comparable)}.
+ */
+ public List<XcodeprojPath<T>> fromPaths(Iterable<? extends T> pbxprojFiles) {
+ ImmutableList.Builder<XcodeprojPath<T>> result = new ImmutableList.Builder<>();
+ for (T pbxprojFile : pbxprojFiles) {
+ result.add(fromPath(pbxprojFile));
+ }
+ return result.build();
+ }
+ }
+
+ private final T xcodeprojDirectory;
+ private final PathTransformer<T> transformer;
+
+ public XcodeprojPath(T xcodeprojDirectory, PathTransformer<T> transformer) {
+ super(xcodeprojDirectory);
+ checkArgument(transformer.name(xcodeprojDirectory).endsWith(XCODEPROJ_DIRECTORY_SUFFIX),
+ "xcodeprojDirectory should end with %s, but it is '%s'", XCODEPROJ_DIRECTORY_SUFFIX,
+ xcodeprojDirectory);
+ this.xcodeprojDirectory = xcodeprojDirectory;
+ this.transformer = transformer;
+ }
+
+ /** Returns a converter which works for the Java {@code Path} class. */
+ public static Converter<Path> converter() {
+ return new Converter<>(PathTransformer.FOR_JAVA_PATH);
+ }
+
+ public final T getXcodeprojDirectory() {
+ return xcodeprojDirectory;
+ }
+
+ public final T getPbxprojFile() {
+ return transformer.join(xcodeprojDirectory, PBXPROJ_FILE_NAME);
+ }
+
+ /**
+ * Returns the package or directory in which the project is located. For instance, if the project
+ * file is {@code /foo/bar/App.xcodeproj/project.pbxproj}, then this method returns
+ * {@code /foo/bar}.
+ */
+ public final T getXcodeprojContainerDir() {
+ return transformer.parent(xcodeprojDirectory);
+ }
+
+ /**
+ * Returns the name of the xcodeproj directory without the {@code .xcodeproj} extension or the
+ * containing directory. For instance, for an xcodeproj directory of
+ * {@code /client/foo.xcodeproj}, this method returns {@code "foo"}.
+ */
+ public final String getProjectName() {
+ String pathStr = transformer.name(xcodeprojDirectory);
+ return pathStr.substring(0, pathStr.length() - XCODEPROJ_DIRECTORY_SUFFIX.length());
+ }
+
+ @Override
+ public final int compareTo(XcodeprojPath<T> o) {
+ return getXcodeprojDirectory().compareTo(o.getXcodeprojDirectory());
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Containing.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Containing.java
new file mode 100644
index 0000000..8f2434d
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Containing.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.Multimap;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Provides methods that make checking for the presence of an item in a collection type-safe. For
+ * instance, in {@code foo.containsKey(bar)}, where {@code foo} is a {@code Map<K, V>}, {@code bar}
+ * can be a type other than {@code K} and may be {@code null}, in which case the method will just
+ * return false (collections that allow null references may of course return true in the latter
+ * case).
+ * <p>
+ * The methods in this class, such as {@link #key(Map, Object)}, will cause a compiler error if you
+ * use the wrong type and throw a {@link NullPointerException} if you pass {@code null} for the
+ * object whose presence to check. In the case where you want to check for {@code null} in a
+ * collection, add a new method to this class, use the methods in the plain Collections API
+ * (such as {@link Collection#contains(Object)}), or use the {@code Optional} type as the element of
+ * the collection.
+ * <p>
+ * TODO(bazel-team): This class should either be simplified or eliminated when the
+ * CollectionIncompatibleType feature is available:
+ * https://code.google.com/p/error-prone/wiki/CollectionIncompatibleType
+ */
+public class Containing {
+ private Containing() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ public static <K> boolean key(Map<K, ?> map, K key) {
+ checkNotNull(key);
+ return map.containsKey(key);
+ }
+
+ public static <K> boolean key(Multimap<K, ?> map, K key) {
+ checkNotNull(key);
+ return map.containsKey(key);
+ }
+
+ public static <E> boolean item(Collection<E> collection, E item) {
+ checkNotNull(item);
+ return collection.contains(item);
+ }
+
+ public static <V> boolean value(Map<?, V> map, V value) {
+ checkNotNull(value);
+ return map.containsValue(value);
+ }
+
+ public static <V> boolean value(Multimap<?, V> map, V value) {
+ checkNotNull(value);
+ return map.containsValue(value);
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Difference.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Difference.java
new file mode 100644
index 0000000..54bd567
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Difference.java
@@ -0,0 +1,43 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+/**
+ * Provides utility methods that make difference operations type safe.
+ *
+ * {@link Sets#difference(Set, Set)} requires no type bound on the second set, which has led to
+ * calls which can never subtract any elements because the set being subtracted cannot contain any
+ * elements which may exist in the first set.
+ */
+public class Difference {
+ private Difference() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Returns the elements in set1 which are not in set2. set2 may contain extra elements which will
+ * be ignored.
+ *
+ * @param set1 Set whose elements to return
+ * @param set2 Set whose elements are to be subtracted
+ */
+ public static <T> Set<T> of(Set<T> set1, Set<T> set2) {
+ return Sets.difference(set1, set2);
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Equaling.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Equaling.java
new file mode 100644
index 0000000..c359e57
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Equaling.java
@@ -0,0 +1,80 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.base.Optional;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Provides utility methods that make equality comparison type safe. The
+ * {@link Object#equals(Object)} method usually returns false when the other object is {@code null}
+ * or a different runtime class. These utility methods try to force each object to be of the same
+ * class (with the method signatures) and non-null (and throwing a {@link NullPointerException} if
+ * either one is null.
+ */
+public class Equaling {
+ private Equaling() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ // Note that we always checkNotNull(b) on a separate line from a.equals(b). This is to make it so
+ // the stack trace will tell you exactly which reference is null.
+
+ public static <T extends Value<T>> boolean of(Value<T> a, Value<T> b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static boolean of(String a, String b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static <T> boolean of(Optional<T> a, Optional<T> b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static <T> boolean of(Set<T> a, Set<T> b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static boolean of(File a, File b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static boolean of(Path a, Path b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static <T> boolean of(List<T> a, List<T> b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+
+ public static <T extends Enum<T>> boolean of(Enum<T> a, Enum<T> b) {
+ checkNotNull(b);
+ return a.equals(b);
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Intersection.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Intersection.java
new file mode 100644
index 0000000..7171cde
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Intersection.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+/**
+ * Provides utility methods that make intersections operations type safe.
+ *
+ * {@link Sets#intersection(Set, Set)} requires no type bound on the second set, which could lead to
+ * calls which always return an empty set.
+ */
+public class Intersection {
+ private Intersection() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Returns the intersection of two sets.
+ */
+ public static <T> Set<T> of(Set<T> set1, Set<T> set2) {
+ return Sets.intersection(set1, set2);
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Interspersing.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Interspersing.java
new file mode 100644
index 0000000..89bf487
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Interspersing.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+/**
+ * Utility code for interspersing items into sequences.
+ */
+public class Interspersing {
+ private Interspersing() {}
+
+ /**
+ * Inserts {@code what} before each item in {@code sequence}, returning a lazy sequence of twice
+ * the length.
+ */
+ public static <E> Iterable<E> beforeEach(final E what, Iterable<E> sequence) {
+ Preconditions.checkNotNull(what);
+ return Iterables.concat(
+ Iterables.transform(
+ sequence,
+ new Function<E, Iterable<E>>() {
+ @Override
+ public Iterable<E> apply(E element) {
+ return ImmutableList.of(what, element);
+ }
+ }
+ ));
+ }
+
+ /**
+ * Prepends {@code what} to each string in {@code sequence}, returning a lazy sequence of the
+ * same length.
+ */
+ public static Iterable<String>
+ prependEach(final String what, Iterable<String> sequence) {
+ Preconditions.checkNotNull(what);
+ return Iterables.transform(
+ sequence,
+ new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ return what + input;
+ }
+ });
+ }
+
+ /**
+ * Similar to {@link #prependEach(String, Iterable)}, but also converts each item in the sequence
+ * to a string.
+ */
+ public static <E> Iterable<String>
+ prependEach(String what, Iterable<E> sequence, Function<? super E, String> toString) {
+ return prependEach(what, Iterables.transform(sequence, toString));
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Mapping.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Mapping.java
new file mode 100644
index 0000000..0e5ef53
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Mapping.java
@@ -0,0 +1,39 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import java.util.Map;
+
+/**
+ * Provides utility methods that make map lookup safe.
+ */
+public class Mapping {
+ private Mapping() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Returns the value mapped to the given key for a map. If the mapping is not present, an absent
+ * {@code Optional} is returned.
+ * @throws NullPointerException if the map or key argument is null
+ */
+ public static <K, V> Optional<V> of(Map<K, V> map, K key) {
+ Preconditions.checkNotNull(key);
+ return Optional.fromNullable(map.get(key));
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Value.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Value.java
new file mode 100644
index 0000000..6363af1
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/util/Value.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Represents a type whose equality, hash code, and string representation are defined by a single
+ * immutable array. This class is designed to be extended by a final class, and to pass the member
+ * data to this class's constructor.
+ *
+ * @param <V> the base class that extends {@code Value}
+ */
+public class Value<V extends Value<V>> {
+ private final Object memberData;
+
+ /**
+ * Constructs a new instance with the given member data. Generally, all member data should be
+ * reflected in final fields in the child class.
+ * @throws NullPointerException if any element in {@code memberData} is null
+ */
+ public Value(Object... memberData) {
+ Preconditions.checkArgument(memberData.length > 0);
+ this.memberData = (memberData.length == 1)
+ ? Preconditions.checkNotNull(memberData[0]) : ImmutableList.copyOf(memberData);
+ }
+
+ /**
+ * A type-safe alternative to calling {@code a.equals(b)}. When using {@code a.equals(b)},
+ * {@code b} may accidentally be a different class from {@code a}, in which case there will be no
+ * compiler warning and the result will always be false. This method requires both values to have
+ * compatible types and to be non-null.
+ */
+ public boolean equalsOther(V other) {
+ return equals(Preconditions.checkNotNull(other));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if ((o == null) || (o.getClass() != getClass())) {
+ return false;
+ }
+ Value<?> other = (Value<?>) o;
+ return memberData.equals(other.memberData);
+ }
+
+ @Override
+ public int hashCode() {
+ return memberData.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ":" + memberData.toString();
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/zip/ZipInputEntry.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zip/ZipInputEntry.java
new file mode 100644
index 0000000..01aaad0
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zip/ZipInputEntry.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.zip;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.singlejar.ZipCombiner;
+import com.google.devtools.build.xcode.util.Value;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+
+/**
+ * Describes an entry in a zip file when the zip file is being created.
+ */
+public class ZipInputEntry extends Value<ZipInputEntry> {
+ /**
+ * The external file attribute used for files by default. This indicates a non-executable regular
+ * file that is readable by group and world.
+ */
+ public static final int DEFAULT_EXTERNAL_FILE_ATTRIBUTE = (0100644 << 16);
+
+ /**
+ * An external file attribute that indicates an executable regular file that is readable and
+ * executable by group and world.
+ */
+ public static final int EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE = (0100755 << 16);
+
+ /**
+ * The central directory record information that is used when adding a plain, non-executable file.
+ */
+ public static final ZipCombiner.DirectoryEntryInfo DEFAULT_DIRECTORY_ENTRY_INFO =
+ ZipCombiner.DEFAULT_DIRECTORY_ENTRY_INFO
+ // This is what .ipa files built by Xcode are set to. Upper byte indicates Unix host.
+ // Lower byte indicates version of encoding software
+ // (note that 0x1e = 30 = (3.0 * 10), so 0x1e translates to 3.0).
+ // The Unix host value in the upper byte is what causes the external file attribute to be
+ // interpreted as POSIX permission and file type bits.
+ .withMadeByVersion((short) 0x031e)
+ .withExternalFileAttribute(DEFAULT_EXTERNAL_FILE_ATTRIBUTE);
+
+ private final Path source;
+ private final String zipPath;
+ private final int externalFileAttribute;
+
+ public ZipInputEntry(Path source, String zipPath) {
+ this(source, zipPath, DEFAULT_EXTERNAL_FILE_ATTRIBUTE);
+ }
+
+ public ZipInputEntry(Path source, String zipPath, int externalFileAttribute) {
+ super(source, zipPath, externalFileAttribute);
+ this.source = source;
+ this.zipPath = zipPath;
+ this.externalFileAttribute = externalFileAttribute;
+ }
+
+ /**
+ * The location of the source file to place in the zip.
+ */
+ public Path getSource() {
+ return source;
+ }
+
+ /**
+ * The full path of the item in the zip.
+ */
+ public String getZipPath() {
+ return zipPath;
+ }
+
+ /**
+ * The external file attribute field of the zip entry in the central directory record. On
+ * Unix-originated .zips, this corresponds to the permission bits (e.g. 0755 for an excutable
+ * file).
+ */
+ public int getExternalFileAttribute() {
+ return externalFileAttribute;
+ }
+
+ /**
+ * Adds this entry to a zip using the given {@code ZipCombiner}.
+ */
+ public void add(ZipCombiner combiner) throws IOException {
+ try (InputStream inputStream = Files.newInputStream(source)) {
+ combiner.addFile(zipPath, ZipCombiner.DOS_EPOCH, inputStream,
+ DEFAULT_DIRECTORY_ENTRY_INFO.withExternalFileAttribute(externalFileAttribute));
+ }
+ }
+
+ public static void addAll(ZipCombiner combiner, Iterable<ZipInputEntry> inputs)
+ throws IOException {
+ for (ZipInputEntry input : inputs) {
+ input.add(combiner);
+ }
+ }
+
+ /**
+ * Returns the entries to place in a zip file as if the zip file mirrors some directory structure.
+ * For instance, if {@code rootDirectory} is /tmp/foo, and the following files exist:
+ * <ul>
+ * <li>/tmp/foo/a
+ * <li>/tmp/foo/bar/c
+ * <li>/tmp/foo/baz/d
+ * </ul>
+ * This function will return entries which point to these files and have in-zip paths of:
+ * <ul>
+ * <li>a
+ * <li>bar/c
+ * <li>baz/d
+ * </ul>
+ */
+ public static Iterable<ZipInputEntry> fromDirectory(final Path rootDirectory) throws IOException {
+ final ImmutableList.Builder<ZipInputEntry> zipInputs = new ImmutableList.Builder<>();
+ Files.walkFileTree(rootDirectory, new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ // TODO(bazel-team): Set the external file attribute based on the attribute of the
+ // permissions of the file on-disk.
+ zipInputs.add(new ZipInputEntry(file, rootDirectory.relativize(file).toString()));
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ return zipInputs.build();
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Arguments.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Arguments.java
new file mode 100644
index 0000000..e6bea8c
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Arguments.java
@@ -0,0 +1,57 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.zippingoutput;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.util.Value;
+
+/**
+ * Arguments that have been parsed from a do-something-and-zip-output wrapper.
+ */
+public class Arguments extends Value<Arguments> {
+
+ private final String outputZip;
+ private final String bundleRoot;
+ private final String subtoolCmd;
+ private final ImmutableList<String> subtoolExtraArgs;
+
+ Arguments(
+ String outputZip,
+ String bundleRoot,
+ String subtoolCmd,
+ ImmutableList<String> subtoolExtraArgs) {
+ super(outputZip, bundleRoot, subtoolCmd, subtoolExtraArgs);
+ this.outputZip = outputZip;
+ this.bundleRoot = bundleRoot;
+ this.subtoolCmd = subtoolCmd;
+ this.subtoolExtraArgs = subtoolExtraArgs;
+ }
+
+ public String outputZip() {
+ return outputZip;
+ }
+
+ public String bundleRoot() {
+ return bundleRoot;
+ }
+
+ public String subtoolCmd() {
+ return subtoolCmd;
+ }
+
+ public ImmutableList<String> subtoolExtraArgs() {
+ return subtoolExtraArgs;
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/ArgumentsParsing.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/ArgumentsParsing.java
new file mode 100644
index 0000000..06e0ae6
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/ArgumentsParsing.java
@@ -0,0 +1,99 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.zippingoutput;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.xcode.util.Value;
+
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.util.Locale;
+
+/**
+ * Arguments passed to the do-something-then-zip wrapper tool.
+ */
+class ArgumentsParsing extends Value<ArgumentsParsing> {
+
+ private final Optional<String> error;
+ private final Optional<Arguments> arguments;
+
+ private ArgumentsParsing(Optional<String> error, Optional<Arguments> arguments) {
+ super(error, arguments);
+ this.error = error;
+ this.arguments = arguments;
+ }
+
+ public Optional<String> error() {
+ return error;
+ }
+
+ public Optional<Arguments> arguments() {
+ return arguments;
+ }
+
+ private static final int MIN_ARGS = 3;
+
+ /**
+ * @param args raw arguments passed to wrapper tool through the {@code main} method
+ * @param subtoolName name of the subtool, such as "actool"
+ * @return an instance based on results of parsing the given arguments.
+ */
+ public static ArgumentsParsing parse(FileSystem fileSystem, String[] args, String wrapperName,
+ String subtoolName) {
+ String capitalizedSubtool = subtoolName.toUpperCase(Locale.US);
+ if (args.length < MIN_ARGS) {
+ return new ArgumentsParsing(Optional.of(String.format(
+ "Expected at least %1$d args.\n"
+ + "Usage: java %2$s OUTZIP ARCHIVEROOT (%3$s_CMD %3$s ARGS)\n"
+ + "Runs %4$s and zips the results.\n"
+ + "OUTZIP - the path to place the output zip file.\n"
+ + "ARCHIVEROOT - the path in the zip to place the output, or an empty\n"
+ + " string for the root of the zip. e.g. 'Payload/foo.app'. If\n"
+ + " this tool outputs a single file, ARCHIVEROOT is the name of\n"
+ + " the only file in the zip file.\n"
+ + "%3$s_CMD - path to the subtool.\n"
+ + " e.g. /Applications/Xcode.app/Contents/Developer/usr/bin/actool\n"
+ + "%3$s ARGS - the arguments to pass to %4$s besides the\n"
+ + " one that specifies the output directory.\n",
+ MIN_ARGS, wrapperName, capitalizedSubtool, subtoolName)),
+ Optional.<Arguments>absent());
+ }
+ String outputZip = args[0];
+ String archiveRoot = args[1];
+ String subtoolCmd = args[2];
+ if (archiveRoot.startsWith("/")) {
+ return new ArgumentsParsing(
+ Optional.of(String.format("Archive root cannot start with /: '%s'\n", archiveRoot)),
+ Optional.<Arguments>absent());
+ }
+
+ // TODO(bazel-team): Remove this hack when the released version of Bazel uses the correct momc
+ // path for device builds.
+ subtoolCmd = subtoolCmd.replace("/iPhoneOS.platform/", "/iPhoneSimulator.platform/");
+ if (!Files.isRegularFile(fileSystem.getPath(subtoolCmd))) {
+ return new ArgumentsParsing(
+ Optional.of(String.format(
+ "The given %s_CMD does not exist: '%s'\n", capitalizedSubtool, subtoolCmd)),
+ Optional.<Arguments>absent());
+ }
+ return new ArgumentsParsing(
+ Optional.<String>absent(),
+ Optional.<Arguments>of(
+ new Arguments(
+ outputZip, archiveRoot, subtoolCmd,
+ ImmutableList.copyOf(args).subList(MIN_ARGS, args.length))));
+ }
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Wrapper.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Wrapper.java
new file mode 100644
index 0000000..975cf72
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Wrapper.java
@@ -0,0 +1,55 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.zippingoutput;
+
+/**
+ * Defines a zipping wrapper for a tool. A wrapper is a tool that runs some subcommand that writes
+ * its output (usually multiple files in a non-trivial directory structure) to some directory, and
+ * then zips the output of that subtool into a timestamp-less zip file whose location is specified
+ * on the command line.
+ * <p>
+ * The arguments passed are passed in this form:
+ * <pre>
+ * java {wrapper_name} OUTZIP ARCHIVEROOT (SUBTOOL_CMD SUBTOOL ARGS)
+ * </pre>
+ * Where SUBTOOL ARGS are the arguments to pass to SUBTOOL_CMD unchanged, except the argument that
+ * specifies the output directory. Note that the arguments are positional and do not use flags in
+ * the form of -f or --foo, except for those flags passed directly to and interpreted by the
+ * subtool itself.
+ * <p>
+ * A wrapper has some simple metadata the name of the wrapped tool
+ * and how to transform {@link Arguments} passed to the wrapper tool into arguments for the subtool.
+ */
+public interface Wrapper {
+ /** The name of the wrapper, such as {@code ActoolZip}. */
+ String name();
+
+ /** The subtool name, such as {@code actool}. */
+ String subtoolName();
+
+ /**
+ * Returns the command (i.e. argv), including the executable file, to be executed.
+ * @param arguments the parsed arguments passed to the wrapper tool
+ * @param outputDirectory the output directory which the subtool should write tool
+ */
+ Iterable<String> subCommand(Arguments arguments, String outputDirectory);
+
+ /**
+ * Indicates whether the output directory must exist before invoking the wrapped tool, this being
+ * dependent on the nature of the tool only. Note that the directory immediately containing the
+ * output directory is guaranteed to always exist, regardless of this method's return value.
+ */
+ boolean outputDirectoryMustExist();
+}
diff --git a/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Wrappers.java b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Wrappers.java
new file mode 100644
index 0000000..b4b3203
--- /dev/null
+++ b/src/tools/xcode-common/java/com/google/devtools/build/xcode/zippingoutput/Wrappers.java
@@ -0,0 +1,67 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.xcode.zippingoutput;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.singlejar.ZipCombiner;
+import com.google.devtools.build.xcode.zip.ZipInputEntry;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/** Utility code for working with {@link Wrapper}s. */
+public class Wrappers {
+ private Wrappers() {
+ throw new UnsupportedOperationException("static-only");
+ }
+
+ /**
+ * Runs the given wrapper using command-line arguments passed to the {@code main} method. Calling
+ * this method should be the last thing you do in {@code main}, because it may exit prematurely
+ * with {@link System#exit(int)}.
+ */
+ public static void execute(String[] argsArray, Wrapper wrapper)
+ throws IOException, InterruptedException {
+ ArgumentsParsing argsParsing = ArgumentsParsing.parse(
+ FileSystems.getDefault(), argsArray, wrapper.name(), wrapper.subtoolName());
+ for (String error : argsParsing.error().asSet()) {
+ System.err.printf(error);
+ System.exit(1);
+ }
+ for (Arguments args : argsParsing.arguments().asSet()) {
+ Path outputDir = Files.createTempDirectory("ZippingOutput");
+ Path rootedOutputDir = outputDir.resolve(args.bundleRoot());
+ Files.createDirectories(
+ wrapper.outputDirectoryMustExist() ? rootedOutputDir : rootedOutputDir.getParent());
+
+ ImmutableList<String> subCommandArguments =
+ ImmutableList.copyOf(wrapper.subCommand(args, rootedOutputDir.toString()));
+ Process subProcess = new ProcessBuilder(subCommandArguments).inheritIO().start();
+ int exit = subProcess.waitFor();
+ if (exit != 0) {
+ System.exit(exit);
+ }
+
+ try (OutputStream out = Files.newOutputStream(Paths.get(args.outputZip()));
+ ZipCombiner combiner = new ZipCombiner(out)) {
+ ZipInputEntry.addAll(combiner, ZipInputEntry.fromDirectory(outputDir));
+ }
+ }
+ }
+}
